{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 11.550545185732766,
  "global_step": 500000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "FLOPS loss": 7.404106872854754e-05,
      "L0_d": 21035.61,
      "MLM loss": 8.70656681060791,
      "epoch": 0.01,
      "step": 499
    },
    {
      "epoch": 0.01,
      "learning_rate": 5e-06,
      "loss": 9.4402,
      "step": 500
    },
    {
      "FLOPS loss": 0.0008858467335812747,
      "L0_d": 27735.09,
      "MLM loss": 7.479305267333984,
      "epoch": 0.02,
      "step": 999
    },
    {
      "epoch": 0.02,
      "learning_rate": 1e-05,
      "loss": 7.9955,
      "step": 1000
    },
    {
      "FLOPS loss": 0.002816247520968318,
      "L0_d": 28068.22,
      "MLM loss": 7.293828010559082,
      "epoch": 0.03,
      "step": 1499
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5e-05,
      "loss": 7.3438,
      "step": 1500
    },
    {
      "FLOPS loss": 0.00617683120071888,
      "L0_d": 28429.55,
      "MLM loss": 7.137616157531738,
      "epoch": 0.05,
      "step": 1999
    },
    {
      "epoch": 0.05,
      "learning_rate": 2e-05,
      "loss": 7.1243,
      "step": 2000
    },
    {
      "FLOPS loss": 0.011358148418366909,
      "L0_d": 29154.48,
      "MLM loss": 6.879582405090332,
      "epoch": 0.06,
      "step": 2499
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.5e-05,
      "loss": 6.9824,
      "step": 2500
    },
    {
      "FLOPS loss": 0.018066763877868652,
      "L0_d": 29186.34,
      "MLM loss": 6.785317420959473,
      "epoch": 0.07,
      "step": 2999
    },
    {
      "epoch": 0.07,
      "learning_rate": 3e-05,
      "loss": 6.8578,
      "step": 3000
    },
    {
      "FLOPS loss": 0.02440321072936058,
      "L0_d": 28600.52,
      "MLM loss": 6.300593376159668,
      "epoch": 0.08,
      "step": 3499
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.5e-05,
      "loss": 6.5223,
      "step": 3500
    },
    {
      "FLOPS loss": 0.03402633219957352,
      "L0_d": 28263.14,
      "MLM loss": 5.7942047119140625,
      "epoch": 0.09,
      "step": 3999
    },
    {
      "epoch": 0.09,
      "learning_rate": 4e-05,
      "loss": 6.0331,
      "step": 4000
    },
    {
      "FLOPS loss": 0.04102250933647156,
      "L0_d": 27604.8,
      "MLM loss": 5.463202953338623,
      "epoch": 0.1,
      "step": 4499
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.5e-05,
      "loss": 5.6372,
      "step": 4500
    },
    {
      "FLOPS loss": 0.048070844262838364,
      "L0_d": 27568.05,
      "MLM loss": 5.396732330322266,
      "epoch": 0.12,
      "step": 4999
    },
    {
      "epoch": 0.12,
      "learning_rate": 5e-05,
      "loss": 5.346,
      "step": 5000
    },
    {
      "FLOPS loss": 0.053655628114938736,
      "L0_d": 26272.8,
      "MLM loss": 4.88920783996582,
      "epoch": 0.13,
      "step": 5499
    },
    {
      "epoch": 0.13,
      "learning_rate": 5.500000000000001e-05,
      "loss": 5.1141,
      "step": 5500
    },
    {
      "FLOPS loss": 0.05216597020626068,
      "L0_d": 23949.03,
      "MLM loss": 4.8225998878479,
      "epoch": 0.14,
      "step": 5999
    },
    {
      "epoch": 0.14,
      "learning_rate": 6e-05,
      "loss": 4.9298,
      "step": 6000
    },
    {
      "FLOPS loss": 0.045533325523138046,
      "L0_d": 21281.05,
      "MLM loss": 4.535486221313477,
      "epoch": 0.15,
      "step": 6499
    },
    {
      "epoch": 0.15,
      "learning_rate": 6.499000000000001e-05,
      "loss": 4.7614,
      "step": 6500
    },
    {
      "FLOPS loss": 0.04692893847823143,
      "L0_d": 19877.23,
      "MLM loss": 4.604552268981934,
      "epoch": 0.16,
      "step": 6999
    },
    {
      "epoch": 0.16,
      "learning_rate": 6.999e-05,
      "loss": 4.622,
      "step": 7000
    },
    {
      "FLOPS loss": 0.0333898551762104,
      "L0_d": 15760.62,
      "MLM loss": 4.219477653503418,
      "epoch": 0.17,
      "step": 7499
    },
    {
      "epoch": 0.17,
      "learning_rate": 7.499e-05,
      "loss": 4.4779,
      "step": 7500
    },
    {
      "FLOPS loss": 0.0419338159263134,
      "L0_d": 16492.83,
      "MLM loss": 4.006763458251953,
      "epoch": 0.18,
      "step": 7999
    },
    {
      "epoch": 0.18,
      "learning_rate": 7.999000000000001e-05,
      "loss": 4.3399,
      "step": 8000
    },
    {
      "FLOPS loss": 0.038417913019657135,
      "L0_d": 14914.7,
      "MLM loss": 4.344199180603027,
      "epoch": 0.2,
      "step": 8499
    },
    {
      "epoch": 0.2,
      "learning_rate": 8.499e-05,
      "loss": 4.1982,
      "step": 8500
    },
    {
      "FLOPS loss": 0.0357012003660202,
      "L0_d": 11732.44,
      "MLM loss": 3.9161806106567383,
      "epoch": 0.21,
      "step": 8999
    },
    {
      "epoch": 0.21,
      "learning_rate": 8.999000000000001e-05,
      "loss": 4.0719,
      "step": 9000
    },
    {
      "FLOPS loss": 0.027552934363484383,
      "L0_d": 10824.06,
      "MLM loss": 3.7745988368988037,
      "epoch": 0.22,
      "step": 9499
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.499e-05,
      "loss": 3.9603,
      "step": 9500
    },
    {
      "FLOPS loss": 0.02538445219397545,
      "L0_d": 9813.38,
      "MLM loss": 3.884188175201416,
      "epoch": 0.23,
      "step": 9999
    },
    {
      "epoch": 0.23,
      "learning_rate": 9.999000000000001e-05,
      "loss": 3.876,
      "step": 10000
    },
    {
      "FLOPS loss": 0.025783885270357132,
      "L0_d": 8739.94,
      "MLM loss": 3.7749428749084473,
      "epoch": 0.24,
      "step": 10499
    },
    {
      "epoch": 0.24,
      "learning_rate": 9.989836734693878e-05,
      "loss": 3.795,
      "step": 10500
    },
    {
      "FLOPS loss": 0.030093064531683922,
      "L0_d": 8860.89,
      "MLM loss": 3.7490487098693848,
      "epoch": 0.25,
      "step": 10999
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.979632653061225e-05,
      "loss": 3.721,
      "step": 11000
    },
    {
      "FLOPS loss": 0.028102116659283638,
      "L0_d": 8556.38,
      "MLM loss": 3.7181925773620605,
      "epoch": 0.27,
      "step": 11499
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.969428571428572e-05,
      "loss": 3.6627,
      "step": 11500
    },
    {
      "FLOPS loss": 0.03335424140095711,
      "L0_d": 8712.69,
      "MLM loss": 3.4150214195251465,
      "epoch": 0.28,
      "step": 11999
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.959224489795919e-05,
      "loss": 3.5984,
      "step": 12000
    },
    {
      "FLOPS loss": 0.029852230101823807,
      "L0_d": 7392.5,
      "MLM loss": 3.384998083114624,
      "epoch": 0.29,
      "step": 12499
    },
    {
      "epoch": 0.29,
      "learning_rate": 9.949020408163265e-05,
      "loss": 3.5447,
      "step": 12500
    },
    {
      "FLOPS loss": 0.03067166358232498,
      "L0_d": 7107.91,
      "MLM loss": 3.4410061836242676,
      "epoch": 0.3,
      "step": 12999
    },
    {
      "epoch": 0.3,
      "learning_rate": 9.938836734693879e-05,
      "loss": 3.5081,
      "step": 13000
    },
    {
      "FLOPS loss": 0.030213626101613045,
      "L0_d": 6799.55,
      "MLM loss": 3.4604804515838623,
      "epoch": 0.31,
      "step": 13499
    },
    {
      "epoch": 0.31,
      "learning_rate": 9.928632653061225e-05,
      "loss": 3.4544,
      "step": 13500
    },
    {
      "FLOPS loss": 0.03034152090549469,
      "L0_d": 6092.02,
      "MLM loss": 3.4961204528808594,
      "epoch": 0.32,
      "step": 13999
    },
    {
      "epoch": 0.32,
      "learning_rate": 9.918428571428572e-05,
      "loss": 3.4262,
      "step": 14000
    },
    {
      "FLOPS loss": 0.026655618101358414,
      "L0_d": 5715.41,
      "MLM loss": 3.321794033050537,
      "epoch": 0.33,
      "step": 14499
    },
    {
      "epoch": 0.33,
      "learning_rate": 9.908224489795918e-05,
      "loss": 3.3953,
      "step": 14500
    },
    {
      "FLOPS loss": 0.028114214539527893,
      "L0_d": 5818.91,
      "MLM loss": 3.3361945152282715,
      "epoch": 0.35,
      "step": 14999
    },
    {
      "epoch": 0.35,
      "learning_rate": 9.898040816326532e-05,
      "loss": 3.3651,
      "step": 15000
    },
    {
      "FLOPS loss": 0.02521996572613716,
      "L0_d": 4736.66,
      "MLM loss": 3.268618106842041,
      "epoch": 0.36,
      "step": 15499
    },
    {
      "epoch": 0.36,
      "learning_rate": 9.887836734693878e-05,
      "loss": 3.3403,
      "step": 15500
    },
    {
      "FLOPS loss": 0.029343057423830032,
      "L0_d": 4741.28,
      "MLM loss": 3.2674968242645264,
      "epoch": 0.37,
      "step": 15999
    },
    {
      "epoch": 0.37,
      "learning_rate": 9.877632653061225e-05,
      "loss": 3.3179,
      "step": 16000
    },
    {
      "FLOPS loss": 0.029638290405273438,
      "L0_d": 4553.05,
      "MLM loss": 3.4543089866638184,
      "epoch": 0.38,
      "step": 16499
    },
    {
      "epoch": 0.38,
      "learning_rate": 9.867428571428572e-05,
      "loss": 3.2948,
      "step": 16500
    },
    {
      "FLOPS loss": 0.039912909269332886,
      "L0_d": 5712.59,
      "MLM loss": 3.241018056869507,
      "epoch": 0.39,
      "step": 16999
    },
    {
      "epoch": 0.39,
      "learning_rate": 9.857244897959183e-05,
      "loss": 3.2752,
      "step": 17000
    },
    {
      "FLOPS loss": 0.035952694714069366,
      "L0_d": 4726.56,
      "MLM loss": 3.2455244064331055,
      "epoch": 0.4,
      "step": 17499
    },
    {
      "epoch": 0.4,
      "learning_rate": 9.847040816326531e-05,
      "loss": 3.2526,
      "step": 17500
    },
    {
      "FLOPS loss": 0.027841633185744286,
      "L0_d": 3770.58,
      "MLM loss": 3.104790449142456,
      "epoch": 0.42,
      "step": 17999
    },
    {
      "epoch": 0.42,
      "learning_rate": 9.836836734693879e-05,
      "loss": 3.2298,
      "step": 18000
    },
    {
      "FLOPS loss": 0.042448658496141434,
      "L0_d": 4327.95,
      "MLM loss": 3.2372331619262695,
      "epoch": 0.43,
      "step": 18499
    },
    {
      "epoch": 0.43,
      "learning_rate": 9.826632653061225e-05,
      "loss": 3.218,
      "step": 18500
    },
    {
      "FLOPS loss": 0.03443501517176628,
      "L0_d": 3733.66,
      "MLM loss": 3.439894676208496,
      "epoch": 0.44,
      "step": 18999
    },
    {
      "epoch": 0.44,
      "learning_rate": 9.816448979591837e-05,
      "loss": 3.1942,
      "step": 19000
    },
    {
      "FLOPS loss": 0.03354410454630852,
      "L0_d": 3822.12,
      "MLM loss": 3.2109413146972656,
      "epoch": 0.45,
      "step": 19499
    },
    {
      "epoch": 0.45,
      "learning_rate": 9.806244897959184e-05,
      "loss": 3.183,
      "step": 19500
    },
    {
      "FLOPS loss": 0.0343574658036232,
      "L0_d": 3360.17,
      "MLM loss": 3.143649101257324,
      "epoch": 0.46,
      "step": 19999
    },
    {
      "epoch": 0.46,
      "learning_rate": 9.796040816326532e-05,
      "loss": 3.1615,
      "step": 20000
    },
    {
      "FLOPS loss": 0.032427407801151276,
      "L0_d": 3297.11,
      "MLM loss": 3.3535146713256836,
      "epoch": 0.47,
      "step": 20499
    },
    {
      "epoch": 0.47,
      "learning_rate": 9.785857142857144e-05,
      "loss": 3.1547,
      "step": 20500
    },
    {
      "FLOPS loss": 0.030162997543811798,
      "L0_d": 2771.8,
      "MLM loss": 3.0056474208831787,
      "epoch": 0.49,
      "step": 20999
    },
    {
      "epoch": 0.49,
      "learning_rate": 9.77565306122449e-05,
      "loss": 3.1396,
      "step": 21000
    },
    {
      "FLOPS loss": 0.031429558992385864,
      "L0_d": 2561.09,
      "MLM loss": 2.9690394401550293,
      "epoch": 0.5,
      "step": 21499
    },
    {
      "epoch": 0.5,
      "learning_rate": 9.765448979591837e-05,
      "loss": 3.1262,
      "step": 21500
    },
    {
      "FLOPS loss": 0.033806342631578445,
      "L0_d": 2472.33,
      "MLM loss": 3.2172610759735107,
      "epoch": 0.51,
      "step": 21999
    },
    {
      "epoch": 0.51,
      "learning_rate": 9.755244897959183e-05,
      "loss": 3.1159,
      "step": 22000
    },
    {
      "FLOPS loss": 0.03768898546695709,
      "L0_d": 3027.31,
      "MLM loss": 3.0621213912963867,
      "epoch": 0.52,
      "step": 22499
    },
    {
      "epoch": 0.52,
      "learning_rate": 9.745061224489797e-05,
      "loss": 3.1082,
      "step": 22500
    },
    {
      "FLOPS loss": 0.04951038211584091,
      "L0_d": 3470.08,
      "MLM loss": 3.0145416259765625,
      "epoch": 0.53,
      "step": 22999
    },
    {
      "epoch": 0.53,
      "learning_rate": 9.734857142857143e-05,
      "loss": 3.0992,
      "step": 23000
    },
    {
      "FLOPS loss": 0.032062649726867676,
      "L0_d": 1964.75,
      "MLM loss": 2.9846644401550293,
      "epoch": 0.54,
      "step": 23499
    },
    {
      "epoch": 0.54,
      "learning_rate": 9.724653061224491e-05,
      "loss": 3.0815,
      "step": 23500
    },
    {
      "FLOPS loss": 0.0394759438931942,
      "L0_d": 2687.91,
      "MLM loss": 3.1492552757263184,
      "epoch": 0.55,
      "step": 23999
    },
    {
      "epoch": 0.55,
      "learning_rate": 9.714448979591837e-05,
      "loss": 3.0752,
      "step": 24000
    },
    {
      "FLOPS loss": 0.046340227127075195,
      "L0_d": 2852.02,
      "MLM loss": 2.789968490600586,
      "epoch": 0.57,
      "step": 24499
    },
    {
      "epoch": 0.57,
      "learning_rate": 9.70426530612245e-05,
      "loss": 3.0647,
      "step": 24500
    },
    {
      "FLOPS loss": 0.04598288983106613,
      "L0_d": 2861.27,
      "MLM loss": 2.9972972869873047,
      "epoch": 0.58,
      "step": 24999
    },
    {
      "epoch": 0.58,
      "learning_rate": 9.694061224489797e-05,
      "loss": 3.0544,
      "step": 25000
    },
    {
      "FLOPS loss": 0.04686904326081276,
      "L0_d": 2361.75,
      "MLM loss": 2.7404046058654785,
      "epoch": 0.59,
      "step": 25499
    },
    {
      "epoch": 0.59,
      "learning_rate": 9.683857142857144e-05,
      "loss": 3.0444,
      "step": 25500
    },
    {
      "FLOPS loss": 0.04754837229847908,
      "L0_d": 2383.89,
      "MLM loss": 2.9416728019714355,
      "epoch": 0.6,
      "step": 25999
    },
    {
      "epoch": 0.6,
      "learning_rate": 9.67365306122449e-05,
      "loss": 3.0383,
      "step": 26000
    },
    {
      "FLOPS loss": 0.05846235528588295,
      "L0_d": 2928.42,
      "MLM loss": 2.9113664627075195,
      "epoch": 0.61,
      "step": 26499
    },
    {
      "epoch": 0.61,
      "learning_rate": 9.663448979591837e-05,
      "loss": 3.0277,
      "step": 26500
    },
    {
      "FLOPS loss": 0.04545710235834122,
      "L0_d": 2646.41,
      "MLM loss": 2.850696563720703,
      "epoch": 0.62,
      "step": 26999
    },
    {
      "epoch": 0.62,
      "learning_rate": 9.65326530612245e-05,
      "loss": 3.0261,
      "step": 27000
    },
    {
      "FLOPS loss": 0.04402191564440727,
      "L0_d": 1969.67,
      "MLM loss": 2.861927032470703,
      "epoch": 0.64,
      "step": 27499
    },
    {
      "epoch": 0.64,
      "learning_rate": 9.643061224489796e-05,
      "loss": 3.0153,
      "step": 27500
    },
    {
      "FLOPS loss": 0.04754064232110977,
      "L0_d": 1777.67,
      "MLM loss": 3.047110080718994,
      "epoch": 0.65,
      "step": 27999
    },
    {
      "epoch": 0.65,
      "learning_rate": 9.632857142857143e-05,
      "loss": 3.0075,
      "step": 28000
    },
    {
      "FLOPS loss": 0.042845577001571655,
      "L0_d": 1711.33,
      "MLM loss": 2.8969533443450928,
      "epoch": 0.66,
      "step": 28499
    },
    {
      "epoch": 0.66,
      "learning_rate": 9.622653061224491e-05,
      "loss": 3.0014,
      "step": 28500
    },
    {
      "FLOPS loss": 0.051014166325330734,
      "L0_d": 2422.53,
      "MLM loss": 3.259045362472534,
      "epoch": 0.67,
      "step": 28999
    },
    {
      "epoch": 0.67,
      "learning_rate": 9.612469387755101e-05,
      "loss": 2.9962,
      "step": 29000
    },
    {
      "FLOPS loss": 0.05882557854056358,
      "L0_d": 2171.72,
      "MLM loss": 3.0199313163757324,
      "epoch": 0.68,
      "step": 29499
    },
    {
      "epoch": 0.68,
      "learning_rate": 9.602265306122449e-05,
      "loss": 2.9811,
      "step": 29500
    },
    {
      "FLOPS loss": 0.06353907287120819,
      "L0_d": 2784.38,
      "MLM loss": 2.922330617904663,
      "epoch": 0.69,
      "step": 29999
    },
    {
      "epoch": 0.69,
      "learning_rate": 9.592061224489797e-05,
      "loss": 2.9847,
      "step": 30000
    },
    {
      "FLOPS loss": 0.06058761849999428,
      "L0_d": 2210.28,
      "MLM loss": 2.9217536449432373,
      "epoch": 0.7,
      "step": 30499
    },
    {
      "epoch": 0.7,
      "learning_rate": 9.581857142857144e-05,
      "loss": 2.974,
      "step": 30500
    },
    {
      "FLOPS loss": 0.049193985760211945,
      "L0_d": 1656.2,
      "MLM loss": 2.7800402641296387,
      "epoch": 0.72,
      "step": 30999
    },
    {
      "epoch": 0.72,
      "learning_rate": 9.571673469387756e-05,
      "loss": 2.9664,
      "step": 31000
    },
    {
      "FLOPS loss": 0.06122875213623047,
      "L0_d": 1938.58,
      "MLM loss": 2.921651840209961,
      "epoch": 0.73,
      "step": 31499
    },
    {
      "epoch": 0.73,
      "learning_rate": 9.561469387755102e-05,
      "loss": 2.9618,
      "step": 31500
    },
    {
      "FLOPS loss": 0.061618685722351074,
      "L0_d": 2109.42,
      "MLM loss": 2.9213171005249023,
      "epoch": 0.74,
      "step": 31999
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.55126530612245e-05,
      "loss": 2.9624,
      "step": 32000
    },
    {
      "FLOPS loss": 0.0622367262840271,
      "L0_d": 1870.62,
      "MLM loss": 2.93810772895813,
      "epoch": 0.75,
      "step": 32499
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.541061224489796e-05,
      "loss": 2.9546,
      "step": 32500
    },
    {
      "FLOPS loss": 0.06546282023191452,
      "L0_d": 1941.95,
      "MLM loss": 2.7924461364746094,
      "epoch": 0.76,
      "step": 32999
    },
    {
      "epoch": 0.76,
      "learning_rate": 9.530877551020408e-05,
      "loss": 2.9479,
      "step": 33000
    },
    {
      "FLOPS loss": 0.05119035392999649,
      "L0_d": 1321.45,
      "MLM loss": 3.0193562507629395,
      "epoch": 0.77,
      "step": 33499
    },
    {
      "epoch": 0.77,
      "learning_rate": 9.520673469387755e-05,
      "loss": 2.945,
      "step": 33500
    },
    {
      "FLOPS loss": 0.05955345928668976,
      "L0_d": 1532.5,
      "MLM loss": 2.9778778553009033,
      "epoch": 0.79,
      "step": 33999
    },
    {
      "epoch": 0.79,
      "learning_rate": 9.510469387755101e-05,
      "loss": 2.937,
      "step": 34000
    },
    {
      "FLOPS loss": 0.06750689446926117,
      "L0_d": 1965.52,
      "MLM loss": 2.970550537109375,
      "epoch": 0.8,
      "step": 34499
    },
    {
      "epoch": 0.8,
      "learning_rate": 9.500265306122449e-05,
      "loss": 2.9383,
      "step": 34500
    },
    {
      "FLOPS loss": 0.057339150458574295,
      "L0_d": 1509.08,
      "MLM loss": 3.053955078125,
      "epoch": 0.81,
      "step": 34999
    },
    {
      "epoch": 0.81,
      "learning_rate": 9.490081632653061e-05,
      "loss": 2.9327,
      "step": 35000
    },
    {
      "FLOPS loss": 0.060432739555835724,
      "L0_d": 1440.39,
      "MLM loss": 2.9338531494140625,
      "epoch": 0.82,
      "step": 35499
    },
    {
      "epoch": 0.82,
      "learning_rate": 9.479877551020409e-05,
      "loss": 2.9273,
      "step": 35500
    },
    {
      "FLOPS loss": 0.05953816697001457,
      "L0_d": 1626.62,
      "MLM loss": 2.8334543704986572,
      "epoch": 0.83,
      "step": 35999
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.469673469387756e-05,
      "loss": 2.9226,
      "step": 36000
    },
    {
      "FLOPS loss": 0.0538330040872097,
      "L0_d": 1472.11,
      "MLM loss": 3.0895233154296875,
      "epoch": 0.84,
      "step": 36499
    },
    {
      "epoch": 0.84,
      "learning_rate": 9.459469387755102e-05,
      "loss": 2.9109,
      "step": 36500
    },
    {
      "FLOPS loss": 0.07072566449642181,
      "L0_d": 1756.19,
      "MLM loss": 2.800241470336914,
      "epoch": 0.85,
      "step": 36999
    },
    {
      "epoch": 0.85,
      "learning_rate": 9.449285714285716e-05,
      "loss": 2.9108,
      "step": 37000
    },
    {
      "FLOPS loss": 0.055461227893829346,
      "L0_d": 1410.72,
      "MLM loss": 3.147261381149292,
      "epoch": 0.87,
      "step": 37499
    },
    {
      "epoch": 0.87,
      "learning_rate": 9.439081632653062e-05,
      "loss": 2.9103,
      "step": 37500
    },
    {
      "FLOPS loss": 0.05898945778608322,
      "L0_d": 1273.66,
      "MLM loss": 2.9772825241088867,
      "epoch": 0.88,
      "step": 37999
    },
    {
      "epoch": 0.88,
      "learning_rate": 9.428877551020408e-05,
      "loss": 2.9015,
      "step": 38000
    },
    {
      "FLOPS loss": 0.06924092024564743,
      "L0_d": 1601.7,
      "MLM loss": 2.562720775604248,
      "epoch": 0.89,
      "step": 38499
    },
    {
      "epoch": 0.89,
      "learning_rate": 9.418673469387755e-05,
      "loss": 2.9001,
      "step": 38500
    },
    {
      "FLOPS loss": 0.07076826691627502,
      "L0_d": 1320.19,
      "MLM loss": 2.9359970092773438,
      "epoch": 0.9,
      "step": 38999
    },
    {
      "epoch": 0.9,
      "learning_rate": 9.408489795918368e-05,
      "loss": 2.9028,
      "step": 39000
    },
    {
      "FLOPS loss": 0.06523440033197403,
      "L0_d": 1285.02,
      "MLM loss": 2.816526412963867,
      "epoch": 0.91,
      "step": 39499
    },
    {
      "epoch": 0.91,
      "learning_rate": 9.398285714285715e-05,
      "loss": 2.892,
      "step": 39500
    },
    {
      "FLOPS loss": 0.058553751558065414,
      "L0_d": 1458.62,
      "MLM loss": 2.819638729095459,
      "epoch": 0.92,
      "step": 39999
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.388081632653061e-05,
      "loss": 2.8847,
      "step": 40000
    },
    {
      "FLOPS loss": 0.053785067051649094,
      "L0_d": 1089.17,
      "MLM loss": 2.7337148189544678,
      "epoch": 0.94,
      "step": 40499
    },
    {
      "epoch": 0.94,
      "learning_rate": 9.377877551020409e-05,
      "loss": 2.8918,
      "step": 40500
    },
    {
      "FLOPS loss": 0.06995739042758942,
      "L0_d": 1275.78,
      "MLM loss": 2.9106178283691406,
      "epoch": 0.95,
      "step": 40999
    },
    {
      "epoch": 0.95,
      "learning_rate": 9.367673469387756e-05,
      "loss": 2.8765,
      "step": 41000
    },
    {
      "FLOPS loss": 0.06189718097448349,
      "L0_d": 1194.67,
      "MLM loss": 2.9347684383392334,
      "epoch": 0.96,
      "step": 41499
    },
    {
      "epoch": 0.96,
      "learning_rate": 9.357489795918368e-05,
      "loss": 2.8766,
      "step": 41500
    },
    {
      "FLOPS loss": 0.07404523342847824,
      "L0_d": 1580.86,
      "MLM loss": 2.7358357906341553,
      "epoch": 0.97,
      "step": 41999
    },
    {
      "epoch": 0.97,
      "learning_rate": 9.347285714285715e-05,
      "loss": 2.8695,
      "step": 42000
    },
    {
      "FLOPS loss": 0.07238270342350006,
      "L0_d": 1594.48,
      "MLM loss": 2.8579506874084473,
      "epoch": 0.98,
      "step": 42499
    },
    {
      "epoch": 0.98,
      "learning_rate": 9.337081632653062e-05,
      "loss": 2.8693,
      "step": 42500
    },
    {
      "FLOPS loss": 0.07086482644081116,
      "L0_d": 1330.3,
      "MLM loss": 2.827727794647217,
      "epoch": 0.99,
      "step": 42999
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.326877551020408e-05,
      "loss": 2.8691,
      "step": 43000
    },
    {
      "FLOPS loss": 0.06407759338617325,
      "L0_d": 963.44,
      "MLM loss": 2.902876377105713,
      "epoch": 1.0,
      "step": 43499
    },
    {
      "epoch": 1.0,
      "learning_rate": 9.31669387755102e-05,
      "loss": 2.8607,
      "step": 43500
    },
    {
      "FLOPS loss": 0.07373356819152832,
      "L0_d": 1145.33,
      "MLM loss": 2.8871655464172363,
      "epoch": 1.02,
      "step": 43999
    },
    {
      "epoch": 1.02,
      "learning_rate": 9.306489795918368e-05,
      "loss": 2.8582,
      "step": 44000
    },
    {
      "FLOPS loss": 0.05329473689198494,
      "L0_d": 1014.88,
      "MLM loss": 2.8173437118530273,
      "epoch": 1.03,
      "step": 44499
    },
    {
      "epoch": 1.03,
      "learning_rate": 9.296285714285715e-05,
      "loss": 2.8558,
      "step": 44500
    },
    {
      "FLOPS loss": 0.07186653465032578,
      "L0_d": 1229.8,
      "MLM loss": 2.6114912033081055,
      "epoch": 1.04,
      "step": 44999
    },
    {
      "epoch": 1.04,
      "learning_rate": 9.286081632653063e-05,
      "loss": 2.8566,
      "step": 45000
    },
    {
      "FLOPS loss": 0.06906762719154358,
      "L0_d": 1032.41,
      "MLM loss": 2.6828746795654297,
      "epoch": 1.05,
      "step": 45499
    },
    {
      "epoch": 1.05,
      "learning_rate": 9.275897959183673e-05,
      "loss": 2.8458,
      "step": 45500
    },
    {
      "FLOPS loss": 0.07486148178577423,
      "L0_d": 1279.2,
      "MLM loss": 2.6118578910827637,
      "epoch": 1.06,
      "step": 45999
    },
    {
      "epoch": 1.06,
      "learning_rate": 9.265693877551021e-05,
      "loss": 2.8553,
      "step": 46000
    },
    {
      "FLOPS loss": 0.06602790206670761,
      "L0_d": 826.56,
      "MLM loss": 2.7278029918670654,
      "epoch": 1.07,
      "step": 46499
    },
    {
      "epoch": 1.07,
      "learning_rate": 9.255489795918368e-05,
      "loss": 2.8456,
      "step": 46500
    },
    {
      "FLOPS loss": 0.08286317437887192,
      "L0_d": 1138.0,
      "MLM loss": 2.9847941398620605,
      "epoch": 1.09,
      "step": 46999
    },
    {
      "epoch": 1.09,
      "learning_rate": 9.245285714285715e-05,
      "loss": 2.8397,
      "step": 47000
    },
    {
      "FLOPS loss": 0.07151538878679276,
      "L0_d": 1165.0,
      "MLM loss": 2.9278883934020996,
      "epoch": 1.1,
      "step": 47499
    },
    {
      "epoch": 1.1,
      "learning_rate": 9.235081632653062e-05,
      "loss": 2.839,
      "step": 47500
    },
    {
      "FLOPS loss": 0.06656032800674438,
      "L0_d": 1079.83,
      "MLM loss": 2.819605827331543,
      "epoch": 1.11,
      "step": 47999
    },
    {
      "epoch": 1.11,
      "learning_rate": 9.224897959183674e-05,
      "loss": 2.8352,
      "step": 48000
    },
    {
      "FLOPS loss": 0.056283093988895416,
      "L0_d": 749.2,
      "MLM loss": 2.7207205295562744,
      "epoch": 1.12,
      "step": 48499
    },
    {
      "epoch": 1.12,
      "learning_rate": 9.21469387755102e-05,
      "loss": 2.8386,
      "step": 48500
    },
    {
      "FLOPS loss": 0.0777335837483406,
      "L0_d": 1304.86,
      "MLM loss": 2.6587204933166504,
      "epoch": 1.13,
      "step": 48999
    },
    {
      "epoch": 1.13,
      "learning_rate": 9.204489795918367e-05,
      "loss": 2.8318,
      "step": 49000
    },
    {
      "FLOPS loss": 0.0929664671421051,
      "L0_d": 1032.58,
      "MLM loss": 2.735736846923828,
      "epoch": 1.14,
      "step": 49499
    },
    {
      "epoch": 1.14,
      "learning_rate": 9.194285714285715e-05,
      "loss": 2.8318,
      "step": 49500
    },
    {
      "FLOPS loss": 0.0649925172328949,
      "L0_d": 917.53,
      "MLM loss": 2.6756234169006348,
      "epoch": 1.16,
      "step": 49999
    },
    {
      "epoch": 1.16,
      "learning_rate": 9.184102040816327e-05,
      "loss": 2.8333,
      "step": 50000
    },
    {
      "FLOPS loss": 0.07300473749637604,
      "L0_d": 927.86,
      "MLM loss": 2.652163505554199,
      "epoch": 1.17,
      "step": 50499
    },
    {
      "epoch": 1.17,
      "learning_rate": 9.173897959183673e-05,
      "loss": 2.8214,
      "step": 50500
    },
    {
      "FLOPS loss": 0.07810306549072266,
      "L0_d": 933.62,
      "MLM loss": 2.746854782104492,
      "epoch": 1.18,
      "step": 50999
    },
    {
      "epoch": 1.18,
      "learning_rate": 9.163693877551021e-05,
      "loss": 2.8241,
      "step": 51000
    },
    {
      "FLOPS loss": 0.07342377305030823,
      "L0_d": 1304.92,
      "MLM loss": 2.8114891052246094,
      "epoch": 1.19,
      "step": 51499
    },
    {
      "epoch": 1.19,
      "learning_rate": 9.153489795918368e-05,
      "loss": 2.8203,
      "step": 51500
    },
    {
      "FLOPS loss": 0.07382655888795853,
      "L0_d": 852.91,
      "MLM loss": 2.8023548126220703,
      "epoch": 1.2,
      "step": 51999
    },
    {
      "epoch": 1.2,
      "learning_rate": 9.14330612244898e-05,
      "loss": 2.8119,
      "step": 52000
    },
    {
      "FLOPS loss": 0.07053514569997787,
      "L0_d": 921.94,
      "MLM loss": 2.863825798034668,
      "epoch": 1.21,
      "step": 52499
    },
    {
      "epoch": 1.21,
      "learning_rate": 9.133102040816327e-05,
      "loss": 2.812,
      "step": 52500
    },
    {
      "FLOPS loss": 0.059692882001399994,
      "L0_d": 794.02,
      "MLM loss": 2.5097005367279053,
      "epoch": 1.22,
      "step": 52999
    },
    {
      "epoch": 1.22,
      "learning_rate": 9.122897959183674e-05,
      "loss": 2.8098,
      "step": 53000
    },
    {
      "FLOPS loss": 0.055110666900873184,
      "L0_d": 732.67,
      "MLM loss": 2.9446611404418945,
      "epoch": 1.24,
      "step": 53499
    },
    {
      "epoch": 1.24,
      "learning_rate": 9.11269387755102e-05,
      "loss": 2.8038,
      "step": 53500
    },
    {
      "FLOPS loss": 0.06119615212082863,
      "L0_d": 995.5,
      "MLM loss": 2.774935007095337,
      "epoch": 1.25,
      "step": 53999
    },
    {
      "epoch": 1.25,
      "learning_rate": 9.102489795918367e-05,
      "loss": 2.7985,
      "step": 54000
    },
    {
      "FLOPS loss": 0.07340589165687561,
      "L0_d": 1106.77,
      "MLM loss": 2.5551958084106445,
      "epoch": 1.26,
      "step": 54499
    },
    {
      "epoch": 1.26,
      "learning_rate": 9.09230612244898e-05,
      "loss": 2.8004,
      "step": 54500
    },
    {
      "FLOPS loss": 0.08167064934968948,
      "L0_d": 1180.7,
      "MLM loss": 2.8482956886291504,
      "epoch": 1.27,
      "step": 54999
    },
    {
      "epoch": 1.27,
      "learning_rate": 9.082102040816327e-05,
      "loss": 2.7972,
      "step": 55000
    },
    {
      "FLOPS loss": 0.0676698312163353,
      "L0_d": 831.5,
      "MLM loss": 2.8186306953430176,
      "epoch": 1.28,
      "step": 55499
    },
    {
      "epoch": 1.28,
      "learning_rate": 9.071897959183673e-05,
      "loss": 2.7908,
      "step": 55500
    },
    {
      "FLOPS loss": 0.07911355048418045,
      "L0_d": 1112.89,
      "MLM loss": 2.6768712997436523,
      "epoch": 1.29,
      "step": 55999
    },
    {
      "epoch": 1.29,
      "learning_rate": 9.061693877551021e-05,
      "loss": 2.7891,
      "step": 56000
    },
    {
      "FLOPS loss": 0.08468664437532425,
      "L0_d": 1028.75,
      "MLM loss": 2.5861382484436035,
      "epoch": 1.31,
      "step": 56499
    },
    {
      "epoch": 1.31,
      "learning_rate": 9.051510204081633e-05,
      "loss": 2.7854,
      "step": 56500
    },
    {
      "FLOPS loss": 0.06968092918395996,
      "L0_d": 1092.77,
      "MLM loss": 2.644059181213379,
      "epoch": 1.32,
      "step": 56999
    },
    {
      "epoch": 1.32,
      "learning_rate": 9.041306122448981e-05,
      "loss": 2.7848,
      "step": 57000
    },
    {
      "FLOPS loss": 0.07919490337371826,
      "L0_d": 1015.0,
      "MLM loss": 2.743734359741211,
      "epoch": 1.33,
      "step": 57499
    },
    {
      "epoch": 1.33,
      "learning_rate": 9.031102040816327e-05,
      "loss": 2.7847,
      "step": 57500
    },
    {
      "FLOPS loss": 0.05528505891561508,
      "L0_d": 672.12,
      "MLM loss": 2.7157461643218994,
      "epoch": 1.34,
      "step": 57999
    },
    {
      "epoch": 1.34,
      "learning_rate": 9.020897959183674e-05,
      "loss": 2.7756,
      "step": 58000
    },
    {
      "FLOPS loss": 0.06388474255800247,
      "L0_d": 836.95,
      "MLM loss": 2.740152359008789,
      "epoch": 1.35,
      "step": 58499
    },
    {
      "epoch": 1.35,
      "learning_rate": 9.010714285714286e-05,
      "loss": 2.7753,
      "step": 58500
    },
    {
      "FLOPS loss": 0.05673633888363838,
      "L0_d": 743.12,
      "MLM loss": 2.7288174629211426,
      "epoch": 1.36,
      "step": 58999
    },
    {
      "epoch": 1.36,
      "learning_rate": 9.000510204081634e-05,
      "loss": 2.7682,
      "step": 59000
    },
    {
      "FLOPS loss": 0.060955584049224854,
      "L0_d": 723.3,
      "MLM loss": 2.4606099128723145,
      "epoch": 1.37,
      "step": 59499
    },
    {
      "epoch": 1.37,
      "learning_rate": 8.99030612244898e-05,
      "loss": 2.7721,
      "step": 59500
    },
    {
      "FLOPS loss": 0.07077349722385406,
      "L0_d": 859.67,
      "MLM loss": 2.699582576751709,
      "epoch": 1.39,
      "step": 59999
    },
    {
      "epoch": 1.39,
      "learning_rate": 8.980102040816327e-05,
      "loss": 2.7674,
      "step": 60000
    },
    {
      "FLOPS loss": 0.07387091219425201,
      "L0_d": 1039.08,
      "MLM loss": 2.389226198196411,
      "epoch": 1.4,
      "step": 60499
    },
    {
      "epoch": 1.4,
      "learning_rate": 8.969918367346939e-05,
      "loss": 2.7638,
      "step": 60500
    },
    {
      "FLOPS loss": 0.07885833084583282,
      "L0_d": 1065.38,
      "MLM loss": 2.8349263668060303,
      "epoch": 1.41,
      "step": 60999
    },
    {
      "epoch": 1.41,
      "learning_rate": 8.959714285714285e-05,
      "loss": 2.7619,
      "step": 61000
    },
    {
      "FLOPS loss": 0.053016629070043564,
      "L0_d": 637.61,
      "MLM loss": 2.779146194458008,
      "epoch": 1.42,
      "step": 61499
    },
    {
      "epoch": 1.42,
      "learning_rate": 8.949510204081633e-05,
      "loss": 2.7601,
      "step": 61500
    },
    {
      "FLOPS loss": 0.05506119132041931,
      "L0_d": 491.41,
      "MLM loss": 2.655466079711914,
      "epoch": 1.43,
      "step": 61999
    },
    {
      "epoch": 1.43,
      "learning_rate": 8.939306122448981e-05,
      "loss": 2.7567,
      "step": 62000
    },
    {
      "FLOPS loss": 0.0772499218583107,
      "L0_d": 1390.47,
      "MLM loss": 2.617006301879883,
      "epoch": 1.44,
      "step": 62499
    },
    {
      "epoch": 1.44,
      "learning_rate": 8.929122448979592e-05,
      "loss": 2.752,
      "step": 62500
    },
    {
      "FLOPS loss": 0.07647674530744553,
      "L0_d": 849.38,
      "MLM loss": 2.6698994636535645,
      "epoch": 1.46,
      "step": 62999
    },
    {
      "epoch": 1.46,
      "learning_rate": 8.91891836734694e-05,
      "loss": 2.7522,
      "step": 63000
    },
    {
      "FLOPS loss": 0.0736238956451416,
      "L0_d": 982.06,
      "MLM loss": 2.5969135761260986,
      "epoch": 1.47,
      "step": 63499
    },
    {
      "epoch": 1.47,
      "learning_rate": 8.908714285714286e-05,
      "loss": 2.7454,
      "step": 63500
    },
    {
      "FLOPS loss": 0.08865927159786224,
      "L0_d": 1242.19,
      "MLM loss": 2.737492084503174,
      "epoch": 1.48,
      "step": 63999
    },
    {
      "epoch": 1.48,
      "learning_rate": 8.898510204081632e-05,
      "loss": 2.7504,
      "step": 64000
    },
    {
      "FLOPS loss": 0.062135014683008194,
      "L0_d": 840.81,
      "MLM loss": 2.648526668548584,
      "epoch": 1.49,
      "step": 64499
    },
    {
      "epoch": 1.49,
      "learning_rate": 8.888326530612246e-05,
      "loss": 2.7451,
      "step": 64500
    },
    {
      "FLOPS loss": 0.05925214663147926,
      "L0_d": 926.7,
      "MLM loss": 2.7958133220672607,
      "epoch": 1.5,
      "step": 64999
    },
    {
      "epoch": 1.5,
      "learning_rate": 8.878122448979592e-05,
      "loss": 2.7378,
      "step": 65000
    },
    {
      "FLOPS loss": 0.08276008069515228,
      "L0_d": 976.19,
      "MLM loss": 2.6551318168640137,
      "epoch": 1.51,
      "step": 65499
    },
    {
      "epoch": 1.51,
      "learning_rate": 8.867918367346939e-05,
      "loss": 2.7361,
      "step": 65500
    },
    {
      "FLOPS loss": 0.07621205598115921,
      "L0_d": 1072.14,
      "MLM loss": 2.7889227867126465,
      "epoch": 1.52,
      "step": 65999
    },
    {
      "epoch": 1.52,
      "learning_rate": 8.857714285714285e-05,
      "loss": 2.7355,
      "step": 66000
    },
    {
      "FLOPS loss": 0.07239171862602234,
      "L0_d": 914.05,
      "MLM loss": 2.578742027282715,
      "epoch": 1.54,
      "step": 66499
    },
    {
      "epoch": 1.54,
      "learning_rate": 8.847530612244899e-05,
      "loss": 2.7293,
      "step": 66500
    },
    {
      "FLOPS loss": 0.06520795077085495,
      "L0_d": 836.77,
      "MLM loss": 2.6456704139709473,
      "epoch": 1.55,
      "step": 66999
    },
    {
      "epoch": 1.55,
      "learning_rate": 8.837326530612245e-05,
      "loss": 2.7282,
      "step": 67000
    },
    {
      "FLOPS loss": 0.07171254605054855,
      "L0_d": 850.34,
      "MLM loss": 2.5980749130249023,
      "epoch": 1.56,
      "step": 67499
    },
    {
      "epoch": 1.56,
      "learning_rate": 8.827122448979593e-05,
      "loss": 2.7284,
      "step": 67500
    },
    {
      "FLOPS loss": 0.07132073491811752,
      "L0_d": 785.77,
      "MLM loss": 2.6635169982910156,
      "epoch": 1.57,
      "step": 67999
    },
    {
      "epoch": 1.57,
      "learning_rate": 8.81691836734694e-05,
      "loss": 2.7293,
      "step": 68000
    },
    {
      "FLOPS loss": 0.06449418514966965,
      "L0_d": 750.56,
      "MLM loss": 2.6216351985931396,
      "epoch": 1.58,
      "step": 68499
    },
    {
      "epoch": 1.58,
      "learning_rate": 8.806734693877551e-05,
      "loss": 2.7213,
      "step": 68500
    },
    {
      "FLOPS loss": 0.07830583304166794,
      "L0_d": 1105.78,
      "MLM loss": 2.6381173133850098,
      "epoch": 1.59,
      "step": 68999
    },
    {
      "epoch": 1.59,
      "learning_rate": 8.796530612244899e-05,
      "loss": 2.7196,
      "step": 69000
    },
    {
      "FLOPS loss": 0.06058787554502487,
      "L0_d": 774.34,
      "MLM loss": 2.590695858001709,
      "epoch": 1.61,
      "step": 69499
    },
    {
      "epoch": 1.61,
      "learning_rate": 8.786326530612246e-05,
      "loss": 2.7193,
      "step": 69500
    },
    {
      "FLOPS loss": 0.07099094986915588,
      "L0_d": 1099.67,
      "MLM loss": 2.466123342514038,
      "epoch": 1.62,
      "step": 69999
    },
    {
      "epoch": 1.62,
      "learning_rate": 8.776122448979592e-05,
      "loss": 2.7156,
      "step": 70000
    },
    {
      "FLOPS loss": 0.05859340727329254,
      "L0_d": 635.06,
      "MLM loss": 2.771111488342285,
      "epoch": 1.63,
      "step": 70499
    },
    {
      "epoch": 1.63,
      "learning_rate": 8.765938775510204e-05,
      "loss": 2.7133,
      "step": 70500
    },
    {
      "FLOPS loss": 0.06658073514699936,
      "L0_d": 971.06,
      "MLM loss": 2.656001091003418,
      "epoch": 1.64,
      "step": 70999
    },
    {
      "epoch": 1.64,
      "learning_rate": 8.755734693877552e-05,
      "loss": 2.7096,
      "step": 71000
    },
    {
      "FLOPS loss": 0.10472187399864197,
      "L0_d": 1132.7,
      "MLM loss": 2.534677267074585,
      "epoch": 1.65,
      "step": 71499
    },
    {
      "epoch": 1.65,
      "learning_rate": 8.745530612244899e-05,
      "loss": 2.707,
      "step": 71500
    },
    {
      "FLOPS loss": 0.054848697036504745,
      "L0_d": 609.2,
      "MLM loss": 2.721672534942627,
      "epoch": 1.66,
      "step": 71999
    },
    {
      "epoch": 1.66,
      "learning_rate": 8.735326530612245e-05,
      "loss": 2.7124,
      "step": 72000
    },
    {
      "FLOPS loss": 0.06477601826190948,
      "L0_d": 952.28,
      "MLM loss": 2.4252777099609375,
      "epoch": 1.67,
      "step": 72499
    },
    {
      "epoch": 1.67,
      "learning_rate": 8.725142857142857e-05,
      "loss": 2.7049,
      "step": 72500
    },
    {
      "FLOPS loss": 0.0618317611515522,
      "L0_d": 742.14,
      "MLM loss": 2.300227642059326,
      "epoch": 1.69,
      "step": 72999
    },
    {
      "epoch": 1.69,
      "learning_rate": 8.714938775510204e-05,
      "loss": 2.7044,
      "step": 73000
    },
    {
      "FLOPS loss": 0.06794578582048416,
      "L0_d": 947.11,
      "MLM loss": 2.5354833602905273,
      "epoch": 1.7,
      "step": 73499
    },
    {
      "epoch": 1.7,
      "learning_rate": 8.704734693877551e-05,
      "loss": 2.7013,
      "step": 73500
    },
    {
      "FLOPS loss": 0.05277640372514725,
      "L0_d": 623.72,
      "MLM loss": 2.622762680053711,
      "epoch": 1.71,
      "step": 73999
    },
    {
      "epoch": 1.71,
      "learning_rate": 8.694530612244899e-05,
      "loss": 2.6975,
      "step": 74000
    },
    {
      "FLOPS loss": 0.07353149354457855,
      "L0_d": 848.69,
      "MLM loss": 2.625680446624756,
      "epoch": 1.72,
      "step": 74499
    },
    {
      "epoch": 1.72,
      "learning_rate": 8.68434693877551e-05,
      "loss": 2.6965,
      "step": 74500
    },
    {
      "FLOPS loss": 0.07950403541326523,
      "L0_d": 909.52,
      "MLM loss": 2.6366257667541504,
      "epoch": 1.73,
      "step": 74999
    },
    {
      "epoch": 1.73,
      "learning_rate": 8.674142857142858e-05,
      "loss": 2.6986,
      "step": 75000
    },
    {
      "FLOPS loss": 0.0671720951795578,
      "L0_d": 1059.39,
      "MLM loss": 2.6872715950012207,
      "epoch": 1.74,
      "step": 75499
    },
    {
      "epoch": 1.74,
      "learning_rate": 8.663938775510204e-05,
      "loss": 2.694,
      "step": 75500
    },
    {
      "FLOPS loss": 0.06870570033788681,
      "L0_d": 728.41,
      "MLM loss": 2.6747617721557617,
      "epoch": 1.76,
      "step": 75999
    },
    {
      "epoch": 1.76,
      "learning_rate": 8.653734693877551e-05,
      "loss": 2.6882,
      "step": 76000
    },
    {
      "FLOPS loss": 0.0645112618803978,
      "L0_d": 747.95,
      "MLM loss": 2.559619426727295,
      "epoch": 1.77,
      "step": 76499
    },
    {
      "epoch": 1.77,
      "learning_rate": 8.643551020408164e-05,
      "loss": 2.691,
      "step": 76500
    },
    {
      "FLOPS loss": 0.04983799159526825,
      "L0_d": 720.17,
      "MLM loss": 2.6016077995300293,
      "epoch": 1.78,
      "step": 76999
    },
    {
      "epoch": 1.78,
      "learning_rate": 8.63334693877551e-05,
      "loss": 2.6884,
      "step": 77000
    },
    {
      "FLOPS loss": 0.0694853812456131,
      "L0_d": 955.48,
      "MLM loss": 2.679753303527832,
      "epoch": 1.79,
      "step": 77499
    },
    {
      "epoch": 1.79,
      "learning_rate": 8.623142857142857e-05,
      "loss": 2.6839,
      "step": 77500
    },
    {
      "FLOPS loss": 0.07788027077913284,
      "L0_d": 1066.77,
      "MLM loss": 2.5656166076660156,
      "epoch": 1.8,
      "step": 77999
    },
    {
      "epoch": 1.8,
      "learning_rate": 8.612938775510204e-05,
      "loss": 2.6836,
      "step": 78000
    },
    {
      "FLOPS loss": 0.05925082787871361,
      "L0_d": 885.91,
      "MLM loss": 2.4231157302856445,
      "epoch": 1.81,
      "step": 78499
    },
    {
      "epoch": 1.81,
      "learning_rate": 8.602755102040817e-05,
      "loss": 2.6817,
      "step": 78500
    },
    {
      "FLOPS loss": 0.07589417695999146,
      "L0_d": 827.25,
      "MLM loss": 2.515610933303833,
      "epoch": 1.82,
      "step": 78999
    },
    {
      "epoch": 1.82,
      "learning_rate": 8.592551020408163e-05,
      "loss": 2.6793,
      "step": 79000
    },
    {
      "FLOPS loss": 0.06679127365350723,
      "L0_d": 889.84,
      "MLM loss": 2.479564905166626,
      "epoch": 1.84,
      "step": 79499
    },
    {
      "epoch": 1.84,
      "learning_rate": 8.582346938775511e-05,
      "loss": 2.6738,
      "step": 79500
    },
    {
      "FLOPS loss": 0.06588038057088852,
      "L0_d": 816.66,
      "MLM loss": 2.779053211212158,
      "epoch": 1.85,
      "step": 79999
    },
    {
      "epoch": 1.85,
      "learning_rate": 8.572142857142858e-05,
      "loss": 2.6703,
      "step": 80000
    },
    {
      "FLOPS loss": 0.06606688350439072,
      "L0_d": 742.75,
      "MLM loss": 2.5370850563049316,
      "epoch": 1.86,
      "step": 80499
    },
    {
      "epoch": 1.86,
      "learning_rate": 8.56195918367347e-05,
      "loss": 2.673,
      "step": 80500
    },
    {
      "FLOPS loss": 0.06476680189371109,
      "L0_d": 953.19,
      "MLM loss": 2.5319418907165527,
      "epoch": 1.87,
      "step": 80999
    },
    {
      "epoch": 1.87,
      "learning_rate": 8.551755102040818e-05,
      "loss": 2.6687,
      "step": 81000
    },
    {
      "FLOPS loss": 0.06020694598555565,
      "L0_d": 820.14,
      "MLM loss": 2.550804615020752,
      "epoch": 1.88,
      "step": 81499
    },
    {
      "epoch": 1.88,
      "learning_rate": 8.541551020408164e-05,
      "loss": 2.6729,
      "step": 81500
    },
    {
      "FLOPS loss": 0.06602118909358978,
      "L0_d": 819.67,
      "MLM loss": 2.5585505962371826,
      "epoch": 1.89,
      "step": 81999
    },
    {
      "epoch": 1.89,
      "learning_rate": 8.53134693877551e-05,
      "loss": 2.6738,
      "step": 82000
    },
    {
      "FLOPS loss": 0.06344706565141678,
      "L0_d": 836.28,
      "MLM loss": 2.728942394256592,
      "epoch": 1.91,
      "step": 82499
    },
    {
      "epoch": 1.91,
      "learning_rate": 8.521163265306123e-05,
      "loss": 2.6635,
      "step": 82500
    },
    {
      "FLOPS loss": 0.06816563755273819,
      "L0_d": 860.02,
      "MLM loss": 2.4916951656341553,
      "epoch": 1.92,
      "step": 82999
    },
    {
      "epoch": 1.92,
      "learning_rate": 8.51095918367347e-05,
      "loss": 2.6667,
      "step": 83000
    },
    {
      "FLOPS loss": 0.059945568442344666,
      "L0_d": 667.73,
      "MLM loss": 2.548002004623413,
      "epoch": 1.93,
      "step": 83499
    },
    {
      "epoch": 1.93,
      "learning_rate": 8.500755102040817e-05,
      "loss": 2.6663,
      "step": 83500
    },
    {
      "FLOPS loss": 0.07439987361431122,
      "L0_d": 867.16,
      "MLM loss": 2.7152485847473145,
      "epoch": 1.94,
      "step": 83999
    },
    {
      "epoch": 1.94,
      "learning_rate": 8.490551020408163e-05,
      "loss": 2.6637,
      "step": 84000
    },
    {
      "FLOPS loss": 0.06373114883899689,
      "L0_d": 820.31,
      "MLM loss": 2.5678906440734863,
      "epoch": 1.95,
      "step": 84499
    },
    {
      "epoch": 1.95,
      "learning_rate": 8.480367346938775e-05,
      "loss": 2.6615,
      "step": 84500
    },
    {
      "FLOPS loss": 0.056776098906993866,
      "L0_d": 765.92,
      "MLM loss": 2.439091682434082,
      "epoch": 1.96,
      "step": 84999
    },
    {
      "epoch": 1.96,
      "learning_rate": 8.470163265306122e-05,
      "loss": 2.6571,
      "step": 85000
    },
    {
      "FLOPS loss": 0.06744925677776337,
      "L0_d": 710.83,
      "MLM loss": 2.693312644958496,
      "epoch": 1.98,
      "step": 85499
    },
    {
      "epoch": 1.98,
      "learning_rate": 8.45995918367347e-05,
      "loss": 2.6605,
      "step": 85500
    },
    {
      "FLOPS loss": 0.06582710146903992,
      "L0_d": 778.98,
      "MLM loss": 2.602993965148926,
      "epoch": 1.99,
      "step": 85999
    },
    {
      "epoch": 1.99,
      "learning_rate": 8.449755102040818e-05,
      "loss": 2.6552,
      "step": 86000
    },
    {
      "FLOPS loss": 0.07695449888706207,
      "L0_d": 884.67,
      "MLM loss": 2.4756698608398438,
      "epoch": 2.0,
      "step": 86499
    },
    {
      "epoch": 2.0,
      "learning_rate": 8.439571428571428e-05,
      "loss": 2.6529,
      "step": 86500
    },
    {
      "FLOPS loss": 0.07339202612638474,
      "L0_d": 901.58,
      "MLM loss": 2.5836970806121826,
      "epoch": 2.01,
      "step": 86999
    },
    {
      "epoch": 2.01,
      "learning_rate": 8.429367346938776e-05,
      "loss": 2.6459,
      "step": 87000
    },
    {
      "FLOPS loss": 0.06989730894565582,
      "L0_d": 806.12,
      "MLM loss": 2.558981418609619,
      "epoch": 2.02,
      "step": 87499
    },
    {
      "epoch": 2.02,
      "learning_rate": 8.419163265306123e-05,
      "loss": 2.6459,
      "step": 87500
    },
    {
      "FLOPS loss": 0.06631279736757278,
      "L0_d": 830.09,
      "MLM loss": 2.4851489067077637,
      "epoch": 2.03,
      "step": 87999
    },
    {
      "epoch": 2.03,
      "learning_rate": 8.408959183673469e-05,
      "loss": 2.644,
      "step": 88000
    },
    {
      "FLOPS loss": 0.07449375092983246,
      "L0_d": 906.97,
      "MLM loss": 2.5570497512817383,
      "epoch": 2.04,
      "step": 88499
    },
    {
      "epoch": 2.04,
      "learning_rate": 8.398775510204083e-05,
      "loss": 2.6432,
      "step": 88500
    },
    {
      "FLOPS loss": 0.08294404298067093,
      "L0_d": 1214.31,
      "MLM loss": 2.543278932571411,
      "epoch": 2.06,
      "step": 88999
    },
    {
      "epoch": 2.06,
      "learning_rate": 8.388571428571429e-05,
      "loss": 2.6493,
      "step": 89000
    },
    {
      "FLOPS loss": 0.06517869979143143,
      "L0_d": 899.28,
      "MLM loss": 2.4547858238220215,
      "epoch": 2.07,
      "step": 89499
    },
    {
      "epoch": 2.07,
      "learning_rate": 8.378367346938775e-05,
      "loss": 2.6421,
      "step": 89500
    },
    {
      "FLOPS loss": 0.07665277272462845,
      "L0_d": 713.2,
      "MLM loss": 2.541884422302246,
      "epoch": 2.08,
      "step": 89999
    },
    {
      "epoch": 2.08,
      "learning_rate": 8.368163265306122e-05,
      "loss": 2.641,
      "step": 90000
    },
    {
      "FLOPS loss": 0.08727514743804932,
      "L0_d": 1206.22,
      "MLM loss": 2.6945085525512695,
      "epoch": 2.09,
      "step": 90499
    },
    {
      "epoch": 2.09,
      "learning_rate": 8.357979591836735e-05,
      "loss": 2.6428,
      "step": 90500
    },
    {
      "FLOPS loss": 0.06770230829715729,
      "L0_d": 1007.8,
      "MLM loss": 2.507075309753418,
      "epoch": 2.1,
      "step": 90999
    },
    {
      "epoch": 2.1,
      "learning_rate": 8.347775510204082e-05,
      "loss": 2.6341,
      "step": 91000
    },
    {
      "FLOPS loss": 0.06252799928188324,
      "L0_d": 856.61,
      "MLM loss": 2.5938777923583984,
      "epoch": 2.11,
      "step": 91499
    },
    {
      "epoch": 2.11,
      "learning_rate": 8.33757142857143e-05,
      "loss": 2.6381,
      "step": 91500
    },
    {
      "FLOPS loss": 0.060771603137254715,
      "L0_d": 886.33,
      "MLM loss": 2.4438791275024414,
      "epoch": 2.13,
      "step": 91999
    },
    {
      "epoch": 2.13,
      "learning_rate": 8.327367346938776e-05,
      "loss": 2.6346,
      "step": 92000
    },
    {
      "FLOPS loss": 0.07406602054834366,
      "L0_d": 751.48,
      "MLM loss": 2.4598939418792725,
      "epoch": 2.14,
      "step": 92499
    },
    {
      "epoch": 2.14,
      "learning_rate": 8.317183673469388e-05,
      "loss": 2.639,
      "step": 92500
    },
    {
      "FLOPS loss": 0.0679251030087471,
      "L0_d": 731.48,
      "MLM loss": 2.6497678756713867,
      "epoch": 2.15,
      "step": 92999
    },
    {
      "epoch": 2.15,
      "learning_rate": 8.306979591836736e-05,
      "loss": 2.6292,
      "step": 93000
    },
    {
      "FLOPS loss": 0.05723276361823082,
      "L0_d": 706.5,
      "MLM loss": 2.60920786857605,
      "epoch": 2.16,
      "step": 93499
    },
    {
      "epoch": 2.16,
      "learning_rate": 8.296775510204082e-05,
      "loss": 2.6314,
      "step": 93500
    },
    {
      "FLOPS loss": 0.08006785064935684,
      "L0_d": 1262.91,
      "MLM loss": 2.6934850215911865,
      "epoch": 2.17,
      "step": 93999
    },
    {
      "epoch": 2.17,
      "learning_rate": 8.286571428571429e-05,
      "loss": 2.6246,
      "step": 94000
    },
    {
      "FLOPS loss": 0.07803159207105637,
      "L0_d": 1247.95,
      "MLM loss": 2.520078659057617,
      "epoch": 2.18,
      "step": 94499
    },
    {
      "epoch": 2.18,
      "learning_rate": 8.276387755102041e-05,
      "loss": 2.6303,
      "step": 94500
    },
    {
      "FLOPS loss": 0.06651651114225388,
      "L0_d": 725.97,
      "MLM loss": 2.3245010375976562,
      "epoch": 2.19,
      "step": 94999
    },
    {
      "epoch": 2.19,
      "learning_rate": 8.266183673469387e-05,
      "loss": 2.6238,
      "step": 95000
    },
    {
      "FLOPS loss": 0.052927643060684204,
      "L0_d": 549.06,
      "MLM loss": 2.350855588912964,
      "epoch": 2.21,
      "step": 95499
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.255979591836735e-05,
      "loss": 2.6232,
      "step": 95500
    },
    {
      "FLOPS loss": 0.07090628147125244,
      "L0_d": 782.56,
      "MLM loss": 2.5654730796813965,
      "epoch": 2.22,
      "step": 95999
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.245775510204082e-05,
      "loss": 2.622,
      "step": 96000
    },
    {
      "FLOPS loss": 0.07428272813558578,
      "L0_d": 1075.02,
      "MLM loss": 2.656454563140869,
      "epoch": 2.23,
      "step": 96499
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.235591836734694e-05,
      "loss": 2.6238,
      "step": 96500
    },
    {
      "FLOPS loss": 0.06645943969488144,
      "L0_d": 740.31,
      "MLM loss": 2.509154796600342,
      "epoch": 2.24,
      "step": 96999
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.22538775510204e-05,
      "loss": 2.617,
      "step": 97000
    },
    {
      "FLOPS loss": 0.08122547715902328,
      "L0_d": 959.48,
      "MLM loss": 2.543363094329834,
      "epoch": 2.25,
      "step": 97499
    },
    {
      "epoch": 2.25,
      "learning_rate": 8.215183673469388e-05,
      "loss": 2.6142,
      "step": 97500
    },
    {
      "FLOPS loss": 0.0727510079741478,
      "L0_d": 1239.5,
      "MLM loss": 2.648973226547241,
      "epoch": 2.26,
      "step": 97999
    },
    {
      "epoch": 2.26,
      "learning_rate": 8.204979591836736e-05,
      "loss": 2.6215,
      "step": 98000
    },
    {
      "FLOPS loss": 0.06306217610836029,
      "L0_d": 1277.14,
      "MLM loss": 2.6636743545532227,
      "epoch": 2.28,
      "step": 98499
    },
    {
      "epoch": 2.28,
      "learning_rate": 8.194795918367347e-05,
      "loss": 2.6183,
      "step": 98500
    },
    {
      "FLOPS loss": 0.07064758986234665,
      "L0_d": 1065.84,
      "MLM loss": 2.3864660263061523,
      "epoch": 2.29,
      "step": 98999
    },
    {
      "epoch": 2.29,
      "learning_rate": 8.184591836734695e-05,
      "loss": 2.6213,
      "step": 99000
    },
    {
      "FLOPS loss": 0.05990250036120415,
      "L0_d": 717.38,
      "MLM loss": 2.5958118438720703,
      "epoch": 2.3,
      "step": 99499
    },
    {
      "epoch": 2.3,
      "learning_rate": 8.174387755102041e-05,
      "loss": 2.6102,
      "step": 99500
    },
    {
      "FLOPS loss": 0.09523125737905502,
      "L0_d": 1146.02,
      "MLM loss": 2.806295394897461,
      "epoch": 2.31,
      "step": 99999
    },
    {
      "epoch": 2.31,
      "learning_rate": 8.164204081632653e-05,
      "loss": 2.6184,
      "step": 100000
    },
    {
      "FLOPS loss": 0.07538797706365585,
      "L0_d": 1017.2,
      "MLM loss": 2.494988441467285,
      "epoch": 2.32,
      "step": 100499
    },
    {
      "epoch": 2.32,
      "learning_rate": 8.154000000000001e-05,
      "loss": 2.6124,
      "step": 100500
    },
    {
      "FLOPS loss": 0.07495392858982086,
      "L0_d": 927.31,
      "MLM loss": 2.38315749168396,
      "epoch": 2.33,
      "step": 100999
    },
    {
      "epoch": 2.33,
      "learning_rate": 8.143795918367347e-05,
      "loss": 2.6134,
      "step": 101000
    },
    {
      "FLOPS loss": 0.06713060289621353,
      "L0_d": 897.39,
      "MLM loss": 2.548612356185913,
      "epoch": 2.34,
      "step": 101499
    },
    {
      "epoch": 2.34,
      "learning_rate": 8.133591836734694e-05,
      "loss": 2.6139,
      "step": 101500
    },
    {
      "FLOPS loss": 0.0808817520737648,
      "L0_d": 1192.45,
      "MLM loss": 2.7223920822143555,
      "epoch": 2.36,
      "step": 101999
    },
    {
      "epoch": 2.36,
      "learning_rate": 8.123387755102042e-05,
      "loss": 2.6094,
      "step": 102000
    },
    {
      "FLOPS loss": 0.07406562566757202,
      "L0_d": 1115.09,
      "MLM loss": 2.659451961517334,
      "epoch": 2.37,
      "step": 102499
    },
    {
      "epoch": 2.37,
      "learning_rate": 8.113183673469388e-05,
      "loss": 2.6038,
      "step": 102500
    },
    {
      "FLOPS loss": 0.0725577175617218,
      "L0_d": 778.3,
      "MLM loss": 2.425353527069092,
      "epoch": 2.38,
      "step": 102999
    },
    {
      "epoch": 2.38,
      "learning_rate": 8.102979591836735e-05,
      "loss": 2.6091,
      "step": 103000
    },
    {
      "FLOPS loss": 0.07214351743459702,
      "L0_d": 974.78,
      "MLM loss": 2.6830689907073975,
      "epoch": 2.39,
      "step": 103499
    },
    {
      "epoch": 2.39,
      "learning_rate": 8.092775510204082e-05,
      "loss": 2.6073,
      "step": 103500
    },
    {
      "FLOPS loss": 0.07022420316934586,
      "L0_d": 763.67,
      "MLM loss": 2.4790377616882324,
      "epoch": 2.4,
      "step": 103999
    },
    {
      "epoch": 2.4,
      "learning_rate": 8.082612244897959e-05,
      "loss": 2.6051,
      "step": 104000
    },
    {
      "FLOPS loss": 0.08411877602338791,
      "L0_d": 1579.39,
      "MLM loss": 2.727245807647705,
      "epoch": 2.41,
      "step": 104499
    },
    {
      "epoch": 2.41,
      "learning_rate": 8.072408163265307e-05,
      "loss": 2.602,
      "step": 104500
    },
    {
      "FLOPS loss": 0.07292833179235458,
      "L0_d": 985.05,
      "MLM loss": 2.465031385421753,
      "epoch": 2.43,
      "step": 104999
    },
    {
      "epoch": 2.43,
      "learning_rate": 8.062204081632654e-05,
      "loss": 2.6007,
      "step": 105000
    },
    {
      "FLOPS loss": 0.06477630138397217,
      "L0_d": 1046.03,
      "MLM loss": 2.657435894012451,
      "epoch": 2.44,
      "step": 105499
    },
    {
      "epoch": 2.44,
      "learning_rate": 8.052000000000001e-05,
      "loss": 2.6033,
      "step": 105500
    },
    {
      "FLOPS loss": 0.07536941021680832,
      "L0_d": 947.55,
      "MLM loss": 2.5360069274902344,
      "epoch": 2.45,
      "step": 105999
    },
    {
      "epoch": 2.45,
      "learning_rate": 8.041795918367347e-05,
      "loss": 2.5996,
      "step": 106000
    },
    {
      "FLOPS loss": 0.05159715563058853,
      "L0_d": 753.66,
      "MLM loss": 2.666386842727661,
      "epoch": 2.46,
      "step": 106499
    },
    {
      "epoch": 2.46,
      "learning_rate": 8.031591836734694e-05,
      "loss": 2.596,
      "step": 106500
    },
    {
      "FLOPS loss": 0.05982593074440956,
      "L0_d": 697.56,
      "MLM loss": 2.532409906387329,
      "epoch": 2.47,
      "step": 106999
    },
    {
      "epoch": 2.47,
      "learning_rate": 8.021387755102042e-05,
      "loss": 2.5964,
      "step": 107000
    },
    {
      "FLOPS loss": 0.05745324492454529,
      "L0_d": 811.14,
      "MLM loss": 2.605365037918091,
      "epoch": 2.48,
      "step": 107499
    },
    {
      "epoch": 2.48,
      "learning_rate": 8.011183673469388e-05,
      "loss": 2.5926,
      "step": 107500
    },
    {
      "FLOPS loss": 0.05868089944124222,
      "L0_d": 651.72,
      "MLM loss": 2.6506195068359375,
      "epoch": 2.49,
      "step": 107999
    },
    {
      "epoch": 2.49,
      "learning_rate": 8.001e-05,
      "loss": 2.5949,
      "step": 108000
    },
    {
      "FLOPS loss": 0.07431881874799728,
      "L0_d": 1089.89,
      "MLM loss": 2.470217704772949,
      "epoch": 2.51,
      "step": 108499
    },
    {
      "epoch": 2.51,
      "learning_rate": 7.990795918367348e-05,
      "loss": 2.5869,
      "step": 108500
    },
    {
      "FLOPS loss": 0.0629262775182724,
      "L0_d": 616.03,
      "MLM loss": 2.6411805152893066,
      "epoch": 2.52,
      "step": 108999
    },
    {
      "epoch": 2.52,
      "learning_rate": 7.980591836734694e-05,
      "loss": 2.5961,
      "step": 109000
    },
    {
      "FLOPS loss": 0.06784447282552719,
      "L0_d": 863.77,
      "MLM loss": 2.5637335777282715,
      "epoch": 2.53,
      "step": 109499
    },
    {
      "epoch": 2.53,
      "learning_rate": 7.970387755102041e-05,
      "loss": 2.5915,
      "step": 109500
    },
    {
      "FLOPS loss": 0.07220463454723358,
      "L0_d": 896.98,
      "MLM loss": 2.5497515201568604,
      "epoch": 2.54,
      "step": 109999
    },
    {
      "epoch": 2.54,
      "learning_rate": 7.960204081632654e-05,
      "loss": 2.5891,
      "step": 110000
    },
    {
      "FLOPS loss": 0.06754712760448456,
      "L0_d": 956.58,
      "MLM loss": 2.623528003692627,
      "epoch": 2.55,
      "step": 110499
    },
    {
      "epoch": 2.55,
      "learning_rate": 7.950000000000001e-05,
      "loss": 2.5828,
      "step": 110500
    },
    {
      "FLOPS loss": 0.07457764446735382,
      "L0_d": 910.52,
      "MLM loss": 2.426210880279541,
      "epoch": 2.56,
      "step": 110999
    },
    {
      "epoch": 2.56,
      "learning_rate": 7.939795918367347e-05,
      "loss": 2.5891,
      "step": 111000
    },
    {
      "FLOPS loss": 0.05750475823879242,
      "L0_d": 984.28,
      "MLM loss": 2.6451473236083984,
      "epoch": 2.58,
      "step": 111499
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.929591836734695e-05,
      "loss": 2.5836,
      "step": 111500
    },
    {
      "FLOPS loss": 0.0681430771946907,
      "L0_d": 805.7,
      "MLM loss": 2.4090609550476074,
      "epoch": 2.59,
      "step": 111999
    },
    {
      "epoch": 2.59,
      "learning_rate": 7.919408163265306e-05,
      "loss": 2.5878,
      "step": 112000
    },
    {
      "FLOPS loss": 0.06985077261924744,
      "L0_d": 780.66,
      "MLM loss": 2.6808598041534424,
      "epoch": 2.6,
      "step": 112499
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.909204081632652e-05,
      "loss": 2.5829,
      "step": 112500
    },
    {
      "FLOPS loss": 0.05487761273980141,
      "L0_d": 690.89,
      "MLM loss": 2.610987424850464,
      "epoch": 2.61,
      "step": 112999
    },
    {
      "epoch": 2.61,
      "learning_rate": 7.899000000000001e-05,
      "loss": 2.581,
      "step": 113000
    },
    {
      "FLOPS loss": 0.07488895207643509,
      "L0_d": 1201.34,
      "MLM loss": 2.655109405517578,
      "epoch": 2.62,
      "step": 113499
    },
    {
      "epoch": 2.62,
      "learning_rate": 7.888795918367348e-05,
      "loss": 2.5752,
      "step": 113500
    },
    {
      "FLOPS loss": 0.0709005743265152,
      "L0_d": 991.5,
      "MLM loss": 2.626437187194824,
      "epoch": 2.63,
      "step": 113999
    },
    {
      "epoch": 2.63,
      "learning_rate": 7.87861224489796e-05,
      "loss": 2.5814,
      "step": 114000
    },
    {
      "FLOPS loss": 0.08310511708259583,
      "L0_d": 945.17,
      "MLM loss": 2.42806077003479,
      "epoch": 2.65,
      "step": 114499
    },
    {
      "epoch": 2.65,
      "learning_rate": 7.868408163265306e-05,
      "loss": 2.5759,
      "step": 114500
    },
    {
      "FLOPS loss": 0.06651800125837326,
      "L0_d": 752.34,
      "MLM loss": 2.439891815185547,
      "epoch": 2.66,
      "step": 114999
    },
    {
      "epoch": 2.66,
      "learning_rate": 7.858204081632653e-05,
      "loss": 2.5783,
      "step": 115000
    },
    {
      "FLOPS loss": 0.06491895765066147,
      "L0_d": 878.84,
      "MLM loss": 2.401111125946045,
      "epoch": 2.67,
      "step": 115499
    },
    {
      "epoch": 2.67,
      "learning_rate": 7.848000000000001e-05,
      "loss": 2.5778,
      "step": 115500
    },
    {
      "FLOPS loss": 0.07317697256803513,
      "L0_d": 854.97,
      "MLM loss": 2.4240808486938477,
      "epoch": 2.68,
      "step": 115999
    },
    {
      "epoch": 2.68,
      "learning_rate": 7.837816326530613e-05,
      "loss": 2.5771,
      "step": 116000
    },
    {
      "FLOPS loss": 0.07135210931301117,
      "L0_d": 901.19,
      "MLM loss": 2.5726559162139893,
      "epoch": 2.69,
      "step": 116499
    },
    {
      "epoch": 2.69,
      "learning_rate": 7.827612244897959e-05,
      "loss": 2.5728,
      "step": 116500
    },
    {
      "FLOPS loss": 0.07425196468830109,
      "L0_d": 984.05,
      "MLM loss": 2.447499990463257,
      "epoch": 2.7,
      "step": 116999
    },
    {
      "epoch": 2.7,
      "learning_rate": 7.817408163265306e-05,
      "loss": 2.5697,
      "step": 117000
    },
    {
      "FLOPS loss": 0.057068273425102234,
      "L0_d": 810.42,
      "MLM loss": 2.51332950592041,
      "epoch": 2.71,
      "step": 117499
    },
    {
      "epoch": 2.71,
      "learning_rate": 7.807204081632654e-05,
      "loss": 2.5769,
      "step": 117500
    },
    {
      "FLOPS loss": 0.07609079033136368,
      "L0_d": 943.66,
      "MLM loss": 2.5173351764678955,
      "epoch": 2.73,
      "step": 117999
    },
    {
      "epoch": 2.73,
      "learning_rate": 7.797020408163266e-05,
      "loss": 2.5737,
      "step": 118000
    },
    {
      "FLOPS loss": 0.07194855064153671,
      "L0_d": 938.45,
      "MLM loss": 2.5522868633270264,
      "epoch": 2.74,
      "step": 118499
    },
    {
      "epoch": 2.74,
      "learning_rate": 7.786816326530612e-05,
      "loss": 2.5668,
      "step": 118500
    },
    {
      "FLOPS loss": 0.08181776106357574,
      "L0_d": 1071.69,
      "MLM loss": 2.3746166229248047,
      "epoch": 2.75,
      "step": 118999
    },
    {
      "epoch": 2.75,
      "learning_rate": 7.77661224489796e-05,
      "loss": 2.5687,
      "step": 119000
    },
    {
      "FLOPS loss": 0.07504246383905411,
      "L0_d": 901.45,
      "MLM loss": 2.3082871437072754,
      "epoch": 2.76,
      "step": 119499
    },
    {
      "epoch": 2.76,
      "learning_rate": 7.766408163265306e-05,
      "loss": 2.5651,
      "step": 119500
    },
    {
      "FLOPS loss": 0.07117310911417007,
      "L0_d": 729.27,
      "MLM loss": 2.4190526008605957,
      "epoch": 2.77,
      "step": 119999
    },
    {
      "epoch": 2.77,
      "learning_rate": 7.756224489795918e-05,
      "loss": 2.5636,
      "step": 120000
    },
    {
      "FLOPS loss": 0.057962577790021896,
      "L0_d": 845.22,
      "MLM loss": 2.624579429626465,
      "epoch": 2.78,
      "step": 120499
    },
    {
      "epoch": 2.78,
      "learning_rate": 7.746020408163266e-05,
      "loss": 2.5709,
      "step": 120500
    },
    {
      "FLOPS loss": 0.07766810059547424,
      "L0_d": 963.77,
      "MLM loss": 2.475480079650879,
      "epoch": 2.8,
      "step": 120999
    },
    {
      "epoch": 2.8,
      "learning_rate": 7.735816326530613e-05,
      "loss": 2.5629,
      "step": 121000
    },
    {
      "FLOPS loss": 0.0626688003540039,
      "L0_d": 1204.02,
      "MLM loss": 2.631937026977539,
      "epoch": 2.81,
      "step": 121499
    },
    {
      "epoch": 2.81,
      "learning_rate": 7.725612244897959e-05,
      "loss": 2.5651,
      "step": 121500
    },
    {
      "FLOPS loss": 0.05523882806301117,
      "L0_d": 606.53,
      "MLM loss": 2.1938161849975586,
      "epoch": 2.82,
      "step": 121999
    },
    {
      "epoch": 2.82,
      "learning_rate": 7.715428571428573e-05,
      "loss": 2.5658,
      "step": 122000
    },
    {
      "FLOPS loss": 0.0693681538105011,
      "L0_d": 941.92,
      "MLM loss": 2.559535503387451,
      "epoch": 2.83,
      "step": 122499
    },
    {
      "epoch": 2.83,
      "learning_rate": 7.705224489795919e-05,
      "loss": 2.5655,
      "step": 122500
    },
    {
      "FLOPS loss": 0.059378039091825485,
      "L0_d": 838.56,
      "MLM loss": 2.5249712467193604,
      "epoch": 2.84,
      "step": 122999
    },
    {
      "epoch": 2.84,
      "learning_rate": 7.695020408163266e-05,
      "loss": 2.5654,
      "step": 123000
    },
    {
      "FLOPS loss": 0.060634613037109375,
      "L0_d": 911.55,
      "MLM loss": 2.4866700172424316,
      "epoch": 2.85,
      "step": 123499
    },
    {
      "epoch": 2.85,
      "learning_rate": 7.684816326530613e-05,
      "loss": 2.5592,
      "step": 123500
    },
    {
      "FLOPS loss": 0.07920261472463608,
      "L0_d": 879.38,
      "MLM loss": 2.4505066871643066,
      "epoch": 2.86,
      "step": 123999
    },
    {
      "epoch": 2.86,
      "learning_rate": 7.674653061224491e-05,
      "loss": 2.5641,
      "step": 124000
    },
    {
      "FLOPS loss": 0.07321295887231827,
      "L0_d": 1098.55,
      "MLM loss": 2.629206657409668,
      "epoch": 2.88,
      "step": 124499
    },
    {
      "epoch": 2.88,
      "learning_rate": 7.664448979591838e-05,
      "loss": 2.562,
      "step": 124500
    },
    {
      "FLOPS loss": 0.06171201169490814,
      "L0_d": 766.72,
      "MLM loss": 2.5414156913757324,
      "epoch": 2.89,
      "step": 124999
    },
    {
      "epoch": 2.89,
      "learning_rate": 7.654244897959184e-05,
      "loss": 2.5537,
      "step": 125000
    },
    {
      "FLOPS loss": 0.06980007141828537,
      "L0_d": 844.75,
      "MLM loss": 2.4977681636810303,
      "epoch": 2.9,
      "step": 125499
    },
    {
      "epoch": 2.9,
      "learning_rate": 7.64404081632653e-05,
      "loss": 2.5586,
      "step": 125500
    },
    {
      "FLOPS loss": 0.06336244940757751,
      "L0_d": 737.97,
      "MLM loss": 2.600965976715088,
      "epoch": 2.91,
      "step": 125999
    },
    {
      "epoch": 2.91,
      "learning_rate": 7.633836734693878e-05,
      "loss": 2.5583,
      "step": 126000
    },
    {
      "FLOPS loss": 0.0698259174823761,
      "L0_d": 1158.59,
      "MLM loss": 2.584460496902466,
      "epoch": 2.92,
      "step": 126499
    },
    {
      "epoch": 2.92,
      "learning_rate": 7.623632653061225e-05,
      "loss": 2.5576,
      "step": 126500
    },
    {
      "FLOPS loss": 0.061494987457990646,
      "L0_d": 814.61,
      "MLM loss": 2.4188783168792725,
      "epoch": 2.93,
      "step": 126999
    },
    {
      "epoch": 2.93,
      "learning_rate": 7.613428571428571e-05,
      "loss": 2.5567,
      "step": 127000
    },
    {
      "FLOPS loss": 0.06094374880194664,
      "L0_d": 864.25,
      "MLM loss": 2.5430715084075928,
      "epoch": 2.95,
      "step": 127499
    },
    {
      "epoch": 2.95,
      "learning_rate": 7.603244897959185e-05,
      "loss": 2.5541,
      "step": 127500
    },
    {
      "FLOPS loss": 0.06037312000989914,
      "L0_d": 794.36,
      "MLM loss": 2.5025737285614014,
      "epoch": 2.96,
      "step": 127999
    },
    {
      "epoch": 2.96,
      "learning_rate": 7.593040816326531e-05,
      "loss": 2.5528,
      "step": 128000
    },
    {
      "FLOPS loss": 0.06420066207647324,
      "L0_d": 810.97,
      "MLM loss": 2.546041488647461,
      "epoch": 2.97,
      "step": 128499
    },
    {
      "epoch": 2.97,
      "learning_rate": 7.582836734693878e-05,
      "loss": 2.5518,
      "step": 128500
    },
    {
      "FLOPS loss": 0.058551397174596786,
      "L0_d": 718.36,
      "MLM loss": 2.5001282691955566,
      "epoch": 2.98,
      "step": 128999
    },
    {
      "epoch": 2.98,
      "learning_rate": 7.572632653061224e-05,
      "loss": 2.554,
      "step": 129000
    },
    {
      "FLOPS loss": 0.06057830527424812,
      "L0_d": 939.95,
      "MLM loss": 2.5786666870117188,
      "epoch": 2.99,
      "step": 129499
    },
    {
      "epoch": 2.99,
      "learning_rate": 7.562428571428572e-05,
      "loss": 2.5472,
      "step": 129500
    },
    {
      "FLOPS loss": 0.06409164518117905,
      "L0_d": 823.48,
      "MLM loss": 2.62949800491333,
      "epoch": 3.0,
      "step": 129999
    },
    {
      "epoch": 3.0,
      "learning_rate": 7.552224489795918e-05,
      "loss": 2.5478,
      "step": 130000
    },
    {
      "FLOPS loss": 0.08524499088525772,
      "L0_d": 1072.58,
      "MLM loss": 2.638741970062256,
      "epoch": 3.01,
      "step": 130499
    },
    {
      "epoch": 3.01,
      "learning_rate": 7.54204081632653e-05,
      "loss": 2.5418,
      "step": 130500
    },
    {
      "FLOPS loss": 0.06733393669128418,
      "L0_d": 877.78,
      "MLM loss": 2.2196545600891113,
      "epoch": 3.03,
      "step": 130999
    },
    {
      "epoch": 3.03,
      "learning_rate": 7.531836734693878e-05,
      "loss": 2.5425,
      "step": 131000
    },
    {
      "FLOPS loss": 0.066806860268116,
      "L0_d": 858.41,
      "MLM loss": 2.543398857116699,
      "epoch": 3.04,
      "step": 131499
    },
    {
      "epoch": 3.04,
      "learning_rate": 7.521632653061225e-05,
      "loss": 2.5362,
      "step": 131500
    },
    {
      "FLOPS loss": 0.08286657929420471,
      "L0_d": 1106.33,
      "MLM loss": 2.3611695766448975,
      "epoch": 3.05,
      "step": 131999
    },
    {
      "epoch": 3.05,
      "learning_rate": 7.511428571428571e-05,
      "loss": 2.5454,
      "step": 132000
    },
    {
      "FLOPS loss": 0.0694253146648407,
      "L0_d": 733.84,
      "MLM loss": 2.368508815765381,
      "epoch": 3.06,
      "step": 132499
    },
    {
      "epoch": 3.06,
      "learning_rate": 7.501224489795918e-05,
      "loss": 2.5423,
      "step": 132500
    },
    {
      "FLOPS loss": 0.06028592213988304,
      "L0_d": 672.62,
      "MLM loss": 2.5725514888763428,
      "epoch": 3.07,
      "step": 132999
    },
    {
      "epoch": 3.07,
      "learning_rate": 7.491040816326531e-05,
      "loss": 2.5393,
      "step": 133000
    },
    {
      "FLOPS loss": 0.0720820501446724,
      "L0_d": 866.38,
      "MLM loss": 2.5013206005096436,
      "epoch": 3.08,
      "step": 133499
    },
    {
      "epoch": 3.08,
      "learning_rate": 7.480836734693878e-05,
      "loss": 2.5438,
      "step": 133500
    },
    {
      "FLOPS loss": 0.05794764682650566,
      "L0_d": 776.61,
      "MLM loss": 2.3504562377929688,
      "epoch": 3.1,
      "step": 133999
    },
    {
      "epoch": 3.1,
      "learning_rate": 7.470632653061224e-05,
      "loss": 2.5376,
      "step": 134000
    },
    {
      "FLOPS loss": 0.06785812228918076,
      "L0_d": 818.0,
      "MLM loss": 2.5081028938293457,
      "epoch": 3.11,
      "step": 134499
    },
    {
      "epoch": 3.11,
      "learning_rate": 7.460428571428572e-05,
      "loss": 2.5382,
      "step": 134500
    },
    {
      "FLOPS loss": 0.060359518975019455,
      "L0_d": 841.02,
      "MLM loss": 2.4709324836730957,
      "epoch": 3.12,
      "step": 134999
    },
    {
      "epoch": 3.12,
      "learning_rate": 7.450224489795918e-05,
      "loss": 2.5437,
      "step": 135000
    },
    {
      "FLOPS loss": 0.06685052067041397,
      "L0_d": 615.05,
      "MLM loss": 2.397055149078369,
      "epoch": 3.13,
      "step": 135499
    },
    {
      "epoch": 3.13,
      "learning_rate": 7.440020408163265e-05,
      "loss": 2.5368,
      "step": 135500
    },
    {
      "FLOPS loss": 0.06463826447725296,
      "L0_d": 1002.53,
      "MLM loss": 2.415133476257324,
      "epoch": 3.14,
      "step": 135999
    },
    {
      "epoch": 3.14,
      "learning_rate": 7.429836734693878e-05,
      "loss": 2.5313,
      "step": 136000
    },
    {
      "FLOPS loss": 0.05908415466547012,
      "L0_d": 812.0,
      "MLM loss": 2.544156789779663,
      "epoch": 3.15,
      "step": 136499
    },
    {
      "epoch": 3.15,
      "learning_rate": 7.419632653061225e-05,
      "loss": 2.5364,
      "step": 136500
    },
    {
      "FLOPS loss": 0.05779504403471947,
      "L0_d": 698.34,
      "MLM loss": 2.5028882026672363,
      "epoch": 3.16,
      "step": 136999
    },
    {
      "epoch": 3.16,
      "learning_rate": 7.409428571428571e-05,
      "loss": 2.539,
      "step": 137000
    },
    {
      "FLOPS loss": 0.06667586416006088,
      "L0_d": 976.53,
      "MLM loss": 2.46866512298584,
      "epoch": 3.18,
      "step": 137499
    },
    {
      "epoch": 3.18,
      "learning_rate": 7.399224489795919e-05,
      "loss": 2.5344,
      "step": 137500
    },
    {
      "FLOPS loss": 0.0754542201757431,
      "L0_d": 902.7,
      "MLM loss": 2.4399266242980957,
      "epoch": 3.19,
      "step": 137999
    },
    {
      "epoch": 3.19,
      "learning_rate": 7.389020408163265e-05,
      "loss": 2.5295,
      "step": 138000
    },
    {
      "FLOPS loss": 0.05262158066034317,
      "L0_d": 840.47,
      "MLM loss": 2.5828146934509277,
      "epoch": 3.2,
      "step": 138499
    },
    {
      "epoch": 3.2,
      "learning_rate": 7.378816326530612e-05,
      "loss": 2.532,
      "step": 138500
    },
    {
      "FLOPS loss": 0.06721633672714233,
      "L0_d": 794.09,
      "MLM loss": 2.4749674797058105,
      "epoch": 3.21,
      "step": 138999
    },
    {
      "epoch": 3.21,
      "learning_rate": 7.36861224489796e-05,
      "loss": 2.531,
      "step": 139000
    },
    {
      "FLOPS loss": 0.06448085606098175,
      "L0_d": 764.8,
      "MLM loss": 2.1966323852539062,
      "epoch": 3.22,
      "step": 139499
    },
    {
      "epoch": 3.22,
      "learning_rate": 7.358408163265306e-05,
      "loss": 2.5345,
      "step": 139500
    },
    {
      "FLOPS loss": 0.0996355339884758,
      "L0_d": 1485.3,
      "MLM loss": 2.5816798210144043,
      "epoch": 3.23,
      "step": 139999
    },
    {
      "epoch": 3.23,
      "learning_rate": 7.348224489795918e-05,
      "loss": 2.531,
      "step": 140000
    },
    {
      "FLOPS loss": 0.057376861572265625,
      "L0_d": 625.52,
      "MLM loss": 2.390540599822998,
      "epoch": 3.25,
      "step": 140499
    },
    {
      "epoch": 3.25,
      "learning_rate": 7.338020408163265e-05,
      "loss": 2.5293,
      "step": 140500
    },
    {
      "FLOPS loss": 0.0580020509660244,
      "L0_d": 743.03,
      "MLM loss": 2.636993885040283,
      "epoch": 3.26,
      "step": 140999
    },
    {
      "epoch": 3.26,
      "learning_rate": 7.327816326530613e-05,
      "loss": 2.5274,
      "step": 141000
    },
    {
      "FLOPS loss": 0.06330183893442154,
      "L0_d": 684.05,
      "MLM loss": 2.500195264816284,
      "epoch": 3.27,
      "step": 141499
    },
    {
      "epoch": 3.27,
      "learning_rate": 7.31761224489796e-05,
      "loss": 2.522,
      "step": 141500
    },
    {
      "FLOPS loss": 0.07066886126995087,
      "L0_d": 913.33,
      "MLM loss": 2.485964298248291,
      "epoch": 3.28,
      "step": 141999
    },
    {
      "epoch": 3.28,
      "learning_rate": 7.307408163265307e-05,
      "loss": 2.5279,
      "step": 142000
    },
    {
      "FLOPS loss": 0.06484290212392807,
      "L0_d": 708.78,
      "MLM loss": 2.480302333831787,
      "epoch": 3.29,
      "step": 142499
    },
    {
      "epoch": 3.29,
      "learning_rate": 7.297204081632653e-05,
      "loss": 2.5259,
      "step": 142500
    },
    {
      "FLOPS loss": 0.07047165185213089,
      "L0_d": 1122.41,
      "MLM loss": 2.410188674926758,
      "epoch": 3.3,
      "step": 142999
    },
    {
      "epoch": 3.3,
      "learning_rate": 7.287e-05,
      "loss": 2.5247,
      "step": 143000
    },
    {
      "FLOPS loss": 0.07310927659273148,
      "L0_d": 933.52,
      "MLM loss": 2.3634700775146484,
      "epoch": 3.31,
      "step": 143499
    },
    {
      "epoch": 3.32,
      "learning_rate": 7.276795918367348e-05,
      "loss": 2.5253,
      "step": 143500
    },
    {
      "FLOPS loss": 0.09060615301132202,
      "L0_d": 973.42,
      "MLM loss": 2.3984951972961426,
      "epoch": 3.33,
      "step": 143999
    },
    {
      "epoch": 3.33,
      "learning_rate": 7.266612244897958e-05,
      "loss": 2.521,
      "step": 144000
    },
    {
      "FLOPS loss": 0.07206370681524277,
      "L0_d": 1249.56,
      "MLM loss": 2.3524210453033447,
      "epoch": 3.34,
      "step": 144499
    },
    {
      "epoch": 3.34,
      "learning_rate": 7.256408163265308e-05,
      "loss": 2.5234,
      "step": 144500
    },
    {
      "FLOPS loss": 0.06724249571561813,
      "L0_d": 814.05,
      "MLM loss": 2.5358521938323975,
      "epoch": 3.35,
      "step": 144999
    },
    {
      "epoch": 3.35,
      "learning_rate": 7.246204081632654e-05,
      "loss": 2.5236,
      "step": 145000
    },
    {
      "FLOPS loss": 0.06638313829898834,
      "L0_d": 966.23,
      "MLM loss": 2.3402137756347656,
      "epoch": 3.36,
      "step": 145499
    },
    {
      "epoch": 3.36,
      "learning_rate": 7.236e-05,
      "loss": 2.5146,
      "step": 145500
    },
    {
      "FLOPS loss": 0.05941477045416832,
      "L0_d": 764.61,
      "MLM loss": 2.6106839179992676,
      "epoch": 3.37,
      "step": 145999
    },
    {
      "epoch": 3.37,
      "learning_rate": 7.225816326530613e-05,
      "loss": 2.5194,
      "step": 146000
    },
    {
      "FLOPS loss": 0.08146113157272339,
      "L0_d": 932.0,
      "MLM loss": 2.321835994720459,
      "epoch": 3.38,
      "step": 146499
    },
    {
      "epoch": 3.38,
      "learning_rate": 7.215612244897959e-05,
      "loss": 2.5239,
      "step": 146500
    },
    {
      "FLOPS loss": 0.063787080347538,
      "L0_d": 868.5,
      "MLM loss": 2.428879976272583,
      "epoch": 3.4,
      "step": 146999
    },
    {
      "epoch": 3.4,
      "learning_rate": 7.205408163265307e-05,
      "loss": 2.5189,
      "step": 147000
    },
    {
      "FLOPS loss": 0.0798846185207367,
      "L0_d": 1102.03,
      "MLM loss": 2.4849820137023926,
      "epoch": 3.41,
      "step": 147499
    },
    {
      "epoch": 3.41,
      "learning_rate": 7.195204081632653e-05,
      "loss": 2.5158,
      "step": 147500
    },
    {
      "FLOPS loss": 0.060002878308296204,
      "L0_d": 809.48,
      "MLM loss": 2.5779366493225098,
      "epoch": 3.42,
      "step": 147999
    },
    {
      "epoch": 3.42,
      "learning_rate": 7.185040816326531e-05,
      "loss": 2.5199,
      "step": 148000
    },
    {
      "FLOPS loss": 0.07338414341211319,
      "L0_d": 946.84,
      "MLM loss": 2.385986089706421,
      "epoch": 3.43,
      "step": 148499
    },
    {
      "epoch": 3.43,
      "learning_rate": 7.174836734693879e-05,
      "loss": 2.519,
      "step": 148500
    },
    {
      "FLOPS loss": 0.07006336748600006,
      "L0_d": 939.03,
      "MLM loss": 2.53456711769104,
      "epoch": 3.44,
      "step": 148999
    },
    {
      "epoch": 3.44,
      "learning_rate": 7.164632653061225e-05,
      "loss": 2.5185,
      "step": 149000
    },
    {
      "FLOPS loss": 0.0663251280784607,
      "L0_d": 868.73,
      "MLM loss": 2.4946329593658447,
      "epoch": 3.45,
      "step": 149499
    },
    {
      "epoch": 3.45,
      "learning_rate": 7.154428571428572e-05,
      "loss": 2.5158,
      "step": 149500
    },
    {
      "FLOPS loss": 0.05879246070981026,
      "L0_d": 841.36,
      "MLM loss": 2.4197614192962646,
      "epoch": 3.47,
      "step": 149999
    },
    {
      "epoch": 3.47,
      "learning_rate": 7.144224489795918e-05,
      "loss": 2.516,
      "step": 150000
    },
    {
      "FLOPS loss": 0.06893225014209747,
      "L0_d": 1096.61,
      "MLM loss": 2.459559202194214,
      "epoch": 3.48,
      "step": 150499
    },
    {
      "epoch": 3.48,
      "learning_rate": 7.134020408163266e-05,
      "loss": 2.516,
      "step": 150500
    },
    {
      "FLOPS loss": 0.06730066984891891,
      "L0_d": 1061.92,
      "MLM loss": 2.5642447471618652,
      "epoch": 3.49,
      "step": 150999
    },
    {
      "epoch": 3.49,
      "learning_rate": 7.123836734693878e-05,
      "loss": 2.5121,
      "step": 151000
    },
    {
      "FLOPS loss": 0.0661296620965004,
      "L0_d": 755.22,
      "MLM loss": 2.353799819946289,
      "epoch": 3.5,
      "step": 151499
    },
    {
      "epoch": 3.5,
      "learning_rate": 7.113632653061225e-05,
      "loss": 2.5118,
      "step": 151500
    },
    {
      "FLOPS loss": 0.07206888496875763,
      "L0_d": 981.41,
      "MLM loss": 2.3166513442993164,
      "epoch": 3.51,
      "step": 151999
    },
    {
      "epoch": 3.51,
      "learning_rate": 7.103428571428572e-05,
      "loss": 2.5139,
      "step": 152000
    },
    {
      "FLOPS loss": 0.06955097615718842,
      "L0_d": 1074.69,
      "MLM loss": 2.5231192111968994,
      "epoch": 3.52,
      "step": 152499
    },
    {
      "epoch": 3.52,
      "learning_rate": 7.093224489795919e-05,
      "loss": 2.5161,
      "step": 152500
    },
    {
      "FLOPS loss": 0.06898514181375504,
      "L0_d": 981.22,
      "MLM loss": 2.3581669330596924,
      "epoch": 3.53,
      "step": 152999
    },
    {
      "epoch": 3.53,
      "learning_rate": 7.083040816326531e-05,
      "loss": 2.5105,
      "step": 153000
    },
    {
      "FLOPS loss": 0.061563555151224136,
      "L0_d": 751.62,
      "MLM loss": 2.4208712577819824,
      "epoch": 3.55,
      "step": 153499
    },
    {
      "epoch": 3.55,
      "learning_rate": 7.072836734693879e-05,
      "loss": 2.5089,
      "step": 153500
    },
    {
      "FLOPS loss": 0.0783623531460762,
      "L0_d": 980.56,
      "MLM loss": 2.6533374786376953,
      "epoch": 3.56,
      "step": 153999
    },
    {
      "epoch": 3.56,
      "learning_rate": 7.062632653061225e-05,
      "loss": 2.5099,
      "step": 154000
    },
    {
      "FLOPS loss": 0.06331176310777664,
      "L0_d": 770.83,
      "MLM loss": 2.3688929080963135,
      "epoch": 3.57,
      "step": 154499
    },
    {
      "epoch": 3.57,
      "learning_rate": 7.052428571428572e-05,
      "loss": 2.508,
      "step": 154500
    },
    {
      "FLOPS loss": 0.06306309252977371,
      "L0_d": 811.0,
      "MLM loss": 2.386118173599243,
      "epoch": 3.58,
      "step": 154999
    },
    {
      "epoch": 3.58,
      "learning_rate": 7.042224489795918e-05,
      "loss": 2.5072,
      "step": 155000
    },
    {
      "FLOPS loss": 0.06929371505975723,
      "L0_d": 981.34,
      "MLM loss": 2.2670648097991943,
      "epoch": 3.59,
      "step": 155499
    },
    {
      "epoch": 3.59,
      "learning_rate": 7.032020408163266e-05,
      "loss": 2.5088,
      "step": 155500
    },
    {
      "FLOPS loss": 0.07412055879831314,
      "L0_d": 1162.94,
      "MLM loss": 2.541748523712158,
      "epoch": 3.6,
      "step": 155999
    },
    {
      "epoch": 3.6,
      "learning_rate": 7.021836734693877e-05,
      "loss": 2.5025,
      "step": 156000
    },
    {
      "FLOPS loss": 0.06029690429568291,
      "L0_d": 692.28,
      "MLM loss": 2.228192090988159,
      "epoch": 3.62,
      "step": 156499
    },
    {
      "epoch": 3.62,
      "learning_rate": 7.011632653061226e-05,
      "loss": 2.5077,
      "step": 156500
    },
    {
      "FLOPS loss": 0.07891872525215149,
      "L0_d": 1194.77,
      "MLM loss": 2.641678810119629,
      "epoch": 3.63,
      "step": 156999
    },
    {
      "epoch": 3.63,
      "learning_rate": 7.001428571428572e-05,
      "loss": 2.5104,
      "step": 157000
    },
    {
      "FLOPS loss": 0.07251445204019547,
      "L0_d": 1016.83,
      "MLM loss": 2.535757303237915,
      "epoch": 3.64,
      "step": 157499
    },
    {
      "epoch": 3.64,
      "learning_rate": 6.991224489795919e-05,
      "loss": 2.5036,
      "step": 157500
    },
    {
      "FLOPS loss": 0.06319019943475723,
      "L0_d": 923.69,
      "MLM loss": 2.5910186767578125,
      "epoch": 3.65,
      "step": 157999
    },
    {
      "epoch": 3.65,
      "learning_rate": 6.981020408163265e-05,
      "loss": 2.5074,
      "step": 158000
    },
    {
      "FLOPS loss": 0.0667872279882431,
      "L0_d": 888.39,
      "MLM loss": 2.5367581844329834,
      "epoch": 3.66,
      "step": 158499
    },
    {
      "epoch": 3.66,
      "learning_rate": 6.970816326530613e-05,
      "loss": 2.5059,
      "step": 158500
    },
    {
      "FLOPS loss": 0.07012256979942322,
      "L0_d": 811.02,
      "MLM loss": 2.6520166397094727,
      "epoch": 3.67,
      "step": 158999
    },
    {
      "epoch": 3.67,
      "learning_rate": 6.960632653061225e-05,
      "loss": 2.5025,
      "step": 159000
    },
    {
      "FLOPS loss": 0.06510796397924423,
      "L0_d": 698.33,
      "MLM loss": 2.357128858566284,
      "epoch": 3.68,
      "step": 159499
    },
    {
      "epoch": 3.68,
      "learning_rate": 6.950428571428572e-05,
      "loss": 2.5018,
      "step": 159500
    },
    {
      "FLOPS loss": 0.06573283672332764,
      "L0_d": 768.05,
      "MLM loss": 2.6148369312286377,
      "epoch": 3.7,
      "step": 159999
    },
    {
      "epoch": 3.7,
      "learning_rate": 6.94022448979592e-05,
      "loss": 2.503,
      "step": 160000
    },
    {
      "FLOPS loss": 0.07524820417165756,
      "L0_d": 890.48,
      "MLM loss": 2.1664788722991943,
      "epoch": 3.71,
      "step": 160499
    },
    {
      "epoch": 3.71,
      "learning_rate": 6.930020408163266e-05,
      "loss": 2.505,
      "step": 160500
    },
    {
      "FLOPS loss": 0.08785947412252426,
      "L0_d": 948.19,
      "MLM loss": 2.360063076019287,
      "epoch": 3.72,
      "step": 160999
    },
    {
      "epoch": 3.72,
      "learning_rate": 6.919816326530612e-05,
      "loss": 2.4969,
      "step": 161000
    },
    {
      "FLOPS loss": 0.056488048285245895,
      "L0_d": 841.88,
      "MLM loss": 2.2539350986480713,
      "epoch": 3.73,
      "step": 161499
    },
    {
      "epoch": 3.73,
      "learning_rate": 6.909612244897959e-05,
      "loss": 2.4992,
      "step": 161500
    },
    {
      "FLOPS loss": 0.06215016916394234,
      "L0_d": 829.36,
      "MLM loss": 2.4596550464630127,
      "epoch": 3.74,
      "step": 161999
    },
    {
      "epoch": 3.74,
      "learning_rate": 6.899408163265307e-05,
      "loss": 2.499,
      "step": 162000
    },
    {
      "FLOPS loss": 0.05874626338481903,
      "L0_d": 774.12,
      "MLM loss": 2.417466640472412,
      "epoch": 3.75,
      "step": 162499
    },
    {
      "epoch": 3.75,
      "learning_rate": 6.889224489795919e-05,
      "loss": 2.4987,
      "step": 162500
    },
    {
      "FLOPS loss": 0.06776445358991623,
      "L0_d": 798.8,
      "MLM loss": 2.3213367462158203,
      "epoch": 3.77,
      "step": 162999
    },
    {
      "epoch": 3.77,
      "learning_rate": 6.879020408163265e-05,
      "loss": 2.4924,
      "step": 163000
    },
    {
      "FLOPS loss": 0.06742731481790543,
      "L0_d": 967.53,
      "MLM loss": 2.482436418533325,
      "epoch": 3.78,
      "step": 163499
    },
    {
      "epoch": 3.78,
      "learning_rate": 6.868816326530613e-05,
      "loss": 2.4984,
      "step": 163500
    },
    {
      "FLOPS loss": 0.06478297710418701,
      "L0_d": 887.28,
      "MLM loss": 2.4176084995269775,
      "epoch": 3.79,
      "step": 163999
    },
    {
      "epoch": 3.79,
      "learning_rate": 6.85861224489796e-05,
      "loss": 2.4956,
      "step": 164000
    },
    {
      "FLOPS loss": 0.06537578254938126,
      "L0_d": 739.81,
      "MLM loss": 2.1967082023620605,
      "epoch": 3.8,
      "step": 164499
    },
    {
      "epoch": 3.8,
      "learning_rate": 6.848408163265306e-05,
      "loss": 2.4918,
      "step": 164500
    },
    {
      "FLOPS loss": 0.06188281252980232,
      "L0_d": 738.86,
      "MLM loss": 2.5005993843078613,
      "epoch": 3.81,
      "step": 164999
    },
    {
      "epoch": 3.81,
      "learning_rate": 6.838204081632653e-05,
      "loss": 2.4994,
      "step": 165000
    },
    {
      "FLOPS loss": 0.07663436233997345,
      "L0_d": 922.31,
      "MLM loss": 2.429055690765381,
      "epoch": 3.82,
      "step": 165499
    },
    {
      "epoch": 3.82,
      "learning_rate": 6.828020408163266e-05,
      "loss": 2.4945,
      "step": 165500
    },
    {
      "FLOPS loss": 0.0570794902741909,
      "L0_d": 576.69,
      "MLM loss": 2.1654653549194336,
      "epoch": 3.83,
      "step": 165999
    },
    {
      "epoch": 3.83,
      "learning_rate": 6.817816326530612e-05,
      "loss": 2.4976,
      "step": 166000
    },
    {
      "FLOPS loss": 0.07364285737276077,
      "L0_d": 984.09,
      "MLM loss": 2.361241579055786,
      "epoch": 3.85,
      "step": 166499
    },
    {
      "epoch": 3.85,
      "learning_rate": 6.807612244897959e-05,
      "loss": 2.4919,
      "step": 166500
    },
    {
      "FLOPS loss": 0.06903867423534393,
      "L0_d": 704.95,
      "MLM loss": 2.1658754348754883,
      "epoch": 3.86,
      "step": 166999
    },
    {
      "epoch": 3.86,
      "learning_rate": 6.797408163265307e-05,
      "loss": 2.49,
      "step": 167000
    },
    {
      "FLOPS loss": 0.05657253786921501,
      "L0_d": 679.52,
      "MLM loss": 2.4498727321624756,
      "epoch": 3.87,
      "step": 167499
    },
    {
      "epoch": 3.87,
      "learning_rate": 6.787204081632653e-05,
      "loss": 2.4901,
      "step": 167500
    },
    {
      "FLOPS loss": 0.0767899677157402,
      "L0_d": 971.0,
      "MLM loss": 2.4667670726776123,
      "epoch": 3.88,
      "step": 167999
    },
    {
      "epoch": 3.88,
      "learning_rate": 6.777020408163267e-05,
      "loss": 2.4973,
      "step": 168000
    },
    {
      "FLOPS loss": 0.05883647873997688,
      "L0_d": 711.62,
      "MLM loss": 2.3792476654052734,
      "epoch": 3.89,
      "step": 168499
    },
    {
      "epoch": 3.89,
      "learning_rate": 6.766816326530613e-05,
      "loss": 2.4897,
      "step": 168500
    },
    {
      "FLOPS loss": 0.06558485329151154,
      "L0_d": 781.28,
      "MLM loss": 2.2171566486358643,
      "epoch": 3.9,
      "step": 168999
    },
    {
      "epoch": 3.9,
      "learning_rate": 6.75661224489796e-05,
      "loss": 2.4888,
      "step": 169000
    },
    {
      "FLOPS loss": 0.06537246704101562,
      "L0_d": 792.88,
      "MLM loss": 2.549509286880493,
      "epoch": 3.92,
      "step": 169499
    },
    {
      "epoch": 3.92,
      "learning_rate": 6.746408163265306e-05,
      "loss": 2.4841,
      "step": 169500
    },
    {
      "FLOPS loss": 0.06672216951847076,
      "L0_d": 1022.78,
      "MLM loss": 2.375237464904785,
      "epoch": 3.93,
      "step": 169999
    },
    {
      "epoch": 3.93,
      "learning_rate": 6.736224489795918e-05,
      "loss": 2.4857,
      "step": 170000
    },
    {
      "FLOPS loss": 0.06473018229007721,
      "L0_d": 891.12,
      "MLM loss": 2.572554349899292,
      "epoch": 3.94,
      "step": 170499
    },
    {
      "epoch": 3.94,
      "learning_rate": 6.726020408163266e-05,
      "loss": 2.4904,
      "step": 170500
    },
    {
      "FLOPS loss": 0.06366889923810959,
      "L0_d": 627.78,
      "MLM loss": 2.4447898864746094,
      "epoch": 3.95,
      "step": 170999
    },
    {
      "epoch": 3.95,
      "learning_rate": 6.715816326530612e-05,
      "loss": 2.4877,
      "step": 171000
    },
    {
      "FLOPS loss": 0.06390350311994553,
      "L0_d": 1048.25,
      "MLM loss": 2.48956561088562,
      "epoch": 3.96,
      "step": 171499
    },
    {
      "epoch": 3.96,
      "learning_rate": 6.70561224489796e-05,
      "loss": 2.4863,
      "step": 171500
    },
    {
      "FLOPS loss": 0.07498548924922943,
      "L0_d": 1041.67,
      "MLM loss": 2.488102674484253,
      "epoch": 3.97,
      "step": 171999
    },
    {
      "epoch": 3.97,
      "learning_rate": 6.695428571428571e-05,
      "loss": 2.4854,
      "step": 172000
    },
    {
      "FLOPS loss": 0.08240804076194763,
      "L0_d": 914.86,
      "MLM loss": 2.6523680686950684,
      "epoch": 3.98,
      "step": 172499
    },
    {
      "epoch": 3.98,
      "learning_rate": 6.685224489795919e-05,
      "loss": 2.484,
      "step": 172500
    },
    {
      "FLOPS loss": 0.06099969893693924,
      "L0_d": 978.86,
      "MLM loss": 2.64540433883667,
      "epoch": 4.0,
      "step": 172999
    },
    {
      "epoch": 4.0,
      "learning_rate": 6.675020408163267e-05,
      "loss": 2.4841,
      "step": 173000
    },
    {
      "FLOPS loss": 0.06223977357149124,
      "L0_d": 757.42,
      "MLM loss": 2.6576099395751953,
      "epoch": 4.01,
      "step": 173499
    },
    {
      "epoch": 4.01,
      "learning_rate": 6.664816326530613e-05,
      "loss": 2.4863,
      "step": 173500
    },
    {
      "FLOPS loss": 0.06578174233436584,
      "L0_d": 1087.84,
      "MLM loss": 2.409296989440918,
      "epoch": 4.02,
      "step": 173999
    },
    {
      "epoch": 4.02,
      "learning_rate": 6.65461224489796e-05,
      "loss": 2.4807,
      "step": 174000
    },
    {
      "FLOPS loss": 0.07225192338228226,
      "L0_d": 849.7,
      "MLM loss": 2.235647678375244,
      "epoch": 4.03,
      "step": 174499
    },
    {
      "epoch": 4.03,
      "learning_rate": 6.644408163265306e-05,
      "loss": 2.4749,
      "step": 174500
    },
    {
      "FLOPS loss": 0.06585001945495605,
      "L0_d": 951.81,
      "MLM loss": 2.3266072273254395,
      "epoch": 4.04,
      "step": 174999
    },
    {
      "epoch": 4.04,
      "learning_rate": 6.634224489795918e-05,
      "loss": 2.4813,
      "step": 175000
    },
    {
      "FLOPS loss": 0.06246158108115196,
      "L0_d": 697.7,
      "MLM loss": 2.5024335384368896,
      "epoch": 4.05,
      "step": 175499
    },
    {
      "epoch": 4.05,
      "learning_rate": 6.624020408163265e-05,
      "loss": 2.4757,
      "step": 175500
    },
    {
      "FLOPS loss": 0.071586973965168,
      "L0_d": 943.44,
      "MLM loss": 2.432471752166748,
      "epoch": 4.07,
      "step": 175999
    },
    {
      "epoch": 4.07,
      "learning_rate": 6.613816326530612e-05,
      "loss": 2.4804,
      "step": 176000
    },
    {
      "FLOPS loss": 0.07770819962024689,
      "L0_d": 967.03,
      "MLM loss": 2.4807729721069336,
      "epoch": 4.08,
      "step": 176499
    },
    {
      "epoch": 4.08,
      "learning_rate": 6.60361224489796e-05,
      "loss": 2.4792,
      "step": 176500
    },
    {
      "FLOPS loss": 0.06387147307395935,
      "L0_d": 819.58,
      "MLM loss": 2.364795207977295,
      "epoch": 4.09,
      "step": 176999
    },
    {
      "epoch": 4.09,
      "learning_rate": 6.593408163265307e-05,
      "loss": 2.4746,
      "step": 177000
    },
    {
      "FLOPS loss": 0.061465099453926086,
      "L0_d": 921.8,
      "MLM loss": 2.4672608375549316,
      "epoch": 4.1,
      "step": 177499
    },
    {
      "epoch": 4.1,
      "learning_rate": 6.583224489795919e-05,
      "loss": 2.4807,
      "step": 177500
    },
    {
      "FLOPS loss": 0.06799270212650299,
      "L0_d": 828.22,
      "MLM loss": 2.5122363567352295,
      "epoch": 4.11,
      "step": 177999
    },
    {
      "epoch": 4.11,
      "learning_rate": 6.573020408163265e-05,
      "loss": 2.4813,
      "step": 178000
    },
    {
      "FLOPS loss": 0.06093809753656387,
      "L0_d": 871.48,
      "MLM loss": 2.2750113010406494,
      "epoch": 4.12,
      "step": 178499
    },
    {
      "epoch": 4.12,
      "learning_rate": 6.562816326530613e-05,
      "loss": 2.477,
      "step": 178500
    },
    {
      "FLOPS loss": 0.05775380879640579,
      "L0_d": 925.05,
      "MLM loss": 2.4610414505004883,
      "epoch": 4.14,
      "step": 178999
    },
    {
      "epoch": 4.14,
      "learning_rate": 6.55261224489796e-05,
      "loss": 2.478,
      "step": 179000
    },
    {
      "FLOPS loss": 0.05679761990904808,
      "L0_d": 599.97,
      "MLM loss": 2.371889114379883,
      "epoch": 4.15,
      "step": 179499
    },
    {
      "epoch": 4.15,
      "learning_rate": 6.542408163265307e-05,
      "loss": 2.4746,
      "step": 179500
    },
    {
      "FLOPS loss": 0.07693447172641754,
      "L0_d": 914.91,
      "MLM loss": 2.4633376598358154,
      "epoch": 4.16,
      "step": 179999
    },
    {
      "epoch": 4.16,
      "learning_rate": 6.532204081632654e-05,
      "loss": 2.4724,
      "step": 180000
    },
    {
      "FLOPS loss": 0.086503766477108,
      "L0_d": 987.11,
      "MLM loss": 2.541353464126587,
      "epoch": 4.17,
      "step": 180499
    },
    {
      "epoch": 4.17,
      "learning_rate": 6.522e-05,
      "loss": 2.4715,
      "step": 180500
    },
    {
      "FLOPS loss": 0.07103153318166733,
      "L0_d": 972.25,
      "MLM loss": 2.412931442260742,
      "epoch": 4.18,
      "step": 180999
    },
    {
      "epoch": 4.18,
      "learning_rate": 6.511795918367347e-05,
      "loss": 2.4691,
      "step": 181000
    },
    {
      "FLOPS loss": 0.0597836896777153,
      "L0_d": 771.88,
      "MLM loss": 2.3725850582122803,
      "epoch": 4.19,
      "step": 181499
    },
    {
      "epoch": 4.19,
      "learning_rate": 6.50161224489796e-05,
      "loss": 2.4685,
      "step": 181500
    },
    {
      "FLOPS loss": 0.06964124739170074,
      "L0_d": 835.75,
      "MLM loss": 2.400667190551758,
      "epoch": 4.2,
      "step": 181999
    },
    {
      "epoch": 4.2,
      "learning_rate": 6.491428571428572e-05,
      "loss": 2.4715,
      "step": 182000
    },
    {
      "FLOPS loss": 0.07146003842353821,
      "L0_d": 1029.83,
      "MLM loss": 2.122744560241699,
      "epoch": 4.22,
      "step": 182499
    },
    {
      "epoch": 4.22,
      "learning_rate": 6.481224489795919e-05,
      "loss": 2.4713,
      "step": 182500
    },
    {
      "FLOPS loss": 0.06947391480207443,
      "L0_d": 910.61,
      "MLM loss": 2.2833240032196045,
      "epoch": 4.23,
      "step": 182999
    },
    {
      "epoch": 4.23,
      "learning_rate": 6.471020408163265e-05,
      "loss": 2.4702,
      "step": 183000
    },
    {
      "FLOPS loss": 0.06873425096273422,
      "L0_d": 746.22,
      "MLM loss": 2.384280204772949,
      "epoch": 4.24,
      "step": 183499
    },
    {
      "epoch": 4.24,
      "learning_rate": 6.460816326530612e-05,
      "loss": 2.4663,
      "step": 183500
    },
    {
      "FLOPS loss": 0.07945630699396133,
      "L0_d": 962.33,
      "MLM loss": 2.3059887886047363,
      "epoch": 4.25,
      "step": 183999
    },
    {
      "epoch": 4.25,
      "learning_rate": 6.45061224489796e-05,
      "loss": 2.4725,
      "step": 184000
    },
    {
      "FLOPS loss": 0.07592689990997314,
      "L0_d": 860.5,
      "MLM loss": 2.4063377380371094,
      "epoch": 4.26,
      "step": 184499
    },
    {
      "epoch": 4.26,
      "learning_rate": 6.440408163265307e-05,
      "loss": 2.466,
      "step": 184500
    },
    {
      "FLOPS loss": 0.0667710080742836,
      "L0_d": 990.41,
      "MLM loss": 2.3277931213378906,
      "epoch": 4.27,
      "step": 184999
    },
    {
      "epoch": 4.27,
      "learning_rate": 6.430204081632654e-05,
      "loss": 2.4706,
      "step": 185000
    },
    {
      "FLOPS loss": 0.059676315635442734,
      "L0_d": 775.2,
      "MLM loss": 2.447890281677246,
      "epoch": 4.29,
      "step": 185499
    },
    {
      "epoch": 4.29,
      "learning_rate": 6.42e-05,
      "loss": 2.4713,
      "step": 185500
    },
    {
      "FLOPS loss": 0.04730994626879692,
      "L0_d": 562.97,
      "MLM loss": 2.41772723197937,
      "epoch": 4.3,
      "step": 185999
    },
    {
      "epoch": 4.3,
      "learning_rate": 6.409816326530612e-05,
      "loss": 2.4665,
      "step": 186000
    },
    {
      "FLOPS loss": 0.07394890487194061,
      "L0_d": 839.94,
      "MLM loss": 2.3134212493896484,
      "epoch": 4.31,
      "step": 186499
    },
    {
      "epoch": 4.31,
      "learning_rate": 6.399612244897959e-05,
      "loss": 2.466,
      "step": 186500
    },
    {
      "FLOPS loss": 0.06606470048427582,
      "L0_d": 914.36,
      "MLM loss": 2.381443500518799,
      "epoch": 4.32,
      "step": 186999
    },
    {
      "epoch": 4.32,
      "learning_rate": 6.389428571428572e-05,
      "loss": 2.4618,
      "step": 187000
    },
    {
      "FLOPS loss": 0.05293360352516174,
      "L0_d": 687.72,
      "MLM loss": 2.465054512023926,
      "epoch": 4.33,
      "step": 187499
    },
    {
      "epoch": 4.33,
      "learning_rate": 6.379224489795919e-05,
      "loss": 2.4642,
      "step": 187500
    },
    {
      "FLOPS loss": 0.07550951093435287,
      "L0_d": 1098.16,
      "MLM loss": 2.3925881385803223,
      "epoch": 4.34,
      "step": 187999
    },
    {
      "epoch": 4.34,
      "learning_rate": 6.369020408163265e-05,
      "loss": 2.4604,
      "step": 188000
    },
    {
      "FLOPS loss": 0.07207269966602325,
      "L0_d": 883.66,
      "MLM loss": 2.3470559120178223,
      "epoch": 4.35,
      "step": 188499
    },
    {
      "epoch": 4.35,
      "learning_rate": 6.358816326530612e-05,
      "loss": 2.4648,
      "step": 188500
    },
    {
      "FLOPS loss": 0.06928683072328568,
      "L0_d": 1007.05,
      "MLM loss": 2.087768077850342,
      "epoch": 4.37,
      "step": 188999
    },
    {
      "epoch": 4.37,
      "learning_rate": 6.348632653061225e-05,
      "loss": 2.4638,
      "step": 189000
    },
    {
      "FLOPS loss": 0.07827793061733246,
      "L0_d": 1170.62,
      "MLM loss": 2.2623305320739746,
      "epoch": 4.38,
      "step": 189499
    },
    {
      "epoch": 4.38,
      "learning_rate": 6.338428571428571e-05,
      "loss": 2.4648,
      "step": 189500
    },
    {
      "FLOPS loss": 0.07797204703092575,
      "L0_d": 805.08,
      "MLM loss": 2.424680233001709,
      "epoch": 4.39,
      "step": 189999
    },
    {
      "epoch": 4.39,
      "learning_rate": 6.328224489795919e-05,
      "loss": 2.4647,
      "step": 190000
    },
    {
      "FLOPS loss": 0.07720698416233063,
      "L0_d": 1047.55,
      "MLM loss": 2.3045454025268555,
      "epoch": 4.4,
      "step": 190499
    },
    {
      "epoch": 4.4,
      "learning_rate": 6.318020408163266e-05,
      "loss": 2.4631,
      "step": 190500
    },
    {
      "FLOPS loss": 0.060575131326913834,
      "L0_d": 751.7,
      "MLM loss": 2.3671631813049316,
      "epoch": 4.41,
      "step": 190999
    },
    {
      "epoch": 4.41,
      "learning_rate": 6.307836734693878e-05,
      "loss": 2.4619,
      "step": 191000
    },
    {
      "FLOPS loss": 0.06604286283254623,
      "L0_d": 847.69,
      "MLM loss": 2.46954083442688,
      "epoch": 4.42,
      "step": 191499
    },
    {
      "epoch": 4.42,
      "learning_rate": 6.297632653061226e-05,
      "loss": 2.4616,
      "step": 191500
    },
    {
      "FLOPS loss": 0.04703153297305107,
      "L0_d": 675.36,
      "MLM loss": 2.5719079971313477,
      "epoch": 4.44,
      "step": 191999
    },
    {
      "epoch": 4.44,
      "learning_rate": 6.287428571428572e-05,
      "loss": 2.4656,
      "step": 192000
    },
    {
      "FLOPS loss": 0.060653429478406906,
      "L0_d": 787.83,
      "MLM loss": 2.260763645172119,
      "epoch": 4.45,
      "step": 192499
    },
    {
      "epoch": 4.45,
      "learning_rate": 6.277224489795919e-05,
      "loss": 2.4611,
      "step": 192500
    },
    {
      "FLOPS loss": 0.06292744725942612,
      "L0_d": 938.81,
      "MLM loss": 2.5848846435546875,
      "epoch": 4.46,
      "step": 192999
    },
    {
      "epoch": 4.46,
      "learning_rate": 6.26704081632653e-05,
      "loss": 2.4647,
      "step": 193000
    },
    {
      "FLOPS loss": 0.0530436746776104,
      "L0_d": 707.17,
      "MLM loss": 2.472729206085205,
      "epoch": 4.47,
      "step": 193499
    },
    {
      "epoch": 4.47,
      "learning_rate": 6.256836734693878e-05,
      "loss": 2.4558,
      "step": 193500
    },
    {
      "FLOPS loss": 0.07420609146356583,
      "L0_d": 895.44,
      "MLM loss": 2.316894292831421,
      "epoch": 4.48,
      "step": 193999
    },
    {
      "epoch": 4.48,
      "learning_rate": 6.246632653061225e-05,
      "loss": 2.4583,
      "step": 194000
    },
    {
      "FLOPS loss": 0.0655120238661766,
      "L0_d": 853.75,
      "MLM loss": 2.3448705673217773,
      "epoch": 4.49,
      "step": 194499
    },
    {
      "epoch": 4.49,
      "learning_rate": 6.236428571428571e-05,
      "loss": 2.4601,
      "step": 194500
    },
    {
      "FLOPS loss": 0.07075578719377518,
      "L0_d": 1198.17,
      "MLM loss": 2.44946551322937,
      "epoch": 4.5,
      "step": 194999
    },
    {
      "epoch": 4.5,
      "learning_rate": 6.226224489795919e-05,
      "loss": 2.4558,
      "step": 195000
    },
    {
      "FLOPS loss": 0.07796045392751694,
      "L0_d": 940.88,
      "MLM loss": 2.329540491104126,
      "epoch": 4.52,
      "step": 195499
    },
    {
      "epoch": 4.52,
      "learning_rate": 6.216020408163266e-05,
      "loss": 2.4581,
      "step": 195500
    },
    {
      "FLOPS loss": 0.06250694394111633,
      "L0_d": 813.84,
      "MLM loss": 2.342804431915283,
      "epoch": 4.53,
      "step": 195999
    },
    {
      "epoch": 4.53,
      "learning_rate": 6.205836734693878e-05,
      "loss": 2.4541,
      "step": 196000
    },
    {
      "FLOPS loss": 0.059746935963630676,
      "L0_d": 639.66,
      "MLM loss": 2.2123336791992188,
      "epoch": 4.54,
      "step": 196499
    },
    {
      "epoch": 4.54,
      "learning_rate": 6.195632653061226e-05,
      "loss": 2.4565,
      "step": 196500
    },
    {
      "FLOPS loss": 0.05932118371129036,
      "L0_d": 833.75,
      "MLM loss": 2.4198126792907715,
      "epoch": 4.55,
      "step": 196999
    },
    {
      "epoch": 4.55,
      "learning_rate": 6.185428571428572e-05,
      "loss": 2.4549,
      "step": 197000
    },
    {
      "FLOPS loss": 0.06285431236028671,
      "L0_d": 736.86,
      "MLM loss": 2.3848378658294678,
      "epoch": 4.56,
      "step": 197499
    },
    {
      "epoch": 4.56,
      "learning_rate": 6.175224489795919e-05,
      "loss": 2.4597,
      "step": 197500
    },
    {
      "FLOPS loss": 0.07099881768226624,
      "L0_d": 950.34,
      "MLM loss": 2.4628701210021973,
      "epoch": 4.57,
      "step": 197999
    },
    {
      "epoch": 4.57,
      "learning_rate": 6.16504081632653e-05,
      "loss": 2.4559,
      "step": 198000
    },
    {
      "FLOPS loss": 0.05927359312772751,
      "L0_d": 951.58,
      "MLM loss": 2.459381103515625,
      "epoch": 4.59,
      "step": 198499
    },
    {
      "epoch": 4.59,
      "learning_rate": 6.154836734693877e-05,
      "loss": 2.4561,
      "step": 198500
    },
    {
      "FLOPS loss": 0.07203791290521622,
      "L0_d": 927.34,
      "MLM loss": 2.253113269805908,
      "epoch": 4.6,
      "step": 198999
    },
    {
      "epoch": 4.6,
      "learning_rate": 6.144632653061225e-05,
      "loss": 2.4606,
      "step": 199000
    },
    {
      "FLOPS loss": 0.055465567857027054,
      "L0_d": 823.09,
      "MLM loss": 2.26115345954895,
      "epoch": 4.61,
      "step": 199499
    },
    {
      "epoch": 4.61,
      "learning_rate": 6.134428571428573e-05,
      "loss": 2.449,
      "step": 199500
    },
    {
      "FLOPS loss": 0.07690739631652832,
      "L0_d": 980.25,
      "MLM loss": 2.428745746612549,
      "epoch": 4.62,
      "step": 199999
    },
    {
      "epoch": 4.62,
      "learning_rate": 6.124224489795919e-05,
      "loss": 2.4557,
      "step": 200000
    },
    {
      "FLOPS loss": 0.0735900029540062,
      "L0_d": 1076.41,
      "MLM loss": 2.3314719200134277,
      "epoch": 4.63,
      "step": 200499
    },
    {
      "epoch": 4.63,
      "learning_rate": 6.114020408163266e-05,
      "loss": 2.4534,
      "step": 200500
    },
    {
      "FLOPS loss": 0.06763220578432083,
      "L0_d": 796.91,
      "MLM loss": 2.3352127075195312,
      "epoch": 4.64,
      "step": 200999
    },
    {
      "epoch": 4.64,
      "learning_rate": 6.103836734693878e-05,
      "loss": 2.4505,
      "step": 201000
    },
    {
      "FLOPS loss": 0.05880173668265343,
      "L0_d": 761.19,
      "MLM loss": 2.3325793743133545,
      "epoch": 4.65,
      "step": 201499
    },
    {
      "epoch": 4.65,
      "learning_rate": 6.0936326530612256e-05,
      "loss": 2.4529,
      "step": 201500
    },
    {
      "FLOPS loss": 0.07583904266357422,
      "L0_d": 1133.97,
      "MLM loss": 2.347179889678955,
      "epoch": 4.67,
      "step": 201999
    },
    {
      "epoch": 4.67,
      "learning_rate": 6.083428571428572e-05,
      "loss": 2.448,
      "step": 202000
    },
    {
      "FLOPS loss": 0.05725431814789772,
      "L0_d": 652.91,
      "MLM loss": 2.454516887664795,
      "epoch": 4.68,
      "step": 202499
    },
    {
      "epoch": 4.68,
      "learning_rate": 6.073224489795919e-05,
      "loss": 2.4514,
      "step": 202500
    },
    {
      "FLOPS loss": 0.062322214245796204,
      "L0_d": 775.33,
      "MLM loss": 2.613377094268799,
      "epoch": 4.69,
      "step": 202999
    },
    {
      "epoch": 4.69,
      "learning_rate": 6.0630408163265306e-05,
      "loss": 2.4488,
      "step": 203000
    },
    {
      "FLOPS loss": 0.06807457655668259,
      "L0_d": 692.81,
      "MLM loss": 2.4366507530212402,
      "epoch": 4.7,
      "step": 203499
    },
    {
      "epoch": 4.7,
      "learning_rate": 6.052836734693878e-05,
      "loss": 2.4506,
      "step": 203500
    },
    {
      "FLOPS loss": 0.07732067257165909,
      "L0_d": 895.83,
      "MLM loss": 2.3486175537109375,
      "epoch": 4.71,
      "step": 203999
    },
    {
      "epoch": 4.71,
      "learning_rate": 6.042632653061224e-05,
      "loss": 2.4488,
      "step": 204000
    },
    {
      "FLOPS loss": 0.071961410343647,
      "L0_d": 880.2,
      "MLM loss": 2.502091407775879,
      "epoch": 4.72,
      "step": 204499
    },
    {
      "epoch": 4.72,
      "learning_rate": 6.032428571428572e-05,
      "loss": 2.452,
      "step": 204500
    },
    {
      "FLOPS loss": 0.06560243666172028,
      "L0_d": 830.52,
      "MLM loss": 2.345327138900757,
      "epoch": 4.74,
      "step": 204999
    },
    {
      "epoch": 4.74,
      "learning_rate": 6.022244897959184e-05,
      "loss": 2.4487,
      "step": 205000
    },
    {
      "FLOPS loss": 0.06014092266559601,
      "L0_d": 812.0,
      "MLM loss": 2.3884024620056152,
      "epoch": 4.75,
      "step": 205499
    },
    {
      "epoch": 4.75,
      "learning_rate": 6.0120408163265306e-05,
      "loss": 2.4475,
      "step": 205500
    },
    {
      "FLOPS loss": 0.06797828525304794,
      "L0_d": 832.78,
      "MLM loss": 2.3767597675323486,
      "epoch": 4.76,
      "step": 205999
    },
    {
      "epoch": 4.76,
      "learning_rate": 6.001836734693878e-05,
      "loss": 2.4496,
      "step": 206000
    },
    {
      "FLOPS loss": 0.06602929532527924,
      "L0_d": 895.05,
      "MLM loss": 2.238253355026245,
      "epoch": 4.77,
      "step": 206499
    },
    {
      "epoch": 4.77,
      "learning_rate": 5.991632653061224e-05,
      "loss": 2.4493,
      "step": 206500
    },
    {
      "FLOPS loss": 0.062009185552597046,
      "L0_d": 832.91,
      "MLM loss": 2.436962604522705,
      "epoch": 4.78,
      "step": 206999
    },
    {
      "epoch": 4.78,
      "learning_rate": 5.981428571428572e-05,
      "loss": 2.4444,
      "step": 207000
    },
    {
      "FLOPS loss": 0.07398274540901184,
      "L0_d": 1058.25,
      "MLM loss": 2.415109157562256,
      "epoch": 4.79,
      "step": 207499
    },
    {
      "epoch": 4.79,
      "learning_rate": 5.971244897959184e-05,
      "loss": 2.4436,
      "step": 207500
    },
    {
      "FLOPS loss": 0.06136331707239151,
      "L0_d": 744.12,
      "MLM loss": 2.3068037033081055,
      "epoch": 4.81,
      "step": 207999
    },
    {
      "epoch": 4.81,
      "learning_rate": 5.9610408163265305e-05,
      "loss": 2.4464,
      "step": 208000
    },
    {
      "FLOPS loss": 0.06158547103404999,
      "L0_d": 929.0,
      "MLM loss": 2.406006336212158,
      "epoch": 4.82,
      "step": 208499
    },
    {
      "epoch": 4.82,
      "learning_rate": 5.950836734693878e-05,
      "loss": 2.446,
      "step": 208500
    },
    {
      "FLOPS loss": 0.06520802527666092,
      "L0_d": 732.05,
      "MLM loss": 2.3168740272521973,
      "epoch": 4.83,
      "step": 208999
    },
    {
      "epoch": 4.83,
      "learning_rate": 5.940632653061224e-05,
      "loss": 2.4432,
      "step": 209000
    },
    {
      "FLOPS loss": 0.08313161134719849,
      "L0_d": 1113.67,
      "MLM loss": 2.4824371337890625,
      "epoch": 4.84,
      "step": 209499
    },
    {
      "epoch": 4.84,
      "learning_rate": 5.930448979591837e-05,
      "loss": 2.4441,
      "step": 209500
    },
    {
      "FLOPS loss": 0.06880463659763336,
      "L0_d": 1170.94,
      "MLM loss": 2.373368978500366,
      "epoch": 4.85,
      "step": 209999
    },
    {
      "epoch": 4.85,
      "learning_rate": 5.920244897959184e-05,
      "loss": 2.4453,
      "step": 210000
    },
    {
      "FLOPS loss": 0.06296434253454208,
      "L0_d": 1000.72,
      "MLM loss": 2.4930760860443115,
      "epoch": 4.86,
      "step": 210499
    },
    {
      "epoch": 4.86,
      "learning_rate": 5.9100408163265305e-05,
      "loss": 2.4431,
      "step": 210500
    },
    {
      "FLOPS loss": 0.0654015988111496,
      "L0_d": 783.62,
      "MLM loss": 2.3720173835754395,
      "epoch": 4.87,
      "step": 210999
    },
    {
      "epoch": 4.87,
      "learning_rate": 5.899836734693878e-05,
      "loss": 2.4443,
      "step": 211000
    },
    {
      "FLOPS loss": 0.06743564456701279,
      "L0_d": 929.95,
      "MLM loss": 2.36538028717041,
      "epoch": 4.89,
      "step": 211499
    },
    {
      "epoch": 4.89,
      "learning_rate": 5.8896530612244904e-05,
      "loss": 2.4434,
      "step": 211500
    },
    {
      "FLOPS loss": 0.06421862542629242,
      "L0_d": 922.92,
      "MLM loss": 2.4841079711914062,
      "epoch": 4.9,
      "step": 211999
    },
    {
      "epoch": 4.9,
      "learning_rate": 5.8794489795918376e-05,
      "loss": 2.44,
      "step": 212000
    },
    {
      "FLOPS loss": 0.06765349954366684,
      "L0_d": 860.09,
      "MLM loss": 2.574281692504883,
      "epoch": 4.91,
      "step": 212499
    },
    {
      "epoch": 4.91,
      "learning_rate": 5.869244897959184e-05,
      "loss": 2.4354,
      "step": 212500
    },
    {
      "FLOPS loss": 0.08442234247922897,
      "L0_d": 1016.08,
      "MLM loss": 2.45365047454834,
      "epoch": 4.92,
      "step": 212999
    },
    {
      "epoch": 4.92,
      "learning_rate": 5.859040816326531e-05,
      "loss": 2.4416,
      "step": 213000
    },
    {
      "FLOPS loss": 0.06505122035741806,
      "L0_d": 833.3,
      "MLM loss": 2.5009758472442627,
      "epoch": 4.93,
      "step": 213499
    },
    {
      "epoch": 4.93,
      "learning_rate": 5.8488367346938776e-05,
      "loss": 2.438,
      "step": 213500
    },
    {
      "FLOPS loss": 0.054140783846378326,
      "L0_d": 705.83,
      "MLM loss": 2.4696202278137207,
      "epoch": 4.94,
      "step": 213999
    },
    {
      "epoch": 4.94,
      "learning_rate": 5.8386530612244904e-05,
      "loss": 2.4395,
      "step": 214000
    },
    {
      "FLOPS loss": 0.07031732052564621,
      "L0_d": 888.97,
      "MLM loss": 2.2769343852996826,
      "epoch": 4.96,
      "step": 214499
    },
    {
      "epoch": 4.96,
      "learning_rate": 5.8284489795918375e-05,
      "loss": 2.4403,
      "step": 214500
    },
    {
      "FLOPS loss": 0.06005926430225372,
      "L0_d": 835.02,
      "MLM loss": 2.513908863067627,
      "epoch": 4.97,
      "step": 214999
    },
    {
      "epoch": 4.97,
      "learning_rate": 5.818244897959184e-05,
      "loss": 2.4433,
      "step": 215000
    },
    {
      "FLOPS loss": 0.08554711937904358,
      "L0_d": 1060.28,
      "MLM loss": 2.2464144229888916,
      "epoch": 4.98,
      "step": 215499
    },
    {
      "epoch": 4.98,
      "learning_rate": 5.808040816326531e-05,
      "loss": 2.434,
      "step": 215500
    },
    {
      "FLOPS loss": 0.07612525671720505,
      "L0_d": 1262.25,
      "MLM loss": 2.3883957862854004,
      "epoch": 4.99,
      "step": 215999
    },
    {
      "epoch": 4.99,
      "learning_rate": 5.7978367346938776e-05,
      "loss": 2.4383,
      "step": 216000
    },
    {
      "FLOPS loss": 0.07493530958890915,
      "L0_d": 963.84,
      "MLM loss": 2.5143649578094482,
      "epoch": 5.0,
      "step": 216499
    },
    {
      "epoch": 5.0,
      "learning_rate": 5.7876530612244904e-05,
      "loss": 2.4388,
      "step": 216500
    },
    {
      "FLOPS loss": 0.05979205295443535,
      "L0_d": 788.12,
      "MLM loss": 2.4131922721862793,
      "epoch": 5.01,
      "step": 216999
    },
    {
      "epoch": 5.01,
      "learning_rate": 5.7774489795918375e-05,
      "loss": 2.4333,
      "step": 217000
    },
    {
      "FLOPS loss": 0.0720650851726532,
      "L0_d": 978.7,
      "MLM loss": 2.429448366165161,
      "epoch": 5.02,
      "step": 217499
    },
    {
      "epoch": 5.02,
      "learning_rate": 5.767244897959184e-05,
      "loss": 2.4342,
      "step": 217500
    },
    {
      "FLOPS loss": 0.07064210623502731,
      "L0_d": 862.91,
      "MLM loss": 2.4350945949554443,
      "epoch": 5.04,
      "step": 217999
    },
    {
      "epoch": 5.04,
      "learning_rate": 5.757040816326531e-05,
      "loss": 2.4375,
      "step": 218000
    },
    {
      "FLOPS loss": 0.062107719480991364,
      "L0_d": 719.12,
      "MLM loss": 2.218329668045044,
      "epoch": 5.05,
      "step": 218499
    },
    {
      "epoch": 5.05,
      "learning_rate": 5.7468367346938776e-05,
      "loss": 2.4326,
      "step": 218500
    },
    {
      "FLOPS loss": 0.0665595754981041,
      "L0_d": 793.11,
      "MLM loss": 2.3749895095825195,
      "epoch": 5.06,
      "step": 218999
    },
    {
      "epoch": 5.06,
      "learning_rate": 5.73665306122449e-05,
      "loss": 2.4326,
      "step": 219000
    },
    {
      "FLOPS loss": 0.05444691702723503,
      "L0_d": 645.83,
      "MLM loss": 2.405651330947876,
      "epoch": 5.07,
      "step": 219499
    },
    {
      "epoch": 5.07,
      "learning_rate": 5.7264489795918375e-05,
      "loss": 2.4248,
      "step": 219500
    },
    {
      "FLOPS loss": 0.051724907010793686,
      "L0_d": 658.75,
      "MLM loss": 2.5104503631591797,
      "epoch": 5.08,
      "step": 219999
    },
    {
      "epoch": 5.08,
      "learning_rate": 5.716244897959184e-05,
      "loss": 2.4326,
      "step": 220000
    },
    {
      "FLOPS loss": 0.05557708442211151,
      "L0_d": 765.36,
      "MLM loss": 2.4535903930664062,
      "epoch": 5.09,
      "step": 220499
    },
    {
      "epoch": 5.09,
      "learning_rate": 5.706040816326531e-05,
      "loss": 2.4311,
      "step": 220500
    },
    {
      "FLOPS loss": 0.05966062471270561,
      "L0_d": 719.16,
      "MLM loss": 2.3905961513519287,
      "epoch": 5.11,
      "step": 220999
    },
    {
      "epoch": 5.11,
      "learning_rate": 5.6958571428571425e-05,
      "loss": 2.4261,
      "step": 221000
    },
    {
      "FLOPS loss": 0.0590512789785862,
      "L0_d": 834.92,
      "MLM loss": 2.4592933654785156,
      "epoch": 5.12,
      "step": 221499
    },
    {
      "epoch": 5.12,
      "learning_rate": 5.6856530612244897e-05,
      "loss": 2.4283,
      "step": 221500
    },
    {
      "FLOPS loss": 0.06647661328315735,
      "L0_d": 777.38,
      "MLM loss": 2.529719829559326,
      "epoch": 5.13,
      "step": 221999
    },
    {
      "epoch": 5.13,
      "learning_rate": 5.6754489795918375e-05,
      "loss": 2.4262,
      "step": 222000
    },
    {
      "FLOPS loss": 0.059614311903715134,
      "L0_d": 668.86,
      "MLM loss": 2.487185001373291,
      "epoch": 5.14,
      "step": 222499
    },
    {
      "epoch": 5.14,
      "learning_rate": 5.6652448979591846e-05,
      "loss": 2.4292,
      "step": 222500
    },
    {
      "FLOPS loss": 0.061933524906635284,
      "L0_d": 705.98,
      "MLM loss": 2.349621295928955,
      "epoch": 5.15,
      "step": 222999
    },
    {
      "epoch": 5.15,
      "learning_rate": 5.655040816326531e-05,
      "loss": 2.4285,
      "step": 223000
    },
    {
      "FLOPS loss": 0.07037528604269028,
      "L0_d": 1112.05,
      "MLM loss": 2.242041826248169,
      "epoch": 5.16,
      "step": 223499
    },
    {
      "epoch": 5.16,
      "learning_rate": 5.644857142857143e-05,
      "loss": 2.4274,
      "step": 223500
    },
    {
      "FLOPS loss": 0.06243540346622467,
      "L0_d": 933.86,
      "MLM loss": 2.3377318382263184,
      "epoch": 5.17,
      "step": 223999
    },
    {
      "epoch": 5.17,
      "learning_rate": 5.6346530612244896e-05,
      "loss": 2.4293,
      "step": 224000
    },
    {
      "FLOPS loss": 0.07176888734102249,
      "L0_d": 1007.48,
      "MLM loss": 2.4397826194763184,
      "epoch": 5.19,
      "step": 224499
    },
    {
      "epoch": 5.19,
      "learning_rate": 5.624448979591837e-05,
      "loss": 2.4249,
      "step": 224500
    },
    {
      "FLOPS loss": 0.06889332085847855,
      "L0_d": 973.16,
      "MLM loss": 2.3823471069335938,
      "epoch": 5.2,
      "step": 224999
    },
    {
      "epoch": 5.2,
      "learning_rate": 5.6142448979591846e-05,
      "loss": 2.4297,
      "step": 225000
    },
    {
      "FLOPS loss": 0.06415066123008728,
      "L0_d": 1091.34,
      "MLM loss": 2.323089361190796,
      "epoch": 5.21,
      "step": 225499
    },
    {
      "epoch": 5.21,
      "learning_rate": 5.604061224489796e-05,
      "loss": 2.4279,
      "step": 225500
    },
    {
      "FLOPS loss": 0.0608639270067215,
      "L0_d": 789.3,
      "MLM loss": 2.265446901321411,
      "epoch": 5.22,
      "step": 225999
    },
    {
      "epoch": 5.22,
      "learning_rate": 5.593857142857143e-05,
      "loss": 2.4256,
      "step": 226000
    },
    {
      "FLOPS loss": 0.06591716408729553,
      "L0_d": 893.92,
      "MLM loss": 2.2293763160705566,
      "epoch": 5.23,
      "step": 226499
    },
    {
      "epoch": 5.23,
      "learning_rate": 5.5836530612244896e-05,
      "loss": 2.4313,
      "step": 226500
    },
    {
      "FLOPS loss": 0.05170826241374016,
      "L0_d": 763.05,
      "MLM loss": 2.5374226570129395,
      "epoch": 5.24,
      "step": 226999
    },
    {
      "epoch": 5.24,
      "learning_rate": 5.573448979591837e-05,
      "loss": 2.4269,
      "step": 227000
    },
    {
      "FLOPS loss": 0.059597332030534744,
      "L0_d": 982.97,
      "MLM loss": 2.4384584426879883,
      "epoch": 5.26,
      "step": 227499
    },
    {
      "epoch": 5.26,
      "learning_rate": 5.5632653061224495e-05,
      "loss": 2.4275,
      "step": 227500
    },
    {
      "FLOPS loss": 0.06841108202934265,
      "L0_d": 841.12,
      "MLM loss": 2.26102876663208,
      "epoch": 5.27,
      "step": 227999
    },
    {
      "epoch": 5.27,
      "learning_rate": 5.553061224489796e-05,
      "loss": 2.4256,
      "step": 228000
    },
    {
      "FLOPS loss": 0.06574366986751556,
      "L0_d": 772.81,
      "MLM loss": 2.215580701828003,
      "epoch": 5.28,
      "step": 228499
    },
    {
      "epoch": 5.28,
      "learning_rate": 5.542857142857143e-05,
      "loss": 2.4234,
      "step": 228500
    },
    {
      "FLOPS loss": 0.05697168409824371,
      "L0_d": 804.02,
      "MLM loss": 2.3969311714172363,
      "epoch": 5.29,
      "step": 228999
    },
    {
      "epoch": 5.29,
      "learning_rate": 5.5326530612244896e-05,
      "loss": 2.4208,
      "step": 229000
    },
    {
      "FLOPS loss": 0.07022450119256973,
      "L0_d": 812.52,
      "MLM loss": 2.4542322158813477,
      "epoch": 5.3,
      "step": 229499
    },
    {
      "epoch": 5.3,
      "learning_rate": 5.522448979591837e-05,
      "loss": 2.423,
      "step": 229500
    },
    {
      "FLOPS loss": 0.06159794330596924,
      "L0_d": 787.3,
      "MLM loss": 2.3856964111328125,
      "epoch": 5.31,
      "step": 229999
    },
    {
      "epoch": 5.31,
      "learning_rate": 5.5122653061224495e-05,
      "loss": 2.4217,
      "step": 230000
    },
    {
      "FLOPS loss": 0.06548204272985458,
      "L0_d": 681.08,
      "MLM loss": 2.3479115962982178,
      "epoch": 5.32,
      "step": 230499
    },
    {
      "epoch": 5.32,
      "learning_rate": 5.502061224489796e-05,
      "loss": 2.4257,
      "step": 230500
    },
    {
      "FLOPS loss": 0.06918176263570786,
      "L0_d": 956.61,
      "MLM loss": 2.3696184158325195,
      "epoch": 5.34,
      "step": 230999
    },
    {
      "epoch": 5.34,
      "learning_rate": 5.491857142857143e-05,
      "loss": 2.42,
      "step": 231000
    },
    {
      "FLOPS loss": 0.05903591215610504,
      "L0_d": 914.56,
      "MLM loss": 2.588008403778076,
      "epoch": 5.35,
      "step": 231499
    },
    {
      "epoch": 5.35,
      "learning_rate": 5.4816530612244896e-05,
      "loss": 2.421,
      "step": 231500
    },
    {
      "FLOPS loss": 0.06530603766441345,
      "L0_d": 738.28,
      "MLM loss": 2.2901949882507324,
      "epoch": 5.36,
      "step": 231999
    },
    {
      "epoch": 5.36,
      "learning_rate": 5.471469387755102e-05,
      "loss": 2.4197,
      "step": 232000
    },
    {
      "FLOPS loss": 0.0630945935845375,
      "L0_d": 786.75,
      "MLM loss": 2.353754758834839,
      "epoch": 5.37,
      "step": 232499
    },
    {
      "epoch": 5.37,
      "learning_rate": 5.4612653061224495e-05,
      "loss": 2.4212,
      "step": 232500
    },
    {
      "FLOPS loss": 0.06529512256383896,
      "L0_d": 911.2,
      "MLM loss": 2.315972328186035,
      "epoch": 5.38,
      "step": 232999
    },
    {
      "epoch": 5.38,
      "learning_rate": 5.4510612244897966e-05,
      "loss": 2.4228,
      "step": 233000
    },
    {
      "FLOPS loss": 0.06647701561450958,
      "L0_d": 1022.75,
      "MLM loss": 2.321000814437866,
      "epoch": 5.39,
      "step": 233499
    },
    {
      "epoch": 5.39,
      "learning_rate": 5.440857142857143e-05,
      "loss": 2.4218,
      "step": 233500
    },
    {
      "FLOPS loss": 0.060300182551145554,
      "L0_d": 700.78,
      "MLM loss": 2.3008556365966797,
      "epoch": 5.41,
      "step": 233999
    },
    {
      "epoch": 5.41,
      "learning_rate": 5.430673469387756e-05,
      "loss": 2.4196,
      "step": 234000
    },
    {
      "FLOPS loss": 0.057080261409282684,
      "L0_d": 816.3,
      "MLM loss": 2.3866772651672363,
      "epoch": 5.42,
      "step": 234499
    },
    {
      "epoch": 5.42,
      "learning_rate": 5.420469387755103e-05,
      "loss": 2.4166,
      "step": 234500
    },
    {
      "FLOPS loss": 0.07618782669305801,
      "L0_d": 1020.66,
      "MLM loss": 2.3357157707214355,
      "epoch": 5.43,
      "step": 234999
    },
    {
      "epoch": 5.43,
      "learning_rate": 5.4102653061224495e-05,
      "loss": 2.4206,
      "step": 235000
    },
    {
      "FLOPS loss": 0.0620071217417717,
      "L0_d": 738.75,
      "MLM loss": 2.4523963928222656,
      "epoch": 5.44,
      "step": 235499
    },
    {
      "epoch": 5.44,
      "learning_rate": 5.4000612244897966e-05,
      "loss": 2.4208,
      "step": 235500
    },
    {
      "FLOPS loss": 0.0646725594997406,
      "L0_d": 1076.83,
      "MLM loss": 2.353867292404175,
      "epoch": 5.45,
      "step": 235999
    },
    {
      "epoch": 5.45,
      "learning_rate": 5.389857142857143e-05,
      "loss": 2.4229,
      "step": 236000
    },
    {
      "FLOPS loss": 0.06458298116922379,
      "L0_d": 860.25,
      "MLM loss": 2.3973288536071777,
      "epoch": 5.46,
      "step": 236499
    },
    {
      "epoch": 5.46,
      "learning_rate": 5.37965306122449e-05,
      "loss": 2.4211,
      "step": 236500
    },
    {
      "FLOPS loss": 0.07283961027860641,
      "L0_d": 744.12,
      "MLM loss": 2.4385931491851807,
      "epoch": 5.47,
      "step": 236999
    },
    {
      "epoch": 5.47,
      "learning_rate": 5.369469387755103e-05,
      "loss": 2.4195,
      "step": 237000
    },
    {
      "FLOPS loss": 0.0643695816397667,
      "L0_d": 817.92,
      "MLM loss": 2.379582643508911,
      "epoch": 5.49,
      "step": 237499
    },
    {
      "epoch": 5.49,
      "learning_rate": 5.3592653061224494e-05,
      "loss": 2.4165,
      "step": 237500
    },
    {
      "FLOPS loss": 0.07279926538467407,
      "L0_d": 962.19,
      "MLM loss": 2.22440767288208,
      "epoch": 5.5,
      "step": 237999
    },
    {
      "epoch": 5.5,
      "learning_rate": 5.3490612244897966e-05,
      "loss": 2.4156,
      "step": 238000
    },
    {
      "FLOPS loss": 0.05908575281500816,
      "L0_d": 686.64,
      "MLM loss": 2.48474383354187,
      "epoch": 5.51,
      "step": 238499
    },
    {
      "epoch": 5.51,
      "learning_rate": 5.338857142857143e-05,
      "loss": 2.416,
      "step": 238500
    },
    {
      "FLOPS loss": 0.07741891592741013,
      "L0_d": 1033.06,
      "MLM loss": 2.2399086952209473,
      "epoch": 5.52,
      "step": 238999
    },
    {
      "epoch": 5.52,
      "learning_rate": 5.32865306122449e-05,
      "loss": 2.4167,
      "step": 239000
    },
    {
      "FLOPS loss": 0.05061428248882294,
      "L0_d": 519.08,
      "MLM loss": 2.3229198455810547,
      "epoch": 5.53,
      "step": 239499
    },
    {
      "epoch": 5.53,
      "learning_rate": 5.318448979591837e-05,
      "loss": 2.4118,
      "step": 239500
    },
    {
      "FLOPS loss": 0.07955832034349442,
      "L0_d": 1008.81,
      "MLM loss": 2.3691508769989014,
      "epoch": 5.54,
      "step": 239999
    },
    {
      "epoch": 5.54,
      "learning_rate": 5.3082653061224494e-05,
      "loss": 2.4098,
      "step": 240000
    },
    {
      "FLOPS loss": 0.061550572514534,
      "L0_d": 968.66,
      "MLM loss": 2.6222047805786133,
      "epoch": 5.56,
      "step": 240499
    },
    {
      "epoch": 5.56,
      "learning_rate": 5.2980612244897966e-05,
      "loss": 2.4183,
      "step": 240500
    },
    {
      "FLOPS loss": 0.08182385563850403,
      "L0_d": 1048.91,
      "MLM loss": 2.4917473793029785,
      "epoch": 5.57,
      "step": 240999
    },
    {
      "epoch": 5.57,
      "learning_rate": 5.287857142857143e-05,
      "loss": 2.4134,
      "step": 241000
    },
    {
      "FLOPS loss": 0.058586906641721725,
      "L0_d": 719.7,
      "MLM loss": 2.3518967628479004,
      "epoch": 5.58,
      "step": 241499
    },
    {
      "epoch": 5.58,
      "learning_rate": 5.27765306122449e-05,
      "loss": 2.4185,
      "step": 241500
    },
    {
      "FLOPS loss": 0.08758343011140823,
      "L0_d": 1149.27,
      "MLM loss": 2.231182336807251,
      "epoch": 5.59,
      "step": 241999
    },
    {
      "epoch": 5.59,
      "learning_rate": 5.2674489795918366e-05,
      "loss": 2.4137,
      "step": 242000
    },
    {
      "FLOPS loss": 0.07074569165706635,
      "L0_d": 842.36,
      "MLM loss": 2.271498441696167,
      "epoch": 5.6,
      "step": 242499
    },
    {
      "epoch": 5.6,
      "learning_rate": 5.2572653061224494e-05,
      "loss": 2.414,
      "step": 242500
    },
    {
      "FLOPS loss": 0.06920843571424484,
      "L0_d": 959.58,
      "MLM loss": 2.2417149543762207,
      "epoch": 5.61,
      "step": 242999
    },
    {
      "epoch": 5.61,
      "learning_rate": 5.2470612244897965e-05,
      "loss": 2.4152,
      "step": 243000
    },
    {
      "FLOPS loss": 0.06951159238815308,
      "L0_d": 879.59,
      "MLM loss": 2.3914523124694824,
      "epoch": 5.63,
      "step": 243499
    },
    {
      "epoch": 5.63,
      "learning_rate": 5.236857142857144e-05,
      "loss": 2.4165,
      "step": 243500
    },
    {
      "FLOPS loss": 0.06333808600902557,
      "L0_d": 845.09,
      "MLM loss": 2.3757832050323486,
      "epoch": 5.64,
      "step": 243999
    },
    {
      "epoch": 5.64,
      "learning_rate": 5.22665306122449e-05,
      "loss": 2.4089,
      "step": 244000
    },
    {
      "FLOPS loss": 0.06202933192253113,
      "L0_d": 826.56,
      "MLM loss": 2.427809476852417,
      "epoch": 5.65,
      "step": 244499
    },
    {
      "epoch": 5.65,
      "learning_rate": 5.2164693877551015e-05,
      "loss": 2.412,
      "step": 244500
    },
    {
      "FLOPS loss": 0.06342844665050507,
      "L0_d": 1106.88,
      "MLM loss": 2.4152002334594727,
      "epoch": 5.66,
      "step": 244999
    },
    {
      "epoch": 5.66,
      "learning_rate": 5.20626530612245e-05,
      "loss": 2.4117,
      "step": 245000
    },
    {
      "FLOPS loss": 0.0744723230600357,
      "L0_d": 1098.58,
      "MLM loss": 2.1651837825775146,
      "epoch": 5.67,
      "step": 245499
    },
    {
      "epoch": 5.67,
      "learning_rate": 5.1960612244897965e-05,
      "loss": 2.4073,
      "step": 245500
    },
    {
      "FLOPS loss": 0.058694589883089066,
      "L0_d": 815.58,
      "MLM loss": 2.2264058589935303,
      "epoch": 5.68,
      "step": 245999
    },
    {
      "epoch": 5.68,
      "learning_rate": 5.185857142857144e-05,
      "loss": 2.4112,
      "step": 246000
    },
    {
      "FLOPS loss": 0.0438019260764122,
      "L0_d": 610.3,
      "MLM loss": 2.3642964363098145,
      "epoch": 5.69,
      "step": 246499
    },
    {
      "epoch": 5.69,
      "learning_rate": 5.17565306122449e-05,
      "loss": 2.4093,
      "step": 246500
    },
    {
      "FLOPS loss": 0.06571359187364578,
      "L0_d": 906.2,
      "MLM loss": 2.217068910598755,
      "epoch": 5.71,
      "step": 246999
    },
    {
      "epoch": 5.71,
      "learning_rate": 5.165469387755102e-05,
      "loss": 2.4076,
      "step": 247000
    },
    {
      "FLOPS loss": 0.057677898555994034,
      "L0_d": 782.92,
      "MLM loss": 2.2437782287597656,
      "epoch": 5.72,
      "step": 247499
    },
    {
      "epoch": 5.72,
      "learning_rate": 5.155265306122449e-05,
      "loss": 2.4119,
      "step": 247500
    },
    {
      "FLOPS loss": 0.0648793950676918,
      "L0_d": 905.28,
      "MLM loss": 2.2714266777038574,
      "epoch": 5.73,
      "step": 247999
    },
    {
      "epoch": 5.73,
      "learning_rate": 5.1450612244897965e-05,
      "loss": 2.4116,
      "step": 248000
    },
    {
      "FLOPS loss": 0.08464235067367554,
      "L0_d": 992.86,
      "MLM loss": 2.3772592544555664,
      "epoch": 5.74,
      "step": 248499
    },
    {
      "epoch": 5.74,
      "learning_rate": 5.1348571428571436e-05,
      "loss": 2.4087,
      "step": 248500
    },
    {
      "FLOPS loss": 0.06436370313167572,
      "L0_d": 712.66,
      "MLM loss": 2.6194450855255127,
      "epoch": 5.75,
      "step": 248999
    },
    {
      "epoch": 5.75,
      "learning_rate": 5.12465306122449e-05,
      "loss": 2.4129,
      "step": 249000
    },
    {
      "FLOPS loss": 0.056343041360378265,
      "L0_d": 690.97,
      "MLM loss": 2.426379680633545,
      "epoch": 5.76,
      "step": 249499
    },
    {
      "epoch": 5.76,
      "learning_rate": 5.114469387755102e-05,
      "loss": 2.4098,
      "step": 249500
    },
    {
      "FLOPS loss": 0.0622674897313118,
      "L0_d": 1193.97,
      "MLM loss": 2.275702476501465,
      "epoch": 5.78,
      "step": 249999
    },
    {
      "epoch": 5.78,
      "learning_rate": 5.1042653061224487e-05,
      "loss": 2.4088,
      "step": 250000
    },
    {
      "FLOPS loss": 0.06920263171195984,
      "L0_d": 1360.25,
      "MLM loss": 2.4793238639831543,
      "epoch": 5.79,
      "step": 250499
    },
    {
      "epoch": 5.79,
      "learning_rate": 5.094061224489796e-05,
      "loss": 2.4015,
      "step": 250500
    },
    {
      "FLOPS loss": 0.061475012451410294,
      "L0_d": 759.97,
      "MLM loss": 2.1970982551574707,
      "epoch": 5.8,
      "step": 250999
    },
    {
      "epoch": 5.8,
      "learning_rate": 5.0838571428571436e-05,
      "loss": 2.4074,
      "step": 251000
    },
    {
      "FLOPS loss": 0.0693504810333252,
      "L0_d": 747.38,
      "MLM loss": 2.315021514892578,
      "epoch": 5.81,
      "step": 251499
    },
    {
      "epoch": 5.81,
      "learning_rate": 5.07365306122449e-05,
      "loss": 2.411,
      "step": 251500
    },
    {
      "FLOPS loss": 0.054750021547079086,
      "L0_d": 565.3,
      "MLM loss": 2.175985336303711,
      "epoch": 5.82,
      "step": 251999
    },
    {
      "epoch": 5.82,
      "learning_rate": 5.063469387755102e-05,
      "loss": 2.4061,
      "step": 252000
    },
    {
      "FLOPS loss": 0.05239395797252655,
      "L0_d": 543.91,
      "MLM loss": 2.3537206649780273,
      "epoch": 5.83,
      "step": 252499
    },
    {
      "epoch": 5.83,
      "learning_rate": 5.0532653061224486e-05,
      "loss": 2.4028,
      "step": 252500
    },
    {
      "FLOPS loss": 0.06858605146408081,
      "L0_d": 982.17,
      "MLM loss": 2.4172556400299072,
      "epoch": 5.84,
      "step": 252999
    },
    {
      "epoch": 5.84,
      "learning_rate": 5.043061224489796e-05,
      "loss": 2.405,
      "step": 253000
    },
    {
      "FLOPS loss": 0.06169794127345085,
      "L0_d": 818.91,
      "MLM loss": 2.324985980987549,
      "epoch": 5.86,
      "step": 253499
    },
    {
      "epoch": 5.86,
      "learning_rate": 5.0328571428571436e-05,
      "loss": 2.4049,
      "step": 253500
    },
    {
      "FLOPS loss": 0.057725466787815094,
      "L0_d": 805.22,
      "MLM loss": 2.414187431335449,
      "epoch": 5.87,
      "step": 253999
    },
    {
      "epoch": 5.87,
      "learning_rate": 5.022653061224491e-05,
      "loss": 2.4093,
      "step": 254000
    },
    {
      "FLOPS loss": 0.06406831741333008,
      "L0_d": 830.33,
      "MLM loss": 2.3183207511901855,
      "epoch": 5.88,
      "step": 254499
    },
    {
      "epoch": 5.88,
      "learning_rate": 5.012469387755102e-05,
      "loss": 2.4061,
      "step": 254500
    },
    {
      "FLOPS loss": 0.07106481492519379,
      "L0_d": 870.17,
      "MLM loss": 2.090304374694824,
      "epoch": 5.89,
      "step": 254999
    },
    {
      "epoch": 5.89,
      "learning_rate": 5.0022653061224486e-05,
      "loss": 2.3991,
      "step": 255000
    },
    {
      "FLOPS loss": 0.05545003339648247,
      "L0_d": 826.73,
      "MLM loss": 2.331218957901001,
      "epoch": 5.9,
      "step": 255499
    },
    {
      "epoch": 5.9,
      "learning_rate": 4.9920612244897964e-05,
      "loss": 2.3988,
      "step": 255500
    },
    {
      "FLOPS loss": 0.05424296855926514,
      "L0_d": 608.84,
      "MLM loss": 2.439082145690918,
      "epoch": 5.91,
      "step": 255999
    },
    {
      "epoch": 5.91,
      "learning_rate": 4.981857142857143e-05,
      "loss": 2.4051,
      "step": 256000
    },
    {
      "FLOPS loss": 0.06380561739206314,
      "L0_d": 683.2,
      "MLM loss": 2.3173537254333496,
      "epoch": 5.93,
      "step": 256499
    },
    {
      "epoch": 5.93,
      "learning_rate": 4.97165306122449e-05,
      "loss": 2.4055,
      "step": 256500
    },
    {
      "FLOPS loss": 0.05620687082409859,
      "L0_d": 766.11,
      "MLM loss": 2.4309253692626953,
      "epoch": 5.94,
      "step": 256999
    },
    {
      "epoch": 5.94,
      "learning_rate": 4.961469387755102e-05,
      "loss": 2.4048,
      "step": 257000
    },
    {
      "FLOPS loss": 0.07935933768749237,
      "L0_d": 1019.94,
      "MLM loss": 2.3071842193603516,
      "epoch": 5.95,
      "step": 257499
    },
    {
      "epoch": 5.95,
      "learning_rate": 4.951265306122449e-05,
      "loss": 2.406,
      "step": 257500
    },
    {
      "FLOPS loss": 0.05459493398666382,
      "L0_d": 696.48,
      "MLM loss": 2.4048876762390137,
      "epoch": 5.96,
      "step": 257999
    },
    {
      "epoch": 5.96,
      "learning_rate": 4.9410612244897964e-05,
      "loss": 2.403,
      "step": 258000
    },
    {
      "FLOPS loss": 0.07205364853143692,
      "L0_d": 837.02,
      "MLM loss": 2.238459825515747,
      "epoch": 5.97,
      "step": 258499
    },
    {
      "epoch": 5.97,
      "learning_rate": 4.930857142857143e-05,
      "loss": 2.404,
      "step": 258500
    },
    {
      "FLOPS loss": 0.07362333685159683,
      "L0_d": 850.81,
      "MLM loss": 2.3778529167175293,
      "epoch": 5.98,
      "step": 258999
    },
    {
      "epoch": 5.98,
      "learning_rate": 4.9206734693877556e-05,
      "loss": 2.4001,
      "step": 259000
    },
    {
      "FLOPS loss": 0.058844566345214844,
      "L0_d": 1018.61,
      "MLM loss": 2.3497064113616943,
      "epoch": 5.99,
      "step": 259499
    },
    {
      "epoch": 5.99,
      "learning_rate": 4.910469387755102e-05,
      "loss": 2.4037,
      "step": 259500
    },
    {
      "FLOPS loss": 0.05885095149278641,
      "L0_d": 664.66,
      "MLM loss": 2.633054494857788,
      "epoch": 6.01,
      "step": 259999
    },
    {
      "epoch": 6.01,
      "learning_rate": 4.900265306122449e-05,
      "loss": 2.3968,
      "step": 260000
    },
    {
      "FLOPS loss": 0.06290178000926971,
      "L0_d": 697.53,
      "MLM loss": 2.379453659057617,
      "epoch": 6.02,
      "step": 260499
    },
    {
      "epoch": 6.02,
      "learning_rate": 4.8900612244897964e-05,
      "loss": 2.3957,
      "step": 260500
    },
    {
      "FLOPS loss": 0.08343258500099182,
      "L0_d": 978.75,
      "MLM loss": 2.326571464538574,
      "epoch": 6.03,
      "step": 260999
    },
    {
      "epoch": 6.03,
      "learning_rate": 4.8798571428571435e-05,
      "loss": 2.3938,
      "step": 261000
    },
    {
      "FLOPS loss": 0.06951142847537994,
      "L0_d": 843.02,
      "MLM loss": 2.1549177169799805,
      "epoch": 6.04,
      "step": 261499
    },
    {
      "epoch": 6.04,
      "learning_rate": 4.8696734693877556e-05,
      "loss": 2.3976,
      "step": 261500
    },
    {
      "FLOPS loss": 0.05598827823996544,
      "L0_d": 773.36,
      "MLM loss": 2.3736531734466553,
      "epoch": 6.05,
      "step": 261999
    },
    {
      "epoch": 6.05,
      "learning_rate": 4.859469387755102e-05,
      "loss": 2.3956,
      "step": 262000
    },
    {
      "FLOPS loss": 0.06191984564065933,
      "L0_d": 748.77,
      "MLM loss": 2.327627182006836,
      "epoch": 6.06,
      "step": 262499
    },
    {
      "epoch": 6.06,
      "learning_rate": 4.849265306122449e-05,
      "loss": 2.3923,
      "step": 262500
    },
    {
      "FLOPS loss": 0.06821770966053009,
      "L0_d": 875.41,
      "MLM loss": 2.2674753665924072,
      "epoch": 6.08,
      "step": 262999
    },
    {
      "epoch": 6.08,
      "learning_rate": 4.839061224489796e-05,
      "loss": 2.3949,
      "step": 263000
    },
    {
      "FLOPS loss": 0.06579490751028061,
      "L0_d": 807.78,
      "MLM loss": 2.183204412460327,
      "epoch": 6.09,
      "step": 263499
    },
    {
      "epoch": 6.09,
      "learning_rate": 4.8288775510204084e-05,
      "loss": 2.3952,
      "step": 263500
    },
    {
      "FLOPS loss": 0.07654042541980743,
      "L0_d": 1016.47,
      "MLM loss": 2.3310086727142334,
      "epoch": 6.1,
      "step": 263999
    },
    {
      "epoch": 6.1,
      "learning_rate": 4.8186734693877556e-05,
      "loss": 2.3956,
      "step": 264000
    },
    {
      "FLOPS loss": 0.0669398382306099,
      "L0_d": 1012.41,
      "MLM loss": 2.40228533744812,
      "epoch": 6.11,
      "step": 264499
    },
    {
      "epoch": 6.11,
      "learning_rate": 4.808469387755102e-05,
      "loss": 2.3932,
      "step": 264500
    },
    {
      "FLOPS loss": 0.06311013549566269,
      "L0_d": 763.83,
      "MLM loss": 2.2010045051574707,
      "epoch": 6.12,
      "step": 264999
    },
    {
      "epoch": 6.12,
      "learning_rate": 4.798265306122449e-05,
      "loss": 2.3955,
      "step": 265000
    },
    {
      "FLOPS loss": 0.0659794807434082,
      "L0_d": 844.52,
      "MLM loss": 2.238668441772461,
      "epoch": 6.13,
      "step": 265499
    },
    {
      "epoch": 6.13,
      "learning_rate": 4.788061224489796e-05,
      "loss": 2.39,
      "step": 265500
    },
    {
      "FLOPS loss": 0.06289741396903992,
      "L0_d": 979.72,
      "MLM loss": 2.186093807220459,
      "epoch": 6.14,
      "step": 265999
    },
    {
      "epoch": 6.14,
      "learning_rate": 4.7778775510204084e-05,
      "loss": 2.3908,
      "step": 266000
    },
    {
      "FLOPS loss": 0.07499533146619797,
      "L0_d": 857.14,
      "MLM loss": 2.23008131980896,
      "epoch": 6.16,
      "step": 266499
    },
    {
      "epoch": 6.16,
      "learning_rate": 4.767673469387755e-05,
      "loss": 2.3931,
      "step": 266500
    },
    {
      "FLOPS loss": 0.06307825446128845,
      "L0_d": 879.86,
      "MLM loss": 2.5647735595703125,
      "epoch": 6.17,
      "step": 266999
    },
    {
      "epoch": 6.17,
      "learning_rate": 4.757469387755103e-05,
      "loss": 2.3905,
      "step": 267000
    },
    {
      "FLOPS loss": 0.06346144527196884,
      "L0_d": 887.95,
      "MLM loss": 2.338149070739746,
      "epoch": 6.18,
      "step": 267499
    },
    {
      "epoch": 6.18,
      "learning_rate": 4.747265306122449e-05,
      "loss": 2.3893,
      "step": 267500
    },
    {
      "FLOPS loss": 0.05799337476491928,
      "L0_d": 830.47,
      "MLM loss": 2.1602370738983154,
      "epoch": 6.19,
      "step": 267999
    },
    {
      "epoch": 6.19,
      "learning_rate": 4.737081632653061e-05,
      "loss": 2.3964,
      "step": 268000
    },
    {
      "FLOPS loss": 0.05315548554062843,
      "L0_d": 583.66,
      "MLM loss": 2.4497416019439697,
      "epoch": 6.2,
      "step": 268499
    },
    {
      "epoch": 6.2,
      "learning_rate": 4.7268775510204084e-05,
      "loss": 2.3877,
      "step": 268500
    },
    {
      "FLOPS loss": 0.07155217230319977,
      "L0_d": 950.55,
      "MLM loss": 2.4541220664978027,
      "epoch": 6.21,
      "step": 268999
    },
    {
      "epoch": 6.21,
      "learning_rate": 4.716673469387755e-05,
      "loss": 2.3935,
      "step": 269000
    },
    {
      "FLOPS loss": 0.060407768934965134,
      "L0_d": 864.31,
      "MLM loss": 2.3954501152038574,
      "epoch": 6.23,
      "step": 269499
    },
    {
      "epoch": 6.23,
      "learning_rate": 4.706469387755103e-05,
      "loss": 2.387,
      "step": 269500
    },
    {
      "FLOPS loss": 0.08911103010177612,
      "L0_d": 947.52,
      "MLM loss": 2.317873477935791,
      "epoch": 6.24,
      "step": 269999
    },
    {
      "epoch": 6.24,
      "learning_rate": 4.696265306122449e-05,
      "loss": 2.3911,
      "step": 270000
    },
    {
      "FLOPS loss": 0.06322460621595383,
      "L0_d": 747.06,
      "MLM loss": 2.447556257247925,
      "epoch": 6.25,
      "step": 270499
    },
    {
      "epoch": 6.25,
      "learning_rate": 4.686061224489796e-05,
      "loss": 2.3851,
      "step": 270500
    },
    {
      "FLOPS loss": 0.06245068833231926,
      "L0_d": 842.75,
      "MLM loss": 2.0686545372009277,
      "epoch": 6.26,
      "step": 270999
    },
    {
      "epoch": 6.26,
      "learning_rate": 4.6758775510204084e-05,
      "loss": 2.3861,
      "step": 271000
    },
    {
      "FLOPS loss": 0.06610134243965149,
      "L0_d": 788.77,
      "MLM loss": 2.152571201324463,
      "epoch": 6.27,
      "step": 271499
    },
    {
      "epoch": 6.27,
      "learning_rate": 4.665673469387755e-05,
      "loss": 2.3904,
      "step": 271500
    },
    {
      "FLOPS loss": 0.052447445690631866,
      "L0_d": 803.31,
      "MLM loss": 2.2232165336608887,
      "epoch": 6.28,
      "step": 271999
    },
    {
      "epoch": 6.28,
      "learning_rate": 4.655469387755102e-05,
      "loss": 2.3908,
      "step": 272000
    },
    {
      "FLOPS loss": 0.056923095136880875,
      "L0_d": 617.55,
      "MLM loss": 2.160806655883789,
      "epoch": 6.3,
      "step": 272499
    },
    {
      "epoch": 6.3,
      "learning_rate": 4.645265306122449e-05,
      "loss": 2.3868,
      "step": 272500
    },
    {
      "FLOPS loss": 0.06665085256099701,
      "L0_d": 807.03,
      "MLM loss": 2.246150493621826,
      "epoch": 6.31,
      "step": 272999
    },
    {
      "epoch": 6.31,
      "learning_rate": 4.635081632653062e-05,
      "loss": 2.3861,
      "step": 273000
    },
    {
      "FLOPS loss": 0.07849167287349701,
      "L0_d": 1440.08,
      "MLM loss": 2.3028604984283447,
      "epoch": 6.32,
      "step": 273499
    },
    {
      "epoch": 6.32,
      "learning_rate": 4.6248775510204084e-05,
      "loss": 2.3854,
      "step": 273500
    },
    {
      "FLOPS loss": 0.055762071162462234,
      "L0_d": 688.75,
      "MLM loss": 2.46593976020813,
      "epoch": 6.33,
      "step": 273999
    },
    {
      "epoch": 6.33,
      "learning_rate": 4.6146734693877555e-05,
      "loss": 2.3891,
      "step": 274000
    },
    {
      "FLOPS loss": 0.06459101289510727,
      "L0_d": 755.36,
      "MLM loss": 2.303806781768799,
      "epoch": 6.34,
      "step": 274499
    },
    {
      "epoch": 6.34,
      "learning_rate": 4.604469387755102e-05,
      "loss": 2.3882,
      "step": 274500
    },
    {
      "FLOPS loss": 0.055319491773843765,
      "L0_d": 676.47,
      "MLM loss": 2.383361577987671,
      "epoch": 6.35,
      "step": 274999
    },
    {
      "epoch": 6.35,
      "learning_rate": 4.594265306122449e-05,
      "loss": 2.3903,
      "step": 275000
    },
    {
      "FLOPS loss": 0.057988785207271576,
      "L0_d": 867.92,
      "MLM loss": 2.260382652282715,
      "epoch": 6.36,
      "step": 275499
    },
    {
      "epoch": 6.36,
      "learning_rate": 4.584081632653062e-05,
      "loss": 2.3875,
      "step": 275500
    },
    {
      "FLOPS loss": 0.068933866918087,
      "L0_d": 709.81,
      "MLM loss": 2.305701494216919,
      "epoch": 6.38,
      "step": 275999
    },
    {
      "epoch": 6.38,
      "learning_rate": 4.5738775510204083e-05,
      "loss": 2.3868,
      "step": 276000
    },
    {
      "FLOPS loss": 0.054191600531339645,
      "L0_d": 662.23,
      "MLM loss": 2.4663784503936768,
      "epoch": 6.39,
      "step": 276499
    },
    {
      "epoch": 6.39,
      "learning_rate": 4.5636734693877555e-05,
      "loss": 2.3901,
      "step": 276500
    },
    {
      "FLOPS loss": 0.06299523264169693,
      "L0_d": 715.62,
      "MLM loss": 2.32785964012146,
      "epoch": 6.4,
      "step": 276999
    },
    {
      "epoch": 6.4,
      "learning_rate": 4.553469387755102e-05,
      "loss": 2.3904,
      "step": 277000
    },
    {
      "FLOPS loss": 0.0625939816236496,
      "L0_d": 768.23,
      "MLM loss": 2.37985897064209,
      "epoch": 6.41,
      "step": 277499
    },
    {
      "epoch": 6.41,
      "learning_rate": 4.543265306122449e-05,
      "loss": 2.3827,
      "step": 277500
    },
    {
      "FLOPS loss": 0.07297017425298691,
      "L0_d": 933.28,
      "MLM loss": 2.275392770767212,
      "epoch": 6.42,
      "step": 277999
    },
    {
      "epoch": 6.42,
      "learning_rate": 4.533061224489796e-05,
      "loss": 2.3839,
      "step": 278000
    },
    {
      "FLOPS loss": 0.07424743473529816,
      "L0_d": 1015.55,
      "MLM loss": 2.41300106048584,
      "epoch": 6.43,
      "step": 278499
    },
    {
      "epoch": 6.43,
      "learning_rate": 4.5228571428571434e-05,
      "loss": 2.3819,
      "step": 278500
    },
    {
      "FLOPS loss": 0.06222039461135864,
      "L0_d": 778.44,
      "MLM loss": 2.3484959602355957,
      "epoch": 6.45,
      "step": 278999
    },
    {
      "epoch": 6.45,
      "learning_rate": 4.5126734693877555e-05,
      "loss": 2.3801,
      "step": 279000
    },
    {
      "FLOPS loss": 0.06579253822565079,
      "L0_d": 879.73,
      "MLM loss": 2.43776273727417,
      "epoch": 6.46,
      "step": 279499
    },
    {
      "epoch": 6.46,
      "learning_rate": 4.502469387755102e-05,
      "loss": 2.3828,
      "step": 279500
    },
    {
      "FLOPS loss": 0.061226122081279755,
      "L0_d": 738.83,
      "MLM loss": 2.400214433670044,
      "epoch": 6.47,
      "step": 279999
    },
    {
      "epoch": 6.47,
      "learning_rate": 4.492265306122449e-05,
      "loss": 2.3852,
      "step": 280000
    },
    {
      "FLOPS loss": 0.05953386053442955,
      "L0_d": 837.47,
      "MLM loss": 2.478689193725586,
      "epoch": 6.48,
      "step": 280499
    },
    {
      "epoch": 6.48,
      "learning_rate": 4.482061224489796e-05,
      "loss": 2.3826,
      "step": 280500
    },
    {
      "FLOPS loss": 0.05504816770553589,
      "L0_d": 672.55,
      "MLM loss": 2.2007439136505127,
      "epoch": 6.49,
      "step": 280999
    },
    {
      "epoch": 6.49,
      "learning_rate": 4.471877551020408e-05,
      "loss": 2.3818,
      "step": 281000
    },
    {
      "FLOPS loss": 0.05578654631972313,
      "L0_d": 575.17,
      "MLM loss": 2.4325294494628906,
      "epoch": 6.5,
      "step": 281499
    },
    {
      "epoch": 6.5,
      "learning_rate": 4.4616734693877555e-05,
      "loss": 2.3869,
      "step": 281500
    },
    {
      "FLOPS loss": 0.06520844995975494,
      "L0_d": 792.69,
      "MLM loss": 2.347710132598877,
      "epoch": 6.51,
      "step": 281999
    },
    {
      "epoch": 6.51,
      "learning_rate": 4.451469387755102e-05,
      "loss": 2.3808,
      "step": 282000
    },
    {
      "FLOPS loss": 0.06950800865888596,
      "L0_d": 819.52,
      "MLM loss": 2.4303078651428223,
      "epoch": 6.53,
      "step": 282499
    },
    {
      "epoch": 6.53,
      "learning_rate": 4.441265306122449e-05,
      "loss": 2.3811,
      "step": 282500
    },
    {
      "FLOPS loss": 0.05945185571908951,
      "L0_d": 691.12,
      "MLM loss": 2.215554714202881,
      "epoch": 6.54,
      "step": 282999
    },
    {
      "epoch": 6.54,
      "learning_rate": 4.431061224489796e-05,
      "loss": 2.3854,
      "step": 283000
    },
    {
      "FLOPS loss": 0.0663042813539505,
      "L0_d": 842.08,
      "MLM loss": 2.335991859436035,
      "epoch": 6.55,
      "step": 283499
    },
    {
      "epoch": 6.55,
      "learning_rate": 4.4208571428571434e-05,
      "loss": 2.3834,
      "step": 283500
    },
    {
      "FLOPS loss": 0.05819135159254074,
      "L0_d": 824.59,
      "MLM loss": 2.4544339179992676,
      "epoch": 6.56,
      "step": 283999
    },
    {
      "epoch": 6.56,
      "learning_rate": 4.4106734693877554e-05,
      "loss": 2.3796,
      "step": 284000
    },
    {
      "FLOPS loss": 0.056056853383779526,
      "L0_d": 739.22,
      "MLM loss": 2.5581278800964355,
      "epoch": 6.57,
      "step": 284499
    },
    {
      "epoch": 6.57,
      "learning_rate": 4.4004693877551026e-05,
      "loss": 2.3826,
      "step": 284500
    },
    {
      "FLOPS loss": 0.07030254602432251,
      "L0_d": 863.52,
      "MLM loss": 2.2112503051757812,
      "epoch": 6.58,
      "step": 284999
    },
    {
      "epoch": 6.58,
      "learning_rate": 4.390265306122449e-05,
      "loss": 2.3774,
      "step": 285000
    },
    {
      "FLOPS loss": 0.0722346305847168,
      "L0_d": 921.95,
      "MLM loss": 2.2126243114471436,
      "epoch": 6.6,
      "step": 285499
    },
    {
      "epoch": 6.6,
      "learning_rate": 4.380061224489796e-05,
      "loss": 2.3796,
      "step": 285500
    },
    {
      "FLOPS loss": 0.0708499550819397,
      "L0_d": 987.69,
      "MLM loss": 2.3058786392211914,
      "epoch": 6.61,
      "step": 285999
    },
    {
      "epoch": 6.61,
      "learning_rate": 4.3698571428571433e-05,
      "loss": 2.3796,
      "step": 286000
    },
    {
      "FLOPS loss": 0.0859348475933075,
      "L0_d": 1135.59,
      "MLM loss": 2.3978452682495117,
      "epoch": 6.62,
      "step": 286499
    },
    {
      "epoch": 6.62,
      "learning_rate": 4.3596734693877554e-05,
      "loss": 2.3804,
      "step": 286500
    },
    {
      "FLOPS loss": 0.058340977877378464,
      "L0_d": 1036.36,
      "MLM loss": 2.3695602416992188,
      "epoch": 6.63,
      "step": 286999
    },
    {
      "epoch": 6.63,
      "learning_rate": 4.3494693877551026e-05,
      "loss": 2.3782,
      "step": 287000
    },
    {
      "FLOPS loss": 0.07051306217908859,
      "L0_d": 870.98,
      "MLM loss": 2.3154542446136475,
      "epoch": 6.64,
      "step": 287499
    },
    {
      "epoch": 6.64,
      "learning_rate": 4.339265306122449e-05,
      "loss": 2.3804,
      "step": 287500
    },
    {
      "FLOPS loss": 0.06338206678628922,
      "L0_d": 833.42,
      "MLM loss": 2.339228868484497,
      "epoch": 6.65,
      "step": 287999
    },
    {
      "epoch": 6.65,
      "learning_rate": 4.329061224489796e-05,
      "loss": 2.3813,
      "step": 288000
    },
    {
      "FLOPS loss": 0.0613948330283165,
      "L0_d": 653.11,
      "MLM loss": 2.249403715133667,
      "epoch": 6.66,
      "step": 288499
    },
    {
      "epoch": 6.66,
      "learning_rate": 4.318877551020408e-05,
      "loss": 2.3809,
      "step": 288500
    },
    {
      "FLOPS loss": 0.07161176204681396,
      "L0_d": 1040.56,
      "MLM loss": 2.434735059738159,
      "epoch": 6.68,
      "step": 288999
    },
    {
      "epoch": 6.68,
      "learning_rate": 4.3086734693877554e-05,
      "loss": 2.3818,
      "step": 289000
    },
    {
      "FLOPS loss": 0.0638524666428566,
      "L0_d": 653.52,
      "MLM loss": 2.3446688652038574,
      "epoch": 6.69,
      "step": 289499
    },
    {
      "epoch": 6.69,
      "learning_rate": 4.2984693877551025e-05,
      "loss": 2.3795,
      "step": 289500
    },
    {
      "FLOPS loss": 0.061977677047252655,
      "L0_d": 847.8,
      "MLM loss": 2.28072190284729,
      "epoch": 6.7,
      "step": 289999
    },
    {
      "epoch": 6.7,
      "learning_rate": 4.288265306122449e-05,
      "loss": 2.3776,
      "step": 290000
    },
    {
      "FLOPS loss": 0.07041425257921219,
      "L0_d": 862.5,
      "MLM loss": 2.322789192199707,
      "epoch": 6.71,
      "step": 290499
    },
    {
      "epoch": 6.71,
      "learning_rate": 4.278061224489796e-05,
      "loss": 2.3828,
      "step": 290500
    },
    {
      "FLOPS loss": 0.06982388347387314,
      "L0_d": 939.08,
      "MLM loss": 2.4443392753601074,
      "epoch": 6.72,
      "step": 290999
    },
    {
      "epoch": 6.72,
      "learning_rate": 4.267877551020408e-05,
      "loss": 2.3784,
      "step": 291000
    },
    {
      "FLOPS loss": 0.07364574819803238,
      "L0_d": 973.33,
      "MLM loss": 2.296755313873291,
      "epoch": 6.73,
      "step": 291499
    },
    {
      "epoch": 6.73,
      "learning_rate": 4.2576734693877554e-05,
      "loss": 2.3769,
      "step": 291500
    },
    {
      "FLOPS loss": 0.06386933475732803,
      "L0_d": 815.48,
      "MLM loss": 2.282402276992798,
      "epoch": 6.75,
      "step": 291999
    },
    {
      "epoch": 6.75,
      "learning_rate": 4.2474693877551025e-05,
      "loss": 2.3798,
      "step": 292000
    },
    {
      "FLOPS loss": 0.07132063060998917,
      "L0_d": 1084.69,
      "MLM loss": 2.1212337017059326,
      "epoch": 6.76,
      "step": 292499
    },
    {
      "epoch": 6.76,
      "learning_rate": 4.237265306122449e-05,
      "loss": 2.3731,
      "step": 292500
    },
    {
      "FLOPS loss": 0.06481712311506271,
      "L0_d": 1047.08,
      "MLM loss": 2.2411792278289795,
      "epoch": 6.77,
      "step": 292999
    },
    {
      "epoch": 6.77,
      "learning_rate": 4.227081632653062e-05,
      "loss": 2.3806,
      "step": 293000
    },
    {
      "FLOPS loss": 0.07611627876758575,
      "L0_d": 935.75,
      "MLM loss": 2.4179129600524902,
      "epoch": 6.78,
      "step": 293499
    },
    {
      "epoch": 6.78,
      "learning_rate": 4.216877551020408e-05,
      "loss": 2.3761,
      "step": 293500
    },
    {
      "FLOPS loss": 0.07053264230489731,
      "L0_d": 918.11,
      "MLM loss": 2.527155876159668,
      "epoch": 6.79,
      "step": 293999
    },
    {
      "epoch": 6.79,
      "learning_rate": 4.2066734693877554e-05,
      "loss": 2.3755,
      "step": 294000
    },
    {
      "FLOPS loss": 0.05917475000023842,
      "L0_d": 761.69,
      "MLM loss": 2.6052803993225098,
      "epoch": 6.8,
      "step": 294499
    },
    {
      "epoch": 6.8,
      "learning_rate": 4.196469387755102e-05,
      "loss": 2.3729,
      "step": 294500
    },
    {
      "FLOPS loss": 0.05868987366557121,
      "L0_d": 839.08,
      "MLM loss": 2.366290330886841,
      "epoch": 6.81,
      "step": 294999
    },
    {
      "epoch": 6.81,
      "learning_rate": 4.1862857142857146e-05,
      "loss": 2.376,
      "step": 295000
    },
    {
      "FLOPS loss": 0.05811339616775513,
      "L0_d": 758.78,
      "MLM loss": 2.499417543411255,
      "epoch": 6.83,
      "step": 295499
    },
    {
      "epoch": 6.83,
      "learning_rate": 4.176081632653062e-05,
      "loss": 2.3766,
      "step": 295500
    },
    {
      "FLOPS loss": 0.06532425433397293,
      "L0_d": 995.02,
      "MLM loss": 2.3556973934173584,
      "epoch": 6.84,
      "step": 295999
    },
    {
      "epoch": 6.84,
      "learning_rate": 4.165877551020408e-05,
      "loss": 2.3794,
      "step": 296000
    },
    {
      "FLOPS loss": 0.05370713397860527,
      "L0_d": 829.56,
      "MLM loss": 2.430586338043213,
      "epoch": 6.85,
      "step": 296499
    },
    {
      "epoch": 6.85,
      "learning_rate": 4.155673469387755e-05,
      "loss": 2.3737,
      "step": 296500
    },
    {
      "FLOPS loss": 0.059047866612672806,
      "L0_d": 831.78,
      "MLM loss": 2.066843271255493,
      "epoch": 6.86,
      "step": 296999
    },
    {
      "epoch": 6.86,
      "learning_rate": 4.145469387755102e-05,
      "loss": 2.3742,
      "step": 297000
    },
    {
      "FLOPS loss": 0.059049107134342194,
      "L0_d": 662.92,
      "MLM loss": 2.134557008743286,
      "epoch": 6.87,
      "step": 297499
    },
    {
      "epoch": 6.87,
      "learning_rate": 4.1352857142857146e-05,
      "loss": 2.3729,
      "step": 297500
    },
    {
      "FLOPS loss": 0.05946721136569977,
      "L0_d": 851.62,
      "MLM loss": 2.1986374855041504,
      "epoch": 6.88,
      "step": 297999
    },
    {
      "epoch": 6.88,
      "learning_rate": 4.125081632653062e-05,
      "loss": 2.3735,
      "step": 298000
    },
    {
      "FLOPS loss": 0.0655231922864914,
      "L0_d": 763.11,
      "MLM loss": 2.2446515560150146,
      "epoch": 6.9,
      "step": 298499
    },
    {
      "epoch": 6.9,
      "learning_rate": 4.114877551020408e-05,
      "loss": 2.3695,
      "step": 298500
    },
    {
      "FLOPS loss": 0.0701904147863388,
      "L0_d": 867.0,
      "MLM loss": 2.29551100730896,
      "epoch": 6.91,
      "step": 298999
    },
    {
      "epoch": 6.91,
      "learning_rate": 4.104673469387755e-05,
      "loss": 2.3733,
      "step": 299000
    },
    {
      "FLOPS loss": 0.06681898236274719,
      "L0_d": 761.53,
      "MLM loss": 2.2125775814056396,
      "epoch": 6.92,
      "step": 299499
    },
    {
      "epoch": 6.92,
      "learning_rate": 4.0944693877551025e-05,
      "loss": 2.3744,
      "step": 299500
    },
    {
      "FLOPS loss": 0.08913971483707428,
      "L0_d": 1045.88,
      "MLM loss": 2.3854105472564697,
      "epoch": 6.93,
      "step": 299999
    },
    {
      "epoch": 6.93,
      "learning_rate": 4.084265306122449e-05,
      "loss": 2.3712,
      "step": 300000
    },
    {
      "FLOPS loss": 0.04997643828392029,
      "L0_d": 551.39,
      "MLM loss": 2.3603034019470215,
      "epoch": 6.94,
      "step": 300499
    },
    {
      "epoch": 6.94,
      "learning_rate": 4.074081632653061e-05,
      "loss": 2.3726,
      "step": 300500
    },
    {
      "FLOPS loss": 0.06283554434776306,
      "L0_d": 954.12,
      "MLM loss": 2.101763963699341,
      "epoch": 6.95,
      "step": 300999
    },
    {
      "epoch": 6.95,
      "learning_rate": 4.063877551020409e-05,
      "loss": 2.3714,
      "step": 301000
    },
    {
      "FLOPS loss": 0.0683046281337738,
      "L0_d": 769.17,
      "MLM loss": 2.2551887035369873,
      "epoch": 6.96,
      "step": 301499
    },
    {
      "epoch": 6.96,
      "learning_rate": 4.053673469387755e-05,
      "loss": 2.37,
      "step": 301500
    },
    {
      "FLOPS loss": 0.06603013724088669,
      "L0_d": 890.02,
      "MLM loss": 2.2235827445983887,
      "epoch": 6.98,
      "step": 301999
    },
    {
      "epoch": 6.98,
      "learning_rate": 4.0434693877551024e-05,
      "loss": 2.3691,
      "step": 302000
    },
    {
      "FLOPS loss": 0.06309976428747177,
      "L0_d": 1070.56,
      "MLM loss": 2.1997439861297607,
      "epoch": 6.99,
      "step": 302499
    },
    {
      "epoch": 6.99,
      "learning_rate": 4.033265306122449e-05,
      "loss": 2.3694,
      "step": 302500
    },
    {
      "FLOPS loss": 0.06498320400714874,
      "L0_d": 823.11,
      "MLM loss": 2.5637640953063965,
      "epoch": 7.0,
      "step": 302999
    },
    {
      "epoch": 7.0,
      "learning_rate": 4.023081632653061e-05,
      "loss": 2.3683,
      "step": 303000
    },
    {
      "FLOPS loss": 0.06452923268079758,
      "L0_d": 732.8,
      "MLM loss": 2.0735771656036377,
      "epoch": 7.01,
      "step": 303499
    },
    {
      "epoch": 7.01,
      "learning_rate": 4.012877551020409e-05,
      "loss": 2.3687,
      "step": 303500
    },
    {
      "FLOPS loss": 0.06292728334665298,
      "L0_d": 832.94,
      "MLM loss": 2.518744945526123,
      "epoch": 7.02,
      "step": 303999
    },
    {
      "epoch": 7.02,
      "learning_rate": 4.002673469387755e-05,
      "loss": 2.3689,
      "step": 304000
    },
    {
      "FLOPS loss": 0.0746520385146141,
      "L0_d": 937.45,
      "MLM loss": 2.2790918350219727,
      "epoch": 7.03,
      "step": 304499
    },
    {
      "epoch": 7.03,
      "learning_rate": 3.9924693877551024e-05,
      "loss": 2.3638,
      "step": 304500
    },
    {
      "FLOPS loss": 0.07710359990596771,
      "L0_d": 1098.89,
      "MLM loss": 2.253662347793579,
      "epoch": 7.05,
      "step": 304999
    },
    {
      "epoch": 7.05,
      "learning_rate": 3.982265306122449e-05,
      "loss": 2.3666,
      "step": 305000
    },
    {
      "FLOPS loss": 0.07163328677415848,
      "L0_d": 895.14,
      "MLM loss": 2.1064743995666504,
      "epoch": 7.06,
      "step": 305499
    },
    {
      "epoch": 7.06,
      "learning_rate": 3.972061224489796e-05,
      "loss": 2.3634,
      "step": 305500
    },
    {
      "FLOPS loss": 0.05717089772224426,
      "L0_d": 643.52,
      "MLM loss": 2.196295976638794,
      "epoch": 7.07,
      "step": 305999
    },
    {
      "epoch": 7.07,
      "learning_rate": 3.961857142857143e-05,
      "loss": 2.3612,
      "step": 306000
    },
    {
      "FLOPS loss": 0.07105471938848495,
      "L0_d": 981.28,
      "MLM loss": 2.2125749588012695,
      "epoch": 7.08,
      "step": 306499
    },
    {
      "epoch": 7.08,
      "learning_rate": 3.951673469387755e-05,
      "loss": 2.3651,
      "step": 306500
    },
    {
      "FLOPS loss": 0.07635489851236343,
      "L0_d": 1220.39,
      "MLM loss": 2.259152412414551,
      "epoch": 7.09,
      "step": 306999
    },
    {
      "epoch": 7.09,
      "learning_rate": 3.9414693877551024e-05,
      "loss": 2.3585,
      "step": 307000
    },
    {
      "FLOPS loss": 0.056164611130952835,
      "L0_d": 959.52,
      "MLM loss": 2.4167141914367676,
      "epoch": 7.1,
      "step": 307499
    },
    {
      "epoch": 7.1,
      "learning_rate": 3.931265306122449e-05,
      "loss": 2.3656,
      "step": 307500
    },
    {
      "FLOPS loss": 0.05773361772298813,
      "L0_d": 784.94,
      "MLM loss": 2.4181439876556396,
      "epoch": 7.12,
      "step": 307999
    },
    {
      "epoch": 7.12,
      "learning_rate": 3.921061224489796e-05,
      "loss": 2.3638,
      "step": 308000
    },
    {
      "FLOPS loss": 0.06944171339273453,
      "L0_d": 838.94,
      "MLM loss": 2.3983397483825684,
      "epoch": 7.13,
      "step": 308499
    },
    {
      "epoch": 7.13,
      "learning_rate": 3.910877551020408e-05,
      "loss": 2.3641,
      "step": 308500
    },
    {
      "FLOPS loss": 0.08090784400701523,
      "L0_d": 1157.02,
      "MLM loss": 2.3439295291900635,
      "epoch": 7.14,
      "step": 308999
    },
    {
      "epoch": 7.14,
      "learning_rate": 3.900673469387755e-05,
      "loss": 2.3621,
      "step": 309000
    },
    {
      "FLOPS loss": 0.06606020033359528,
      "L0_d": 867.73,
      "MLM loss": 2.192032814025879,
      "epoch": 7.15,
      "step": 309499
    },
    {
      "epoch": 7.15,
      "learning_rate": 3.8904693877551024e-05,
      "loss": 2.364,
      "step": 309500
    },
    {
      "FLOPS loss": 0.06273338198661804,
      "L0_d": 827.27,
      "MLM loss": 2.3575186729431152,
      "epoch": 7.16,
      "step": 309999
    },
    {
      "epoch": 7.16,
      "learning_rate": 3.8802653061224495e-05,
      "loss": 2.3614,
      "step": 310000
    },
    {
      "FLOPS loss": 0.06386949867010117,
      "L0_d": 798.56,
      "MLM loss": 2.291250705718994,
      "epoch": 7.17,
      "step": 310499
    },
    {
      "epoch": 7.17,
      "learning_rate": 3.8700816326530616e-05,
      "loss": 2.3663,
      "step": 310500
    },
    {
      "FLOPS loss": 0.0672420784831047,
      "L0_d": 785.33,
      "MLM loss": 2.1985931396484375,
      "epoch": 7.18,
      "step": 310999
    },
    {
      "epoch": 7.18,
      "learning_rate": 3.859877551020408e-05,
      "loss": 2.3613,
      "step": 311000
    },
    {
      "FLOPS loss": 0.05241331830620766,
      "L0_d": 664.06,
      "MLM loss": 2.3520312309265137,
      "epoch": 7.2,
      "step": 311499
    },
    {
      "epoch": 7.2,
      "learning_rate": 3.849673469387755e-05,
      "loss": 2.3652,
      "step": 311500
    },
    {
      "FLOPS loss": 0.07527154684066772,
      "L0_d": 1110.08,
      "MLM loss": 2.1942214965820312,
      "epoch": 7.21,
      "step": 311999
    },
    {
      "epoch": 7.21,
      "learning_rate": 3.8394693877551024e-05,
      "loss": 2.367,
      "step": 312000
    },
    {
      "FLOPS loss": 0.07647871971130371,
      "L0_d": 1271.92,
      "MLM loss": 2.137394428253174,
      "epoch": 7.22,
      "step": 312499
    },
    {
      "epoch": 7.22,
      "learning_rate": 3.8292857142857144e-05,
      "loss": 2.3624,
      "step": 312500
    },
    {
      "FLOPS loss": 0.06865822523832321,
      "L0_d": 896.69,
      "MLM loss": 2.2419652938842773,
      "epoch": 7.23,
      "step": 312999
    },
    {
      "epoch": 7.23,
      "learning_rate": 3.8190816326530616e-05,
      "loss": 2.3603,
      "step": 313000
    },
    {
      "FLOPS loss": 0.061193566769361496,
      "L0_d": 932.89,
      "MLM loss": 2.2621870040893555,
      "epoch": 7.24,
      "step": 313499
    },
    {
      "epoch": 7.24,
      "learning_rate": 3.808877551020408e-05,
      "loss": 2.363,
      "step": 313500
    },
    {
      "FLOPS loss": 0.06347547471523285,
      "L0_d": 807.72,
      "MLM loss": 2.4694080352783203,
      "epoch": 7.25,
      "step": 313999
    },
    {
      "epoch": 7.25,
      "learning_rate": 3.798673469387755e-05,
      "loss": 2.3612,
      "step": 314000
    },
    {
      "FLOPS loss": 0.059469565749168396,
      "L0_d": 726.23,
      "MLM loss": 2.3274638652801514,
      "epoch": 7.27,
      "step": 314499
    },
    {
      "epoch": 7.27,
      "learning_rate": 3.788489795918367e-05,
      "loss": 2.3589,
      "step": 314500
    },
    {
      "FLOPS loss": 0.056309543550014496,
      "L0_d": 703.62,
      "MLM loss": 2.3300890922546387,
      "epoch": 7.28,
      "step": 314999
    },
    {
      "epoch": 7.28,
      "learning_rate": 3.7782857142857144e-05,
      "loss": 2.3569,
      "step": 315000
    },
    {
      "FLOPS loss": 0.06213608756661415,
      "L0_d": 892.17,
      "MLM loss": 2.4156923294067383,
      "epoch": 7.29,
      "step": 315499
    },
    {
      "epoch": 7.29,
      "learning_rate": 3.7680816326530616e-05,
      "loss": 2.3593,
      "step": 315500
    },
    {
      "FLOPS loss": 0.05070233717560768,
      "L0_d": 730.11,
      "MLM loss": 2.285841703414917,
      "epoch": 7.3,
      "step": 315999
    },
    {
      "epoch": 7.3,
      "learning_rate": 3.757877551020409e-05,
      "loss": 2.3574,
      "step": 316000
    },
    {
      "FLOPS loss": 0.05257587879896164,
      "L0_d": 803.59,
      "MLM loss": 2.432795524597168,
      "epoch": 7.31,
      "step": 316499
    },
    {
      "epoch": 7.31,
      "learning_rate": 3.747693877551021e-05,
      "loss": 2.3581,
      "step": 316500
    },
    {
      "FLOPS loss": 0.08210153877735138,
      "L0_d": 1165.5,
      "MLM loss": 2.3357038497924805,
      "epoch": 7.32,
      "step": 316999
    },
    {
      "epoch": 7.32,
      "learning_rate": 3.737489795918367e-05,
      "loss": 2.362,
      "step": 317000
    },
    {
      "FLOPS loss": 0.07381358742713928,
      "L0_d": 1099.89,
      "MLM loss": 2.410074234008789,
      "epoch": 7.33,
      "step": 317499
    },
    {
      "epoch": 7.33,
      "learning_rate": 3.7272857142857144e-05,
      "loss": 2.3552,
      "step": 317500
    },
    {
      "FLOPS loss": 0.06650281697511673,
      "L0_d": 752.72,
      "MLM loss": 2.2369513511657715,
      "epoch": 7.35,
      "step": 317999
    },
    {
      "epoch": 7.35,
      "learning_rate": 3.7170816326530615e-05,
      "loss": 2.3599,
      "step": 318000
    },
    {
      "FLOPS loss": 0.05438707023859024,
      "L0_d": 769.48,
      "MLM loss": 2.160310745239258,
      "epoch": 7.36,
      "step": 318499
    },
    {
      "epoch": 7.36,
      "learning_rate": 3.706877551020409e-05,
      "loss": 2.36,
      "step": 318500
    },
    {
      "FLOPS loss": 0.06458336114883423,
      "L0_d": 743.03,
      "MLM loss": 2.202986240386963,
      "epoch": 7.37,
      "step": 318999
    },
    {
      "epoch": 7.37,
      "learning_rate": 3.696693877551021e-05,
      "loss": 2.3602,
      "step": 319000
    },
    {
      "FLOPS loss": 0.08015909790992737,
      "L0_d": 981.23,
      "MLM loss": 2.1715171337127686,
      "epoch": 7.38,
      "step": 319499
    },
    {
      "epoch": 7.38,
      "learning_rate": 3.686489795918367e-05,
      "loss": 2.3598,
      "step": 319500
    },
    {
      "FLOPS loss": 0.0799102857708931,
      "L0_d": 1328.16,
      "MLM loss": 2.2394657135009766,
      "epoch": 7.39,
      "step": 319999
    },
    {
      "epoch": 7.39,
      "learning_rate": 3.6762857142857144e-05,
      "loss": 2.3564,
      "step": 320000
    },
    {
      "FLOPS loss": 0.05967347323894501,
      "L0_d": 700.19,
      "MLM loss": 2.1987807750701904,
      "epoch": 7.4,
      "step": 320499
    },
    {
      "epoch": 7.4,
      "learning_rate": 3.666081632653061e-05,
      "loss": 2.3597,
      "step": 320500
    },
    {
      "FLOPS loss": 0.060638427734375,
      "L0_d": 749.98,
      "MLM loss": 2.3340020179748535,
      "epoch": 7.42,
      "step": 320999
    },
    {
      "epoch": 7.42,
      "learning_rate": 3.655877551020409e-05,
      "loss": 2.3565,
      "step": 321000
    },
    {
      "FLOPS loss": 0.06832794100046158,
      "L0_d": 881.05,
      "MLM loss": 2.452484369277954,
      "epoch": 7.43,
      "step": 321499
    },
    {
      "epoch": 7.43,
      "learning_rate": 3.645693877551021e-05,
      "loss": 2.3592,
      "step": 321500
    },
    {
      "FLOPS loss": 0.06505056470632553,
      "L0_d": 859.77,
      "MLM loss": 2.3983802795410156,
      "epoch": 7.44,
      "step": 321999
    },
    {
      "epoch": 7.44,
      "learning_rate": 3.635489795918368e-05,
      "loss": 2.3595,
      "step": 322000
    },
    {
      "FLOPS loss": 0.06231040507555008,
      "L0_d": 796.2,
      "MLM loss": 2.3952202796936035,
      "epoch": 7.45,
      "step": 322499
    },
    {
      "epoch": 7.45,
      "learning_rate": 3.6252857142857144e-05,
      "loss": 2.3584,
      "step": 322500
    },
    {
      "FLOPS loss": 0.05062004178762436,
      "L0_d": 647.19,
      "MLM loss": 2.355503559112549,
      "epoch": 7.46,
      "step": 322999
    },
    {
      "epoch": 7.46,
      "learning_rate": 3.6150816326530615e-05,
      "loss": 2.3588,
      "step": 323000
    },
    {
      "FLOPS loss": 0.05564975365996361,
      "L0_d": 723.34,
      "MLM loss": 2.186389207839966,
      "epoch": 7.47,
      "step": 323499
    },
    {
      "epoch": 7.47,
      "learning_rate": 3.6048775510204086e-05,
      "loss": 2.35,
      "step": 323500
    },
    {
      "FLOPS loss": 0.06510195136070251,
      "L0_d": 705.69,
      "MLM loss": 2.2318332195281982,
      "epoch": 7.48,
      "step": 323999
    },
    {
      "epoch": 7.48,
      "learning_rate": 3.594693877551021e-05,
      "loss": 2.3493,
      "step": 324000
    },
    {
      "FLOPS loss": 0.07114795595407486,
      "L0_d": 1248.36,
      "MLM loss": 2.2258198261260986,
      "epoch": 7.5,
      "step": 324499
    },
    {
      "epoch": 7.5,
      "learning_rate": 3.584489795918368e-05,
      "loss": 2.3566,
      "step": 324500
    },
    {
      "FLOPS loss": 0.06501613557338715,
      "L0_d": 806.12,
      "MLM loss": 2.154510021209717,
      "epoch": 7.51,
      "step": 324999
    },
    {
      "epoch": 7.51,
      "learning_rate": 3.574285714285714e-05,
      "loss": 2.3544,
      "step": 325000
    },
    {
      "FLOPS loss": 0.06274013221263885,
      "L0_d": 906.39,
      "MLM loss": 2.1443748474121094,
      "epoch": 7.52,
      "step": 325499
    },
    {
      "epoch": 7.52,
      "learning_rate": 3.5640816326530615e-05,
      "loss": 2.3583,
      "step": 325500
    },
    {
      "FLOPS loss": 0.06960269808769226,
      "L0_d": 1022.05,
      "MLM loss": 2.3154430389404297,
      "epoch": 7.53,
      "step": 325999
    },
    {
      "epoch": 7.53,
      "learning_rate": 3.5538979591836735e-05,
      "loss": 2.3535,
      "step": 326000
    },
    {
      "FLOPS loss": 0.05796834081411362,
      "L0_d": 738.72,
      "MLM loss": 2.11289119720459,
      "epoch": 7.54,
      "step": 326499
    },
    {
      "epoch": 7.54,
      "learning_rate": 3.54369387755102e-05,
      "loss": 2.3555,
      "step": 326500
    },
    {
      "FLOPS loss": 0.08279315382242203,
      "L0_d": 936.34,
      "MLM loss": 2.1696484088897705,
      "epoch": 7.55,
      "step": 326999
    },
    {
      "epoch": 7.55,
      "learning_rate": 3.533489795918368e-05,
      "loss": 2.3557,
      "step": 327000
    },
    {
      "FLOPS loss": 0.07448771595954895,
      "L0_d": 977.88,
      "MLM loss": 2.3000853061676025,
      "epoch": 7.57,
      "step": 327499
    },
    {
      "epoch": 7.57,
      "learning_rate": 3.523285714285714e-05,
      "loss": 2.3518,
      "step": 327500
    },
    {
      "FLOPS loss": 0.06067269667983055,
      "L0_d": 716.92,
      "MLM loss": 2.23938250541687,
      "epoch": 7.58,
      "step": 327999
    },
    {
      "epoch": 7.58,
      "learning_rate": 3.5130816326530615e-05,
      "loss": 2.3565,
      "step": 328000
    },
    {
      "FLOPS loss": 0.056932076811790466,
      "L0_d": 786.47,
      "MLM loss": 2.4538161754608154,
      "epoch": 7.59,
      "step": 328499
    },
    {
      "epoch": 7.59,
      "learning_rate": 3.5028979591836735e-05,
      "loss": 2.3513,
      "step": 328500
    },
    {
      "FLOPS loss": 0.0752127468585968,
      "L0_d": 935.5,
      "MLM loss": 2.1751489639282227,
      "epoch": 7.6,
      "step": 328999
    },
    {
      "epoch": 7.6,
      "learning_rate": 3.492693877551021e-05,
      "loss": 2.3496,
      "step": 329000
    },
    {
      "FLOPS loss": 0.055434972047805786,
      "L0_d": 711.95,
      "MLM loss": 2.378598690032959,
      "epoch": 7.61,
      "step": 329499
    },
    {
      "epoch": 7.61,
      "learning_rate": 3.482489795918368e-05,
      "loss": 2.3567,
      "step": 329500
    },
    {
      "FLOPS loss": 0.07803203910589218,
      "L0_d": 961.14,
      "MLM loss": 2.332076072692871,
      "epoch": 7.62,
      "step": 329999
    },
    {
      "epoch": 7.62,
      "learning_rate": 3.472285714285714e-05,
      "loss": 2.3558,
      "step": 330000
    },
    {
      "FLOPS loss": 0.0613698773086071,
      "L0_d": 854.08,
      "MLM loss": 2.2925326824188232,
      "epoch": 7.63,
      "step": 330499
    },
    {
      "epoch": 7.63,
      "learning_rate": 3.4620816326530614e-05,
      "loss": 2.3536,
      "step": 330500
    },
    {
      "FLOPS loss": 0.0583004504442215,
      "L0_d": 613.38,
      "MLM loss": 2.266010284423828,
      "epoch": 7.65,
      "step": 330999
    },
    {
      "epoch": 7.65,
      "learning_rate": 3.4518979591836735e-05,
      "loss": 2.3546,
      "step": 331000
    },
    {
      "FLOPS loss": 0.06375960260629654,
      "L0_d": 875.72,
      "MLM loss": 2.4512927532196045,
      "epoch": 7.66,
      "step": 331499
    },
    {
      "epoch": 7.66,
      "learning_rate": 3.4416938775510207e-05,
      "loss": 2.3504,
      "step": 331500
    },
    {
      "FLOPS loss": 0.06886997818946838,
      "L0_d": 880.61,
      "MLM loss": 2.2343506813049316,
      "epoch": 7.67,
      "step": 331999
    },
    {
      "epoch": 7.67,
      "learning_rate": 3.431489795918367e-05,
      "loss": 2.3526,
      "step": 332000
    },
    {
      "FLOPS loss": 0.06465931236743927,
      "L0_d": 1193.62,
      "MLM loss": 2.2527096271514893,
      "epoch": 7.68,
      "step": 332499
    },
    {
      "epoch": 7.68,
      "learning_rate": 3.421285714285715e-05,
      "loss": 2.3474,
      "step": 332500
    },
    {
      "FLOPS loss": 0.05739612132310867,
      "L0_d": 848.39,
      "MLM loss": 2.299609422683716,
      "epoch": 7.69,
      "step": 332999
    },
    {
      "epoch": 7.69,
      "learning_rate": 3.411102040816327e-05,
      "loss": 2.3476,
      "step": 333000
    },
    {
      "FLOPS loss": 0.05512355640530586,
      "L0_d": 688.33,
      "MLM loss": 2.35714054107666,
      "epoch": 7.7,
      "step": 333499
    },
    {
      "epoch": 7.7,
      "learning_rate": 3.4008979591836735e-05,
      "loss": 2.3566,
      "step": 333500
    },
    {
      "FLOPS loss": 0.07219377160072327,
      "L0_d": 1120.45,
      "MLM loss": 2.3413336277008057,
      "epoch": 7.72,
      "step": 333999
    },
    {
      "epoch": 7.72,
      "learning_rate": 3.3906938775510206e-05,
      "loss": 2.3516,
      "step": 334000
    },
    {
      "FLOPS loss": 0.059764862060546875,
      "L0_d": 846.83,
      "MLM loss": 2.2598955631256104,
      "epoch": 7.73,
      "step": 334499
    },
    {
      "epoch": 7.73,
      "learning_rate": 3.380489795918367e-05,
      "loss": 2.3526,
      "step": 334500
    },
    {
      "FLOPS loss": 0.057846374809741974,
      "L0_d": 834.73,
      "MLM loss": 2.469571113586426,
      "epoch": 7.74,
      "step": 334999
    },
    {
      "epoch": 7.74,
      "learning_rate": 3.37030612244898e-05,
      "loss": 2.3504,
      "step": 335000
    },
    {
      "FLOPS loss": 0.06122131273150444,
      "L0_d": 781.16,
      "MLM loss": 2.2299857139587402,
      "epoch": 7.75,
      "step": 335499
    },
    {
      "epoch": 7.75,
      "learning_rate": 3.360102040816327e-05,
      "loss": 2.3489,
      "step": 335500
    },
    {
      "FLOPS loss": 0.057046886533498764,
      "L0_d": 740.44,
      "MLM loss": 2.0816891193389893,
      "epoch": 7.76,
      "step": 335999
    },
    {
      "epoch": 7.76,
      "learning_rate": 3.3498979591836735e-05,
      "loss": 2.3499,
      "step": 336000
    },
    {
      "FLOPS loss": 0.06181034818291664,
      "L0_d": 745.03,
      "MLM loss": 2.248274087905884,
      "epoch": 7.77,
      "step": 336499
    },
    {
      "epoch": 7.77,
      "learning_rate": 3.3396938775510206e-05,
      "loss": 2.3468,
      "step": 336500
    },
    {
      "FLOPS loss": 0.07484866678714752,
      "L0_d": 1056.61,
      "MLM loss": 2.207113027572632,
      "epoch": 7.79,
      "step": 336999
    },
    {
      "epoch": 7.79,
      "learning_rate": 3.329489795918367e-05,
      "loss": 2.3452,
      "step": 337000
    },
    {
      "FLOPS loss": 0.06287125498056412,
      "L0_d": 827.58,
      "MLM loss": 2.3059585094451904,
      "epoch": 7.8,
      "step": 337499
    },
    {
      "epoch": 7.8,
      "learning_rate": 3.31930612244898e-05,
      "loss": 2.3487,
      "step": 337500
    },
    {
      "FLOPS loss": 0.06856507062911987,
      "L0_d": 1210.05,
      "MLM loss": 2.2399425506591797,
      "epoch": 7.81,
      "step": 337999
    },
    {
      "epoch": 7.81,
      "learning_rate": 3.309102040816326e-05,
      "loss": 2.3499,
      "step": 338000
    },
    {
      "FLOPS loss": 0.06494159996509552,
      "L0_d": 690.55,
      "MLM loss": 2.3285300731658936,
      "epoch": 7.82,
      "step": 338499
    },
    {
      "epoch": 7.82,
      "learning_rate": 3.298897959183674e-05,
      "loss": 2.349,
      "step": 338500
    },
    {
      "FLOPS loss": 0.06229567155241966,
      "L0_d": 870.64,
      "MLM loss": 2.2471065521240234,
      "epoch": 7.83,
      "step": 338999
    },
    {
      "epoch": 7.83,
      "learning_rate": 3.2886938775510206e-05,
      "loss": 2.3463,
      "step": 339000
    },
    {
      "FLOPS loss": 0.07021673768758774,
      "L0_d": 757.09,
      "MLM loss": 2.137368679046631,
      "epoch": 7.84,
      "step": 339499
    },
    {
      "epoch": 7.84,
      "learning_rate": 3.278510204081633e-05,
      "loss": 2.3485,
      "step": 339500
    },
    {
      "FLOPS loss": 0.058832865208387375,
      "L0_d": 844.09,
      "MLM loss": 2.342620849609375,
      "epoch": 7.85,
      "step": 339999
    },
    {
      "epoch": 7.85,
      "learning_rate": 3.26830612244898e-05,
      "loss": 2.3447,
      "step": 340000
    },
    {
      "FLOPS loss": 0.07662883400917053,
      "L0_d": 809.62,
      "MLM loss": 2.345280408859253,
      "epoch": 7.87,
      "step": 340499
    },
    {
      "epoch": 7.87,
      "learning_rate": 3.258102040816326e-05,
      "loss": 2.3478,
      "step": 340500
    },
    {
      "FLOPS loss": 0.06886345148086548,
      "L0_d": 936.47,
      "MLM loss": 2.2469725608825684,
      "epoch": 7.88,
      "step": 340999
    },
    {
      "epoch": 7.88,
      "learning_rate": 3.247918367346939e-05,
      "loss": 2.3451,
      "step": 341000
    },
    {
      "FLOPS loss": 0.07367608696222305,
      "L0_d": 989.2,
      "MLM loss": 2.325202465057373,
      "epoch": 7.89,
      "step": 341499
    },
    {
      "epoch": 7.89,
      "learning_rate": 3.237714285714286e-05,
      "loss": 2.3468,
      "step": 341500
    },
    {
      "FLOPS loss": 0.0521760955452919,
      "L0_d": 719.47,
      "MLM loss": 2.2234766483306885,
      "epoch": 7.9,
      "step": 341999
    },
    {
      "epoch": 7.9,
      "learning_rate": 3.2275102040816326e-05,
      "loss": 2.3465,
      "step": 342000
    },
    {
      "FLOPS loss": 0.04794573411345482,
      "L0_d": 775.98,
      "MLM loss": 2.3546948432922363,
      "epoch": 7.91,
      "step": 342499
    },
    {
      "epoch": 7.91,
      "learning_rate": 3.21730612244898e-05,
      "loss": 2.3463,
      "step": 342500
    },
    {
      "FLOPS loss": 0.07742329686880112,
      "L0_d": 1222.39,
      "MLM loss": 2.299515962600708,
      "epoch": 7.92,
      "step": 342999
    },
    {
      "epoch": 7.92,
      "learning_rate": 3.207102040816326e-05,
      "loss": 2.3515,
      "step": 343000
    },
    {
      "FLOPS loss": 0.06620312482118607,
      "L0_d": 818.12,
      "MLM loss": 2.2101948261260986,
      "epoch": 7.94,
      "step": 343499
    },
    {
      "epoch": 7.94,
      "learning_rate": 3.1968979591836734e-05,
      "loss": 2.3442,
      "step": 343500
    },
    {
      "FLOPS loss": 0.06726676225662231,
      "L0_d": 865.52,
      "MLM loss": 2.2596335411071777,
      "epoch": 7.95,
      "step": 343999
    },
    {
      "epoch": 7.95,
      "learning_rate": 3.1866938775510206e-05,
      "loss": 2.343,
      "step": 344000
    },
    {
      "FLOPS loss": 0.07519533485174179,
      "L0_d": 1003.86,
      "MLM loss": 2.146808385848999,
      "epoch": 7.96,
      "step": 344499
    },
    {
      "epoch": 7.96,
      "learning_rate": 3.176489795918368e-05,
      "loss": 2.3454,
      "step": 344500
    },
    {
      "FLOPS loss": 0.05884729325771332,
      "L0_d": 785.52,
      "MLM loss": 2.313364267349243,
      "epoch": 7.97,
      "step": 344999
    },
    {
      "epoch": 7.97,
      "learning_rate": 3.16630612244898e-05,
      "loss": 2.3491,
      "step": 345000
    },
    {
      "FLOPS loss": 0.06093428656458855,
      "L0_d": 809.14,
      "MLM loss": 2.34356689453125,
      "epoch": 7.98,
      "step": 345499
    },
    {
      "epoch": 7.98,
      "learning_rate": 3.156102040816327e-05,
      "loss": 2.3459,
      "step": 345500
    },
    {
      "FLOPS loss": 0.06816817075014114,
      "L0_d": 1047.36,
      "MLM loss": 2.244194507598877,
      "epoch": 7.99,
      "step": 345999
    },
    {
      "epoch": 7.99,
      "learning_rate": 3.1458979591836734e-05,
      "loss": 2.3464,
      "step": 346000
    },
    {
      "FLOPS loss": 0.05808025971055031,
      "L0_d": 676.52,
      "MLM loss": 2.2438483238220215,
      "epoch": 8.0,
      "step": 346499
    },
    {
      "epoch": 8.0,
      "learning_rate": 3.1356938775510205e-05,
      "loss": 2.345,
      "step": 346500
    },
    {
      "FLOPS loss": 0.06837411969900131,
      "L0_d": 1041.8,
      "MLM loss": 2.158121347427368,
      "epoch": 8.02,
      "step": 346999
    },
    {
      "epoch": 8.02,
      "learning_rate": 3.125489795918368e-05,
      "loss": 2.3381,
      "step": 347000
    },
    {
      "FLOPS loss": 0.0699104368686676,
      "L0_d": 1003.94,
      "MLM loss": 2.311729907989502,
      "epoch": 8.03,
      "step": 347499
    },
    {
      "epoch": 8.03,
      "learning_rate": 3.11530612244898e-05,
      "loss": 2.341,
      "step": 347500
    },
    {
      "FLOPS loss": 0.06548525393009186,
      "L0_d": 820.19,
      "MLM loss": 2.227398633956909,
      "epoch": 8.04,
      "step": 347999
    },
    {
      "epoch": 8.04,
      "learning_rate": 3.105102040816327e-05,
      "loss": 2.344,
      "step": 348000
    },
    {
      "FLOPS loss": 0.054751232266426086,
      "L0_d": 688.12,
      "MLM loss": 1.976632833480835,
      "epoch": 8.05,
      "step": 348499
    },
    {
      "epoch": 8.05,
      "learning_rate": 3.0948979591836734e-05,
      "loss": 2.338,
      "step": 348500
    },
    {
      "FLOPS loss": 0.0631544440984726,
      "L0_d": 868.38,
      "MLM loss": 2.347715377807617,
      "epoch": 8.06,
      "step": 348999
    },
    {
      "epoch": 8.06,
      "learning_rate": 3.0846938775510205e-05,
      "loss": 2.3413,
      "step": 349000
    },
    {
      "FLOPS loss": 0.065920390188694,
      "L0_d": 734.41,
      "MLM loss": 2.276221990585327,
      "epoch": 8.07,
      "step": 349499
    },
    {
      "epoch": 8.07,
      "learning_rate": 3.074489795918368e-05,
      "loss": 2.3381,
      "step": 349500
    },
    {
      "FLOPS loss": 0.06035129353404045,
      "L0_d": 884.69,
      "MLM loss": 2.314701557159424,
      "epoch": 8.09,
      "step": 349999
    },
    {
      "epoch": 8.09,
      "learning_rate": 3.06430612244898e-05,
      "loss": 2.3396,
      "step": 350000
    },
    {
      "FLOPS loss": 0.06962145864963531,
      "L0_d": 1142.48,
      "MLM loss": 2.310885190963745,
      "epoch": 8.1,
      "step": 350499
    },
    {
      "epoch": 8.1,
      "learning_rate": 3.054102040816327e-05,
      "loss": 2.3369,
      "step": 350500
    },
    {
      "FLOPS loss": 0.06199546530842781,
      "L0_d": 866.83,
      "MLM loss": 2.0542640686035156,
      "epoch": 8.11,
      "step": 350999
    },
    {
      "epoch": 8.11,
      "learning_rate": 3.0438979591836737e-05,
      "loss": 2.3395,
      "step": 351000
    },
    {
      "FLOPS loss": 0.0696890726685524,
      "L0_d": 1098.45,
      "MLM loss": 2.324911117553711,
      "epoch": 8.12,
      "step": 351499
    },
    {
      "epoch": 8.12,
      "learning_rate": 3.0336938775510205e-05,
      "loss": 2.344,
      "step": 351500
    },
    {
      "FLOPS loss": 0.0821695551276207,
      "L0_d": 939.31,
      "MLM loss": 2.1598875522613525,
      "epoch": 8.13,
      "step": 351999
    },
    {
      "epoch": 8.13,
      "learning_rate": 3.0235102040816326e-05,
      "loss": 2.337,
      "step": 352000
    },
    {
      "FLOPS loss": 0.07730408757925034,
      "L0_d": 957.91,
      "MLM loss": 2.1525473594665527,
      "epoch": 8.14,
      "step": 352499
    },
    {
      "epoch": 8.14,
      "learning_rate": 3.0133061224489794e-05,
      "loss": 2.3394,
      "step": 352500
    },
    {
      "FLOPS loss": 0.066657654941082,
      "L0_d": 874.75,
      "MLM loss": 2.327643871307373,
      "epoch": 8.15,
      "step": 352999
    },
    {
      "epoch": 8.15,
      "learning_rate": 3.003102040816327e-05,
      "loss": 2.331,
      "step": 353000
    },
    {
      "FLOPS loss": 0.05348766967654228,
      "L0_d": 813.14,
      "MLM loss": 2.308769702911377,
      "epoch": 8.17,
      "step": 353499
    },
    {
      "epoch": 8.17,
      "learning_rate": 2.9928979591836737e-05,
      "loss": 2.3411,
      "step": 353500
    },
    {
      "FLOPS loss": 0.06494653224945068,
      "L0_d": 882.59,
      "MLM loss": 2.115314245223999,
      "epoch": 8.18,
      "step": 353999
    },
    {
      "epoch": 8.18,
      "learning_rate": 2.9826938775510205e-05,
      "loss": 2.3423,
      "step": 354000
    },
    {
      "FLOPS loss": 0.0652025118470192,
      "L0_d": 910.12,
      "MLM loss": 2.3116602897644043,
      "epoch": 8.19,
      "step": 354499
    },
    {
      "epoch": 8.19,
      "learning_rate": 2.972510204081633e-05,
      "loss": 2.3361,
      "step": 354500
    },
    {
      "FLOPS loss": 0.05392623692750931,
      "L0_d": 715.48,
      "MLM loss": 2.3743042945861816,
      "epoch": 8.2,
      "step": 354999
    },
    {
      "epoch": 8.2,
      "learning_rate": 2.9623061224489797e-05,
      "loss": 2.3366,
      "step": 355000
    },
    {
      "FLOPS loss": 0.061027780175209045,
      "L0_d": 796.05,
      "MLM loss": 2.131939172744751,
      "epoch": 8.21,
      "step": 355499
    },
    {
      "epoch": 8.21,
      "learning_rate": 2.9521224489795918e-05,
      "loss": 2.3333,
      "step": 355500
    },
    {
      "FLOPS loss": 0.05769571289420128,
      "L0_d": 797.92,
      "MLM loss": 2.1019766330718994,
      "epoch": 8.22,
      "step": 355999
    },
    {
      "epoch": 8.22,
      "learning_rate": 2.9419183673469392e-05,
      "loss": 2.3366,
      "step": 356000
    },
    {
      "FLOPS loss": 0.0655391588807106,
      "L0_d": 1163.17,
      "MLM loss": 2.2659382820129395,
      "epoch": 8.24,
      "step": 356499
    },
    {
      "epoch": 8.24,
      "learning_rate": 2.931714285714286e-05,
      "loss": 2.331,
      "step": 356500
    },
    {
      "FLOPS loss": 0.06379566341638565,
      "L0_d": 839.8,
      "MLM loss": 2.1597399711608887,
      "epoch": 8.25,
      "step": 356999
    },
    {
      "epoch": 8.25,
      "learning_rate": 2.921510204081633e-05,
      "loss": 2.335,
      "step": 357000
    },
    {
      "FLOPS loss": 0.06905535608530045,
      "L0_d": 840.22,
      "MLM loss": 2.254056215286255,
      "epoch": 8.26,
      "step": 357499
    },
    {
      "epoch": 8.26,
      "learning_rate": 2.9113061224489797e-05,
      "loss": 2.3367,
      "step": 357500
    },
    {
      "FLOPS loss": 0.07997802644968033,
      "L0_d": 798.45,
      "MLM loss": 2.198784828186035,
      "epoch": 8.27,
      "step": 357999
    },
    {
      "epoch": 8.27,
      "learning_rate": 2.9011020408163265e-05,
      "loss": 2.3338,
      "step": 358000
    },
    {
      "FLOPS loss": 0.05796641856431961,
      "L0_d": 737.81,
      "MLM loss": 2.426778793334961,
      "epoch": 8.28,
      "step": 358499
    },
    {
      "epoch": 8.28,
      "learning_rate": 2.8908979591836736e-05,
      "loss": 2.3367,
      "step": 358500
    },
    {
      "FLOPS loss": 0.06763458997011185,
      "L0_d": 1093.58,
      "MLM loss": 2.2339916229248047,
      "epoch": 8.29,
      "step": 358999
    },
    {
      "epoch": 8.29,
      "learning_rate": 2.8806938775510204e-05,
      "loss": 2.3366,
      "step": 359000
    },
    {
      "FLOPS loss": 0.07436629384756088,
      "L0_d": 956.61,
      "MLM loss": 2.32150936126709,
      "epoch": 8.3,
      "step": 359499
    },
    {
      "epoch": 8.3,
      "learning_rate": 2.870510204081633e-05,
      "loss": 2.3345,
      "step": 359500
    },
    {
      "FLOPS loss": 0.06429058313369751,
      "L0_d": 748.19,
      "MLM loss": 2.3280255794525146,
      "epoch": 8.32,
      "step": 359999
    },
    {
      "epoch": 8.32,
      "learning_rate": 2.8603061224489797e-05,
      "loss": 2.3306,
      "step": 360000
    },
    {
      "FLOPS loss": 0.05643421411514282,
      "L0_d": 732.22,
      "MLM loss": 2.1442980766296387,
      "epoch": 8.33,
      "step": 360499
    },
    {
      "epoch": 8.33,
      "learning_rate": 2.8501020408163265e-05,
      "loss": 2.3343,
      "step": 360500
    },
    {
      "FLOPS loss": 0.05485772714018822,
      "L0_d": 830.95,
      "MLM loss": 2.435837507247925,
      "epoch": 8.34,
      "step": 360999
    },
    {
      "epoch": 8.34,
      "learning_rate": 2.839897959183674e-05,
      "loss": 2.3321,
      "step": 361000
    },
    {
      "FLOPS loss": 0.05842866748571396,
      "L0_d": 631.92,
      "MLM loss": 2.117673397064209,
      "epoch": 8.35,
      "step": 361499
    },
    {
      "epoch": 8.35,
      "learning_rate": 2.829714285714286e-05,
      "loss": 2.3324,
      "step": 361500
    },
    {
      "FLOPS loss": 0.05581725761294365,
      "L0_d": 723.44,
      "MLM loss": 2.2919936180114746,
      "epoch": 8.36,
      "step": 361999
    },
    {
      "epoch": 8.36,
      "learning_rate": 2.8195102040816328e-05,
      "loss": 2.3353,
      "step": 362000
    },
    {
      "FLOPS loss": 0.06750074028968811,
      "L0_d": 1013.06,
      "MLM loss": 2.2367117404937744,
      "epoch": 8.37,
      "step": 362499
    },
    {
      "epoch": 8.37,
      "learning_rate": 2.8093061224489796e-05,
      "loss": 2.3355,
      "step": 362500
    },
    {
      "FLOPS loss": 0.06170298904180527,
      "L0_d": 883.92,
      "MLM loss": 2.2028086185455322,
      "epoch": 8.39,
      "step": 362999
    },
    {
      "epoch": 8.39,
      "learning_rate": 2.7991020408163264e-05,
      "loss": 2.3373,
      "step": 363000
    },
    {
      "FLOPS loss": 0.07157032936811447,
      "L0_d": 1057.62,
      "MLM loss": 2.5820722579956055,
      "epoch": 8.4,
      "step": 363499
    },
    {
      "epoch": 8.4,
      "learning_rate": 2.788918367346939e-05,
      "loss": 2.3328,
      "step": 363500
    },
    {
      "FLOPS loss": 0.06726321578025818,
      "L0_d": 904.16,
      "MLM loss": 2.196951389312744,
      "epoch": 8.41,
      "step": 363999
    },
    {
      "epoch": 8.41,
      "learning_rate": 2.7787142857142857e-05,
      "loss": 2.3322,
      "step": 364000
    },
    {
      "FLOPS loss": 0.06428459286689758,
      "L0_d": 955.66,
      "MLM loss": 2.373371124267578,
      "epoch": 8.42,
      "step": 364499
    },
    {
      "epoch": 8.42,
      "learning_rate": 2.7685102040816328e-05,
      "loss": 2.3342,
      "step": 364500
    },
    {
      "FLOPS loss": 0.07280469685792923,
      "L0_d": 1063.44,
      "MLM loss": 2.370555877685547,
      "epoch": 8.43,
      "step": 364999
    },
    {
      "epoch": 8.43,
      "learning_rate": 2.75830612244898e-05,
      "loss": 2.3317,
      "step": 365000
    },
    {
      "FLOPS loss": 0.08263550698757172,
      "L0_d": 976.44,
      "MLM loss": 2.2560606002807617,
      "epoch": 8.44,
      "step": 365499
    },
    {
      "epoch": 8.44,
      "learning_rate": 2.7481020408163268e-05,
      "loss": 2.3333,
      "step": 365500
    },
    {
      "FLOPS loss": 0.06574293226003647,
      "L0_d": 815.53,
      "MLM loss": 2.3375725746154785,
      "epoch": 8.45,
      "step": 365999
    },
    {
      "epoch": 8.45,
      "learning_rate": 2.7379183673469388e-05,
      "loss": 2.3365,
      "step": 366000
    },
    {
      "FLOPS loss": 0.06853322684764862,
      "L0_d": 1019.62,
      "MLM loss": 2.271479606628418,
      "epoch": 8.47,
      "step": 366499
    },
    {
      "epoch": 8.47,
      "learning_rate": 2.7277142857142856e-05,
      "loss": 2.3327,
      "step": 366500
    },
    {
      "FLOPS loss": 0.05954744666814804,
      "L0_d": 780.14,
      "MLM loss": 2.1975862979888916,
      "epoch": 8.48,
      "step": 366999
    },
    {
      "epoch": 8.48,
      "learning_rate": 2.717510204081633e-05,
      "loss": 2.3286,
      "step": 367000
    },
    {
      "FLOPS loss": 0.059766821563243866,
      "L0_d": 682.48,
      "MLM loss": 2.3481855392456055,
      "epoch": 8.49,
      "step": 367499
    },
    {
      "epoch": 8.49,
      "learning_rate": 2.70730612244898e-05,
      "loss": 2.3316,
      "step": 367500
    },
    {
      "FLOPS loss": 0.06330662965774536,
      "L0_d": 932.05,
      "MLM loss": 2.3544859886169434,
      "epoch": 8.5,
      "step": 367999
    },
    {
      "epoch": 8.5,
      "learning_rate": 2.6971020408163267e-05,
      "loss": 2.3291,
      "step": 368000
    },
    {
      "FLOPS loss": 0.06118059530854225,
      "L0_d": 1056.3,
      "MLM loss": 2.271243095397949,
      "epoch": 8.51,
      "step": 368499
    },
    {
      "epoch": 8.51,
      "learning_rate": 2.6869183673469388e-05,
      "loss": 2.3322,
      "step": 368500
    },
    {
      "FLOPS loss": 0.05249141529202461,
      "L0_d": 698.84,
      "MLM loss": 2.2780275344848633,
      "epoch": 8.52,
      "step": 368999
    },
    {
      "epoch": 8.52,
      "learning_rate": 2.6767142857142856e-05,
      "loss": 2.3301,
      "step": 369000
    },
    {
      "FLOPS loss": 0.06862546503543854,
      "L0_d": 836.22,
      "MLM loss": 2.4001739025115967,
      "epoch": 8.54,
      "step": 369499
    },
    {
      "epoch": 8.54,
      "learning_rate": 2.6665102040816324e-05,
      "loss": 2.3289,
      "step": 369500
    },
    {
      "FLOPS loss": 0.05359656736254692,
      "L0_d": 714.91,
      "MLM loss": 2.456220865249634,
      "epoch": 8.55,
      "step": 369999
    },
    {
      "epoch": 8.55,
      "learning_rate": 2.65630612244898e-05,
      "loss": 2.3307,
      "step": 370000
    },
    {
      "FLOPS loss": 0.06717686355113983,
      "L0_d": 1018.62,
      "MLM loss": 2.279381275177002,
      "epoch": 8.56,
      "step": 370499
    },
    {
      "epoch": 8.56,
      "learning_rate": 2.6461020408163267e-05,
      "loss": 2.3312,
      "step": 370500
    },
    {
      "FLOPS loss": 0.059827450662851334,
      "L0_d": 758.89,
      "MLM loss": 2.482081174850464,
      "epoch": 8.57,
      "step": 370999
    },
    {
      "epoch": 8.57,
      "learning_rate": 2.635918367346939e-05,
      "loss": 2.3335,
      "step": 371000
    },
    {
      "FLOPS loss": 0.08209186792373657,
      "L0_d": 1095.8,
      "MLM loss": 2.0456771850585938,
      "epoch": 8.58,
      "step": 371499
    },
    {
      "epoch": 8.58,
      "learning_rate": 2.625714285714286e-05,
      "loss": 2.327,
      "step": 371500
    },
    {
      "FLOPS loss": 0.0643707886338234,
      "L0_d": 944.06,
      "MLM loss": 2.3206868171691895,
      "epoch": 8.59,
      "step": 371999
    },
    {
      "epoch": 8.59,
      "learning_rate": 2.6155102040816327e-05,
      "loss": 2.3267,
      "step": 372000
    },
    {
      "FLOPS loss": 0.06913337111473083,
      "L0_d": 1096.0,
      "MLM loss": 2.271151065826416,
      "epoch": 8.61,
      "step": 372499
    },
    {
      "epoch": 8.61,
      "learning_rate": 2.60530612244898e-05,
      "loss": 2.33,
      "step": 372500
    },
    {
      "FLOPS loss": 0.07182160764932632,
      "L0_d": 845.78,
      "MLM loss": 2.113877773284912,
      "epoch": 8.62,
      "step": 372999
    },
    {
      "epoch": 8.62,
      "learning_rate": 2.5951224489795923e-05,
      "loss": 2.3322,
      "step": 373000
    },
    {
      "FLOPS loss": 0.06277167797088623,
      "L0_d": 864.33,
      "MLM loss": 2.166372060775757,
      "epoch": 8.63,
      "step": 373499
    },
    {
      "epoch": 8.63,
      "learning_rate": 2.584918367346939e-05,
      "loss": 2.3296,
      "step": 373500
    },
    {
      "FLOPS loss": 0.07612694054841995,
      "L0_d": 1035.11,
      "MLM loss": 2.098829507827759,
      "epoch": 8.64,
      "step": 373999
    },
    {
      "epoch": 8.64,
      "learning_rate": 2.574714285714286e-05,
      "loss": 2.3319,
      "step": 374000
    },
    {
      "FLOPS loss": 0.07450399547815323,
      "L0_d": 1027.17,
      "MLM loss": 2.2074460983276367,
      "epoch": 8.65,
      "step": 374499
    },
    {
      "epoch": 8.65,
      "learning_rate": 2.5645102040816327e-05,
      "loss": 2.3302,
      "step": 374500
    },
    {
      "FLOPS loss": 0.05644816905260086,
      "L0_d": 777.75,
      "MLM loss": 2.3011317253112793,
      "epoch": 8.66,
      "step": 374999
    },
    {
      "epoch": 8.66,
      "learning_rate": 2.5543265306122448e-05,
      "loss": 2.328,
      "step": 375000
    },
    {
      "FLOPS loss": 0.07566533237695694,
      "L0_d": 1021.97,
      "MLM loss": 2.2594408988952637,
      "epoch": 8.67,
      "step": 375499
    },
    {
      "epoch": 8.67,
      "learning_rate": 2.5441224489795916e-05,
      "loss": 2.3286,
      "step": 375500
    },
    {
      "FLOPS loss": 0.07324004918336868,
      "L0_d": 980.56,
      "MLM loss": 2.3295974731445312,
      "epoch": 8.69,
      "step": 375999
    },
    {
      "epoch": 8.69,
      "learning_rate": 2.533918367346939e-05,
      "loss": 2.329,
      "step": 376000
    },
    {
      "FLOPS loss": 0.06034684181213379,
      "L0_d": 641.38,
      "MLM loss": 2.2357232570648193,
      "epoch": 8.7,
      "step": 376499
    },
    {
      "epoch": 8.7,
      "learning_rate": 2.523714285714286e-05,
      "loss": 2.3332,
      "step": 376500
    },
    {
      "FLOPS loss": 0.06770546734333038,
      "L0_d": 754.91,
      "MLM loss": 2.13252329826355,
      "epoch": 8.71,
      "step": 376999
    },
    {
      "epoch": 8.71,
      "learning_rate": 2.5135102040816327e-05,
      "loss": 2.3253,
      "step": 377000
    },
    {
      "FLOPS loss": 0.06255783885717392,
      "L0_d": 936.42,
      "MLM loss": 2.3120524883270264,
      "epoch": 8.72,
      "step": 377499
    },
    {
      "epoch": 8.72,
      "learning_rate": 2.5033061224489795e-05,
      "loss": 2.3254,
      "step": 377500
    },
    {
      "FLOPS loss": 0.05889131501317024,
      "L0_d": 851.02,
      "MLM loss": 2.2504754066467285,
      "epoch": 8.73,
      "step": 377999
    },
    {
      "epoch": 8.73,
      "learning_rate": 2.4931020408163267e-05,
      "loss": 2.3329,
      "step": 378000
    },
    {
      "FLOPS loss": 0.07563289999961853,
      "L0_d": 977.06,
      "MLM loss": 2.1566548347473145,
      "epoch": 8.74,
      "step": 378499
    },
    {
      "epoch": 8.74,
      "learning_rate": 2.4828979591836735e-05,
      "loss": 2.3283,
      "step": 378500
    },
    {
      "FLOPS loss": 0.062325071543455124,
      "L0_d": 886.58,
      "MLM loss": 2.2504539489746094,
      "epoch": 8.76,
      "step": 378999
    },
    {
      "epoch": 8.76,
      "learning_rate": 2.4727142857142855e-05,
      "loss": 2.3259,
      "step": 379000
    },
    {
      "FLOPS loss": 0.06517352163791656,
      "L0_d": 938.56,
      "MLM loss": 2.065889358520508,
      "epoch": 8.77,
      "step": 379499
    },
    {
      "epoch": 8.77,
      "learning_rate": 2.4625102040816327e-05,
      "loss": 2.3258,
      "step": 379500
    },
    {
      "FLOPS loss": 0.0534166656434536,
      "L0_d": 696.45,
      "MLM loss": 2.1012320518493652,
      "epoch": 8.78,
      "step": 379999
    },
    {
      "epoch": 8.78,
      "learning_rate": 2.4523061224489795e-05,
      "loss": 2.3226,
      "step": 380000
    },
    {
      "FLOPS loss": 0.07024350017309189,
      "L0_d": 941.0,
      "MLM loss": 2.2870864868164062,
      "epoch": 8.79,
      "step": 380499
    },
    {
      "epoch": 8.79,
      "learning_rate": 2.4421020408163266e-05,
      "loss": 2.3299,
      "step": 380500
    },
    {
      "FLOPS loss": 0.06589783728122711,
      "L0_d": 771.78,
      "MLM loss": 2.293532371520996,
      "epoch": 8.8,
      "step": 380999
    },
    {
      "epoch": 8.8,
      "learning_rate": 2.4318979591836734e-05,
      "loss": 2.3282,
      "step": 381000
    },
    {
      "FLOPS loss": 0.06442509591579437,
      "L0_d": 754.53,
      "MLM loss": 2.274405002593994,
      "epoch": 8.81,
      "step": 381499
    },
    {
      "epoch": 8.81,
      "learning_rate": 2.421714285714286e-05,
      "loss": 2.3228,
      "step": 381500
    },
    {
      "FLOPS loss": 0.061390411108732224,
      "L0_d": 938.61,
      "MLM loss": 2.2242677211761475,
      "epoch": 8.82,
      "step": 381999
    },
    {
      "epoch": 8.82,
      "learning_rate": 2.411510204081633e-05,
      "loss": 2.3263,
      "step": 382000
    },
    {
      "FLOPS loss": 0.053517624735832214,
      "L0_d": 791.31,
      "MLM loss": 2.1186835765838623,
      "epoch": 8.84,
      "step": 382499
    },
    {
      "epoch": 8.84,
      "learning_rate": 2.4013061224489798e-05,
      "loss": 2.3184,
      "step": 382500
    },
    {
      "FLOPS loss": 0.07673989981412888,
      "L0_d": 1034.72,
      "MLM loss": 2.1242787837982178,
      "epoch": 8.85,
      "step": 382999
    },
    {
      "epoch": 8.85,
      "learning_rate": 2.3911020408163266e-05,
      "loss": 2.3287,
      "step": 383000
    },
    {
      "FLOPS loss": 0.05768826603889465,
      "L0_d": 748.44,
      "MLM loss": 2.2013959884643555,
      "epoch": 8.86,
      "step": 383499
    },
    {
      "epoch": 8.86,
      "learning_rate": 2.380918367346939e-05,
      "loss": 2.3237,
      "step": 383500
    },
    {
      "FLOPS loss": 0.07118546217679977,
      "L0_d": 847.38,
      "MLM loss": 2.2173261642456055,
      "epoch": 8.87,
      "step": 383999
    },
    {
      "epoch": 8.87,
      "learning_rate": 2.370714285714286e-05,
      "loss": 2.3261,
      "step": 384000
    },
    {
      "FLOPS loss": 0.07411494851112366,
      "L0_d": 881.77,
      "MLM loss": 2.3400588035583496,
      "epoch": 8.88,
      "step": 384499
    },
    {
      "epoch": 8.88,
      "learning_rate": 2.3605102040816326e-05,
      "loss": 2.3276,
      "step": 384500
    },
    {
      "FLOPS loss": 0.05761401355266571,
      "L0_d": 958.17,
      "MLM loss": 2.2552809715270996,
      "epoch": 8.89,
      "step": 384999
    },
    {
      "epoch": 8.89,
      "learning_rate": 2.3503061224489798e-05,
      "loss": 2.3257,
      "step": 385000
    },
    {
      "FLOPS loss": 0.0631277784705162,
      "L0_d": 898.61,
      "MLM loss": 2.143202781677246,
      "epoch": 8.91,
      "step": 385499
    },
    {
      "epoch": 8.91,
      "learning_rate": 2.340122448979592e-05,
      "loss": 2.3234,
      "step": 385500
    },
    {
      "FLOPS loss": 0.05597534030675888,
      "L0_d": 722.88,
      "MLM loss": 2.2366416454315186,
      "epoch": 8.92,
      "step": 385999
    },
    {
      "epoch": 8.92,
      "learning_rate": 2.3299183673469387e-05,
      "loss": 2.3207,
      "step": 386000
    },
    {
      "FLOPS loss": 0.05632057785987854,
      "L0_d": 992.81,
      "MLM loss": 2.2561912536621094,
      "epoch": 8.93,
      "step": 386499
    },
    {
      "epoch": 8.93,
      "learning_rate": 2.3197142857142858e-05,
      "loss": 2.3251,
      "step": 386500
    },
    {
      "FLOPS loss": 0.061308152973651886,
      "L0_d": 797.61,
      "MLM loss": 2.1445047855377197,
      "epoch": 8.94,
      "step": 386999
    },
    {
      "epoch": 8.94,
      "learning_rate": 2.3095102040816326e-05,
      "loss": 2.3227,
      "step": 387000
    },
    {
      "FLOPS loss": 0.06084592267870903,
      "L0_d": 671.08,
      "MLM loss": 2.295675754547119,
      "epoch": 8.95,
      "step": 387499
    },
    {
      "epoch": 8.95,
      "learning_rate": 2.299326530612245e-05,
      "loss": 2.3214,
      "step": 387500
    },
    {
      "FLOPS loss": 0.06371425837278366,
      "L0_d": 816.7,
      "MLM loss": 2.254873037338257,
      "epoch": 8.96,
      "step": 387999
    },
    {
      "epoch": 8.96,
      "learning_rate": 2.2891224489795922e-05,
      "loss": 2.3197,
      "step": 388000
    },
    {
      "FLOPS loss": 0.054075125604867935,
      "L0_d": 661.2,
      "MLM loss": 2.2977006435394287,
      "epoch": 8.97,
      "step": 388499
    },
    {
      "epoch": 8.97,
      "learning_rate": 2.278918367346939e-05,
      "loss": 2.3265,
      "step": 388500
    },
    {
      "FLOPS loss": 0.06039130687713623,
      "L0_d": 934.31,
      "MLM loss": 2.118281126022339,
      "epoch": 8.99,
      "step": 388999
    },
    {
      "epoch": 8.99,
      "learning_rate": 2.2687142857142858e-05,
      "loss": 2.3191,
      "step": 389000
    },
    {
      "FLOPS loss": 0.060361187905073166,
      "L0_d": 1027.16,
      "MLM loss": 2.3029537200927734,
      "epoch": 9.0,
      "step": 389499
    },
    {
      "epoch": 9.0,
      "learning_rate": 2.2585306122448982e-05,
      "loss": 2.3242,
      "step": 389500
    },
    {
      "FLOPS loss": 0.05790635943412781,
      "L0_d": 642.02,
      "MLM loss": 2.3230996131896973,
      "epoch": 9.01,
      "step": 389999
    },
    {
      "epoch": 9.01,
      "learning_rate": 2.248326530612245e-05,
      "loss": 2.3203,
      "step": 390000
    },
    {
      "FLOPS loss": 0.06403976678848267,
      "L0_d": 807.78,
      "MLM loss": 2.265406370162964,
      "epoch": 9.02,
      "step": 390499
    },
    {
      "epoch": 9.02,
      "learning_rate": 2.2381224489795918e-05,
      "loss": 2.3134,
      "step": 390500
    },
    {
      "FLOPS loss": 0.05303754657506943,
      "L0_d": 676.48,
      "MLM loss": 2.174380302429199,
      "epoch": 9.03,
      "step": 390999
    },
    {
      "epoch": 9.03,
      "learning_rate": 2.227918367346939e-05,
      "loss": 2.3196,
      "step": 391000
    },
    {
      "FLOPS loss": 0.05659378692507744,
      "L0_d": 665.75,
      "MLM loss": 2.2747392654418945,
      "epoch": 9.04,
      "step": 391499
    },
    {
      "epoch": 9.04,
      "learning_rate": 2.2177142857142858e-05,
      "loss": 2.315,
      "step": 391500
    },
    {
      "FLOPS loss": 0.05577780678868294,
      "L0_d": 716.7,
      "MLM loss": 2.3399062156677246,
      "epoch": 9.06,
      "step": 391999
    },
    {
      "epoch": 9.06,
      "learning_rate": 2.207530612244898e-05,
      "loss": 2.3206,
      "step": 392000
    },
    {
      "FLOPS loss": 0.07858218997716904,
      "L0_d": 882.84,
      "MLM loss": 2.1923105716705322,
      "epoch": 9.07,
      "step": 392499
    },
    {
      "epoch": 9.07,
      "learning_rate": 2.197326530612245e-05,
      "loss": 2.3193,
      "step": 392500
    },
    {
      "FLOPS loss": 0.06561288982629776,
      "L0_d": 1004.69,
      "MLM loss": 2.4301981925964355,
      "epoch": 9.08,
      "step": 392999
    },
    {
      "epoch": 9.08,
      "learning_rate": 2.1871224489795918e-05,
      "loss": 2.3172,
      "step": 393000
    },
    {
      "FLOPS loss": 0.05799808353185654,
      "L0_d": 736.95,
      "MLM loss": 2.291177988052368,
      "epoch": 9.09,
      "step": 393499
    },
    {
      "epoch": 9.09,
      "learning_rate": 2.176918367346939e-05,
      "loss": 2.3225,
      "step": 393500
    },
    {
      "FLOPS loss": 0.06807775795459747,
      "L0_d": 922.12,
      "MLM loss": 2.2046947479248047,
      "epoch": 9.1,
      "step": 393999
    },
    {
      "epoch": 9.1,
      "learning_rate": 2.1667142857142858e-05,
      "loss": 2.3189,
      "step": 394000
    },
    {
      "FLOPS loss": 0.07165581732988358,
      "L0_d": 943.2,
      "MLM loss": 2.2551870346069336,
      "epoch": 9.11,
      "step": 394499
    },
    {
      "epoch": 9.11,
      "learning_rate": 2.1565102040816326e-05,
      "loss": 2.3178,
      "step": 394500
    },
    {
      "FLOPS loss": 0.05095088481903076,
      "L0_d": 776.98,
      "MLM loss": 2.3392550945281982,
      "epoch": 9.12,
      "step": 394999
    },
    {
      "epoch": 9.12,
      "learning_rate": 2.146326530612245e-05,
      "loss": 2.3171,
      "step": 395000
    },
    {
      "FLOPS loss": 0.06212861090898514,
      "L0_d": 827.08,
      "MLM loss": 2.1800572872161865,
      "epoch": 9.14,
      "step": 395499
    },
    {
      "epoch": 9.14,
      "learning_rate": 2.136122448979592e-05,
      "loss": 2.3185,
      "step": 395500
    },
    {
      "FLOPS loss": 0.06836603581905365,
      "L0_d": 1011.97,
      "MLM loss": 2.172955274581909,
      "epoch": 9.15,
      "step": 395999
    },
    {
      "epoch": 9.15,
      "learning_rate": 2.125918367346939e-05,
      "loss": 2.3153,
      "step": 396000
    },
    {
      "FLOPS loss": 0.05926735699176788,
      "L0_d": 921.53,
      "MLM loss": 2.3053805828094482,
      "epoch": 9.16,
      "step": 396499
    },
    {
      "epoch": 9.16,
      "learning_rate": 2.1157142857142857e-05,
      "loss": 2.3188,
      "step": 396500
    },
    {
      "FLOPS loss": 0.07315704971551895,
      "L0_d": 921.8,
      "MLM loss": 2.3621654510498047,
      "epoch": 9.17,
      "step": 396999
    },
    {
      "epoch": 9.17,
      "learning_rate": 2.1055102040816325e-05,
      "loss": 2.3167,
      "step": 397000
    },
    {
      "FLOPS loss": 0.062325432896614075,
      "L0_d": 829.0,
      "MLM loss": 2.2659590244293213,
      "epoch": 9.18,
      "step": 397499
    },
    {
      "epoch": 9.18,
      "learning_rate": 2.095326530612245e-05,
      "loss": 2.3158,
      "step": 397500
    },
    {
      "FLOPS loss": 0.061964452266693115,
      "L0_d": 868.09,
      "MLM loss": 2.215615749359131,
      "epoch": 9.19,
      "step": 397999
    },
    {
      "epoch": 9.19,
      "learning_rate": 2.085122448979592e-05,
      "loss": 2.315,
      "step": 398000
    },
    {
      "FLOPS loss": 0.05633864551782608,
      "L0_d": 734.2,
      "MLM loss": 2.322652816772461,
      "epoch": 9.21,
      "step": 398499
    },
    {
      "epoch": 9.21,
      "learning_rate": 2.074918367346939e-05,
      "loss": 2.319,
      "step": 398500
    },
    {
      "FLOPS loss": 0.0727355107665062,
      "L0_d": 1047.61,
      "MLM loss": 2.278925657272339,
      "epoch": 9.22,
      "step": 398999
    },
    {
      "epoch": 9.22,
      "learning_rate": 2.064714285714286e-05,
      "loss": 2.3147,
      "step": 399000
    },
    {
      "FLOPS loss": 0.052611831575632095,
      "L0_d": 711.98,
      "MLM loss": 2.3901736736297607,
      "epoch": 9.23,
      "step": 399499
    },
    {
      "epoch": 9.23,
      "learning_rate": 2.054530612244898e-05,
      "loss": 2.3119,
      "step": 399500
    },
    {
      "FLOPS loss": 0.05829920992255211,
      "L0_d": 716.16,
      "MLM loss": 2.512298583984375,
      "epoch": 9.24,
      "step": 399999
    },
    {
      "epoch": 9.24,
      "learning_rate": 2.044326530612245e-05,
      "loss": 2.313,
      "step": 400000
    },
    {
      "FLOPS loss": 0.059103552252054214,
      "L0_d": 747.2,
      "MLM loss": 2.4075937271118164,
      "epoch": 9.25,
      "step": 400499
    },
    {
      "epoch": 9.25,
      "learning_rate": 2.0341224489795917e-05,
      "loss": 2.3155,
      "step": 400500
    },
    {
      "FLOPS loss": 0.06063724309206009,
      "L0_d": 813.27,
      "MLM loss": 2.265842914581299,
      "epoch": 9.26,
      "step": 400999
    },
    {
      "epoch": 9.26,
      "learning_rate": 2.023918367346939e-05,
      "loss": 2.3103,
      "step": 401000
    },
    {
      "FLOPS loss": 0.0646892711520195,
      "L0_d": 869.34,
      "MLM loss": 2.1556687355041504,
      "epoch": 9.28,
      "step": 401499
    },
    {
      "epoch": 9.28,
      "learning_rate": 2.0137346938775513e-05,
      "loss": 2.3162,
      "step": 401500
    },
    {
      "FLOPS loss": 0.06266539543867111,
      "L0_d": 819.53,
      "MLM loss": 2.1329922676086426,
      "epoch": 9.29,
      "step": 401999
    },
    {
      "epoch": 9.29,
      "learning_rate": 2.0035510204081634e-05,
      "loss": 2.3117,
      "step": 402000
    },
    {
      "FLOPS loss": 0.0540151484310627,
      "L0_d": 850.94,
      "MLM loss": 2.3327624797821045,
      "epoch": 9.3,
      "step": 402499
    },
    {
      "epoch": 9.3,
      "learning_rate": 1.9933469387755102e-05,
      "loss": 2.3149,
      "step": 402500
    },
    {
      "FLOPS loss": 0.061880338937044144,
      "L0_d": 796.36,
      "MLM loss": 2.212796688079834,
      "epoch": 9.31,
      "step": 402999
    },
    {
      "epoch": 9.31,
      "learning_rate": 1.9831428571428573e-05,
      "loss": 2.3146,
      "step": 403000
    },
    {
      "FLOPS loss": 0.060214437544345856,
      "L0_d": 981.72,
      "MLM loss": 2.1206552982330322,
      "epoch": 9.32,
      "step": 403499
    },
    {
      "epoch": 9.32,
      "learning_rate": 1.972938775510204e-05,
      "loss": 2.3135,
      "step": 403500
    },
    {
      "FLOPS loss": 0.06157975271344185,
      "L0_d": 777.95,
      "MLM loss": 2.2471861839294434,
      "epoch": 9.33,
      "step": 403999
    },
    {
      "epoch": 9.33,
      "learning_rate": 1.9627346938775513e-05,
      "loss": 2.3132,
      "step": 404000
    },
    {
      "FLOPS loss": 0.06235690787434578,
      "L0_d": 889.75,
      "MLM loss": 2.3554129600524902,
      "epoch": 9.34,
      "step": 404499
    },
    {
      "epoch": 9.34,
      "learning_rate": 1.952530612244898e-05,
      "loss": 2.3109,
      "step": 404500
    },
    {
      "FLOPS loss": 0.06397629529237747,
      "L0_d": 991.97,
      "MLM loss": 2.151730537414551,
      "epoch": 9.36,
      "step": 404999
    },
    {
      "epoch": 9.36,
      "learning_rate": 1.9423265306122452e-05,
      "loss": 2.3159,
      "step": 405000
    },
    {
      "FLOPS loss": 0.10298942774534225,
      "L0_d": 1378.95,
      "MLM loss": 2.2610526084899902,
      "epoch": 9.37,
      "step": 405499
    },
    {
      "epoch": 9.37,
      "learning_rate": 1.932122448979592e-05,
      "loss": 2.3077,
      "step": 405500
    },
    {
      "FLOPS loss": 0.07055822759866714,
      "L0_d": 773.44,
      "MLM loss": 2.2387983798980713,
      "epoch": 9.38,
      "step": 405999
    },
    {
      "epoch": 9.38,
      "learning_rate": 1.921938775510204e-05,
      "loss": 2.3157,
      "step": 406000
    },
    {
      "FLOPS loss": 0.07199456542730331,
      "L0_d": 1106.86,
      "MLM loss": 2.1209793090820312,
      "epoch": 9.39,
      "step": 406499
    },
    {
      "epoch": 9.39,
      "learning_rate": 1.911734693877551e-05,
      "loss": 2.3132,
      "step": 406500
    },
    {
      "FLOPS loss": 0.0611785389482975,
      "L0_d": 818.05,
      "MLM loss": 2.2507667541503906,
      "epoch": 9.4,
      "step": 406999
    },
    {
      "epoch": 9.4,
      "learning_rate": 1.901530612244898e-05,
      "loss": 2.3099,
      "step": 407000
    },
    {
      "FLOPS loss": 0.07224996387958527,
      "L0_d": 808.06,
      "MLM loss": 2.213388204574585,
      "epoch": 9.41,
      "step": 407499
    },
    {
      "epoch": 9.41,
      "learning_rate": 1.891326530612245e-05,
      "loss": 2.3127,
      "step": 407500
    },
    {
      "FLOPS loss": 0.08076408505439758,
      "L0_d": 990.81,
      "MLM loss": 2.188027858734131,
      "epoch": 9.43,
      "step": 407999
    },
    {
      "epoch": 9.43,
      "learning_rate": 1.881122448979592e-05,
      "loss": 2.3148,
      "step": 408000
    },
    {
      "FLOPS loss": 0.06700234115123749,
      "L0_d": 1071.58,
      "MLM loss": 2.417141914367676,
      "epoch": 9.44,
      "step": 408499
    },
    {
      "epoch": 9.44,
      "learning_rate": 1.870938775510204e-05,
      "loss": 2.3142,
      "step": 408500
    },
    {
      "FLOPS loss": 0.05891914665699005,
      "L0_d": 648.22,
      "MLM loss": 2.059519052505493,
      "epoch": 9.45,
      "step": 408999
    },
    {
      "epoch": 9.45,
      "learning_rate": 1.860734693877551e-05,
      "loss": 2.3107,
      "step": 409000
    },
    {
      "FLOPS loss": 0.060197729617357254,
      "L0_d": 740.95,
      "MLM loss": 2.3801562786102295,
      "epoch": 9.46,
      "step": 409499
    },
    {
      "epoch": 9.46,
      "learning_rate": 1.850530612244898e-05,
      "loss": 2.3118,
      "step": 409500
    },
    {
      "FLOPS loss": 0.07512973248958588,
      "L0_d": 730.03,
      "MLM loss": 2.2139153480529785,
      "epoch": 9.47,
      "step": 409999
    },
    {
      "epoch": 9.47,
      "learning_rate": 1.840326530612245e-05,
      "loss": 2.3114,
      "step": 410000
    },
    {
      "FLOPS loss": 0.061726443469524384,
      "L0_d": 945.25,
      "MLM loss": 2.4507713317871094,
      "epoch": 9.48,
      "step": 410499
    },
    {
      "epoch": 9.48,
      "learning_rate": 1.8301428571428573e-05,
      "loss": 2.3125,
      "step": 410500
    },
    {
      "FLOPS loss": 0.07949730008840561,
      "L0_d": 1145.23,
      "MLM loss": 2.116482734680176,
      "epoch": 9.49,
      "step": 410999
    },
    {
      "epoch": 9.49,
      "learning_rate": 1.819938775510204e-05,
      "loss": 2.3099,
      "step": 411000
    },
    {
      "FLOPS loss": 0.07129678875207901,
      "L0_d": 848.55,
      "MLM loss": 2.09537672996521,
      "epoch": 9.51,
      "step": 411499
    },
    {
      "epoch": 9.51,
      "learning_rate": 1.8097346938775512e-05,
      "loss": 2.3094,
      "step": 411500
    },
    {
      "FLOPS loss": 0.07410188019275665,
      "L0_d": 856.41,
      "MLM loss": 2.248734712600708,
      "epoch": 9.52,
      "step": 411999
    },
    {
      "epoch": 9.52,
      "learning_rate": 1.799530612244898e-05,
      "loss": 2.3142,
      "step": 412000
    },
    {
      "FLOPS loss": 0.06592919677495956,
      "L0_d": 784.77,
      "MLM loss": 2.303936719894409,
      "epoch": 9.53,
      "step": 412499
    },
    {
      "epoch": 9.53,
      "learning_rate": 1.7893265306122452e-05,
      "loss": 2.3109,
      "step": 412500
    },
    {
      "FLOPS loss": 0.06576629728078842,
      "L0_d": 809.38,
      "MLM loss": 2.3034827709198,
      "epoch": 9.54,
      "step": 412999
    },
    {
      "epoch": 9.54,
      "learning_rate": 1.7791428571428572e-05,
      "loss": 2.3131,
      "step": 413000
    },
    {
      "FLOPS loss": 0.06040722504258156,
      "L0_d": 905.58,
      "MLM loss": 2.0183334350585938,
      "epoch": 9.55,
      "step": 413499
    },
    {
      "epoch": 9.55,
      "learning_rate": 1.768938775510204e-05,
      "loss": 2.3093,
      "step": 413500
    },
    {
      "FLOPS loss": 0.069266177713871,
      "L0_d": 861.3,
      "MLM loss": 2.247541904449463,
      "epoch": 9.56,
      "step": 413999
    },
    {
      "epoch": 9.56,
      "learning_rate": 1.7587346938775512e-05,
      "loss": 2.3099,
      "step": 414000
    },
    {
      "FLOPS loss": 0.059506528079509735,
      "L0_d": 794.53,
      "MLM loss": 2.3322439193725586,
      "epoch": 9.58,
      "step": 414499
    },
    {
      "epoch": 9.58,
      "learning_rate": 1.748530612244898e-05,
      "loss": 2.3096,
      "step": 414500
    },
    {
      "FLOPS loss": 0.06200943514704704,
      "L0_d": 938.34,
      "MLM loss": 2.3402085304260254,
      "epoch": 9.59,
      "step": 414999
    },
    {
      "epoch": 9.59,
      "learning_rate": 1.7383469387755104e-05,
      "loss": 2.3089,
      "step": 415000
    },
    {
      "FLOPS loss": 0.06389506906270981,
      "L0_d": 851.53,
      "MLM loss": 2.2279720306396484,
      "epoch": 9.6,
      "step": 415499
    },
    {
      "epoch": 9.6,
      "learning_rate": 1.7281428571428572e-05,
      "loss": 2.3068,
      "step": 415500
    },
    {
      "FLOPS loss": 0.0726497620344162,
      "L0_d": 1118.03,
      "MLM loss": 2.2254295349121094,
      "epoch": 9.61,
      "step": 415999
    },
    {
      "epoch": 9.61,
      "learning_rate": 1.717938775510204e-05,
      "loss": 2.3155,
      "step": 416000
    },
    {
      "FLOPS loss": 0.0513957142829895,
      "L0_d": 812.94,
      "MLM loss": 2.3883609771728516,
      "epoch": 9.62,
      "step": 416499
    },
    {
      "epoch": 9.62,
      "learning_rate": 1.707734693877551e-05,
      "loss": 2.3086,
      "step": 416500
    },
    {
      "FLOPS loss": 0.06384290754795074,
      "L0_d": 716.12,
      "MLM loss": 2.221311092376709,
      "epoch": 9.63,
      "step": 416999
    },
    {
      "epoch": 9.63,
      "learning_rate": 1.6975510204081632e-05,
      "loss": 2.3098,
      "step": 417000
    },
    {
      "FLOPS loss": 0.05616021156311035,
      "L0_d": 742.58,
      "MLM loss": 2.1164889335632324,
      "epoch": 9.64,
      "step": 417499
    },
    {
      "epoch": 9.64,
      "learning_rate": 1.6873469387755104e-05,
      "loss": 2.3073,
      "step": 417500
    },
    {
      "FLOPS loss": 0.06069912016391754,
      "L0_d": 945.22,
      "MLM loss": 2.2470812797546387,
      "epoch": 9.66,
      "step": 417999
    },
    {
      "epoch": 9.66,
      "learning_rate": 1.6771428571428572e-05,
      "loss": 2.3097,
      "step": 418000
    },
    {
      "FLOPS loss": 0.05973891541361809,
      "L0_d": 870.66,
      "MLM loss": 2.144036054611206,
      "epoch": 9.67,
      "step": 418499
    },
    {
      "epoch": 9.67,
      "learning_rate": 1.6669387755102044e-05,
      "loss": 2.3065,
      "step": 418500
    },
    {
      "FLOPS loss": 0.06439421325922012,
      "L0_d": 986.64,
      "MLM loss": 1.9868640899658203,
      "epoch": 9.68,
      "step": 418999
    },
    {
      "epoch": 9.68,
      "learning_rate": 1.656734693877551e-05,
      "loss": 2.308,
      "step": 419000
    },
    {
      "FLOPS loss": 0.06345753371715546,
      "L0_d": 728.62,
      "MLM loss": 2.2475640773773193,
      "epoch": 9.69,
      "step": 419499
    },
    {
      "epoch": 9.69,
      "learning_rate": 1.6465510204081632e-05,
      "loss": 2.3098,
      "step": 419500
    },
    {
      "FLOPS loss": 0.05375149846076965,
      "L0_d": 691.36,
      "MLM loss": 2.2957427501678467,
      "epoch": 9.7,
      "step": 419999
    },
    {
      "epoch": 9.7,
      "learning_rate": 1.6363469387755104e-05,
      "loss": 2.3042,
      "step": 420000
    },
    {
      "FLOPS loss": 0.06595911830663681,
      "L0_d": 857.59,
      "MLM loss": 2.0994808673858643,
      "epoch": 9.71,
      "step": 420499
    },
    {
      "epoch": 9.71,
      "learning_rate": 1.6261428571428572e-05,
      "loss": 2.3052,
      "step": 420500
    },
    {
      "FLOPS loss": 0.062460485845804214,
      "L0_d": 906.17,
      "MLM loss": 2.2028586864471436,
      "epoch": 9.73,
      "step": 420999
    },
    {
      "epoch": 9.73,
      "learning_rate": 1.6159387755102043e-05,
      "loss": 2.3014,
      "step": 421000
    },
    {
      "FLOPS loss": 0.07285495102405548,
      "L0_d": 1217.08,
      "MLM loss": 2.313466787338257,
      "epoch": 9.74,
      "step": 421499
    },
    {
      "epoch": 9.74,
      "learning_rate": 1.6057551020408164e-05,
      "loss": 2.3095,
      "step": 421500
    },
    {
      "FLOPS loss": 0.08623205125331879,
      "L0_d": 1016.34,
      "MLM loss": 2.080904960632324,
      "epoch": 9.75,
      "step": 421999
    },
    {
      "epoch": 9.75,
      "learning_rate": 1.5955510204081632e-05,
      "loss": 2.3081,
      "step": 422000
    },
    {
      "FLOPS loss": 0.06952735781669617,
      "L0_d": 831.94,
      "MLM loss": 2.1841421127319336,
      "epoch": 9.76,
      "step": 422499
    },
    {
      "epoch": 9.76,
      "learning_rate": 1.58534693877551e-05,
      "loss": 2.3092,
      "step": 422500
    },
    {
      "FLOPS loss": 0.06762253493070602,
      "L0_d": 919.55,
      "MLM loss": 2.146679639816284,
      "epoch": 9.77,
      "step": 422999
    },
    {
      "epoch": 9.77,
      "learning_rate": 1.575142857142857e-05,
      "loss": 2.3091,
      "step": 423000
    },
    {
      "FLOPS loss": 0.05603819340467453,
      "L0_d": 740.88,
      "MLM loss": 2.2936301231384277,
      "epoch": 9.78,
      "step": 423499
    },
    {
      "epoch": 9.78,
      "learning_rate": 1.564938775510204e-05,
      "loss": 2.3056,
      "step": 423500
    },
    {
      "FLOPS loss": 0.06984943896532059,
      "L0_d": 789.66,
      "MLM loss": 2.2071163654327393,
      "epoch": 9.79,
      "step": 423999
    },
    {
      "epoch": 9.79,
      "learning_rate": 1.5547551020408164e-05,
      "loss": 2.3048,
      "step": 424000
    },
    {
      "FLOPS loss": 0.05715217813849449,
      "L0_d": 874.83,
      "MLM loss": 2.1982874870300293,
      "epoch": 9.81,
      "step": 424499
    },
    {
      "epoch": 9.81,
      "learning_rate": 1.5445510204081635e-05,
      "loss": 2.3063,
      "step": 424500
    },
    {
      "FLOPS loss": 0.07178690284490585,
      "L0_d": 923.66,
      "MLM loss": 2.177065849304199,
      "epoch": 9.82,
      "step": 424999
    },
    {
      "epoch": 9.82,
      "learning_rate": 1.5343469387755103e-05,
      "loss": 2.3086,
      "step": 425000
    },
    {
      "FLOPS loss": 0.0699949711561203,
      "L0_d": 1009.09,
      "MLM loss": 2.053196430206299,
      "epoch": 9.83,
      "step": 425499
    },
    {
      "epoch": 9.83,
      "learning_rate": 1.5241632653061224e-05,
      "loss": 2.3076,
      "step": 425500
    },
    {
      "FLOPS loss": 0.06477265805006027,
      "L0_d": 866.89,
      "MLM loss": 2.051081657409668,
      "epoch": 9.84,
      "step": 425999
    },
    {
      "epoch": 9.84,
      "learning_rate": 1.5139591836734696e-05,
      "loss": 2.305,
      "step": 426000
    },
    {
      "FLOPS loss": 0.06138918921351433,
      "L0_d": 841.31,
      "MLM loss": 2.3111472129821777,
      "epoch": 9.85,
      "step": 426499
    },
    {
      "epoch": 9.85,
      "learning_rate": 1.5037551020408164e-05,
      "loss": 2.3028,
      "step": 426500
    },
    {
      "FLOPS loss": 0.056554101407527924,
      "L0_d": 701.53,
      "MLM loss": 2.369089365005493,
      "epoch": 9.86,
      "step": 426999
    },
    {
      "epoch": 9.86,
      "learning_rate": 1.4935510204081635e-05,
      "loss": 2.3083,
      "step": 427000
    },
    {
      "FLOPS loss": 0.06950782984495163,
      "L0_d": 992.92,
      "MLM loss": 2.152656316757202,
      "epoch": 9.88,
      "step": 427499
    },
    {
      "epoch": 9.88,
      "learning_rate": 1.4833469387755103e-05,
      "loss": 2.3074,
      "step": 427500
    },
    {
      "FLOPS loss": 0.06892652064561844,
      "L0_d": 788.14,
      "MLM loss": 2.2031593322753906,
      "epoch": 9.89,
      "step": 427999
    },
    {
      "epoch": 9.89,
      "learning_rate": 1.4731428571428571e-05,
      "loss": 2.3043,
      "step": 428000
    },
    {
      "FLOPS loss": 0.06714266538619995,
      "L0_d": 781.45,
      "MLM loss": 2.1796374320983887,
      "epoch": 9.9,
      "step": 428499
    },
    {
      "epoch": 9.9,
      "learning_rate": 1.4629387755102043e-05,
      "loss": 2.3016,
      "step": 428500
    },
    {
      "FLOPS loss": 0.06731442362070084,
      "L0_d": 892.73,
      "MLM loss": 2.282862663269043,
      "epoch": 9.91,
      "step": 428999
    },
    {
      "epoch": 9.91,
      "learning_rate": 1.452734693877551e-05,
      "loss": 2.3041,
      "step": 429000
    },
    {
      "FLOPS loss": 0.06822635233402252,
      "L0_d": 978.84,
      "MLM loss": 2.432333469390869,
      "epoch": 9.92,
      "step": 429499
    },
    {
      "epoch": 9.92,
      "learning_rate": 1.442530612244898e-05,
      "loss": 2.3024,
      "step": 429500
    },
    {
      "FLOPS loss": 0.06567897647619247,
      "L0_d": 1059.81,
      "MLM loss": 2.2899770736694336,
      "epoch": 9.93,
      "step": 429999
    },
    {
      "epoch": 9.93,
      "learning_rate": 1.4323469387755103e-05,
      "loss": 2.3042,
      "step": 430000
    },
    {
      "FLOPS loss": 0.06040837988257408,
      "L0_d": 883.02,
      "MLM loss": 2.23317289352417,
      "epoch": 9.94,
      "step": 430499
    },
    {
      "epoch": 9.95,
      "learning_rate": 1.4221428571428571e-05,
      "loss": 2.2991,
      "step": 430500
    },
    {
      "FLOPS loss": 0.054521579295396805,
      "L0_d": 714.81,
      "MLM loss": 2.4254093170166016,
      "epoch": 9.96,
      "step": 430999
    },
    {
      "epoch": 9.96,
      "learning_rate": 1.4119387755102043e-05,
      "loss": 2.302,
      "step": 431000
    },
    {
      "FLOPS loss": 0.0858311727643013,
      "L0_d": 1071.42,
      "MLM loss": 2.1953415870666504,
      "epoch": 9.97,
      "step": 431499
    },
    {
      "epoch": 9.97,
      "learning_rate": 1.401734693877551e-05,
      "loss": 2.3025,
      "step": 431500
    },
    {
      "FLOPS loss": 0.05570479482412338,
      "L0_d": 688.91,
      "MLM loss": 2.1967906951904297,
      "epoch": 9.98,
      "step": 431999
    },
    {
      "epoch": 9.98,
      "learning_rate": 1.3915510204081633e-05,
      "loss": 2.3029,
      "step": 432000
    },
    {
      "FLOPS loss": 0.0755709856748581,
      "L0_d": 966.83,
      "MLM loss": 2.218096971511841,
      "epoch": 9.99,
      "step": 432499
    },
    {
      "epoch": 9.99,
      "learning_rate": 1.3813469387755101e-05,
      "loss": 2.3022,
      "step": 432500
    },
    {
      "FLOPS loss": 0.06049109250307083,
      "L0_d": 864.84,
      "MLM loss": 2.365525245666504,
      "epoch": 10.0,
      "step": 432999
    },
    {
      "epoch": 10.0,
      "learning_rate": 1.3711428571428573e-05,
      "loss": 2.302,
      "step": 433000
    },
    {
      "FLOPS loss": 0.05764416232705116,
      "L0_d": 792.52,
      "MLM loss": 2.233534812927246,
      "epoch": 10.01,
      "step": 433499
    },
    {
      "epoch": 10.01,
      "learning_rate": 1.360938775510204e-05,
      "loss": 2.3035,
      "step": 433500
    },
    {
      "FLOPS loss": 0.07010982185602188,
      "L0_d": 886.73,
      "MLM loss": 2.5419046878814697,
      "epoch": 10.03,
      "step": 433999
    },
    {
      "epoch": 10.03,
      "learning_rate": 1.3507551020408163e-05,
      "loss": 2.3015,
      "step": 434000
    },
    {
      "FLOPS loss": 0.06383447349071503,
      "L0_d": 905.14,
      "MLM loss": 2.1592020988464355,
      "epoch": 10.04,
      "step": 434499
    },
    {
      "epoch": 10.04,
      "learning_rate": 1.3405510204081634e-05,
      "loss": 2.2996,
      "step": 434500
    },
    {
      "FLOPS loss": 0.06052009016275406,
      "L0_d": 726.97,
      "MLM loss": 2.085092306137085,
      "epoch": 10.05,
      "step": 434999
    },
    {
      "epoch": 10.05,
      "learning_rate": 1.3303469387755103e-05,
      "loss": 2.2996,
      "step": 435000
    },
    {
      "FLOPS loss": 0.06284129619598389,
      "L0_d": 836.41,
      "MLM loss": 2.291759967803955,
      "epoch": 10.06,
      "step": 435499
    },
    {
      "epoch": 10.06,
      "learning_rate": 1.3201632653061225e-05,
      "loss": 2.2995,
      "step": 435500
    },
    {
      "FLOPS loss": 0.07444918155670166,
      "L0_d": 872.38,
      "MLM loss": 2.4256529808044434,
      "epoch": 10.07,
      "step": 435999
    },
    {
      "epoch": 10.07,
      "learning_rate": 1.3099591836734695e-05,
      "loss": 2.3009,
      "step": 436000
    },
    {
      "FLOPS loss": 0.06040747091174126,
      "L0_d": 693.17,
      "MLM loss": 2.163949966430664,
      "epoch": 10.08,
      "step": 436499
    },
    {
      "epoch": 10.08,
      "learning_rate": 1.2997551020408163e-05,
      "loss": 2.3036,
      "step": 436500
    },
    {
      "FLOPS loss": 0.062491338700056076,
      "L0_d": 836.11,
      "MLM loss": 2.4563846588134766,
      "epoch": 10.1,
      "step": 436999
    },
    {
      "epoch": 10.1,
      "learning_rate": 1.2895510204081634e-05,
      "loss": 2.2969,
      "step": 437000
    },
    {
      "FLOPS loss": 0.06521119922399521,
      "L0_d": 845.45,
      "MLM loss": 2.5408596992492676,
      "epoch": 10.11,
      "step": 437499
    },
    {
      "epoch": 10.11,
      "learning_rate": 1.2793469387755102e-05,
      "loss": 2.299,
      "step": 437500
    },
    {
      "FLOPS loss": 0.06396574527025223,
      "L0_d": 834.34,
      "MLM loss": 2.319390296936035,
      "epoch": 10.12,
      "step": 437999
    },
    {
      "epoch": 10.12,
      "learning_rate": 1.269142857142857e-05,
      "loss": 2.3018,
      "step": 438000
    },
    {
      "FLOPS loss": 0.07069003582000732,
      "L0_d": 950.38,
      "MLM loss": 2.2196543216705322,
      "epoch": 10.13,
      "step": 438499
    },
    {
      "epoch": 10.13,
      "learning_rate": 1.2589387755102042e-05,
      "loss": 2.3015,
      "step": 438500
    },
    {
      "FLOPS loss": 0.06846750527620316,
      "L0_d": 1224.31,
      "MLM loss": 2.2630181312561035,
      "epoch": 10.14,
      "step": 438999
    },
    {
      "epoch": 10.14,
      "learning_rate": 1.2487346938775512e-05,
      "loss": 2.2973,
      "step": 439000
    },
    {
      "FLOPS loss": 0.05883738771080971,
      "L0_d": 825.19,
      "MLM loss": 2.2000060081481934,
      "epoch": 10.15,
      "step": 439499
    },
    {
      "epoch": 10.15,
      "learning_rate": 1.2385510204081634e-05,
      "loss": 2.2976,
      "step": 439500
    },
    {
      "FLOPS loss": 0.0656290054321289,
      "L0_d": 852.75,
      "MLM loss": 2.289665937423706,
      "epoch": 10.16,
      "step": 439999
    },
    {
      "epoch": 10.16,
      "learning_rate": 1.2283469387755104e-05,
      "loss": 2.2963,
      "step": 440000
    },
    {
      "FLOPS loss": 0.06831687688827515,
      "L0_d": 893.62,
      "MLM loss": 2.349213123321533,
      "epoch": 10.18,
      "step": 440499
    },
    {
      "epoch": 10.18,
      "learning_rate": 1.2181428571428572e-05,
      "loss": 2.2919,
      "step": 440500
    },
    {
      "FLOPS loss": 0.06690240651369095,
      "L0_d": 897.55,
      "MLM loss": 2.0150651931762695,
      "epoch": 10.19,
      "step": 440999
    },
    {
      "epoch": 10.19,
      "learning_rate": 1.2079591836734694e-05,
      "loss": 2.2969,
      "step": 441000
    },
    {
      "FLOPS loss": 0.06606750190258026,
      "L0_d": 1001.64,
      "MLM loss": 2.107846975326538,
      "epoch": 10.2,
      "step": 441499
    },
    {
      "epoch": 10.2,
      "learning_rate": 1.1977551020408164e-05,
      "loss": 2.2954,
      "step": 441500
    },
    {
      "FLOPS loss": 0.06601283699274063,
      "L0_d": 1012.19,
      "MLM loss": 2.3590779304504395,
      "epoch": 10.21,
      "step": 441999
    },
    {
      "epoch": 10.21,
      "learning_rate": 1.1875510204081632e-05,
      "loss": 2.3013,
      "step": 442000
    },
    {
      "FLOPS loss": 0.056500278413295746,
      "L0_d": 760.25,
      "MLM loss": 2.1543731689453125,
      "epoch": 10.22,
      "step": 442499
    },
    {
      "epoch": 10.22,
      "learning_rate": 1.1773469387755102e-05,
      "loss": 2.3026,
      "step": 442500
    },
    {
      "FLOPS loss": 0.07678142189979553,
      "L0_d": 1012.47,
      "MLM loss": 2.2874748706817627,
      "epoch": 10.23,
      "step": 442999
    },
    {
      "epoch": 10.23,
      "learning_rate": 1.1671428571428572e-05,
      "loss": 2.2946,
      "step": 443000
    },
    {
      "FLOPS loss": 0.0532967634499073,
      "L0_d": 725.55,
      "MLM loss": 2.112173318862915,
      "epoch": 10.25,
      "step": 443499
    },
    {
      "epoch": 10.25,
      "learning_rate": 1.1569387755102042e-05,
      "loss": 2.2919,
      "step": 443500
    },
    {
      "FLOPS loss": 0.07034281641244888,
      "L0_d": 948.98,
      "MLM loss": 2.24344539642334,
      "epoch": 10.26,
      "step": 443999
    },
    {
      "epoch": 10.26,
      "learning_rate": 1.1467346938775511e-05,
      "loss": 2.296,
      "step": 444000
    },
    {
      "FLOPS loss": 0.05562940984964371,
      "L0_d": 747.36,
      "MLM loss": 2.1856186389923096,
      "epoch": 10.27,
      "step": 444499
    },
    {
      "epoch": 10.27,
      "learning_rate": 1.1365306122448981e-05,
      "loss": 2.296,
      "step": 444500
    },
    {
      "FLOPS loss": 0.0713595598936081,
      "L0_d": 809.14,
      "MLM loss": 2.2879340648651123,
      "epoch": 10.28,
      "step": 444999
    },
    {
      "epoch": 10.28,
      "learning_rate": 1.1263469387755103e-05,
      "loss": 2.2967,
      "step": 445000
    },
    {
      "FLOPS loss": 0.062412329018116,
      "L0_d": 759.11,
      "MLM loss": 2.179746627807617,
      "epoch": 10.29,
      "step": 445499
    },
    {
      "epoch": 10.29,
      "learning_rate": 1.1161428571428572e-05,
      "loss": 2.2951,
      "step": 445500
    },
    {
      "FLOPS loss": 0.060674265027046204,
      "L0_d": 851.06,
      "MLM loss": 2.2669856548309326,
      "epoch": 10.3,
      "step": 445999
    },
    {
      "epoch": 10.3,
      "learning_rate": 1.1059387755102041e-05,
      "loss": 2.2965,
      "step": 446000
    },
    {
      "FLOPS loss": 0.06142638996243477,
      "L0_d": 834.64,
      "MLM loss": 2.347390651702881,
      "epoch": 10.31,
      "step": 446499
    },
    {
      "epoch": 10.31,
      "learning_rate": 1.0957346938775511e-05,
      "loss": 2.2897,
      "step": 446500
    },
    {
      "FLOPS loss": 0.06716670095920563,
      "L0_d": 921.64,
      "MLM loss": 2.276606798171997,
      "epoch": 10.33,
      "step": 446999
    },
    {
      "epoch": 10.33,
      "learning_rate": 1.0855510204081633e-05,
      "loss": 2.295,
      "step": 447000
    },
    {
      "FLOPS loss": 0.06482750177383423,
      "L0_d": 959.17,
      "MLM loss": 2.3427610397338867,
      "epoch": 10.34,
      "step": 447499
    },
    {
      "epoch": 10.34,
      "learning_rate": 1.0753469387755102e-05,
      "loss": 2.2996,
      "step": 447500
    },
    {
      "FLOPS loss": 0.06260459870100021,
      "L0_d": 833.97,
      "MLM loss": 2.090470790863037,
      "epoch": 10.35,
      "step": 447999
    },
    {
      "epoch": 10.35,
      "learning_rate": 1.0651428571428571e-05,
      "loss": 2.2926,
      "step": 448000
    },
    {
      "FLOPS loss": 0.06450529396533966,
      "L0_d": 851.77,
      "MLM loss": 2.002493143081665,
      "epoch": 10.36,
      "step": 448499
    },
    {
      "epoch": 10.36,
      "learning_rate": 1.0549387755102041e-05,
      "loss": 2.2925,
      "step": 448500
    },
    {
      "FLOPS loss": 0.07590842247009277,
      "L0_d": 1052.16,
      "MLM loss": 2.1307899951934814,
      "epoch": 10.37,
      "step": 448999
    },
    {
      "epoch": 10.37,
      "learning_rate": 1.0447346938775511e-05,
      "loss": 2.2996,
      "step": 449000
    },
    {
      "FLOPS loss": 0.07303966581821442,
      "L0_d": 1016.38,
      "MLM loss": 2.1589527130126953,
      "epoch": 10.38,
      "step": 449499
    },
    {
      "epoch": 10.38,
      "learning_rate": 1.0345510204081633e-05,
      "loss": 2.2974,
      "step": 449500
    },
    {
      "FLOPS loss": 0.053986258804798126,
      "L0_d": 647.03,
      "MLM loss": 2.321556568145752,
      "epoch": 10.4,
      "step": 449999
    },
    {
      "epoch": 10.4,
      "learning_rate": 1.0243469387755103e-05,
      "loss": 2.2924,
      "step": 450000
    },
    {
      "FLOPS loss": 0.06856471300125122,
      "L0_d": 924.59,
      "MLM loss": 2.3103318214416504,
      "epoch": 10.41,
      "step": 450499
    },
    {
      "epoch": 10.41,
      "learning_rate": 1.0141428571428573e-05,
      "loss": 2.2974,
      "step": 450500
    },
    {
      "FLOPS loss": 0.07135065644979477,
      "L0_d": 973.39,
      "MLM loss": 2.135502338409424,
      "epoch": 10.42,
      "step": 450999
    },
    {
      "epoch": 10.42,
      "learning_rate": 1.0039387755102041e-05,
      "loss": 2.2953,
      "step": 451000
    },
    {
      "FLOPS loss": 0.058795761317014694,
      "L0_d": 866.27,
      "MLM loss": 2.257445812225342,
      "epoch": 10.43,
      "step": 451499
    },
    {
      "epoch": 10.43,
      "learning_rate": 9.937551020408163e-06,
      "loss": 2.2953,
      "step": 451500
    },
    {
      "FLOPS loss": 0.06395602971315384,
      "L0_d": 891.98,
      "MLM loss": 2.267632246017456,
      "epoch": 10.44,
      "step": 451999
    },
    {
      "epoch": 10.44,
      "learning_rate": 9.835510204081633e-06,
      "loss": 2.2962,
      "step": 452000
    },
    {
      "FLOPS loss": 0.05605367198586464,
      "L0_d": 919.94,
      "MLM loss": 2.185353994369507,
      "epoch": 10.45,
      "step": 452499
    },
    {
      "epoch": 10.45,
      "learning_rate": 9.733469387755103e-06,
      "loss": 2.291,
      "step": 452500
    },
    {
      "FLOPS loss": 0.056460727006196976,
      "L0_d": 827.66,
      "MLM loss": 2.2944674491882324,
      "epoch": 10.46,
      "step": 452999
    },
    {
      "epoch": 10.46,
      "learning_rate": 9.631428571428573e-06,
      "loss": 2.2941,
      "step": 453000
    },
    {
      "FLOPS loss": 0.06284444779157639,
      "L0_d": 758.91,
      "MLM loss": 2.1061954498291016,
      "epoch": 10.48,
      "step": 453499
    },
    {
      "epoch": 10.48,
      "learning_rate": 9.52938775510204e-06,
      "loss": 2.2939,
      "step": 453500
    },
    {
      "FLOPS loss": 0.054719168692827225,
      "L0_d": 649.95,
      "MLM loss": 2.1994714736938477,
      "epoch": 10.49,
      "step": 453999
    },
    {
      "epoch": 10.49,
      "learning_rate": 9.427551020408163e-06,
      "loss": 2.292,
      "step": 454000
    },
    {
      "FLOPS loss": 0.06368663907051086,
      "L0_d": 844.17,
      "MLM loss": 2.1803951263427734,
      "epoch": 10.5,
      "step": 454499
    },
    {
      "epoch": 10.5,
      "learning_rate": 9.325510204081633e-06,
      "loss": 2.2931,
      "step": 454500
    },
    {
      "FLOPS loss": 0.08477547019720078,
      "L0_d": 1192.67,
      "MLM loss": 2.25724458694458,
      "epoch": 10.51,
      "step": 454999
    },
    {
      "epoch": 10.51,
      "learning_rate": 9.223469387755103e-06,
      "loss": 2.2921,
      "step": 455000
    },
    {
      "FLOPS loss": 0.05767339468002319,
      "L0_d": 670.22,
      "MLM loss": 2.289708137512207,
      "epoch": 10.52,
      "step": 455499
    },
    {
      "epoch": 10.52,
      "learning_rate": 9.121428571428572e-06,
      "loss": 2.2942,
      "step": 455500
    },
    {
      "FLOPS loss": 0.055055826902389526,
      "L0_d": 861.41,
      "MLM loss": 2.225193738937378,
      "epoch": 10.53,
      "step": 455999
    },
    {
      "epoch": 10.53,
      "learning_rate": 9.019591836734695e-06,
      "loss": 2.296,
      "step": 456000
    },
    {
      "FLOPS loss": 0.06768262386322021,
      "L0_d": 778.47,
      "MLM loss": 2.0815377235412598,
      "epoch": 10.55,
      "step": 456499
    },
    {
      "epoch": 10.55,
      "learning_rate": 8.917551020408163e-06,
      "loss": 2.2913,
      "step": 456500
    },
    {
      "FLOPS loss": 0.06390358507633209,
      "L0_d": 767.75,
      "MLM loss": 2.110469341278076,
      "epoch": 10.56,
      "step": 456999
    },
    {
      "epoch": 10.56,
      "learning_rate": 8.815510204081633e-06,
      "loss": 2.2901,
      "step": 457000
    },
    {
      "FLOPS loss": 0.05460204929113388,
      "L0_d": 885.66,
      "MLM loss": 2.217503070831299,
      "epoch": 10.57,
      "step": 457499
    },
    {
      "epoch": 10.57,
      "learning_rate": 8.713469387755102e-06,
      "loss": 2.2921,
      "step": 457500
    },
    {
      "FLOPS loss": 0.060463305562734604,
      "L0_d": 938.72,
      "MLM loss": 2.1308157444000244,
      "epoch": 10.58,
      "step": 457999
    },
    {
      "epoch": 10.58,
      "learning_rate": 8.611428571428572e-06,
      "loss": 2.2944,
      "step": 458000
    },
    {
      "FLOPS loss": 0.06594925373792648,
      "L0_d": 911.19,
      "MLM loss": 2.3237662315368652,
      "epoch": 10.59,
      "step": 458499
    },
    {
      "epoch": 10.59,
      "learning_rate": 8.509591836734695e-06,
      "loss": 2.2917,
      "step": 458500
    },
    {
      "FLOPS loss": 0.08003830164670944,
      "L0_d": 973.66,
      "MLM loss": 2.3158817291259766,
      "epoch": 10.6,
      "step": 458999
    },
    {
      "epoch": 10.6,
      "learning_rate": 8.407755102040817e-06,
      "loss": 2.2932,
      "step": 459000
    },
    {
      "FLOPS loss": 0.06033896282315254,
      "L0_d": 647.02,
      "MLM loss": 2.24031400680542,
      "epoch": 10.61,
      "step": 459499
    },
    {
      "epoch": 10.61,
      "learning_rate": 8.305714285714287e-06,
      "loss": 2.2928,
      "step": 459500
    },
    {
      "FLOPS loss": 0.0587412565946579,
      "L0_d": 741.28,
      "MLM loss": 2.185731887817383,
      "epoch": 10.63,
      "step": 459999
    },
    {
      "epoch": 10.63,
      "learning_rate": 8.203673469387755e-06,
      "loss": 2.2938,
      "step": 460000
    },
    {
      "FLOPS loss": 0.06728052347898483,
      "L0_d": 794.55,
      "MLM loss": 2.4783236980438232,
      "epoch": 10.64,
      "step": 460499
    },
    {
      "epoch": 10.64,
      "learning_rate": 8.101632653061225e-06,
      "loss": 2.2952,
      "step": 460500
    },
    {
      "FLOPS loss": 0.07311265915632248,
      "L0_d": 912.84,
      "MLM loss": 2.2954416275024414,
      "epoch": 10.65,
      "step": 460999
    },
    {
      "epoch": 10.65,
      "learning_rate": 7.999591836734694e-06,
      "loss": 2.2947,
      "step": 461000
    },
    {
      "FLOPS loss": 0.06787005066871643,
      "L0_d": 876.83,
      "MLM loss": 2.2662861347198486,
      "epoch": 10.66,
      "step": 461499
    },
    {
      "epoch": 10.66,
      "learning_rate": 7.897551020408164e-06,
      "loss": 2.2881,
      "step": 461500
    },
    {
      "FLOPS loss": 0.052038803696632385,
      "L0_d": 670.36,
      "MLM loss": 2.4589502811431885,
      "epoch": 10.67,
      "step": 461999
    },
    {
      "epoch": 10.67,
      "learning_rate": 7.795510204081632e-06,
      "loss": 2.2944,
      "step": 462000
    },
    {
      "FLOPS loss": 0.05376419052481651,
      "L0_d": 768.39,
      "MLM loss": 2.2146544456481934,
      "epoch": 10.68,
      "step": 462499
    },
    {
      "epoch": 10.68,
      "learning_rate": 7.693469387755102e-06,
      "loss": 2.2914,
      "step": 462500
    },
    {
      "FLOPS loss": 0.05922848358750343,
      "L0_d": 819.77,
      "MLM loss": 2.292515516281128,
      "epoch": 10.7,
      "step": 462999
    },
    {
      "epoch": 10.7,
      "learning_rate": 7.591428571428572e-06,
      "loss": 2.2913,
      "step": 463000
    },
    {
      "FLOPS loss": 0.05460599809885025,
      "L0_d": 802.72,
      "MLM loss": 2.2540738582611084,
      "epoch": 10.71,
      "step": 463499
    },
    {
      "epoch": 10.71,
      "learning_rate": 7.489591836734694e-06,
      "loss": 2.2922,
      "step": 463500
    },
    {
      "FLOPS loss": 0.06698250025510788,
      "L0_d": 889.16,
      "MLM loss": 2.2473649978637695,
      "epoch": 10.72,
      "step": 463999
    },
    {
      "epoch": 10.72,
      "learning_rate": 7.387551020408163e-06,
      "loss": 2.2892,
      "step": 464000
    },
    {
      "FLOPS loss": 0.057049185037612915,
      "L0_d": 764.62,
      "MLM loss": 2.392953395843506,
      "epoch": 10.73,
      "step": 464499
    },
    {
      "epoch": 10.73,
      "learning_rate": 7.285510204081633e-06,
      "loss": 2.2937,
      "step": 464500
    },
    {
      "FLOPS loss": 0.0670197457075119,
      "L0_d": 987.97,
      "MLM loss": 2.170027732849121,
      "epoch": 10.74,
      "step": 464999
    },
    {
      "epoch": 10.74,
      "learning_rate": 7.183469387755103e-06,
      "loss": 2.2883,
      "step": 465000
    },
    {
      "FLOPS loss": 0.07646064460277557,
      "L0_d": 931.58,
      "MLM loss": 2.1344852447509766,
      "epoch": 10.75,
      "step": 465499
    },
    {
      "epoch": 10.75,
      "learning_rate": 7.081632653061225e-06,
      "loss": 2.2881,
      "step": 465500
    },
    {
      "FLOPS loss": 0.05379510298371315,
      "L0_d": 915.58,
      "MLM loss": 2.1789708137512207,
      "epoch": 10.77,
      "step": 465999
    },
    {
      "epoch": 10.77,
      "learning_rate": 6.979591836734695e-06,
      "loss": 2.2925,
      "step": 466000
    },
    {
      "FLOPS loss": 0.060537777841091156,
      "L0_d": 804.53,
      "MLM loss": 2.200517177581787,
      "epoch": 10.78,
      "step": 466499
    },
    {
      "epoch": 10.78,
      "learning_rate": 6.877551020408164e-06,
      "loss": 2.2956,
      "step": 466500
    },
    {
      "FLOPS loss": 0.06291097402572632,
      "L0_d": 866.05,
      "MLM loss": 2.135246515274048,
      "epoch": 10.79,
      "step": 466999
    },
    {
      "epoch": 10.79,
      "learning_rate": 6.775510204081633e-06,
      "loss": 2.2913,
      "step": 467000
    },
    {
      "FLOPS loss": 0.05674765631556511,
      "L0_d": 680.55,
      "MLM loss": 2.1429038047790527,
      "epoch": 10.8,
      "step": 467499
    },
    {
      "epoch": 10.8,
      "learning_rate": 6.673673469387755e-06,
      "loss": 2.2889,
      "step": 467500
    },
    {
      "FLOPS loss": 0.07323261350393295,
      "L0_d": 945.06,
      "MLM loss": 2.293457508087158,
      "epoch": 10.81,
      "step": 467999
    },
    {
      "epoch": 10.81,
      "learning_rate": 6.571632653061224e-06,
      "loss": 2.2878,
      "step": 468000
    },
    {
      "FLOPS loss": 0.07292532175779343,
      "L0_d": 844.14,
      "MLM loss": 2.3283419609069824,
      "epoch": 10.82,
      "step": 468499
    },
    {
      "epoch": 10.82,
      "learning_rate": 6.469591836734694e-06,
      "loss": 2.2917,
      "step": 468500
    },
    {
      "FLOPS loss": 0.06475947052240372,
      "L0_d": 1061.98,
      "MLM loss": 2.2273287773132324,
      "epoch": 10.83,
      "step": 468999
    },
    {
      "epoch": 10.83,
      "learning_rate": 6.367551020408164e-06,
      "loss": 2.2884,
      "step": 469000
    },
    {
      "FLOPS loss": 0.06330478191375732,
      "L0_d": 812.52,
      "MLM loss": 2.358989953994751,
      "epoch": 10.85,
      "step": 469499
    },
    {
      "epoch": 10.85,
      "learning_rate": 6.265714285714286e-06,
      "loss": 2.2847,
      "step": 469500
    },
    {
      "FLOPS loss": 0.05554909631609917,
      "L0_d": 683.17,
      "MLM loss": 2.204073667526245,
      "epoch": 10.86,
      "step": 469999
    },
    {
      "epoch": 10.86,
      "learning_rate": 6.163673469387756e-06,
      "loss": 2.2977,
      "step": 470000
    },
    {
      "FLOPS loss": 0.07192515581846237,
      "L0_d": 838.58,
      "MLM loss": 2.146226406097412,
      "epoch": 10.87,
      "step": 470499
    },
    {
      "epoch": 10.87,
      "learning_rate": 6.061632653061225e-06,
      "loss": 2.2869,
      "step": 470500
    },
    {
      "FLOPS loss": 0.06218468397855759,
      "L0_d": 894.47,
      "MLM loss": 2.3167052268981934,
      "epoch": 10.88,
      "step": 470999
    },
    {
      "epoch": 10.88,
      "learning_rate": 5.959591836734694e-06,
      "loss": 2.288,
      "step": 471000
    },
    {
      "FLOPS loss": 0.07072613388299942,
      "L0_d": 963.66,
      "MLM loss": 2.1581053733825684,
      "epoch": 10.89,
      "step": 471499
    },
    {
      "epoch": 10.89,
      "learning_rate": 5.857755102040816e-06,
      "loss": 2.2867,
      "step": 471500
    },
    {
      "FLOPS loss": 0.05788072943687439,
      "L0_d": 876.33,
      "MLM loss": 2.335477828979492,
      "epoch": 10.9,
      "step": 471999
    },
    {
      "epoch": 10.9,
      "learning_rate": 5.755714285714286e-06,
      "loss": 2.2861,
      "step": 472000
    },
    {
      "FLOPS loss": 0.05549454689025879,
      "L0_d": 770.08,
      "MLM loss": 2.025320291519165,
      "epoch": 10.92,
      "step": 472499
    },
    {
      "epoch": 10.92,
      "learning_rate": 5.6536734693877556e-06,
      "loss": 2.2834,
      "step": 472500
    },
    {
      "FLOPS loss": 0.06566669046878815,
      "L0_d": 902.3,
      "MLM loss": 2.3450229167938232,
      "epoch": 10.93,
      "step": 472999
    },
    {
      "epoch": 10.93,
      "learning_rate": 5.5516326530612245e-06,
      "loss": 2.287,
      "step": 473000
    },
    {
      "FLOPS loss": 0.05576305836439133,
      "L0_d": 733.09,
      "MLM loss": 2.2725815773010254,
      "epoch": 10.94,
      "step": 473499
    },
    {
      "epoch": 10.94,
      "learning_rate": 5.449591836734694e-06,
      "loss": 2.2889,
      "step": 473500
    },
    {
      "FLOPS loss": 0.05368854105472565,
      "L0_d": 714.02,
      "MLM loss": 2.2860021591186523,
      "epoch": 10.95,
      "step": 473999
    },
    {
      "epoch": 10.95,
      "learning_rate": 5.347755102040817e-06,
      "loss": 2.289,
      "step": 474000
    },
    {
      "FLOPS loss": 0.058568090200424194,
      "L0_d": 872.48,
      "MLM loss": 2.3067092895507812,
      "epoch": 10.96,
      "step": 474499
    },
    {
      "epoch": 10.96,
      "learning_rate": 5.245714285714286e-06,
      "loss": 2.2879,
      "step": 474500
    },
    {
      "FLOPS loss": 0.06257157772779465,
      "L0_d": 826.19,
      "MLM loss": 2.350205421447754,
      "epoch": 10.97,
      "step": 474999
    },
    {
      "epoch": 10.97,
      "learning_rate": 5.143673469387755e-06,
      "loss": 2.2859,
      "step": 475000
    },
    {
      "FLOPS loss": 0.06970212608575821,
      "L0_d": 883.05,
      "MLM loss": 2.2712743282318115,
      "epoch": 10.98,
      "step": 475499
    },
    {
      "epoch": 10.98,
      "learning_rate": 5.041632653061225e-06,
      "loss": 2.2862,
      "step": 475500
    },
    {
      "FLOPS loss": 0.051716398447752,
      "L0_d": 697.44,
      "MLM loss": 2.2224364280700684,
      "epoch": 11.0,
      "step": 475999
    },
    {
      "epoch": 11.0,
      "learning_rate": 4.9397959183673475e-06,
      "loss": 2.2881,
      "step": 476000
    },
    {
      "FLOPS loss": 0.054959673434495926,
      "L0_d": 784.42,
      "MLM loss": 2.4748611450195312,
      "epoch": 11.01,
      "step": 476499
    },
    {
      "epoch": 11.01,
      "learning_rate": 4.8377551020408165e-06,
      "loss": 2.2866,
      "step": 476500
    },
    {
      "FLOPS loss": 0.05612470582127571,
      "L0_d": 797.98,
      "MLM loss": 2.285034418106079,
      "epoch": 11.02,
      "step": 476999
    },
    {
      "epoch": 11.02,
      "learning_rate": 4.735714285714285e-06,
      "loss": 2.2905,
      "step": 477000
    },
    {
      "FLOPS loss": 0.059794358909130096,
      "L0_d": 824.89,
      "MLM loss": 2.137261152267456,
      "epoch": 11.03,
      "step": 477499
    },
    {
      "epoch": 11.03,
      "learning_rate": 4.633673469387755e-06,
      "loss": 2.2846,
      "step": 477500
    },
    {
      "FLOPS loss": 0.06297947466373444,
      "L0_d": 829.75,
      "MLM loss": 2.0144095420837402,
      "epoch": 11.04,
      "step": 477999
    },
    {
      "epoch": 11.04,
      "learning_rate": 4.5318367346938776e-06,
      "loss": 2.2832,
      "step": 478000
    },
    {
      "FLOPS loss": 0.06466788053512573,
      "L0_d": 868.56,
      "MLM loss": 2.188913345336914,
      "epoch": 11.05,
      "step": 478499
    },
    {
      "epoch": 11.05,
      "learning_rate": 4.429795918367347e-06,
      "loss": 2.2858,
      "step": 478500
    },
    {
      "FLOPS loss": 0.06475504487752914,
      "L0_d": 915.92,
      "MLM loss": 2.1946072578430176,
      "epoch": 11.07,
      "step": 478999
    },
    {
      "epoch": 11.07,
      "learning_rate": 4.327755102040817e-06,
      "loss": 2.2838,
      "step": 479000
    },
    {
      "FLOPS loss": 0.06781303882598877,
      "L0_d": 900.3,
      "MLM loss": 2.2967867851257324,
      "epoch": 11.08,
      "step": 479499
    },
    {
      "epoch": 11.08,
      "learning_rate": 4.225714285714286e-06,
      "loss": 2.2855,
      "step": 479500
    },
    {
      "FLOPS loss": 0.06631471961736679,
      "L0_d": 864.19,
      "MLM loss": 2.262704372406006,
      "epoch": 11.09,
      "step": 479999
    },
    {
      "epoch": 11.09,
      "learning_rate": 4.123877551020408e-06,
      "loss": 2.2866,
      "step": 480000
    },
    {
      "FLOPS loss": 0.0421132929623127,
      "L0_d": 671.28,
      "MLM loss": 2.4269192218780518,
      "epoch": 11.1,
      "step": 480499
    },
    {
      "epoch": 11.1,
      "learning_rate": 4.021836734693877e-06,
      "loss": 2.2837,
      "step": 480500
    },
    {
      "FLOPS loss": 0.06580214202404022,
      "L0_d": 907.77,
      "MLM loss": 2.096919536590576,
      "epoch": 11.11,
      "step": 480999
    },
    {
      "epoch": 11.11,
      "learning_rate": 3.919795918367347e-06,
      "loss": 2.2826,
      "step": 481000
    },
    {
      "FLOPS loss": 0.0580022819340229,
      "L0_d": 816.48,
      "MLM loss": 2.2413530349731445,
      "epoch": 11.12,
      "step": 481499
    },
    {
      "epoch": 11.12,
      "learning_rate": 3.817755102040817e-06,
      "loss": 2.2858,
      "step": 481500
    },
    {
      "FLOPS loss": 0.0691581517457962,
      "L0_d": 988.39,
      "MLM loss": 2.397145986557007,
      "epoch": 11.13,
      "step": 481999
    },
    {
      "epoch": 11.13,
      "learning_rate": 3.7159183673469393e-06,
      "loss": 2.2866,
      "step": 482000
    },
    {
      "FLOPS loss": 0.053739335387945175,
      "L0_d": 679.95,
      "MLM loss": 2.0748467445373535,
      "epoch": 11.15,
      "step": 482499
    },
    {
      "epoch": 11.15,
      "learning_rate": 3.613877551020408e-06,
      "loss": 2.2872,
      "step": 482500
    },
    {
      "FLOPS loss": 0.05533215031027794,
      "L0_d": 825.25,
      "MLM loss": 2.2202677726745605,
      "epoch": 11.16,
      "step": 482999
    },
    {
      "epoch": 11.16,
      "learning_rate": 3.5118367346938776e-06,
      "loss": 2.2851,
      "step": 483000
    },
    {
      "FLOPS loss": 0.058010417968034744,
      "L0_d": 726.12,
      "MLM loss": 2.253462314605713,
      "epoch": 11.17,
      "step": 483499
    },
    {
      "epoch": 11.17,
      "learning_rate": 3.409795918367347e-06,
      "loss": 2.2854,
      "step": 483500
    },
    {
      "FLOPS loss": 0.057092875242233276,
      "L0_d": 834.2,
      "MLM loss": 2.214354991912842,
      "epoch": 11.18,
      "step": 483999
    },
    {
      "epoch": 11.18,
      "learning_rate": 3.3079591836734697e-06,
      "loss": 2.2797,
      "step": 484000
    },
    {
      "FLOPS loss": 0.057993967086076736,
      "L0_d": 937.23,
      "MLM loss": 2.2875499725341797,
      "epoch": 11.19,
      "step": 484499
    },
    {
      "epoch": 11.19,
      "learning_rate": 3.205918367346939e-06,
      "loss": 2.2846,
      "step": 484500
    },
    {
      "FLOPS loss": 0.07085564732551575,
      "L0_d": 1015.91,
      "MLM loss": 2.227050304412842,
      "epoch": 11.2,
      "step": 484999
    },
    {
      "epoch": 11.2,
      "learning_rate": 3.1038775510204084e-06,
      "loss": 2.2838,
      "step": 485000
    },
    {
      "FLOPS loss": 0.054904911667108536,
      "L0_d": 781.31,
      "MLM loss": 2.2337186336517334,
      "epoch": 11.22,
      "step": 485499
    },
    {
      "epoch": 11.22,
      "learning_rate": 3.001836734693878e-06,
      "loss": 2.2847,
      "step": 485500
    },
    {
      "FLOPS loss": 0.07090209424495697,
      "L0_d": 930.45,
      "MLM loss": 2.343792676925659,
      "epoch": 11.23,
      "step": 485999
    },
    {
      "epoch": 11.23,
      "learning_rate": 2.899795918367347e-06,
      "loss": 2.2835,
      "step": 486000
    },
    {
      "FLOPS loss": 0.06975448131561279,
      "L0_d": 1004.09,
      "MLM loss": 2.3023698329925537,
      "epoch": 11.24,
      "step": 486499
    },
    {
      "epoch": 11.24,
      "learning_rate": 2.7979591836734695e-06,
      "loss": 2.2819,
      "step": 486500
    },
    {
      "FLOPS loss": 0.07231532782316208,
      "L0_d": 883.98,
      "MLM loss": 2.448282241821289,
      "epoch": 11.25,
      "step": 486999
    },
    {
      "epoch": 11.25,
      "learning_rate": 2.695918367346939e-06,
      "loss": 2.2827,
      "step": 487000
    },
    {
      "FLOPS loss": 0.06464584916830063,
      "L0_d": 878.47,
      "MLM loss": 2.4496078491210938,
      "epoch": 11.26,
      "step": 487499
    },
    {
      "epoch": 11.26,
      "learning_rate": 2.5938775510204082e-06,
      "loss": 2.2875,
      "step": 487500
    },
    {
      "FLOPS loss": 0.06287199258804321,
      "L0_d": 914.27,
      "MLM loss": 2.293144702911377,
      "epoch": 11.27,
      "step": 487999
    },
    {
      "epoch": 11.27,
      "learning_rate": 2.4918367346938776e-06,
      "loss": 2.2859,
      "step": 488000
    },
    {
      "FLOPS loss": 0.06138462945818901,
      "L0_d": 809.25,
      "MLM loss": 2.25559139251709,
      "epoch": 11.28,
      "step": 488499
    },
    {
      "epoch": 11.28,
      "learning_rate": 2.3900000000000004e-06,
      "loss": 2.2864,
      "step": 488500
    },
    {
      "FLOPS loss": 0.0626230463385582,
      "L0_d": 787.83,
      "MLM loss": 2.248427391052246,
      "epoch": 11.3,
      "step": 488999
    },
    {
      "epoch": 11.3,
      "learning_rate": 2.2879591836734693e-06,
      "loss": 2.284,
      "step": 489000
    },
    {
      "FLOPS loss": 0.06445842236280441,
      "L0_d": 913.44,
      "MLM loss": 2.2899110317230225,
      "epoch": 11.31,
      "step": 489499
    },
    {
      "epoch": 11.31,
      "learning_rate": 2.185918367346939e-06,
      "loss": 2.2865,
      "step": 489500
    },
    {
      "FLOPS loss": 0.06436813622713089,
      "L0_d": 843.73,
      "MLM loss": 2.0450210571289062,
      "epoch": 11.32,
      "step": 489999
    },
    {
      "epoch": 11.32,
      "learning_rate": 2.083877551020408e-06,
      "loss": 2.2795,
      "step": 490000
    },
    {
      "FLOPS loss": 0.05308640003204346,
      "L0_d": 662.95,
      "MLM loss": 2.297100305557251,
      "epoch": 11.33,
      "step": 490499
    },
    {
      "epoch": 11.33,
      "learning_rate": 1.981836734693878e-06,
      "loss": 2.2855,
      "step": 490500
    },
    {
      "FLOPS loss": 0.07051984965801239,
      "L0_d": 930.33,
      "MLM loss": 2.173715353012085,
      "epoch": 11.34,
      "step": 490999
    },
    {
      "epoch": 11.34,
      "learning_rate": 1.8800000000000002e-06,
      "loss": 2.2768,
      "step": 491000
    },
    {
      "FLOPS loss": 0.06497853994369507,
      "L0_d": 941.98,
      "MLM loss": 2.1293177604675293,
      "epoch": 11.35,
      "step": 491499
    },
    {
      "epoch": 11.35,
      "learning_rate": 1.7779591836734694e-06,
      "loss": 2.2856,
      "step": 491500
    },
    {
      "FLOPS loss": 0.05521395802497864,
      "L0_d": 848.78,
      "MLM loss": 2.2102599143981934,
      "epoch": 11.37,
      "step": 491999
    },
    {
      "epoch": 11.37,
      "learning_rate": 1.675918367346939e-06,
      "loss": 2.2816,
      "step": 492000
    },
    {
      "FLOPS loss": 0.07123885303735733,
      "L0_d": 958.03,
      "MLM loss": 2.224113702774048,
      "epoch": 11.38,
      "step": 492499
    },
    {
      "epoch": 11.38,
      "learning_rate": 1.573877551020408e-06,
      "loss": 2.2853,
      "step": 492500
    },
    {
      "FLOPS loss": 0.0590461902320385,
      "L0_d": 896.55,
      "MLM loss": 2.306436538696289,
      "epoch": 11.39,
      "step": 492999
    },
    {
      "epoch": 11.39,
      "learning_rate": 1.4720408163265307e-06,
      "loss": 2.2788,
      "step": 493000
    },
    {
      "FLOPS loss": 0.057050079107284546,
      "L0_d": 733.53,
      "MLM loss": 2.209432363510132,
      "epoch": 11.4,
      "step": 493499
    },
    {
      "epoch": 11.4,
      "learning_rate": 1.37e-06,
      "loss": 2.2844,
      "step": 493500
    },
    {
      "FLOPS loss": 0.047397587448358536,
      "L0_d": 655.14,
      "MLM loss": 2.215719699859619,
      "epoch": 11.41,
      "step": 493999
    },
    {
      "epoch": 11.41,
      "learning_rate": 1.2679591836734696e-06,
      "loss": 2.2796,
      "step": 494000
    },
    {
      "FLOPS loss": 0.0556488037109375,
      "L0_d": 752.7,
      "MLM loss": 2.3418068885803223,
      "epoch": 11.42,
      "step": 494499
    },
    {
      "epoch": 11.42,
      "learning_rate": 1.165918367346939e-06,
      "loss": 2.2841,
      "step": 494500
    },
    {
      "FLOPS loss": 0.0699276253581047,
      "L0_d": 918.27,
      "MLM loss": 2.2648048400878906,
      "epoch": 11.44,
      "step": 494999
    },
    {
      "epoch": 11.44,
      "learning_rate": 1.0638775510204083e-06,
      "loss": 2.2847,
      "step": 495000
    },
    {
      "FLOPS loss": 0.06734742224216461,
      "L0_d": 917.17,
      "MLM loss": 2.244868516921997,
      "epoch": 11.45,
      "step": 495499
    },
    {
      "epoch": 11.45,
      "learning_rate": 9.620408163265307e-07,
      "loss": 2.2869,
      "step": 495500
    },
    {
      "FLOPS loss": 0.06317664682865143,
      "L0_d": 786.08,
      "MLM loss": 2.1748099327087402,
      "epoch": 11.46,
      "step": 495999
    },
    {
      "epoch": 11.46,
      "learning_rate": 8.6e-07,
      "loss": 2.2775,
      "step": 496000
    },
    {
      "FLOPS loss": 0.05801856517791748,
      "L0_d": 728.44,
      "MLM loss": 2.228121757507324,
      "epoch": 11.47,
      "step": 496499
    },
    {
      "epoch": 11.47,
      "learning_rate": 7.579591836734694e-07,
      "loss": 2.2885,
      "step": 496500
    },
    {
      "FLOPS loss": 0.06091460585594177,
      "L0_d": 796.03,
      "MLM loss": 2.3856430053710938,
      "epoch": 11.48,
      "step": 496999
    },
    {
      "epoch": 11.48,
      "learning_rate": 6.559183673469388e-07,
      "loss": 2.2859,
      "step": 497000
    },
    {
      "FLOPS loss": 0.0647670179605484,
      "L0_d": 960.53,
      "MLM loss": 2.304900646209717,
      "epoch": 11.49,
      "step": 497499
    },
    {
      "epoch": 11.49,
      "learning_rate": 5.540816326530612e-07,
      "loss": 2.2799,
      "step": 497500
    },
    {
      "FLOPS loss": 0.06450213491916656,
      "L0_d": 1017.69,
      "MLM loss": 2.2775943279266357,
      "epoch": 11.5,
      "step": 497999
    },
    {
      "epoch": 11.5,
      "learning_rate": 4.5204081632653063e-07,
      "loss": 2.2798,
      "step": 498000
    },
    {
      "FLOPS loss": 0.0563599094748497,
      "L0_d": 762.67,
      "MLM loss": 2.046053171157837,
      "epoch": 11.52,
      "step": 498499
    },
    {
      "epoch": 11.52,
      "learning_rate": 3.5000000000000004e-07,
      "loss": 2.2774,
      "step": 498500
    },
    {
      "FLOPS loss": 0.06709830462932587,
      "L0_d": 899.8,
      "MLM loss": 2.305211305618286,
      "epoch": 11.53,
      "step": 498999
    },
    {
      "epoch": 11.53,
      "learning_rate": 2.479591836734694e-07,
      "loss": 2.2826,
      "step": 499000
    },
    {
      "FLOPS loss": 0.08217326551675797,
      "L0_d": 960.14,
      "MLM loss": 2.168755054473877,
      "epoch": 11.54,
      "step": 499499
    },
    {
      "epoch": 11.54,
      "learning_rate": 1.4612244897959183e-07,
      "loss": 2.282,
      "step": 499500
    },
    {
      "FLOPS loss": 0.06401801854372025,
      "L0_d": 961.06,
      "MLM loss": 2.2633349895477295,
      "epoch": 11.55,
      "step": 499999
    },
    {
      "epoch": 11.55,
      "learning_rate": 4.4081632653061224e-08,
      "loss": 2.2808,
      "step": 500000
    },
    {
      "epoch": 11.55,
      "step": 500000,
      "total_flos": 8.483549701629542e+18,
      "train_loss": 2.541904963623047,
      "train_runtime": 186540.8186,
      "train_samples_per_second": 686.177,
      "train_steps_per_second": 2.68
    }
  ],
  "max_steps": 500000,
  "num_train_epochs": 12,
  "total_flos": 8.483549701629542e+18,
  "trial_name": null,
  "trial_params": null
}