gsmyrnis's picture
Training in progress, epoch 3
c2d49d3 verified
{"current_steps": 10, "total_steps": 942, "loss": 0.942, "learning_rate": 5e-06, "epoch": 0.03184713375796178, "percentage": 1.06, "elapsed_time": "0:00:58", "remaining_time": "1:31:09"}
{"current_steps": 20, "total_steps": 942, "loss": 0.8332, "learning_rate": 5e-06, "epoch": 0.06369426751592357, "percentage": 2.12, "elapsed_time": "0:01:56", "remaining_time": "1:29:41"}
{"current_steps": 30, "total_steps": 942, "loss": 0.7857, "learning_rate": 5e-06, "epoch": 0.09554140127388536, "percentage": 3.18, "elapsed_time": "0:02:55", "remaining_time": "1:28:45"}
{"current_steps": 40, "total_steps": 942, "loss": 0.7718, "learning_rate": 5e-06, "epoch": 0.12738853503184713, "percentage": 4.25, "elapsed_time": "0:03:53", "remaining_time": "1:27:51"}
{"current_steps": 50, "total_steps": 942, "loss": 0.7492, "learning_rate": 5e-06, "epoch": 0.1592356687898089, "percentage": 5.31, "elapsed_time": "0:04:51", "remaining_time": "1:26:47"}
{"current_steps": 60, "total_steps": 942, "loss": 0.7413, "learning_rate": 5e-06, "epoch": 0.1910828025477707, "percentage": 6.37, "elapsed_time": "0:05:50", "remaining_time": "1:25:48"}
{"current_steps": 70, "total_steps": 942, "loss": 0.719, "learning_rate": 5e-06, "epoch": 0.2229299363057325, "percentage": 7.43, "elapsed_time": "0:06:48", "remaining_time": "1:24:48"}
{"current_steps": 80, "total_steps": 942, "loss": 0.7184, "learning_rate": 5e-06, "epoch": 0.25477707006369427, "percentage": 8.49, "elapsed_time": "0:07:46", "remaining_time": "1:23:47"}
{"current_steps": 90, "total_steps": 942, "loss": 0.7121, "learning_rate": 5e-06, "epoch": 0.28662420382165604, "percentage": 9.55, "elapsed_time": "0:08:44", "remaining_time": "1:22:49"}
{"current_steps": 100, "total_steps": 942, "loss": 0.7092, "learning_rate": 5e-06, "epoch": 0.3184713375796178, "percentage": 10.62, "elapsed_time": "0:09:43", "remaining_time": "1:21:53"}
{"current_steps": 110, "total_steps": 942, "loss": 0.7013, "learning_rate": 5e-06, "epoch": 0.3503184713375796, "percentage": 11.68, "elapsed_time": "0:10:41", "remaining_time": "1:20:53"}
{"current_steps": 120, "total_steps": 942, "loss": 0.7022, "learning_rate": 5e-06, "epoch": 0.3821656050955414, "percentage": 12.74, "elapsed_time": "0:11:40", "remaining_time": "1:19:56"}
{"current_steps": 130, "total_steps": 942, "loss": 0.7044, "learning_rate": 5e-06, "epoch": 0.4140127388535032, "percentage": 13.8, "elapsed_time": "0:12:38", "remaining_time": "1:18:56"}
{"current_steps": 140, "total_steps": 942, "loss": 0.692, "learning_rate": 5e-06, "epoch": 0.445859872611465, "percentage": 14.86, "elapsed_time": "0:13:36", "remaining_time": "1:17:57"}
{"current_steps": 150, "total_steps": 942, "loss": 0.7023, "learning_rate": 5e-06, "epoch": 0.47770700636942676, "percentage": 15.92, "elapsed_time": "0:14:35", "remaining_time": "1:17:00"}
{"current_steps": 160, "total_steps": 942, "loss": 0.6944, "learning_rate": 5e-06, "epoch": 0.5095541401273885, "percentage": 16.99, "elapsed_time": "0:15:34", "remaining_time": "1:16:05"}
{"current_steps": 170, "total_steps": 942, "loss": 0.6966, "learning_rate": 5e-06, "epoch": 0.5414012738853503, "percentage": 18.05, "elapsed_time": "0:16:32", "remaining_time": "1:15:06"}
{"current_steps": 180, "total_steps": 942, "loss": 0.6876, "learning_rate": 5e-06, "epoch": 0.5732484076433121, "percentage": 19.11, "elapsed_time": "0:17:30", "remaining_time": "1:14:07"}
{"current_steps": 190, "total_steps": 942, "loss": 0.6891, "learning_rate": 5e-06, "epoch": 0.6050955414012739, "percentage": 20.17, "elapsed_time": "0:18:28", "remaining_time": "1:13:09"}
{"current_steps": 200, "total_steps": 942, "loss": 0.6832, "learning_rate": 5e-06, "epoch": 0.6369426751592356, "percentage": 21.23, "elapsed_time": "0:19:27", "remaining_time": "1:12:10"}
{"current_steps": 210, "total_steps": 942, "loss": 0.6803, "learning_rate": 5e-06, "epoch": 0.6687898089171974, "percentage": 22.29, "elapsed_time": "0:20:25", "remaining_time": "1:11:11"}
{"current_steps": 220, "total_steps": 942, "loss": 0.6768, "learning_rate": 5e-06, "epoch": 0.7006369426751592, "percentage": 23.35, "elapsed_time": "0:21:24", "remaining_time": "1:10:14"}
{"current_steps": 230, "total_steps": 942, "loss": 0.6825, "learning_rate": 5e-06, "epoch": 0.732484076433121, "percentage": 24.42, "elapsed_time": "0:22:22", "remaining_time": "1:09:15"}
{"current_steps": 240, "total_steps": 942, "loss": 0.6806, "learning_rate": 5e-06, "epoch": 0.7643312101910829, "percentage": 25.48, "elapsed_time": "0:23:20", "remaining_time": "1:08:17"}
{"current_steps": 250, "total_steps": 942, "loss": 0.6848, "learning_rate": 5e-06, "epoch": 0.7961783439490446, "percentage": 26.54, "elapsed_time": "0:24:19", "remaining_time": "1:07:19"}
{"current_steps": 260, "total_steps": 942, "loss": 0.6842, "learning_rate": 5e-06, "epoch": 0.8280254777070064, "percentage": 27.6, "elapsed_time": "0:25:17", "remaining_time": "1:06:20"}
{"current_steps": 270, "total_steps": 942, "loss": 0.6826, "learning_rate": 5e-06, "epoch": 0.8598726114649682, "percentage": 28.66, "elapsed_time": "0:26:15", "remaining_time": "1:05:21"}
{"current_steps": 280, "total_steps": 942, "loss": 0.6812, "learning_rate": 5e-06, "epoch": 0.89171974522293, "percentage": 29.72, "elapsed_time": "0:27:13", "remaining_time": "1:04:22"}
{"current_steps": 290, "total_steps": 942, "loss": 0.6774, "learning_rate": 5e-06, "epoch": 0.9235668789808917, "percentage": 30.79, "elapsed_time": "0:28:11", "remaining_time": "1:03:23"}
{"current_steps": 300, "total_steps": 942, "loss": 0.6757, "learning_rate": 5e-06, "epoch": 0.9554140127388535, "percentage": 31.85, "elapsed_time": "0:29:10", "remaining_time": "1:02:25"}
{"current_steps": 310, "total_steps": 942, "loss": 0.6774, "learning_rate": 5e-06, "epoch": 0.9872611464968153, "percentage": 32.91, "elapsed_time": "0:30:08", "remaining_time": "1:01:26"}
{"current_steps": 314, "total_steps": 942, "eval_loss": 0.6760347485542297, "epoch": 1.0, "percentage": 33.33, "elapsed_time": "0:31:01", "remaining_time": "1:02:03"}
{"current_steps": 320, "total_steps": 942, "loss": 0.6476, "learning_rate": 5e-06, "epoch": 1.019108280254777, "percentage": 33.97, "elapsed_time": "0:32:51", "remaining_time": "1:03:52"}
{"current_steps": 330, "total_steps": 942, "loss": 0.6341, "learning_rate": 5e-06, "epoch": 1.0509554140127388, "percentage": 35.03, "elapsed_time": "0:33:50", "remaining_time": "1:02:45"}
{"current_steps": 340, "total_steps": 942, "loss": 0.6285, "learning_rate": 5e-06, "epoch": 1.0828025477707006, "percentage": 36.09, "elapsed_time": "0:34:48", "remaining_time": "1:01:37"}
{"current_steps": 350, "total_steps": 942, "loss": 0.6293, "learning_rate": 5e-06, "epoch": 1.1146496815286624, "percentage": 37.15, "elapsed_time": "0:35:46", "remaining_time": "1:00:31"}
{"current_steps": 360, "total_steps": 942, "loss": 0.624, "learning_rate": 5e-06, "epoch": 1.1464968152866242, "percentage": 38.22, "elapsed_time": "0:36:44", "remaining_time": "0:59:24"}
{"current_steps": 370, "total_steps": 942, "loss": 0.628, "learning_rate": 5e-06, "epoch": 1.178343949044586, "percentage": 39.28, "elapsed_time": "0:37:42", "remaining_time": "0:58:18"}
{"current_steps": 380, "total_steps": 942, "loss": 0.6293, "learning_rate": 5e-06, "epoch": 1.2101910828025477, "percentage": 40.34, "elapsed_time": "0:38:41", "remaining_time": "0:57:12"}
{"current_steps": 390, "total_steps": 942, "loss": 0.6272, "learning_rate": 5e-06, "epoch": 1.2420382165605095, "percentage": 41.4, "elapsed_time": "0:39:39", "remaining_time": "0:56:07"}
{"current_steps": 400, "total_steps": 942, "loss": 0.6314, "learning_rate": 5e-06, "epoch": 1.2738853503184713, "percentage": 42.46, "elapsed_time": "0:40:37", "remaining_time": "0:55:02"}
{"current_steps": 410, "total_steps": 942, "loss": 0.6309, "learning_rate": 5e-06, "epoch": 1.305732484076433, "percentage": 43.52, "elapsed_time": "0:41:35", "remaining_time": "0:53:58"}
{"current_steps": 420, "total_steps": 942, "loss": 0.6322, "learning_rate": 5e-06, "epoch": 1.3375796178343948, "percentage": 44.59, "elapsed_time": "0:42:33", "remaining_time": "0:52:54"}
{"current_steps": 430, "total_steps": 942, "loss": 0.6293, "learning_rate": 5e-06, "epoch": 1.3694267515923566, "percentage": 45.65, "elapsed_time": "0:43:32", "remaining_time": "0:51:50"}
{"current_steps": 440, "total_steps": 942, "loss": 0.6231, "learning_rate": 5e-06, "epoch": 1.4012738853503186, "percentage": 46.71, "elapsed_time": "0:44:30", "remaining_time": "0:50:46"}
{"current_steps": 450, "total_steps": 942, "loss": 0.6308, "learning_rate": 5e-06, "epoch": 1.4331210191082802, "percentage": 47.77, "elapsed_time": "0:45:28", "remaining_time": "0:49:43"}
{"current_steps": 460, "total_steps": 942, "loss": 0.6316, "learning_rate": 5e-06, "epoch": 1.4649681528662422, "percentage": 48.83, "elapsed_time": "0:46:26", "remaining_time": "0:48:40"}
{"current_steps": 470, "total_steps": 942, "loss": 0.6216, "learning_rate": 5e-06, "epoch": 1.4968152866242037, "percentage": 49.89, "elapsed_time": "0:47:24", "remaining_time": "0:47:36"}
{"current_steps": 480, "total_steps": 942, "loss": 0.634, "learning_rate": 5e-06, "epoch": 1.5286624203821657, "percentage": 50.96, "elapsed_time": "0:48:22", "remaining_time": "0:46:34"}
{"current_steps": 490, "total_steps": 942, "loss": 0.6211, "learning_rate": 5e-06, "epoch": 1.5605095541401273, "percentage": 52.02, "elapsed_time": "0:49:21", "remaining_time": "0:45:31"}
{"current_steps": 500, "total_steps": 942, "loss": 0.6263, "learning_rate": 5e-06, "epoch": 1.5923566878980893, "percentage": 53.08, "elapsed_time": "0:50:19", "remaining_time": "0:44:29"}
{"current_steps": 510, "total_steps": 942, "loss": 0.6283, "learning_rate": 5e-06, "epoch": 1.6242038216560508, "percentage": 54.14, "elapsed_time": "0:51:17", "remaining_time": "0:43:26"}
{"current_steps": 520, "total_steps": 942, "loss": 0.6296, "learning_rate": 5e-06, "epoch": 1.6560509554140128, "percentage": 55.2, "elapsed_time": "0:52:15", "remaining_time": "0:42:24"}
{"current_steps": 530, "total_steps": 942, "loss": 0.6243, "learning_rate": 5e-06, "epoch": 1.6878980891719744, "percentage": 56.26, "elapsed_time": "0:53:13", "remaining_time": "0:41:22"}
{"current_steps": 540, "total_steps": 942, "loss": 0.6262, "learning_rate": 5e-06, "epoch": 1.7197452229299364, "percentage": 57.32, "elapsed_time": "0:54:11", "remaining_time": "0:40:20"}
{"current_steps": 550, "total_steps": 942, "loss": 0.6233, "learning_rate": 5e-06, "epoch": 1.7515923566878981, "percentage": 58.39, "elapsed_time": "0:55:10", "remaining_time": "0:39:19"}
{"current_steps": 560, "total_steps": 942, "loss": 0.6325, "learning_rate": 5e-06, "epoch": 1.78343949044586, "percentage": 59.45, "elapsed_time": "0:56:08", "remaining_time": "0:38:17"}
{"current_steps": 570, "total_steps": 942, "loss": 0.6262, "learning_rate": 5e-06, "epoch": 1.8152866242038217, "percentage": 60.51, "elapsed_time": "0:57:06", "remaining_time": "0:37:16"}
{"current_steps": 580, "total_steps": 942, "loss": 0.6238, "learning_rate": 5e-06, "epoch": 1.8471337579617835, "percentage": 61.57, "elapsed_time": "0:58:05", "remaining_time": "0:36:15"}
{"current_steps": 590, "total_steps": 942, "loss": 0.6195, "learning_rate": 5e-06, "epoch": 1.8789808917197452, "percentage": 62.63, "elapsed_time": "0:59:03", "remaining_time": "0:35:13"}
{"current_steps": 600, "total_steps": 942, "loss": 0.6242, "learning_rate": 5e-06, "epoch": 1.910828025477707, "percentage": 63.69, "elapsed_time": "1:00:01", "remaining_time": "0:34:12"}
{"current_steps": 610, "total_steps": 942, "loss": 0.6282, "learning_rate": 5e-06, "epoch": 1.9426751592356688, "percentage": 64.76, "elapsed_time": "1:00:59", "remaining_time": "0:33:11"}
{"current_steps": 620, "total_steps": 942, "loss": 0.6192, "learning_rate": 5e-06, "epoch": 1.9745222929936306, "percentage": 65.82, "elapsed_time": "1:01:58", "remaining_time": "0:32:11"}
{"current_steps": 628, "total_steps": 942, "eval_loss": 0.6656389832496643, "epoch": 2.0, "percentage": 66.67, "elapsed_time": "1:03:15", "remaining_time": "0:31:37"}
{"current_steps": 630, "total_steps": 942, "loss": 0.6157, "learning_rate": 5e-06, "epoch": 2.0063694267515926, "percentage": 66.88, "elapsed_time": "1:04:56", "remaining_time": "0:32:09"}
{"current_steps": 640, "total_steps": 942, "loss": 0.5735, "learning_rate": 5e-06, "epoch": 2.038216560509554, "percentage": 67.94, "elapsed_time": "1:05:55", "remaining_time": "0:31:06"}
{"current_steps": 650, "total_steps": 942, "loss": 0.5778, "learning_rate": 5e-06, "epoch": 2.070063694267516, "percentage": 69.0, "elapsed_time": "1:06:53", "remaining_time": "0:30:02"}
{"current_steps": 660, "total_steps": 942, "loss": 0.5784, "learning_rate": 5e-06, "epoch": 2.1019108280254777, "percentage": 70.06, "elapsed_time": "1:07:51", "remaining_time": "0:28:59"}
{"current_steps": 670, "total_steps": 942, "loss": 0.5751, "learning_rate": 5e-06, "epoch": 2.1337579617834397, "percentage": 71.13, "elapsed_time": "1:08:49", "remaining_time": "0:27:56"}
{"current_steps": 680, "total_steps": 942, "loss": 0.5766, "learning_rate": 5e-06, "epoch": 2.1656050955414012, "percentage": 72.19, "elapsed_time": "1:09:47", "remaining_time": "0:26:53"}
{"current_steps": 690, "total_steps": 942, "loss": 0.5798, "learning_rate": 5e-06, "epoch": 2.1974522292993632, "percentage": 73.25, "elapsed_time": "1:10:46", "remaining_time": "0:25:50"}
{"current_steps": 700, "total_steps": 942, "loss": 0.5767, "learning_rate": 5e-06, "epoch": 2.229299363057325, "percentage": 74.31, "elapsed_time": "1:11:44", "remaining_time": "0:24:48"}
{"current_steps": 710, "total_steps": 942, "loss": 0.5773, "learning_rate": 5e-06, "epoch": 2.261146496815287, "percentage": 75.37, "elapsed_time": "1:12:42", "remaining_time": "0:23:45"}
{"current_steps": 720, "total_steps": 942, "loss": 0.579, "learning_rate": 5e-06, "epoch": 2.2929936305732483, "percentage": 76.43, "elapsed_time": "1:13:40", "remaining_time": "0:22:42"}
{"current_steps": 730, "total_steps": 942, "loss": 0.5796, "learning_rate": 5e-06, "epoch": 2.3248407643312103, "percentage": 77.49, "elapsed_time": "1:14:38", "remaining_time": "0:21:40"}
{"current_steps": 740, "total_steps": 942, "loss": 0.5748, "learning_rate": 5e-06, "epoch": 2.356687898089172, "percentage": 78.56, "elapsed_time": "1:15:36", "remaining_time": "0:20:38"}
{"current_steps": 750, "total_steps": 942, "loss": 0.5834, "learning_rate": 5e-06, "epoch": 2.388535031847134, "percentage": 79.62, "elapsed_time": "1:16:34", "remaining_time": "0:19:36"}
{"current_steps": 760, "total_steps": 942, "loss": 0.5758, "learning_rate": 5e-06, "epoch": 2.4203821656050954, "percentage": 80.68, "elapsed_time": "1:17:32", "remaining_time": "0:18:34"}
{"current_steps": 770, "total_steps": 942, "loss": 0.5856, "learning_rate": 5e-06, "epoch": 2.4522292993630574, "percentage": 81.74, "elapsed_time": "1:18:31", "remaining_time": "0:17:32"}
{"current_steps": 780, "total_steps": 942, "loss": 0.5786, "learning_rate": 5e-06, "epoch": 2.484076433121019, "percentage": 82.8, "elapsed_time": "1:19:29", "remaining_time": "0:16:30"}
{"current_steps": 790, "total_steps": 942, "loss": 0.577, "learning_rate": 5e-06, "epoch": 2.515923566878981, "percentage": 83.86, "elapsed_time": "1:20:27", "remaining_time": "0:15:28"}
{"current_steps": 800, "total_steps": 942, "loss": 0.581, "learning_rate": 5e-06, "epoch": 2.5477707006369426, "percentage": 84.93, "elapsed_time": "1:21:25", "remaining_time": "0:14:27"}
{"current_steps": 810, "total_steps": 942, "loss": 0.5809, "learning_rate": 5e-06, "epoch": 2.5796178343949046, "percentage": 85.99, "elapsed_time": "1:22:23", "remaining_time": "0:13:25"}
{"current_steps": 820, "total_steps": 942, "loss": 0.5845, "learning_rate": 5e-06, "epoch": 2.611464968152866, "percentage": 87.05, "elapsed_time": "1:23:21", "remaining_time": "0:12:24"}
{"current_steps": 830, "total_steps": 942, "loss": 0.5837, "learning_rate": 5e-06, "epoch": 2.643312101910828, "percentage": 88.11, "elapsed_time": "1:24:20", "remaining_time": "0:11:22"}
{"current_steps": 840, "total_steps": 942, "loss": 0.5871, "learning_rate": 5e-06, "epoch": 2.6751592356687897, "percentage": 89.17, "elapsed_time": "1:25:18", "remaining_time": "0:10:21"}
{"current_steps": 850, "total_steps": 942, "loss": 0.5797, "learning_rate": 5e-06, "epoch": 2.7070063694267517, "percentage": 90.23, "elapsed_time": "1:26:16", "remaining_time": "0:09:20"}
{"current_steps": 860, "total_steps": 942, "loss": 0.5814, "learning_rate": 5e-06, "epoch": 2.738853503184713, "percentage": 91.3, "elapsed_time": "1:27:14", "remaining_time": "0:08:19"}
{"current_steps": 870, "total_steps": 942, "loss": 0.585, "learning_rate": 5e-06, "epoch": 2.770700636942675, "percentage": 92.36, "elapsed_time": "1:28:12", "remaining_time": "0:07:18"}
{"current_steps": 880, "total_steps": 942, "loss": 0.5813, "learning_rate": 5e-06, "epoch": 2.802547770700637, "percentage": 93.42, "elapsed_time": "1:29:10", "remaining_time": "0:06:16"}
{"current_steps": 890, "total_steps": 942, "loss": 0.5823, "learning_rate": 5e-06, "epoch": 2.8343949044585988, "percentage": 94.48, "elapsed_time": "1:30:08", "remaining_time": "0:05:16"}
{"current_steps": 900, "total_steps": 942, "loss": 0.5799, "learning_rate": 5e-06, "epoch": 2.8662420382165603, "percentage": 95.54, "elapsed_time": "1:31:07", "remaining_time": "0:04:15"}
{"current_steps": 910, "total_steps": 942, "loss": 0.583, "learning_rate": 5e-06, "epoch": 2.8980891719745223, "percentage": 96.6, "elapsed_time": "1:32:05", "remaining_time": "0:03:14"}
{"current_steps": 920, "total_steps": 942, "loss": 0.5816, "learning_rate": 5e-06, "epoch": 2.9299363057324843, "percentage": 97.66, "elapsed_time": "1:33:03", "remaining_time": "0:02:13"}
{"current_steps": 930, "total_steps": 942, "loss": 0.585, "learning_rate": 5e-06, "epoch": 2.961783439490446, "percentage": 98.73, "elapsed_time": "1:34:02", "remaining_time": "0:01:12"}
{"current_steps": 940, "total_steps": 942, "loss": 0.5826, "learning_rate": 5e-06, "epoch": 2.9936305732484074, "percentage": 99.79, "elapsed_time": "1:35:00", "remaining_time": "0:00:12"}
{"current_steps": 942, "total_steps": 942, "eval_loss": 0.6691574454307556, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:37:04", "remaining_time": "0:00:00"}
{"current_steps": 942, "total_steps": 942, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:38:31", "remaining_time": "0:00:00"}