|
[17:10:21] - INFO - absl - Restoring checkpoint from ./checkpoint_10000 |
|
tcmalloc: large alloc 1530273792 bytes == 0x9b524000 @ 0x7f5b75dd5680 0x7f5b75df6824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7f5b75bea0b3 0x5f96de |
|
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed |
|
warnings.warn( |
|
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed |
|
warnings.warn( |
|
Epoch ... (1/2): 0%| | 0/2 [00:00<?, ?it/s] |
|
|
|
|
|
|
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100067/352766 [04:37<16:57:35, 4.14it/s] |
|
|
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100119/352766 [04:57<13:06:22, 5.35it/s] |
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100152/352766 [05:03<26:17:15, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100202/352766 [05:23<26:02:15, 2.69it/s] |
|
|
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100252/352766 [05:43<26:10:16, 2.68it/s] |
|
|
|
|
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100317/352766 [06:17<17:05:12, 4.10it/s] |
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100352/352766 [06:23<26:12:05, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100402/352766 [06:43<26:11:04, 2.68it/s] |
|
|
|
|
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100469/352766 [07:17<13:12:02, 5.31it/s] |
|
|
|
|
|
|
|
Training...: 28%|βββββββββββββββββββββββββββββββββββββββ | 100502/352766 [07:23<26:09:06, 2.68it/s] |
|
|
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 100571/352766 [07:58<10:30:33, 6.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 100619/352766 [08:18<13:30:57, 5.18it/s] |
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 100652/352766 [08:23<25:56:56, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 100702/352766 [08:43<26:11:10, 2.67it/s] |
|
|
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 100771/352766 [09:18<10:27:46, 6.69it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 100821/352766 [09:38<10:21:37, 6.75it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 100873/352766 [09:58<8:37:24, 8.11it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 100923/352766 [10:18<8:39:16, 8.08it/s] |
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 100952/352766 [10:24<25:57:41, 2.69it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101002/352766 [10:44<26:08:39, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101052/352766 [11:04<26:34:55, 2.63it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101102/352766 [11:24<26:07:20, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101152/352766 [11:44<25:00:47, 2.79it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101202/352766 [12:04<26:07:29, 2.67it/s] |
|
|
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101271/352766 [12:38<10:35:05, 6.60it/s] |
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101302/352766 [12:44<25:54:10, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101352/352766 [13:04<26:04:56, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101402/352766 [13:24<25:52:37, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|βββββββββββββββββββββββββββββββββββββββ | 101452/352766 [13:44<26:05:47, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101502/352766 [14:04<25:53:30, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101552/352766 [14:24<26:05:49, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101602/352766 [14:44<25:48:27, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101652/352766 [15:04<26:04:33, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101702/352766 [15:24<25:57:03, 2.69it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101752/352766 [15:44<26:05:09, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101802/352766 [16:04<26:04:34, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101852/352766 [16:24<26:03:08, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101902/352766 [16:44<26:01:53, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 101952/352766 [17:04<26:08:22, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102002/352766 [17:24<26:02:13, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102052/352766 [17:44<25:49:15, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102102/352766 [18:05<26:01:33, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102152/352766 [18:25<25:46:31, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102202/352766 [18:45<26:00:54, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102252/352766 [19:05<25:47:08, 2.70it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102302/352766 [19:25<26:00:27, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102352/352766 [19:45<25:53:04, 2.69it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102402/352766 [20:05<26:01:48, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102452/352766 [20:25<26:00:49, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102502/352766 [20:45<25:58:45, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102552/352766 [21:05<25:58:02, 2.68it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102602/352766 [21:25<25:58:39, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102652/352766 [21:45<25:59:04, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102702/352766 [22:05<25:43:17, 2.70it/s] |
|
|
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102777/352766 [22:40<6:58:15, 9.96it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102831/352766 [23:00<5:31:01, 12.58it/s] |
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102852/352766 [23:05<25:58:26, 2.67it/s] |
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102902/352766 [23:25<25:41:12, 2.70it/s] |
|
|
|
|
|
|
|
|
|
|
|
Training...: 29%|ββββββββββββββββββββββββββββββββββββββββ | 102977/352766 [24:00<6:26:06, 10.78it/s] |
|
|
|
|
|
|
|
|