| { | |
| "dataset_name": "multi_species_final_178G.upper.n.filtered.6_mers.1000_tok_len.overlap_50", | |
| "alphabet": "k-mers", | |
| "k_for_kmers": 6, | |
| "num_warmup_updates": 16000, | |
| "warmup_init_lr": 5e-05, | |
| "warmup_end_lr": 0.0001, | |
| "training_set_proportion": 0.95, | |
| "tokens_per_batch": 1000000.0, | |
| "tokens_per_checkpoint": "3500000000.0", | |
| "masking_ratio": 0.15, | |
| "masking_prob": 0.8, | |
| "batch_size": 2, | |
| "random_token_prob": 0.1, | |
| "dropout_rate": 0.1, | |
| "num_hosts": 8, | |
| "server_address": "3422826-worker-0:1234", | |
| "alphabet_size": 4105, | |
| "pad_token_id": 1, | |
| "mask_token_id": 2, | |
| "class_token_id": 3, | |
| "eos_token_id": -1, | |
| "prepend_bos": true, | |
| "append_eos": false, | |
| "max_positions": 1000, | |
| "emb_layer_norm_before": false, | |
| "attention_heads": 20, | |
| "embed_dim": 2560, | |
| "ffn_embed_dim": 10240, | |
| "num_layers": 32, | |
| "token_dropout": true, | |
| "embed_scale": 1.0, | |
| "use_remat": false, | |
| "architecture": "Vanilla", | |
| "acc_batch_size": 8, | |
| "num_local_devices": 8, | |
| "num_global_devices": 64, | |
| "tokens_length": 1000, | |
| "mixed-precision": true, | |
| "model_num_parameters": "2547800585", | |
| "shift": true, | |
| "overlap": 50 | |
| } |