File size: 536 Bytes
a30d966 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
{
"d_model": 256,
"n_layer": 8,
"d_inner": 1024,
"vocab_size": 12,
"resid_dropout": 0.0,
"embed_dropout": 0.1,
"fused_mlp": false,
"fused_dropout_add_ln": true,
"checkpoint_mixer": true,
"checkpoint_mlp": true,
"residual_in_fp32": true,
"pad_vocab_size_multiple": 8,
"return_hidden_state": true,
"layer": {
"_name_": "hyena",
"emb_dim": 5,
"filter_order": 64,
"local_order": 3,
"l_max": 450002,
"modulate": true,
"w": 10,
"lr": 6e-4,
"wd": 0.0,
"lr_pos_emb": 0.0
}
} |