VISOR-GPT / train /models /megatron /bert_3.9B_config.json
szukevin's picture
upload
7900c16
raw
history blame
343 Bytes
{
"emb_size": 2560,
"feedforward_size": 10240,
"hidden_size": 2560,
"hidden_act": "gelu",
"heads_num": 40,
"layers_num": 48,
"max_seq_length": 512,
"dropout": 0.1,
"data_processor": "bert",
"embedding": ["word", "pos", "seg"],
"encoder": "transformer",
"mask": "fully_visible",
"target": ["mlm", "sp"]
}