mnaylor commited on
Commit
328c6c9
1 Parent(s): ee588b6

updating saved model after name change from encoders to layers in MegaModel

Browse files
Files changed (2) hide show
  1. config.json +5 -2
  2. pytorch_model.bin +2 -2
config.json CHANGED
@@ -12,13 +12,15 @@
12
  "chunk_size": -1,
13
  "classifier_dropout": null,
14
  "dropout_prob": 0.1,
 
 
 
15
  "ema_projection_size": 16,
16
  "eos_token_id": 2,
17
  "hidden_dropout_prob": 0.1,
18
  "hidden_size": 128,
19
  "initializer_range": 0.02,
20
  "intermediate_size": 256,
21
- "layer_norm_eps": 1e-12,
22
  "max_positions": 10000,
23
  "model_type": "mega",
24
  "nffn_activation_dropout_prob": 0.0,
@@ -27,12 +29,13 @@
27
  "normalization_type": "scalenorm",
28
  "normalize_before_ffn": true,
29
  "normalize_before_mega": true,
 
30
  "num_hidden_layers": 4,
31
  "pad_token_id": 1,
32
  "relative_positional_bias": "rotary",
33
  "shared_representation_size": 64,
34
  "torch_dtype": "float32",
35
- "transformers_version": "4.27.0.dev0",
36
  "truncation": null,
37
  "type_vocab_size": 2,
38
  "use_cache": true,
 
12
  "chunk_size": -1,
13
  "classifier_dropout": null,
14
  "dropout_prob": 0.1,
15
+ "ema_beta_range": 0.02,
16
+ "ema_delta_alpha_range": 0.2,
17
+ "ema_gamma_omega_range": 1.0,
18
  "ema_projection_size": 16,
19
  "eos_token_id": 2,
20
  "hidden_dropout_prob": 0.1,
21
  "hidden_size": 128,
22
  "initializer_range": 0.02,
23
  "intermediate_size": 256,
 
24
  "max_positions": 10000,
25
  "model_type": "mega",
26
  "nffn_activation_dropout_prob": 0.0,
 
29
  "normalization_type": "scalenorm",
30
  "normalize_before_ffn": true,
31
  "normalize_before_mega": true,
32
+ "num_attention_heads": 1,
33
  "num_hidden_layers": 4,
34
  "pad_token_id": 1,
35
  "relative_positional_bias": "rotary",
36
  "shared_representation_size": 64,
37
  "torch_dtype": "float32",
38
+ "transformers_version": "4.28.0.dev0",
39
  "truncation": null,
40
  "type_vocab_size": 2,
41
  "use_cache": true,
pytorch_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:148c0df0aca6894b870a3b4ef483148d9131957231b64f6068ba7861b8eb5e29
3
- size 29332668
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69f1dfc6152299b92c6729e6ea7c955b70f07fe63dafc799e0386690e2349df6
3
+ size 29332476