Nitral commited on
Commit
6797b67
1 Parent(s): ecd5a70

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +15 -5
config.json CHANGED
@@ -3,7 +3,10 @@
3
  "architectures": [
4
  "MistralForCausalLM"
5
  ],
6
- "attention_dropout": 0.0,
 
 
 
7
  "bos_token_id": 1,
8
  "eos_token_id": 2,
9
  "hidden_act": "silu",
@@ -11,16 +14,23 @@
11
  "initializer_range": 0.02,
12
  "intermediate_size": 14336,
13
  "max_position_embeddings": 32768,
 
14
  "model_type": "mistral",
15
  "num_attention_heads": 32,
16
  "num_hidden_layers": 32,
17
  "num_key_value_heads": 8,
18
  "rms_norm_eps": 1e-05,
19
- "rope_theta": 1000000.0,
20
- "sliding_window": null,
 
 
 
 
 
 
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
23
- "transformers_version": "4.35.2",
24
- "use_cache": false,
25
  "vocab_size": 32000
26
  }
 
3
  "architectures": [
4
  "MistralForCausalLM"
5
  ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_mistral.MistralConfig",
8
+ "AutoModelForCausalLM": "modeling_mistral_yarn.MistralForCausalLM"
9
+ },
10
  "bos_token_id": 1,
11
  "eos_token_id": 2,
12
  "hidden_act": "silu",
 
14
  "initializer_range": 0.02,
15
  "intermediate_size": 14336,
16
  "max_position_embeddings": 32768,
17
+ "max_sequence_length": 131072,
18
  "model_type": "mistral",
19
  "num_attention_heads": 32,
20
  "num_hidden_layers": 32,
21
  "num_key_value_heads": 8,
22
  "rms_norm_eps": 1e-05,
23
+ "rope_scaling": {
24
+ "factor": 16.0,
25
+ "finetuned": true,
26
+ "original_max_position_embeddings": 8192,
27
+ "type": "yarn"
28
+ },
29
+ "rope_theta": 10000.0,
30
+ "sliding_window": 131072,
31
  "tie_word_embeddings": false,
32
  "torch_dtype": "bfloat16",
33
+ "transformers_version": "4.35.0.dev0",
34
+ "use_cache": true,
35
  "vocab_size": 32000
36
  }