N8Programs
commited on
Commit
•
58b88ed
1
Parent(s):
d02a23d
Upload folder using huggingface_hub
Browse files- config.json +47 -0
- configuration_exaone.py +183 -0
- merges.txt +0 -0
- model.safetensors +3 -0
- model.safetensors.index.json +701 -0
- modeling_exaone.py +1394 -0
- special_tokens_map.json +30 -0
- tokenizer.json +0 -0
- tokenizer_config.json +3221 -0
- vocab.json +0 -0
config.json
ADDED
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"activation_function": "silu",
|
3 |
+
"architectures": [
|
4 |
+
"ExaoneForCausalLM"
|
5 |
+
],
|
6 |
+
"attention_dropout": 0.0,
|
7 |
+
"auto_map": {
|
8 |
+
"AutoConfig": "configuration_exaone.ExaoneConfig",
|
9 |
+
"AutoModelForCausalLM": "modeling_exaone.ExaoneForCausalLM",
|
10 |
+
"AutoModelForSequenceClassification": "modeling_exaone.ExaoneForSequenceClassification"
|
11 |
+
},
|
12 |
+
"bos_token_id": 1,
|
13 |
+
"embed_dropout": 0.0,
|
14 |
+
"eos_token_id": 361,
|
15 |
+
"head_dim": 80,
|
16 |
+
"hidden_size": 2560,
|
17 |
+
"initializer_range": 0.02,
|
18 |
+
"intermediate_size": 7168,
|
19 |
+
"layer_norm_epsilon": 1e-05,
|
20 |
+
"max_position_embeddings": 32768,
|
21 |
+
"model_type": "exaone",
|
22 |
+
"num_attention_heads": 32,
|
23 |
+
"num_key_value_heads": 8,
|
24 |
+
"num_layers": 30,
|
25 |
+
"pad_token_id": 0,
|
26 |
+
"quantization": {
|
27 |
+
"group_size": 64,
|
28 |
+
"bits": 8
|
29 |
+
},
|
30 |
+
"quantization_config": {
|
31 |
+
"group_size": 64,
|
32 |
+
"bits": 8
|
33 |
+
},
|
34 |
+
"rope_scaling": {
|
35 |
+
"factor": 8.0,
|
36 |
+
"high_freq_factor": 4.0,
|
37 |
+
"low_freq_factor": 1.0,
|
38 |
+
"original_max_position_embeddings": 8192,
|
39 |
+
"rope_type": "llama3"
|
40 |
+
},
|
41 |
+
"rope_theta": 1000000,
|
42 |
+
"tie_word_embeddings": true,
|
43 |
+
"torch_dtype": "float32",
|
44 |
+
"transformers_version": "4.43.0",
|
45 |
+
"use_cache": true,
|
46 |
+
"vocab_size": 102400
|
47 |
+
}
|
configuration_exaone.py
ADDED
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# coding=utf-8
|
2 |
+
# Copyright 2021 The LG AI Research EXAONE Lab. All rights reserved.
|
3 |
+
#
|
4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5 |
+
# you may not use this file except in compliance with the License.
|
6 |
+
# You may obtain a copy of the License at
|
7 |
+
#
|
8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9 |
+
#
|
10 |
+
# Unless required by applicable law or agreed to in writing, software
|
11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13 |
+
# See the License for the specific language governing permissions and
|
14 |
+
# limitations under the License.
|
15 |
+
"""EXAONE model configuration"""
|
16 |
+
|
17 |
+
from transformers.configuration_utils import PretrainedConfig
|
18 |
+
from transformers.utils import logging
|
19 |
+
|
20 |
+
|
21 |
+
logger = logging.get_logger(__name__)
|
22 |
+
|
23 |
+
EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
|
24 |
+
|
25 |
+
|
26 |
+
class ExaoneConfig(PretrainedConfig):
|
27 |
+
r"""
|
28 |
+
This is the configuration class to store the configuration of a [`ExaoneModel`]. It is used to
|
29 |
+
instantiate a EXAONE model according to the specified arguments, defining the model architecture. Instantiating a
|
30 |
+
configuration with the defaults will yield a similar configuration to that of the EXAONE-3.0-7.8B-Instruct [LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct)
|
31 |
+
|
32 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
|
33 |
+
outputs. Read the documentation from [`PretrainedConfig`] for more information.
|
34 |
+
|
35 |
+
|
36 |
+
Args:
|
37 |
+
vocab_size (`int`, *optional*, defaults to 102400):
|
38 |
+
Vocabulary size of the EXAONE model. Defines the number of different tokens that can be represented by the
|
39 |
+
`inputs_ids` passed when calling [`ExaoneModel`]. Vocabulary size of the model.
|
40 |
+
Defines the different tokens that can be represented by the `inputs_ids` passed to the forward method of
|
41 |
+
[`ExaoneModel`].
|
42 |
+
max_position_embeddings (`int`, *optional*, defaults to 2048):
|
43 |
+
The maximum sequence length that this model might ever be used with. Typically set this to something large
|
44 |
+
just in case (e.g., 512 or 1024 or 2048).
|
45 |
+
hidden_size (`int`, *optional*, defaults to 2048):
|
46 |
+
Dimensionality of the encoder layers and the pooler layer.
|
47 |
+
num_layers (`int`, *optional*, defaults to 32):
|
48 |
+
Number of hidden layers in the Transformer encoder.
|
49 |
+
num_attention_heads (`int`, *optional*, defaults to 32):
|
50 |
+
Number of attention heads for each attention layer in the Transformer decoder.
|
51 |
+
num_key_value_heads (`int`, *optional*):
|
52 |
+
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
53 |
+
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
54 |
+
`num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
55 |
+
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
56 |
+
by meanpooling all the original heads within that group. For more details checkout [this
|
57 |
+
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
|
58 |
+
`num_attention_heads`.
|
59 |
+
intermediate_size (`int`, *optional*, defaults to `hidden_size * 4`):
|
60 |
+
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
|
61 |
+
activation_function (`str` or `function`, *optional*, defaults to `"silu"`):
|
62 |
+
The non-linear activation function (function or string) in the decoder.
|
63 |
+
rope_theta (`float`, *optional*, defaults to 10000.0):
|
64 |
+
The base period of the RoPE embeddings.
|
65 |
+
rope_scaling (`Dict`, *optional*):
|
66 |
+
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
|
67 |
+
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
|
68 |
+
accordingly.
|
69 |
+
Expected contents:
|
70 |
+
`rope_type` (`str`):
|
71 |
+
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
|
72 |
+
'llama3'], with 'default' being the original RoPE implementation.
|
73 |
+
`factor` (`float`, *optional*):
|
74 |
+
Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
|
75 |
+
most scaling types, a `factor` of x will enable the model to handle sequences of length x *
|
76 |
+
original maximum pre-trained length.
|
77 |
+
`original_max_position_embeddings` (`int`, *optional*):
|
78 |
+
Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
|
79 |
+
pretraining.
|
80 |
+
`attention_factor` (`float`, *optional*):
|
81 |
+
Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
|
82 |
+
computation. If unspecified, it defaults to value recommended by the implementation, using the
|
83 |
+
`factor` field to infer the suggested value.
|
84 |
+
`beta_fast` (`float`, *optional*):
|
85 |
+
Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
|
86 |
+
ramp function. If unspecified, it defaults to 32.
|
87 |
+
`beta_slow` (`float`, *optional*):
|
88 |
+
Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
|
89 |
+
ramp function. If unspecified, it defaults to 1.
|
90 |
+
`short_factor` (`List[float]`, *optional*):
|
91 |
+
Only used with 'longrope'. The scaling factor to be applied to short contexts (<
|
92 |
+
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
93 |
+
size divided by the number of attention heads divided by 2
|
94 |
+
`long_factor` (`List[float]`, *optional*):
|
95 |
+
Only used with 'longrope'. The scaling factor to be applied to long contexts (<
|
96 |
+
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
97 |
+
size divided by the number of attention heads divided by 2
|
98 |
+
`low_freq_factor` (`float`, *optional*):
|
99 |
+
Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
|
100 |
+
`high_freq_factor` (`float`, *optional*):
|
101 |
+
Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
|
102 |
+
embed_dropout (`float`, *optional*, defaults to 0.0):
|
103 |
+
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
|
104 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
105 |
+
The dropout ratio for the attention probabilities.
|
106 |
+
layer_norm_epsilon (`float`, *optional*, defaults to 1e-05):
|
107 |
+
The epsilon used by the layer normalization layers.
|
108 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
109 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
110 |
+
use_cache (`bool`, *optional*, defaults to `True`):
|
111 |
+
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
112 |
+
relevant if ``config.is_decoder=True``.
|
113 |
+
bos_token_id (`int`, *optional*, defaults to 0):
|
114 |
+
Beginning of stream token id.
|
115 |
+
eos_token_id (`int`, *optional*, defaults to 2):
|
116 |
+
End of stream token id.
|
117 |
+
|
118 |
+
Example:
|
119 |
+
|
120 |
+
```python
|
121 |
+
>>> from transformers import EXAONEModel, ExaoneConfig
|
122 |
+
|
123 |
+
>>> # Initializing a EXAONE configuration
|
124 |
+
>>> configuration = ExaoneConfig()
|
125 |
+
|
126 |
+
>>> # Initializing a model from configuration
|
127 |
+
>>> model = EXAONEModel(configuration)
|
128 |
+
|
129 |
+
>>> # Accessing the model configuration
|
130 |
+
>>> configuration = model.config
|
131 |
+
```"""
|
132 |
+
|
133 |
+
model_type = "exaone"
|
134 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
135 |
+
attribute_map = {"num_hidden_layers": "num_layers"}
|
136 |
+
|
137 |
+
def __init__(
|
138 |
+
self,
|
139 |
+
vocab_size=102400,
|
140 |
+
max_position_embeddings=2048,
|
141 |
+
hidden_size=2048,
|
142 |
+
num_layers=32,
|
143 |
+
num_attention_heads=32,
|
144 |
+
num_key_value_heads=None,
|
145 |
+
intermediate_size=None,
|
146 |
+
activation_function="silu",
|
147 |
+
rope_theta=10000.0,
|
148 |
+
rope_scaling=None,
|
149 |
+
embed_dropout=0.0,
|
150 |
+
attention_dropout=0.0,
|
151 |
+
layer_norm_epsilon=1e-5,
|
152 |
+
initializer_range=0.02,
|
153 |
+
use_cache=True,
|
154 |
+
bos_token_id=0,
|
155 |
+
eos_token_id=2,
|
156 |
+
**kwargs,
|
157 |
+
):
|
158 |
+
self.vocab_size = vocab_size
|
159 |
+
self.max_position_embeddings = max_position_embeddings
|
160 |
+
self.hidden_size = hidden_size
|
161 |
+
self.num_layers = num_layers
|
162 |
+
self.num_attention_heads = num_attention_heads
|
163 |
+
self.num_layers = num_layers
|
164 |
+
if num_key_value_heads is None:
|
165 |
+
num_key_value_heads = num_attention_heads
|
166 |
+
self.num_key_value_heads = num_key_value_heads
|
167 |
+
if intermediate_size:
|
168 |
+
self.intermediate_size = intermediate_size
|
169 |
+
else:
|
170 |
+
self.intermediate_size = hidden_size * 4
|
171 |
+
self.activation_function = activation_function
|
172 |
+
self.embed_dropout = embed_dropout
|
173 |
+
self.attention_dropout = attention_dropout
|
174 |
+
self.layer_norm_epsilon = layer_norm_epsilon
|
175 |
+
self.initializer_range = initializer_range
|
176 |
+
self.use_cache = use_cache
|
177 |
+
self.rope_theta = rope_theta
|
178 |
+
self.rope_scaling = rope_scaling
|
179 |
+
|
180 |
+
self.bos_token_id = bos_token_id
|
181 |
+
self.eos_token_id = eos_token_id
|
182 |
+
|
183 |
+
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:76a2cf2dc93b9e93f6c36939e0ca0e24db28e178a835ed6e8bc3d540011c527c
|
3 |
+
size 2555885931
|
model.safetensors.index.json
ADDED
@@ -0,0 +1,701 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"metadata": {
|
3 |
+
"total_size": 2555806720
|
4 |
+
},
|
5 |
+
"weight_map": {
|
6 |
+
"transformer.h.0.attn.attention.k_proj.biases": "model.safetensors",
|
7 |
+
"transformer.h.0.attn.attention.k_proj.scales": "model.safetensors",
|
8 |
+
"transformer.h.0.attn.attention.k_proj.weight": "model.safetensors",
|
9 |
+
"transformer.h.0.attn.attention.out_proj.biases": "model.safetensors",
|
10 |
+
"transformer.h.0.attn.attention.out_proj.scales": "model.safetensors",
|
11 |
+
"transformer.h.0.attn.attention.out_proj.weight": "model.safetensors",
|
12 |
+
"transformer.h.0.attn.attention.q_proj.biases": "model.safetensors",
|
13 |
+
"transformer.h.0.attn.attention.q_proj.scales": "model.safetensors",
|
14 |
+
"transformer.h.0.attn.attention.q_proj.weight": "model.safetensors",
|
15 |
+
"transformer.h.0.attn.attention.v_proj.biases": "model.safetensors",
|
16 |
+
"transformer.h.0.attn.attention.v_proj.scales": "model.safetensors",
|
17 |
+
"transformer.h.0.attn.attention.v_proj.weight": "model.safetensors",
|
18 |
+
"transformer.h.0.ln_1.weight": "model.safetensors",
|
19 |
+
"transformer.h.0.ln_2.weight": "model.safetensors",
|
20 |
+
"transformer.h.0.mlp.c_fc_0.biases": "model.safetensors",
|
21 |
+
"transformer.h.0.mlp.c_fc_0.scales": "model.safetensors",
|
22 |
+
"transformer.h.0.mlp.c_fc_0.weight": "model.safetensors",
|
23 |
+
"transformer.h.0.mlp.c_fc_1.biases": "model.safetensors",
|
24 |
+
"transformer.h.0.mlp.c_fc_1.scales": "model.safetensors",
|
25 |
+
"transformer.h.0.mlp.c_fc_1.weight": "model.safetensors",
|
26 |
+
"transformer.h.0.mlp.c_proj.biases": "model.safetensors",
|
27 |
+
"transformer.h.0.mlp.c_proj.scales": "model.safetensors",
|
28 |
+
"transformer.h.0.mlp.c_proj.weight": "model.safetensors",
|
29 |
+
"transformer.h.1.attn.attention.k_proj.biases": "model.safetensors",
|
30 |
+
"transformer.h.1.attn.attention.k_proj.scales": "model.safetensors",
|
31 |
+
"transformer.h.1.attn.attention.k_proj.weight": "model.safetensors",
|
32 |
+
"transformer.h.1.attn.attention.out_proj.biases": "model.safetensors",
|
33 |
+
"transformer.h.1.attn.attention.out_proj.scales": "model.safetensors",
|
34 |
+
"transformer.h.1.attn.attention.out_proj.weight": "model.safetensors",
|
35 |
+
"transformer.h.1.attn.attention.q_proj.biases": "model.safetensors",
|
36 |
+
"transformer.h.1.attn.attention.q_proj.scales": "model.safetensors",
|
37 |
+
"transformer.h.1.attn.attention.q_proj.weight": "model.safetensors",
|
38 |
+
"transformer.h.1.attn.attention.v_proj.biases": "model.safetensors",
|
39 |
+
"transformer.h.1.attn.attention.v_proj.scales": "model.safetensors",
|
40 |
+
"transformer.h.1.attn.attention.v_proj.weight": "model.safetensors",
|
41 |
+
"transformer.h.1.ln_1.weight": "model.safetensors",
|
42 |
+
"transformer.h.1.ln_2.weight": "model.safetensors",
|
43 |
+
"transformer.h.1.mlp.c_fc_0.biases": "model.safetensors",
|
44 |
+
"transformer.h.1.mlp.c_fc_0.scales": "model.safetensors",
|
45 |
+
"transformer.h.1.mlp.c_fc_0.weight": "model.safetensors",
|
46 |
+
"transformer.h.1.mlp.c_fc_1.biases": "model.safetensors",
|
47 |
+
"transformer.h.1.mlp.c_fc_1.scales": "model.safetensors",
|
48 |
+
"transformer.h.1.mlp.c_fc_1.weight": "model.safetensors",
|
49 |
+
"transformer.h.1.mlp.c_proj.biases": "model.safetensors",
|
50 |
+
"transformer.h.1.mlp.c_proj.scales": "model.safetensors",
|
51 |
+
"transformer.h.1.mlp.c_proj.weight": "model.safetensors",
|
52 |
+
"transformer.h.10.attn.attention.k_proj.biases": "model.safetensors",
|
53 |
+
"transformer.h.10.attn.attention.k_proj.scales": "model.safetensors",
|
54 |
+
"transformer.h.10.attn.attention.k_proj.weight": "model.safetensors",
|
55 |
+
"transformer.h.10.attn.attention.out_proj.biases": "model.safetensors",
|
56 |
+
"transformer.h.10.attn.attention.out_proj.scales": "model.safetensors",
|
57 |
+
"transformer.h.10.attn.attention.out_proj.weight": "model.safetensors",
|
58 |
+
"transformer.h.10.attn.attention.q_proj.biases": "model.safetensors",
|
59 |
+
"transformer.h.10.attn.attention.q_proj.scales": "model.safetensors",
|
60 |
+
"transformer.h.10.attn.attention.q_proj.weight": "model.safetensors",
|
61 |
+
"transformer.h.10.attn.attention.v_proj.biases": "model.safetensors",
|
62 |
+
"transformer.h.10.attn.attention.v_proj.scales": "model.safetensors",
|
63 |
+
"transformer.h.10.attn.attention.v_proj.weight": "model.safetensors",
|
64 |
+
"transformer.h.10.ln_1.weight": "model.safetensors",
|
65 |
+
"transformer.h.10.ln_2.weight": "model.safetensors",
|
66 |
+
"transformer.h.10.mlp.c_fc_0.biases": "model.safetensors",
|
67 |
+
"transformer.h.10.mlp.c_fc_0.scales": "model.safetensors",
|
68 |
+
"transformer.h.10.mlp.c_fc_0.weight": "model.safetensors",
|
69 |
+
"transformer.h.10.mlp.c_fc_1.biases": "model.safetensors",
|
70 |
+
"transformer.h.10.mlp.c_fc_1.scales": "model.safetensors",
|
71 |
+
"transformer.h.10.mlp.c_fc_1.weight": "model.safetensors",
|
72 |
+
"transformer.h.10.mlp.c_proj.biases": "model.safetensors",
|
73 |
+
"transformer.h.10.mlp.c_proj.scales": "model.safetensors",
|
74 |
+
"transformer.h.10.mlp.c_proj.weight": "model.safetensors",
|
75 |
+
"transformer.h.11.attn.attention.k_proj.biases": "model.safetensors",
|
76 |
+
"transformer.h.11.attn.attention.k_proj.scales": "model.safetensors",
|
77 |
+
"transformer.h.11.attn.attention.k_proj.weight": "model.safetensors",
|
78 |
+
"transformer.h.11.attn.attention.out_proj.biases": "model.safetensors",
|
79 |
+
"transformer.h.11.attn.attention.out_proj.scales": "model.safetensors",
|
80 |
+
"transformer.h.11.attn.attention.out_proj.weight": "model.safetensors",
|
81 |
+
"transformer.h.11.attn.attention.q_proj.biases": "model.safetensors",
|
82 |
+
"transformer.h.11.attn.attention.q_proj.scales": "model.safetensors",
|
83 |
+
"transformer.h.11.attn.attention.q_proj.weight": "model.safetensors",
|
84 |
+
"transformer.h.11.attn.attention.v_proj.biases": "model.safetensors",
|
85 |
+
"transformer.h.11.attn.attention.v_proj.scales": "model.safetensors",
|
86 |
+
"transformer.h.11.attn.attention.v_proj.weight": "model.safetensors",
|
87 |
+
"transformer.h.11.ln_1.weight": "model.safetensors",
|
88 |
+
"transformer.h.11.ln_2.weight": "model.safetensors",
|
89 |
+
"transformer.h.11.mlp.c_fc_0.biases": "model.safetensors",
|
90 |
+
"transformer.h.11.mlp.c_fc_0.scales": "model.safetensors",
|
91 |
+
"transformer.h.11.mlp.c_fc_0.weight": "model.safetensors",
|
92 |
+
"transformer.h.11.mlp.c_fc_1.biases": "model.safetensors",
|
93 |
+
"transformer.h.11.mlp.c_fc_1.scales": "model.safetensors",
|
94 |
+
"transformer.h.11.mlp.c_fc_1.weight": "model.safetensors",
|
95 |
+
"transformer.h.11.mlp.c_proj.biases": "model.safetensors",
|
96 |
+
"transformer.h.11.mlp.c_proj.scales": "model.safetensors",
|
97 |
+
"transformer.h.11.mlp.c_proj.weight": "model.safetensors",
|
98 |
+
"transformer.h.12.attn.attention.k_proj.biases": "model.safetensors",
|
99 |
+
"transformer.h.12.attn.attention.k_proj.scales": "model.safetensors",
|
100 |
+
"transformer.h.12.attn.attention.k_proj.weight": "model.safetensors",
|
101 |
+
"transformer.h.12.attn.attention.out_proj.biases": "model.safetensors",
|
102 |
+
"transformer.h.12.attn.attention.out_proj.scales": "model.safetensors",
|
103 |
+
"transformer.h.12.attn.attention.out_proj.weight": "model.safetensors",
|
104 |
+
"transformer.h.12.attn.attention.q_proj.biases": "model.safetensors",
|
105 |
+
"transformer.h.12.attn.attention.q_proj.scales": "model.safetensors",
|
106 |
+
"transformer.h.12.attn.attention.q_proj.weight": "model.safetensors",
|
107 |
+
"transformer.h.12.attn.attention.v_proj.biases": "model.safetensors",
|
108 |
+
"transformer.h.12.attn.attention.v_proj.scales": "model.safetensors",
|
109 |
+
"transformer.h.12.attn.attention.v_proj.weight": "model.safetensors",
|
110 |
+
"transformer.h.12.ln_1.weight": "model.safetensors",
|
111 |
+
"transformer.h.12.ln_2.weight": "model.safetensors",
|
112 |
+
"transformer.h.12.mlp.c_fc_0.biases": "model.safetensors",
|
113 |
+
"transformer.h.12.mlp.c_fc_0.scales": "model.safetensors",
|
114 |
+
"transformer.h.12.mlp.c_fc_0.weight": "model.safetensors",
|
115 |
+
"transformer.h.12.mlp.c_fc_1.biases": "model.safetensors",
|
116 |
+
"transformer.h.12.mlp.c_fc_1.scales": "model.safetensors",
|
117 |
+
"transformer.h.12.mlp.c_fc_1.weight": "model.safetensors",
|
118 |
+
"transformer.h.12.mlp.c_proj.biases": "model.safetensors",
|
119 |
+
"transformer.h.12.mlp.c_proj.scales": "model.safetensors",
|
120 |
+
"transformer.h.12.mlp.c_proj.weight": "model.safetensors",
|
121 |
+
"transformer.h.13.attn.attention.k_proj.biases": "model.safetensors",
|
122 |
+
"transformer.h.13.attn.attention.k_proj.scales": "model.safetensors",
|
123 |
+
"transformer.h.13.attn.attention.k_proj.weight": "model.safetensors",
|
124 |
+
"transformer.h.13.attn.attention.out_proj.biases": "model.safetensors",
|
125 |
+
"transformer.h.13.attn.attention.out_proj.scales": "model.safetensors",
|
126 |
+
"transformer.h.13.attn.attention.out_proj.weight": "model.safetensors",
|
127 |
+
"transformer.h.13.attn.attention.q_proj.biases": "model.safetensors",
|
128 |
+
"transformer.h.13.attn.attention.q_proj.scales": "model.safetensors",
|
129 |
+
"transformer.h.13.attn.attention.q_proj.weight": "model.safetensors",
|
130 |
+
"transformer.h.13.attn.attention.v_proj.biases": "model.safetensors",
|
131 |
+
"transformer.h.13.attn.attention.v_proj.scales": "model.safetensors",
|
132 |
+
"transformer.h.13.attn.attention.v_proj.weight": "model.safetensors",
|
133 |
+
"transformer.h.13.ln_1.weight": "model.safetensors",
|
134 |
+
"transformer.h.13.ln_2.weight": "model.safetensors",
|
135 |
+
"transformer.h.13.mlp.c_fc_0.biases": "model.safetensors",
|
136 |
+
"transformer.h.13.mlp.c_fc_0.scales": "model.safetensors",
|
137 |
+
"transformer.h.13.mlp.c_fc_0.weight": "model.safetensors",
|
138 |
+
"transformer.h.13.mlp.c_fc_1.biases": "model.safetensors",
|
139 |
+
"transformer.h.13.mlp.c_fc_1.scales": "model.safetensors",
|
140 |
+
"transformer.h.13.mlp.c_fc_1.weight": "model.safetensors",
|
141 |
+
"transformer.h.13.mlp.c_proj.biases": "model.safetensors",
|
142 |
+
"transformer.h.13.mlp.c_proj.scales": "model.safetensors",
|
143 |
+
"transformer.h.13.mlp.c_proj.weight": "model.safetensors",
|
144 |
+
"transformer.h.14.attn.attention.k_proj.biases": "model.safetensors",
|
145 |
+
"transformer.h.14.attn.attention.k_proj.scales": "model.safetensors",
|
146 |
+
"transformer.h.14.attn.attention.k_proj.weight": "model.safetensors",
|
147 |
+
"transformer.h.14.attn.attention.out_proj.biases": "model.safetensors",
|
148 |
+
"transformer.h.14.attn.attention.out_proj.scales": "model.safetensors",
|
149 |
+
"transformer.h.14.attn.attention.out_proj.weight": "model.safetensors",
|
150 |
+
"transformer.h.14.attn.attention.q_proj.biases": "model.safetensors",
|
151 |
+
"transformer.h.14.attn.attention.q_proj.scales": "model.safetensors",
|
152 |
+
"transformer.h.14.attn.attention.q_proj.weight": "model.safetensors",
|
153 |
+
"transformer.h.14.attn.attention.v_proj.biases": "model.safetensors",
|
154 |
+
"transformer.h.14.attn.attention.v_proj.scales": "model.safetensors",
|
155 |
+
"transformer.h.14.attn.attention.v_proj.weight": "model.safetensors",
|
156 |
+
"transformer.h.14.ln_1.weight": "model.safetensors",
|
157 |
+
"transformer.h.14.ln_2.weight": "model.safetensors",
|
158 |
+
"transformer.h.14.mlp.c_fc_0.biases": "model.safetensors",
|
159 |
+
"transformer.h.14.mlp.c_fc_0.scales": "model.safetensors",
|
160 |
+
"transformer.h.14.mlp.c_fc_0.weight": "model.safetensors",
|
161 |
+
"transformer.h.14.mlp.c_fc_1.biases": "model.safetensors",
|
162 |
+
"transformer.h.14.mlp.c_fc_1.scales": "model.safetensors",
|
163 |
+
"transformer.h.14.mlp.c_fc_1.weight": "model.safetensors",
|
164 |
+
"transformer.h.14.mlp.c_proj.biases": "model.safetensors",
|
165 |
+
"transformer.h.14.mlp.c_proj.scales": "model.safetensors",
|
166 |
+
"transformer.h.14.mlp.c_proj.weight": "model.safetensors",
|
167 |
+
"transformer.h.15.attn.attention.k_proj.biases": "model.safetensors",
|
168 |
+
"transformer.h.15.attn.attention.k_proj.scales": "model.safetensors",
|
169 |
+
"transformer.h.15.attn.attention.k_proj.weight": "model.safetensors",
|
170 |
+
"transformer.h.15.attn.attention.out_proj.biases": "model.safetensors",
|
171 |
+
"transformer.h.15.attn.attention.out_proj.scales": "model.safetensors",
|
172 |
+
"transformer.h.15.attn.attention.out_proj.weight": "model.safetensors",
|
173 |
+
"transformer.h.15.attn.attention.q_proj.biases": "model.safetensors",
|
174 |
+
"transformer.h.15.attn.attention.q_proj.scales": "model.safetensors",
|
175 |
+
"transformer.h.15.attn.attention.q_proj.weight": "model.safetensors",
|
176 |
+
"transformer.h.15.attn.attention.v_proj.biases": "model.safetensors",
|
177 |
+
"transformer.h.15.attn.attention.v_proj.scales": "model.safetensors",
|
178 |
+
"transformer.h.15.attn.attention.v_proj.weight": "model.safetensors",
|
179 |
+
"transformer.h.15.ln_1.weight": "model.safetensors",
|
180 |
+
"transformer.h.15.ln_2.weight": "model.safetensors",
|
181 |
+
"transformer.h.15.mlp.c_fc_0.biases": "model.safetensors",
|
182 |
+
"transformer.h.15.mlp.c_fc_0.scales": "model.safetensors",
|
183 |
+
"transformer.h.15.mlp.c_fc_0.weight": "model.safetensors",
|
184 |
+
"transformer.h.15.mlp.c_fc_1.biases": "model.safetensors",
|
185 |
+
"transformer.h.15.mlp.c_fc_1.scales": "model.safetensors",
|
186 |
+
"transformer.h.15.mlp.c_fc_1.weight": "model.safetensors",
|
187 |
+
"transformer.h.15.mlp.c_proj.biases": "model.safetensors",
|
188 |
+
"transformer.h.15.mlp.c_proj.scales": "model.safetensors",
|
189 |
+
"transformer.h.15.mlp.c_proj.weight": "model.safetensors",
|
190 |
+
"transformer.h.16.attn.attention.k_proj.biases": "model.safetensors",
|
191 |
+
"transformer.h.16.attn.attention.k_proj.scales": "model.safetensors",
|
192 |
+
"transformer.h.16.attn.attention.k_proj.weight": "model.safetensors",
|
193 |
+
"transformer.h.16.attn.attention.out_proj.biases": "model.safetensors",
|
194 |
+
"transformer.h.16.attn.attention.out_proj.scales": "model.safetensors",
|
195 |
+
"transformer.h.16.attn.attention.out_proj.weight": "model.safetensors",
|
196 |
+
"transformer.h.16.attn.attention.q_proj.biases": "model.safetensors",
|
197 |
+
"transformer.h.16.attn.attention.q_proj.scales": "model.safetensors",
|
198 |
+
"transformer.h.16.attn.attention.q_proj.weight": "model.safetensors",
|
199 |
+
"transformer.h.16.attn.attention.v_proj.biases": "model.safetensors",
|
200 |
+
"transformer.h.16.attn.attention.v_proj.scales": "model.safetensors",
|
201 |
+
"transformer.h.16.attn.attention.v_proj.weight": "model.safetensors",
|
202 |
+
"transformer.h.16.ln_1.weight": "model.safetensors",
|
203 |
+
"transformer.h.16.ln_2.weight": "model.safetensors",
|
204 |
+
"transformer.h.16.mlp.c_fc_0.biases": "model.safetensors",
|
205 |
+
"transformer.h.16.mlp.c_fc_0.scales": "model.safetensors",
|
206 |
+
"transformer.h.16.mlp.c_fc_0.weight": "model.safetensors",
|
207 |
+
"transformer.h.16.mlp.c_fc_1.biases": "model.safetensors",
|
208 |
+
"transformer.h.16.mlp.c_fc_1.scales": "model.safetensors",
|
209 |
+
"transformer.h.16.mlp.c_fc_1.weight": "model.safetensors",
|
210 |
+
"transformer.h.16.mlp.c_proj.biases": "model.safetensors",
|
211 |
+
"transformer.h.16.mlp.c_proj.scales": "model.safetensors",
|
212 |
+
"transformer.h.16.mlp.c_proj.weight": "model.safetensors",
|
213 |
+
"transformer.h.17.attn.attention.k_proj.biases": "model.safetensors",
|
214 |
+
"transformer.h.17.attn.attention.k_proj.scales": "model.safetensors",
|
215 |
+
"transformer.h.17.attn.attention.k_proj.weight": "model.safetensors",
|
216 |
+
"transformer.h.17.attn.attention.out_proj.biases": "model.safetensors",
|
217 |
+
"transformer.h.17.attn.attention.out_proj.scales": "model.safetensors",
|
218 |
+
"transformer.h.17.attn.attention.out_proj.weight": "model.safetensors",
|
219 |
+
"transformer.h.17.attn.attention.q_proj.biases": "model.safetensors",
|
220 |
+
"transformer.h.17.attn.attention.q_proj.scales": "model.safetensors",
|
221 |
+
"transformer.h.17.attn.attention.q_proj.weight": "model.safetensors",
|
222 |
+
"transformer.h.17.attn.attention.v_proj.biases": "model.safetensors",
|
223 |
+
"transformer.h.17.attn.attention.v_proj.scales": "model.safetensors",
|
224 |
+
"transformer.h.17.attn.attention.v_proj.weight": "model.safetensors",
|
225 |
+
"transformer.h.17.ln_1.weight": "model.safetensors",
|
226 |
+
"transformer.h.17.ln_2.weight": "model.safetensors",
|
227 |
+
"transformer.h.17.mlp.c_fc_0.biases": "model.safetensors",
|
228 |
+
"transformer.h.17.mlp.c_fc_0.scales": "model.safetensors",
|
229 |
+
"transformer.h.17.mlp.c_fc_0.weight": "model.safetensors",
|
230 |
+
"transformer.h.17.mlp.c_fc_1.biases": "model.safetensors",
|
231 |
+
"transformer.h.17.mlp.c_fc_1.scales": "model.safetensors",
|
232 |
+
"transformer.h.17.mlp.c_fc_1.weight": "model.safetensors",
|
233 |
+
"transformer.h.17.mlp.c_proj.biases": "model.safetensors",
|
234 |
+
"transformer.h.17.mlp.c_proj.scales": "model.safetensors",
|
235 |
+
"transformer.h.17.mlp.c_proj.weight": "model.safetensors",
|
236 |
+
"transformer.h.18.attn.attention.k_proj.biases": "model.safetensors",
|
237 |
+
"transformer.h.18.attn.attention.k_proj.scales": "model.safetensors",
|
238 |
+
"transformer.h.18.attn.attention.k_proj.weight": "model.safetensors",
|
239 |
+
"transformer.h.18.attn.attention.out_proj.biases": "model.safetensors",
|
240 |
+
"transformer.h.18.attn.attention.out_proj.scales": "model.safetensors",
|
241 |
+
"transformer.h.18.attn.attention.out_proj.weight": "model.safetensors",
|
242 |
+
"transformer.h.18.attn.attention.q_proj.biases": "model.safetensors",
|
243 |
+
"transformer.h.18.attn.attention.q_proj.scales": "model.safetensors",
|
244 |
+
"transformer.h.18.attn.attention.q_proj.weight": "model.safetensors",
|
245 |
+
"transformer.h.18.attn.attention.v_proj.biases": "model.safetensors",
|
246 |
+
"transformer.h.18.attn.attention.v_proj.scales": "model.safetensors",
|
247 |
+
"transformer.h.18.attn.attention.v_proj.weight": "model.safetensors",
|
248 |
+
"transformer.h.18.ln_1.weight": "model.safetensors",
|
249 |
+
"transformer.h.18.ln_2.weight": "model.safetensors",
|
250 |
+
"transformer.h.18.mlp.c_fc_0.biases": "model.safetensors",
|
251 |
+
"transformer.h.18.mlp.c_fc_0.scales": "model.safetensors",
|
252 |
+
"transformer.h.18.mlp.c_fc_0.weight": "model.safetensors",
|
253 |
+
"transformer.h.18.mlp.c_fc_1.biases": "model.safetensors",
|
254 |
+
"transformer.h.18.mlp.c_fc_1.scales": "model.safetensors",
|
255 |
+
"transformer.h.18.mlp.c_fc_1.weight": "model.safetensors",
|
256 |
+
"transformer.h.18.mlp.c_proj.biases": "model.safetensors",
|
257 |
+
"transformer.h.18.mlp.c_proj.scales": "model.safetensors",
|
258 |
+
"transformer.h.18.mlp.c_proj.weight": "model.safetensors",
|
259 |
+
"transformer.h.19.attn.attention.k_proj.biases": "model.safetensors",
|
260 |
+
"transformer.h.19.attn.attention.k_proj.scales": "model.safetensors",
|
261 |
+
"transformer.h.19.attn.attention.k_proj.weight": "model.safetensors",
|
262 |
+
"transformer.h.19.attn.attention.out_proj.biases": "model.safetensors",
|
263 |
+
"transformer.h.19.attn.attention.out_proj.scales": "model.safetensors",
|
264 |
+
"transformer.h.19.attn.attention.out_proj.weight": "model.safetensors",
|
265 |
+
"transformer.h.19.attn.attention.q_proj.biases": "model.safetensors",
|
266 |
+
"transformer.h.19.attn.attention.q_proj.scales": "model.safetensors",
|
267 |
+
"transformer.h.19.attn.attention.q_proj.weight": "model.safetensors",
|
268 |
+
"transformer.h.19.attn.attention.v_proj.biases": "model.safetensors",
|
269 |
+
"transformer.h.19.attn.attention.v_proj.scales": "model.safetensors",
|
270 |
+
"transformer.h.19.attn.attention.v_proj.weight": "model.safetensors",
|
271 |
+
"transformer.h.19.ln_1.weight": "model.safetensors",
|
272 |
+
"transformer.h.19.ln_2.weight": "model.safetensors",
|
273 |
+
"transformer.h.19.mlp.c_fc_0.biases": "model.safetensors",
|
274 |
+
"transformer.h.19.mlp.c_fc_0.scales": "model.safetensors",
|
275 |
+
"transformer.h.19.mlp.c_fc_0.weight": "model.safetensors",
|
276 |
+
"transformer.h.19.mlp.c_fc_1.biases": "model.safetensors",
|
277 |
+
"transformer.h.19.mlp.c_fc_1.scales": "model.safetensors",
|
278 |
+
"transformer.h.19.mlp.c_fc_1.weight": "model.safetensors",
|
279 |
+
"transformer.h.19.mlp.c_proj.biases": "model.safetensors",
|
280 |
+
"transformer.h.19.mlp.c_proj.scales": "model.safetensors",
|
281 |
+
"transformer.h.19.mlp.c_proj.weight": "model.safetensors",
|
282 |
+
"transformer.h.2.attn.attention.k_proj.biases": "model.safetensors",
|
283 |
+
"transformer.h.2.attn.attention.k_proj.scales": "model.safetensors",
|
284 |
+
"transformer.h.2.attn.attention.k_proj.weight": "model.safetensors",
|
285 |
+
"transformer.h.2.attn.attention.out_proj.biases": "model.safetensors",
|
286 |
+
"transformer.h.2.attn.attention.out_proj.scales": "model.safetensors",
|
287 |
+
"transformer.h.2.attn.attention.out_proj.weight": "model.safetensors",
|
288 |
+
"transformer.h.2.attn.attention.q_proj.biases": "model.safetensors",
|
289 |
+
"transformer.h.2.attn.attention.q_proj.scales": "model.safetensors",
|
290 |
+
"transformer.h.2.attn.attention.q_proj.weight": "model.safetensors",
|
291 |
+
"transformer.h.2.attn.attention.v_proj.biases": "model.safetensors",
|
292 |
+
"transformer.h.2.attn.attention.v_proj.scales": "model.safetensors",
|
293 |
+
"transformer.h.2.attn.attention.v_proj.weight": "model.safetensors",
|
294 |
+
"transformer.h.2.ln_1.weight": "model.safetensors",
|
295 |
+
"transformer.h.2.ln_2.weight": "model.safetensors",
|
296 |
+
"transformer.h.2.mlp.c_fc_0.biases": "model.safetensors",
|
297 |
+
"transformer.h.2.mlp.c_fc_0.scales": "model.safetensors",
|
298 |
+
"transformer.h.2.mlp.c_fc_0.weight": "model.safetensors",
|
299 |
+
"transformer.h.2.mlp.c_fc_1.biases": "model.safetensors",
|
300 |
+
"transformer.h.2.mlp.c_fc_1.scales": "model.safetensors",
|
301 |
+
"transformer.h.2.mlp.c_fc_1.weight": "model.safetensors",
|
302 |
+
"transformer.h.2.mlp.c_proj.biases": "model.safetensors",
|
303 |
+
"transformer.h.2.mlp.c_proj.scales": "model.safetensors",
|
304 |
+
"transformer.h.2.mlp.c_proj.weight": "model.safetensors",
|
305 |
+
"transformer.h.20.attn.attention.k_proj.biases": "model.safetensors",
|
306 |
+
"transformer.h.20.attn.attention.k_proj.scales": "model.safetensors",
|
307 |
+
"transformer.h.20.attn.attention.k_proj.weight": "model.safetensors",
|
308 |
+
"transformer.h.20.attn.attention.out_proj.biases": "model.safetensors",
|
309 |
+
"transformer.h.20.attn.attention.out_proj.scales": "model.safetensors",
|
310 |
+
"transformer.h.20.attn.attention.out_proj.weight": "model.safetensors",
|
311 |
+
"transformer.h.20.attn.attention.q_proj.biases": "model.safetensors",
|
312 |
+
"transformer.h.20.attn.attention.q_proj.scales": "model.safetensors",
|
313 |
+
"transformer.h.20.attn.attention.q_proj.weight": "model.safetensors",
|
314 |
+
"transformer.h.20.attn.attention.v_proj.biases": "model.safetensors",
|
315 |
+
"transformer.h.20.attn.attention.v_proj.scales": "model.safetensors",
|
316 |
+
"transformer.h.20.attn.attention.v_proj.weight": "model.safetensors",
|
317 |
+
"transformer.h.20.ln_1.weight": "model.safetensors",
|
318 |
+
"transformer.h.20.ln_2.weight": "model.safetensors",
|
319 |
+
"transformer.h.20.mlp.c_fc_0.biases": "model.safetensors",
|
320 |
+
"transformer.h.20.mlp.c_fc_0.scales": "model.safetensors",
|
321 |
+
"transformer.h.20.mlp.c_fc_0.weight": "model.safetensors",
|
322 |
+
"transformer.h.20.mlp.c_fc_1.biases": "model.safetensors",
|
323 |
+
"transformer.h.20.mlp.c_fc_1.scales": "model.safetensors",
|
324 |
+
"transformer.h.20.mlp.c_fc_1.weight": "model.safetensors",
|
325 |
+
"transformer.h.20.mlp.c_proj.biases": "model.safetensors",
|
326 |
+
"transformer.h.20.mlp.c_proj.scales": "model.safetensors",
|
327 |
+
"transformer.h.20.mlp.c_proj.weight": "model.safetensors",
|
328 |
+
"transformer.h.21.attn.attention.k_proj.biases": "model.safetensors",
|
329 |
+
"transformer.h.21.attn.attention.k_proj.scales": "model.safetensors",
|
330 |
+
"transformer.h.21.attn.attention.k_proj.weight": "model.safetensors",
|
331 |
+
"transformer.h.21.attn.attention.out_proj.biases": "model.safetensors",
|
332 |
+
"transformer.h.21.attn.attention.out_proj.scales": "model.safetensors",
|
333 |
+
"transformer.h.21.attn.attention.out_proj.weight": "model.safetensors",
|
334 |
+
"transformer.h.21.attn.attention.q_proj.biases": "model.safetensors",
|
335 |
+
"transformer.h.21.attn.attention.q_proj.scales": "model.safetensors",
|
336 |
+
"transformer.h.21.attn.attention.q_proj.weight": "model.safetensors",
|
337 |
+
"transformer.h.21.attn.attention.v_proj.biases": "model.safetensors",
|
338 |
+
"transformer.h.21.attn.attention.v_proj.scales": "model.safetensors",
|
339 |
+
"transformer.h.21.attn.attention.v_proj.weight": "model.safetensors",
|
340 |
+
"transformer.h.21.ln_1.weight": "model.safetensors",
|
341 |
+
"transformer.h.21.ln_2.weight": "model.safetensors",
|
342 |
+
"transformer.h.21.mlp.c_fc_0.biases": "model.safetensors",
|
343 |
+
"transformer.h.21.mlp.c_fc_0.scales": "model.safetensors",
|
344 |
+
"transformer.h.21.mlp.c_fc_0.weight": "model.safetensors",
|
345 |
+
"transformer.h.21.mlp.c_fc_1.biases": "model.safetensors",
|
346 |
+
"transformer.h.21.mlp.c_fc_1.scales": "model.safetensors",
|
347 |
+
"transformer.h.21.mlp.c_fc_1.weight": "model.safetensors",
|
348 |
+
"transformer.h.21.mlp.c_proj.biases": "model.safetensors",
|
349 |
+
"transformer.h.21.mlp.c_proj.scales": "model.safetensors",
|
350 |
+
"transformer.h.21.mlp.c_proj.weight": "model.safetensors",
|
351 |
+
"transformer.h.22.attn.attention.k_proj.biases": "model.safetensors",
|
352 |
+
"transformer.h.22.attn.attention.k_proj.scales": "model.safetensors",
|
353 |
+
"transformer.h.22.attn.attention.k_proj.weight": "model.safetensors",
|
354 |
+
"transformer.h.22.attn.attention.out_proj.biases": "model.safetensors",
|
355 |
+
"transformer.h.22.attn.attention.out_proj.scales": "model.safetensors",
|
356 |
+
"transformer.h.22.attn.attention.out_proj.weight": "model.safetensors",
|
357 |
+
"transformer.h.22.attn.attention.q_proj.biases": "model.safetensors",
|
358 |
+
"transformer.h.22.attn.attention.q_proj.scales": "model.safetensors",
|
359 |
+
"transformer.h.22.attn.attention.q_proj.weight": "model.safetensors",
|
360 |
+
"transformer.h.22.attn.attention.v_proj.biases": "model.safetensors",
|
361 |
+
"transformer.h.22.attn.attention.v_proj.scales": "model.safetensors",
|
362 |
+
"transformer.h.22.attn.attention.v_proj.weight": "model.safetensors",
|
363 |
+
"transformer.h.22.ln_1.weight": "model.safetensors",
|
364 |
+
"transformer.h.22.ln_2.weight": "model.safetensors",
|
365 |
+
"transformer.h.22.mlp.c_fc_0.biases": "model.safetensors",
|
366 |
+
"transformer.h.22.mlp.c_fc_0.scales": "model.safetensors",
|
367 |
+
"transformer.h.22.mlp.c_fc_0.weight": "model.safetensors",
|
368 |
+
"transformer.h.22.mlp.c_fc_1.biases": "model.safetensors",
|
369 |
+
"transformer.h.22.mlp.c_fc_1.scales": "model.safetensors",
|
370 |
+
"transformer.h.22.mlp.c_fc_1.weight": "model.safetensors",
|
371 |
+
"transformer.h.22.mlp.c_proj.biases": "model.safetensors",
|
372 |
+
"transformer.h.22.mlp.c_proj.scales": "model.safetensors",
|
373 |
+
"transformer.h.22.mlp.c_proj.weight": "model.safetensors",
|
374 |
+
"transformer.h.23.attn.attention.k_proj.biases": "model.safetensors",
|
375 |
+
"transformer.h.23.attn.attention.k_proj.scales": "model.safetensors",
|
376 |
+
"transformer.h.23.attn.attention.k_proj.weight": "model.safetensors",
|
377 |
+
"transformer.h.23.attn.attention.out_proj.biases": "model.safetensors",
|
378 |
+
"transformer.h.23.attn.attention.out_proj.scales": "model.safetensors",
|
379 |
+
"transformer.h.23.attn.attention.out_proj.weight": "model.safetensors",
|
380 |
+
"transformer.h.23.attn.attention.q_proj.biases": "model.safetensors",
|
381 |
+
"transformer.h.23.attn.attention.q_proj.scales": "model.safetensors",
|
382 |
+
"transformer.h.23.attn.attention.q_proj.weight": "model.safetensors",
|
383 |
+
"transformer.h.23.attn.attention.v_proj.biases": "model.safetensors",
|
384 |
+
"transformer.h.23.attn.attention.v_proj.scales": "model.safetensors",
|
385 |
+
"transformer.h.23.attn.attention.v_proj.weight": "model.safetensors",
|
386 |
+
"transformer.h.23.ln_1.weight": "model.safetensors",
|
387 |
+
"transformer.h.23.ln_2.weight": "model.safetensors",
|
388 |
+
"transformer.h.23.mlp.c_fc_0.biases": "model.safetensors",
|
389 |
+
"transformer.h.23.mlp.c_fc_0.scales": "model.safetensors",
|
390 |
+
"transformer.h.23.mlp.c_fc_0.weight": "model.safetensors",
|
391 |
+
"transformer.h.23.mlp.c_fc_1.biases": "model.safetensors",
|
392 |
+
"transformer.h.23.mlp.c_fc_1.scales": "model.safetensors",
|
393 |
+
"transformer.h.23.mlp.c_fc_1.weight": "model.safetensors",
|
394 |
+
"transformer.h.23.mlp.c_proj.biases": "model.safetensors",
|
395 |
+
"transformer.h.23.mlp.c_proj.scales": "model.safetensors",
|
396 |
+
"transformer.h.23.mlp.c_proj.weight": "model.safetensors",
|
397 |
+
"transformer.h.24.attn.attention.k_proj.biases": "model.safetensors",
|
398 |
+
"transformer.h.24.attn.attention.k_proj.scales": "model.safetensors",
|
399 |
+
"transformer.h.24.attn.attention.k_proj.weight": "model.safetensors",
|
400 |
+
"transformer.h.24.attn.attention.out_proj.biases": "model.safetensors",
|
401 |
+
"transformer.h.24.attn.attention.out_proj.scales": "model.safetensors",
|
402 |
+
"transformer.h.24.attn.attention.out_proj.weight": "model.safetensors",
|
403 |
+
"transformer.h.24.attn.attention.q_proj.biases": "model.safetensors",
|
404 |
+
"transformer.h.24.attn.attention.q_proj.scales": "model.safetensors",
|
405 |
+
"transformer.h.24.attn.attention.q_proj.weight": "model.safetensors",
|
406 |
+
"transformer.h.24.attn.attention.v_proj.biases": "model.safetensors",
|
407 |
+
"transformer.h.24.attn.attention.v_proj.scales": "model.safetensors",
|
408 |
+
"transformer.h.24.attn.attention.v_proj.weight": "model.safetensors",
|
409 |
+
"transformer.h.24.ln_1.weight": "model.safetensors",
|
410 |
+
"transformer.h.24.ln_2.weight": "model.safetensors",
|
411 |
+
"transformer.h.24.mlp.c_fc_0.biases": "model.safetensors",
|
412 |
+
"transformer.h.24.mlp.c_fc_0.scales": "model.safetensors",
|
413 |
+
"transformer.h.24.mlp.c_fc_0.weight": "model.safetensors",
|
414 |
+
"transformer.h.24.mlp.c_fc_1.biases": "model.safetensors",
|
415 |
+
"transformer.h.24.mlp.c_fc_1.scales": "model.safetensors",
|
416 |
+
"transformer.h.24.mlp.c_fc_1.weight": "model.safetensors",
|
417 |
+
"transformer.h.24.mlp.c_proj.biases": "model.safetensors",
|
418 |
+
"transformer.h.24.mlp.c_proj.scales": "model.safetensors",
|
419 |
+
"transformer.h.24.mlp.c_proj.weight": "model.safetensors",
|
420 |
+
"transformer.h.25.attn.attention.k_proj.biases": "model.safetensors",
|
421 |
+
"transformer.h.25.attn.attention.k_proj.scales": "model.safetensors",
|
422 |
+
"transformer.h.25.attn.attention.k_proj.weight": "model.safetensors",
|
423 |
+
"transformer.h.25.attn.attention.out_proj.biases": "model.safetensors",
|
424 |
+
"transformer.h.25.attn.attention.out_proj.scales": "model.safetensors",
|
425 |
+
"transformer.h.25.attn.attention.out_proj.weight": "model.safetensors",
|
426 |
+
"transformer.h.25.attn.attention.q_proj.biases": "model.safetensors",
|
427 |
+
"transformer.h.25.attn.attention.q_proj.scales": "model.safetensors",
|
428 |
+
"transformer.h.25.attn.attention.q_proj.weight": "model.safetensors",
|
429 |
+
"transformer.h.25.attn.attention.v_proj.biases": "model.safetensors",
|
430 |
+
"transformer.h.25.attn.attention.v_proj.scales": "model.safetensors",
|
431 |
+
"transformer.h.25.attn.attention.v_proj.weight": "model.safetensors",
|
432 |
+
"transformer.h.25.ln_1.weight": "model.safetensors",
|
433 |
+
"transformer.h.25.ln_2.weight": "model.safetensors",
|
434 |
+
"transformer.h.25.mlp.c_fc_0.biases": "model.safetensors",
|
435 |
+
"transformer.h.25.mlp.c_fc_0.scales": "model.safetensors",
|
436 |
+
"transformer.h.25.mlp.c_fc_0.weight": "model.safetensors",
|
437 |
+
"transformer.h.25.mlp.c_fc_1.biases": "model.safetensors",
|
438 |
+
"transformer.h.25.mlp.c_fc_1.scales": "model.safetensors",
|
439 |
+
"transformer.h.25.mlp.c_fc_1.weight": "model.safetensors",
|
440 |
+
"transformer.h.25.mlp.c_proj.biases": "model.safetensors",
|
441 |
+
"transformer.h.25.mlp.c_proj.scales": "model.safetensors",
|
442 |
+
"transformer.h.25.mlp.c_proj.weight": "model.safetensors",
|
443 |
+
"transformer.h.26.attn.attention.k_proj.biases": "model.safetensors",
|
444 |
+
"transformer.h.26.attn.attention.k_proj.scales": "model.safetensors",
|
445 |
+
"transformer.h.26.attn.attention.k_proj.weight": "model.safetensors",
|
446 |
+
"transformer.h.26.attn.attention.out_proj.biases": "model.safetensors",
|
447 |
+
"transformer.h.26.attn.attention.out_proj.scales": "model.safetensors",
|
448 |
+
"transformer.h.26.attn.attention.out_proj.weight": "model.safetensors",
|
449 |
+
"transformer.h.26.attn.attention.q_proj.biases": "model.safetensors",
|
450 |
+
"transformer.h.26.attn.attention.q_proj.scales": "model.safetensors",
|
451 |
+
"transformer.h.26.attn.attention.q_proj.weight": "model.safetensors",
|
452 |
+
"transformer.h.26.attn.attention.v_proj.biases": "model.safetensors",
|
453 |
+
"transformer.h.26.attn.attention.v_proj.scales": "model.safetensors",
|
454 |
+
"transformer.h.26.attn.attention.v_proj.weight": "model.safetensors",
|
455 |
+
"transformer.h.26.ln_1.weight": "model.safetensors",
|
456 |
+
"transformer.h.26.ln_2.weight": "model.safetensors",
|
457 |
+
"transformer.h.26.mlp.c_fc_0.biases": "model.safetensors",
|
458 |
+
"transformer.h.26.mlp.c_fc_0.scales": "model.safetensors",
|
459 |
+
"transformer.h.26.mlp.c_fc_0.weight": "model.safetensors",
|
460 |
+
"transformer.h.26.mlp.c_fc_1.biases": "model.safetensors",
|
461 |
+
"transformer.h.26.mlp.c_fc_1.scales": "model.safetensors",
|
462 |
+
"transformer.h.26.mlp.c_fc_1.weight": "model.safetensors",
|
463 |
+
"transformer.h.26.mlp.c_proj.biases": "model.safetensors",
|
464 |
+
"transformer.h.26.mlp.c_proj.scales": "model.safetensors",
|
465 |
+
"transformer.h.26.mlp.c_proj.weight": "model.safetensors",
|
466 |
+
"transformer.h.27.attn.attention.k_proj.biases": "model.safetensors",
|
467 |
+
"transformer.h.27.attn.attention.k_proj.scales": "model.safetensors",
|
468 |
+
"transformer.h.27.attn.attention.k_proj.weight": "model.safetensors",
|
469 |
+
"transformer.h.27.attn.attention.out_proj.biases": "model.safetensors",
|
470 |
+
"transformer.h.27.attn.attention.out_proj.scales": "model.safetensors",
|
471 |
+
"transformer.h.27.attn.attention.out_proj.weight": "model.safetensors",
|
472 |
+
"transformer.h.27.attn.attention.q_proj.biases": "model.safetensors",
|
473 |
+
"transformer.h.27.attn.attention.q_proj.scales": "model.safetensors",
|
474 |
+
"transformer.h.27.attn.attention.q_proj.weight": "model.safetensors",
|
475 |
+
"transformer.h.27.attn.attention.v_proj.biases": "model.safetensors",
|
476 |
+
"transformer.h.27.attn.attention.v_proj.scales": "model.safetensors",
|
477 |
+
"transformer.h.27.attn.attention.v_proj.weight": "model.safetensors",
|
478 |
+
"transformer.h.27.ln_1.weight": "model.safetensors",
|
479 |
+
"transformer.h.27.ln_2.weight": "model.safetensors",
|
480 |
+
"transformer.h.27.mlp.c_fc_0.biases": "model.safetensors",
|
481 |
+
"transformer.h.27.mlp.c_fc_0.scales": "model.safetensors",
|
482 |
+
"transformer.h.27.mlp.c_fc_0.weight": "model.safetensors",
|
483 |
+
"transformer.h.27.mlp.c_fc_1.biases": "model.safetensors",
|
484 |
+
"transformer.h.27.mlp.c_fc_1.scales": "model.safetensors",
|
485 |
+
"transformer.h.27.mlp.c_fc_1.weight": "model.safetensors",
|
486 |
+
"transformer.h.27.mlp.c_proj.biases": "model.safetensors",
|
487 |
+
"transformer.h.27.mlp.c_proj.scales": "model.safetensors",
|
488 |
+
"transformer.h.27.mlp.c_proj.weight": "model.safetensors",
|
489 |
+
"transformer.h.28.attn.attention.k_proj.biases": "model.safetensors",
|
490 |
+
"transformer.h.28.attn.attention.k_proj.scales": "model.safetensors",
|
491 |
+
"transformer.h.28.attn.attention.k_proj.weight": "model.safetensors",
|
492 |
+
"transformer.h.28.attn.attention.out_proj.biases": "model.safetensors",
|
493 |
+
"transformer.h.28.attn.attention.out_proj.scales": "model.safetensors",
|
494 |
+
"transformer.h.28.attn.attention.out_proj.weight": "model.safetensors",
|
495 |
+
"transformer.h.28.attn.attention.q_proj.biases": "model.safetensors",
|
496 |
+
"transformer.h.28.attn.attention.q_proj.scales": "model.safetensors",
|
497 |
+
"transformer.h.28.attn.attention.q_proj.weight": "model.safetensors",
|
498 |
+
"transformer.h.28.attn.attention.v_proj.biases": "model.safetensors",
|
499 |
+
"transformer.h.28.attn.attention.v_proj.scales": "model.safetensors",
|
500 |
+
"transformer.h.28.attn.attention.v_proj.weight": "model.safetensors",
|
501 |
+
"transformer.h.28.ln_1.weight": "model.safetensors",
|
502 |
+
"transformer.h.28.ln_2.weight": "model.safetensors",
|
503 |
+
"transformer.h.28.mlp.c_fc_0.biases": "model.safetensors",
|
504 |
+
"transformer.h.28.mlp.c_fc_0.scales": "model.safetensors",
|
505 |
+
"transformer.h.28.mlp.c_fc_0.weight": "model.safetensors",
|
506 |
+
"transformer.h.28.mlp.c_fc_1.biases": "model.safetensors",
|
507 |
+
"transformer.h.28.mlp.c_fc_1.scales": "model.safetensors",
|
508 |
+
"transformer.h.28.mlp.c_fc_1.weight": "model.safetensors",
|
509 |
+
"transformer.h.28.mlp.c_proj.biases": "model.safetensors",
|
510 |
+
"transformer.h.28.mlp.c_proj.scales": "model.safetensors",
|
511 |
+
"transformer.h.28.mlp.c_proj.weight": "model.safetensors",
|
512 |
+
"transformer.h.29.attn.attention.k_proj.biases": "model.safetensors",
|
513 |
+
"transformer.h.29.attn.attention.k_proj.scales": "model.safetensors",
|
514 |
+
"transformer.h.29.attn.attention.k_proj.weight": "model.safetensors",
|
515 |
+
"transformer.h.29.attn.attention.out_proj.biases": "model.safetensors",
|
516 |
+
"transformer.h.29.attn.attention.out_proj.scales": "model.safetensors",
|
517 |
+
"transformer.h.29.attn.attention.out_proj.weight": "model.safetensors",
|
518 |
+
"transformer.h.29.attn.attention.q_proj.biases": "model.safetensors",
|
519 |
+
"transformer.h.29.attn.attention.q_proj.scales": "model.safetensors",
|
520 |
+
"transformer.h.29.attn.attention.q_proj.weight": "model.safetensors",
|
521 |
+
"transformer.h.29.attn.attention.v_proj.biases": "model.safetensors",
|
522 |
+
"transformer.h.29.attn.attention.v_proj.scales": "model.safetensors",
|
523 |
+
"transformer.h.29.attn.attention.v_proj.weight": "model.safetensors",
|
524 |
+
"transformer.h.29.ln_1.weight": "model.safetensors",
|
525 |
+
"transformer.h.29.ln_2.weight": "model.safetensors",
|
526 |
+
"transformer.h.29.mlp.c_fc_0.biases": "model.safetensors",
|
527 |
+
"transformer.h.29.mlp.c_fc_0.scales": "model.safetensors",
|
528 |
+
"transformer.h.29.mlp.c_fc_0.weight": "model.safetensors",
|
529 |
+
"transformer.h.29.mlp.c_fc_1.biases": "model.safetensors",
|
530 |
+
"transformer.h.29.mlp.c_fc_1.scales": "model.safetensors",
|
531 |
+
"transformer.h.29.mlp.c_fc_1.weight": "model.safetensors",
|
532 |
+
"transformer.h.29.mlp.c_proj.biases": "model.safetensors",
|
533 |
+
"transformer.h.29.mlp.c_proj.scales": "model.safetensors",
|
534 |
+
"transformer.h.29.mlp.c_proj.weight": "model.safetensors",
|
535 |
+
"transformer.h.3.attn.attention.k_proj.biases": "model.safetensors",
|
536 |
+
"transformer.h.3.attn.attention.k_proj.scales": "model.safetensors",
|
537 |
+
"transformer.h.3.attn.attention.k_proj.weight": "model.safetensors",
|
538 |
+
"transformer.h.3.attn.attention.out_proj.biases": "model.safetensors",
|
539 |
+
"transformer.h.3.attn.attention.out_proj.scales": "model.safetensors",
|
540 |
+
"transformer.h.3.attn.attention.out_proj.weight": "model.safetensors",
|
541 |
+
"transformer.h.3.attn.attention.q_proj.biases": "model.safetensors",
|
542 |
+
"transformer.h.3.attn.attention.q_proj.scales": "model.safetensors",
|
543 |
+
"transformer.h.3.attn.attention.q_proj.weight": "model.safetensors",
|
544 |
+
"transformer.h.3.attn.attention.v_proj.biases": "model.safetensors",
|
545 |
+
"transformer.h.3.attn.attention.v_proj.scales": "model.safetensors",
|
546 |
+
"transformer.h.3.attn.attention.v_proj.weight": "model.safetensors",
|
547 |
+
"transformer.h.3.ln_1.weight": "model.safetensors",
|
548 |
+
"transformer.h.3.ln_2.weight": "model.safetensors",
|
549 |
+
"transformer.h.3.mlp.c_fc_0.biases": "model.safetensors",
|
550 |
+
"transformer.h.3.mlp.c_fc_0.scales": "model.safetensors",
|
551 |
+
"transformer.h.3.mlp.c_fc_0.weight": "model.safetensors",
|
552 |
+
"transformer.h.3.mlp.c_fc_1.biases": "model.safetensors",
|
553 |
+
"transformer.h.3.mlp.c_fc_1.scales": "model.safetensors",
|
554 |
+
"transformer.h.3.mlp.c_fc_1.weight": "model.safetensors",
|
555 |
+
"transformer.h.3.mlp.c_proj.biases": "model.safetensors",
|
556 |
+
"transformer.h.3.mlp.c_proj.scales": "model.safetensors",
|
557 |
+
"transformer.h.3.mlp.c_proj.weight": "model.safetensors",
|
558 |
+
"transformer.h.4.attn.attention.k_proj.biases": "model.safetensors",
|
559 |
+
"transformer.h.4.attn.attention.k_proj.scales": "model.safetensors",
|
560 |
+
"transformer.h.4.attn.attention.k_proj.weight": "model.safetensors",
|
561 |
+
"transformer.h.4.attn.attention.out_proj.biases": "model.safetensors",
|
562 |
+
"transformer.h.4.attn.attention.out_proj.scales": "model.safetensors",
|
563 |
+
"transformer.h.4.attn.attention.out_proj.weight": "model.safetensors",
|
564 |
+
"transformer.h.4.attn.attention.q_proj.biases": "model.safetensors",
|
565 |
+
"transformer.h.4.attn.attention.q_proj.scales": "model.safetensors",
|
566 |
+
"transformer.h.4.attn.attention.q_proj.weight": "model.safetensors",
|
567 |
+
"transformer.h.4.attn.attention.v_proj.biases": "model.safetensors",
|
568 |
+
"transformer.h.4.attn.attention.v_proj.scales": "model.safetensors",
|
569 |
+
"transformer.h.4.attn.attention.v_proj.weight": "model.safetensors",
|
570 |
+
"transformer.h.4.ln_1.weight": "model.safetensors",
|
571 |
+
"transformer.h.4.ln_2.weight": "model.safetensors",
|
572 |
+
"transformer.h.4.mlp.c_fc_0.biases": "model.safetensors",
|
573 |
+
"transformer.h.4.mlp.c_fc_0.scales": "model.safetensors",
|
574 |
+
"transformer.h.4.mlp.c_fc_0.weight": "model.safetensors",
|
575 |
+
"transformer.h.4.mlp.c_fc_1.biases": "model.safetensors",
|
576 |
+
"transformer.h.4.mlp.c_fc_1.scales": "model.safetensors",
|
577 |
+
"transformer.h.4.mlp.c_fc_1.weight": "model.safetensors",
|
578 |
+
"transformer.h.4.mlp.c_proj.biases": "model.safetensors",
|
579 |
+
"transformer.h.4.mlp.c_proj.scales": "model.safetensors",
|
580 |
+
"transformer.h.4.mlp.c_proj.weight": "model.safetensors",
|
581 |
+
"transformer.h.5.attn.attention.k_proj.biases": "model.safetensors",
|
582 |
+
"transformer.h.5.attn.attention.k_proj.scales": "model.safetensors",
|
583 |
+
"transformer.h.5.attn.attention.k_proj.weight": "model.safetensors",
|
584 |
+
"transformer.h.5.attn.attention.out_proj.biases": "model.safetensors",
|
585 |
+
"transformer.h.5.attn.attention.out_proj.scales": "model.safetensors",
|
586 |
+
"transformer.h.5.attn.attention.out_proj.weight": "model.safetensors",
|
587 |
+
"transformer.h.5.attn.attention.q_proj.biases": "model.safetensors",
|
588 |
+
"transformer.h.5.attn.attention.q_proj.scales": "model.safetensors",
|
589 |
+
"transformer.h.5.attn.attention.q_proj.weight": "model.safetensors",
|
590 |
+
"transformer.h.5.attn.attention.v_proj.biases": "model.safetensors",
|
591 |
+
"transformer.h.5.attn.attention.v_proj.scales": "model.safetensors",
|
592 |
+
"transformer.h.5.attn.attention.v_proj.weight": "model.safetensors",
|
593 |
+
"transformer.h.5.ln_1.weight": "model.safetensors",
|
594 |
+
"transformer.h.5.ln_2.weight": "model.safetensors",
|
595 |
+
"transformer.h.5.mlp.c_fc_0.biases": "model.safetensors",
|
596 |
+
"transformer.h.5.mlp.c_fc_0.scales": "model.safetensors",
|
597 |
+
"transformer.h.5.mlp.c_fc_0.weight": "model.safetensors",
|
598 |
+
"transformer.h.5.mlp.c_fc_1.biases": "model.safetensors",
|
599 |
+
"transformer.h.5.mlp.c_fc_1.scales": "model.safetensors",
|
600 |
+
"transformer.h.5.mlp.c_fc_1.weight": "model.safetensors",
|
601 |
+
"transformer.h.5.mlp.c_proj.biases": "model.safetensors",
|
602 |
+
"transformer.h.5.mlp.c_proj.scales": "model.safetensors",
|
603 |
+
"transformer.h.5.mlp.c_proj.weight": "model.safetensors",
|
604 |
+
"transformer.h.6.attn.attention.k_proj.biases": "model.safetensors",
|
605 |
+
"transformer.h.6.attn.attention.k_proj.scales": "model.safetensors",
|
606 |
+
"transformer.h.6.attn.attention.k_proj.weight": "model.safetensors",
|
607 |
+
"transformer.h.6.attn.attention.out_proj.biases": "model.safetensors",
|
608 |
+
"transformer.h.6.attn.attention.out_proj.scales": "model.safetensors",
|
609 |
+
"transformer.h.6.attn.attention.out_proj.weight": "model.safetensors",
|
610 |
+
"transformer.h.6.attn.attention.q_proj.biases": "model.safetensors",
|
611 |
+
"transformer.h.6.attn.attention.q_proj.scales": "model.safetensors",
|
612 |
+
"transformer.h.6.attn.attention.q_proj.weight": "model.safetensors",
|
613 |
+
"transformer.h.6.attn.attention.v_proj.biases": "model.safetensors",
|
614 |
+
"transformer.h.6.attn.attention.v_proj.scales": "model.safetensors",
|
615 |
+
"transformer.h.6.attn.attention.v_proj.weight": "model.safetensors",
|
616 |
+
"transformer.h.6.ln_1.weight": "model.safetensors",
|
617 |
+
"transformer.h.6.ln_2.weight": "model.safetensors",
|
618 |
+
"transformer.h.6.mlp.c_fc_0.biases": "model.safetensors",
|
619 |
+
"transformer.h.6.mlp.c_fc_0.scales": "model.safetensors",
|
620 |
+
"transformer.h.6.mlp.c_fc_0.weight": "model.safetensors",
|
621 |
+
"transformer.h.6.mlp.c_fc_1.biases": "model.safetensors",
|
622 |
+
"transformer.h.6.mlp.c_fc_1.scales": "model.safetensors",
|
623 |
+
"transformer.h.6.mlp.c_fc_1.weight": "model.safetensors",
|
624 |
+
"transformer.h.6.mlp.c_proj.biases": "model.safetensors",
|
625 |
+
"transformer.h.6.mlp.c_proj.scales": "model.safetensors",
|
626 |
+
"transformer.h.6.mlp.c_proj.weight": "model.safetensors",
|
627 |
+
"transformer.h.7.attn.attention.k_proj.biases": "model.safetensors",
|
628 |
+
"transformer.h.7.attn.attention.k_proj.scales": "model.safetensors",
|
629 |
+
"transformer.h.7.attn.attention.k_proj.weight": "model.safetensors",
|
630 |
+
"transformer.h.7.attn.attention.out_proj.biases": "model.safetensors",
|
631 |
+
"transformer.h.7.attn.attention.out_proj.scales": "model.safetensors",
|
632 |
+
"transformer.h.7.attn.attention.out_proj.weight": "model.safetensors",
|
633 |
+
"transformer.h.7.attn.attention.q_proj.biases": "model.safetensors",
|
634 |
+
"transformer.h.7.attn.attention.q_proj.scales": "model.safetensors",
|
635 |
+
"transformer.h.7.attn.attention.q_proj.weight": "model.safetensors",
|
636 |
+
"transformer.h.7.attn.attention.v_proj.biases": "model.safetensors",
|
637 |
+
"transformer.h.7.attn.attention.v_proj.scales": "model.safetensors",
|
638 |
+
"transformer.h.7.attn.attention.v_proj.weight": "model.safetensors",
|
639 |
+
"transformer.h.7.ln_1.weight": "model.safetensors",
|
640 |
+
"transformer.h.7.ln_2.weight": "model.safetensors",
|
641 |
+
"transformer.h.7.mlp.c_fc_0.biases": "model.safetensors",
|
642 |
+
"transformer.h.7.mlp.c_fc_0.scales": "model.safetensors",
|
643 |
+
"transformer.h.7.mlp.c_fc_0.weight": "model.safetensors",
|
644 |
+
"transformer.h.7.mlp.c_fc_1.biases": "model.safetensors",
|
645 |
+
"transformer.h.7.mlp.c_fc_1.scales": "model.safetensors",
|
646 |
+
"transformer.h.7.mlp.c_fc_1.weight": "model.safetensors",
|
647 |
+
"transformer.h.7.mlp.c_proj.biases": "model.safetensors",
|
648 |
+
"transformer.h.7.mlp.c_proj.scales": "model.safetensors",
|
649 |
+
"transformer.h.7.mlp.c_proj.weight": "model.safetensors",
|
650 |
+
"transformer.h.8.attn.attention.k_proj.biases": "model.safetensors",
|
651 |
+
"transformer.h.8.attn.attention.k_proj.scales": "model.safetensors",
|
652 |
+
"transformer.h.8.attn.attention.k_proj.weight": "model.safetensors",
|
653 |
+
"transformer.h.8.attn.attention.out_proj.biases": "model.safetensors",
|
654 |
+
"transformer.h.8.attn.attention.out_proj.scales": "model.safetensors",
|
655 |
+
"transformer.h.8.attn.attention.out_proj.weight": "model.safetensors",
|
656 |
+
"transformer.h.8.attn.attention.q_proj.biases": "model.safetensors",
|
657 |
+
"transformer.h.8.attn.attention.q_proj.scales": "model.safetensors",
|
658 |
+
"transformer.h.8.attn.attention.q_proj.weight": "model.safetensors",
|
659 |
+
"transformer.h.8.attn.attention.v_proj.biases": "model.safetensors",
|
660 |
+
"transformer.h.8.attn.attention.v_proj.scales": "model.safetensors",
|
661 |
+
"transformer.h.8.attn.attention.v_proj.weight": "model.safetensors",
|
662 |
+
"transformer.h.8.ln_1.weight": "model.safetensors",
|
663 |
+
"transformer.h.8.ln_2.weight": "model.safetensors",
|
664 |
+
"transformer.h.8.mlp.c_fc_0.biases": "model.safetensors",
|
665 |
+
"transformer.h.8.mlp.c_fc_0.scales": "model.safetensors",
|
666 |
+
"transformer.h.8.mlp.c_fc_0.weight": "model.safetensors",
|
667 |
+
"transformer.h.8.mlp.c_fc_1.biases": "model.safetensors",
|
668 |
+
"transformer.h.8.mlp.c_fc_1.scales": "model.safetensors",
|
669 |
+
"transformer.h.8.mlp.c_fc_1.weight": "model.safetensors",
|
670 |
+
"transformer.h.8.mlp.c_proj.biases": "model.safetensors",
|
671 |
+
"transformer.h.8.mlp.c_proj.scales": "model.safetensors",
|
672 |
+
"transformer.h.8.mlp.c_proj.weight": "model.safetensors",
|
673 |
+
"transformer.h.9.attn.attention.k_proj.biases": "model.safetensors",
|
674 |
+
"transformer.h.9.attn.attention.k_proj.scales": "model.safetensors",
|
675 |
+
"transformer.h.9.attn.attention.k_proj.weight": "model.safetensors",
|
676 |
+
"transformer.h.9.attn.attention.out_proj.biases": "model.safetensors",
|
677 |
+
"transformer.h.9.attn.attention.out_proj.scales": "model.safetensors",
|
678 |
+
"transformer.h.9.attn.attention.out_proj.weight": "model.safetensors",
|
679 |
+
"transformer.h.9.attn.attention.q_proj.biases": "model.safetensors",
|
680 |
+
"transformer.h.9.attn.attention.q_proj.scales": "model.safetensors",
|
681 |
+
"transformer.h.9.attn.attention.q_proj.weight": "model.safetensors",
|
682 |
+
"transformer.h.9.attn.attention.v_proj.biases": "model.safetensors",
|
683 |
+
"transformer.h.9.attn.attention.v_proj.scales": "model.safetensors",
|
684 |
+
"transformer.h.9.attn.attention.v_proj.weight": "model.safetensors",
|
685 |
+
"transformer.h.9.ln_1.weight": "model.safetensors",
|
686 |
+
"transformer.h.9.ln_2.weight": "model.safetensors",
|
687 |
+
"transformer.h.9.mlp.c_fc_0.biases": "model.safetensors",
|
688 |
+
"transformer.h.9.mlp.c_fc_0.scales": "model.safetensors",
|
689 |
+
"transformer.h.9.mlp.c_fc_0.weight": "model.safetensors",
|
690 |
+
"transformer.h.9.mlp.c_fc_1.biases": "model.safetensors",
|
691 |
+
"transformer.h.9.mlp.c_fc_1.scales": "model.safetensors",
|
692 |
+
"transformer.h.9.mlp.c_fc_1.weight": "model.safetensors",
|
693 |
+
"transformer.h.9.mlp.c_proj.biases": "model.safetensors",
|
694 |
+
"transformer.h.9.mlp.c_proj.scales": "model.safetensors",
|
695 |
+
"transformer.h.9.mlp.c_proj.weight": "model.safetensors",
|
696 |
+
"transformer.ln_f.weight": "model.safetensors",
|
697 |
+
"transformer.wte.biases": "model.safetensors",
|
698 |
+
"transformer.wte.scales": "model.safetensors",
|
699 |
+
"transformer.wte.weight": "model.safetensors"
|
700 |
+
}
|
701 |
+
}
|
modeling_exaone.py
ADDED
@@ -0,0 +1,1394 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# coding=utf-8
|
2 |
+
# Copyright 2021 The LG AI Research EXAONE Lab.
|
3 |
+
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
4 |
+
#
|
5 |
+
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
6 |
+
# and OPT implementations in this library. It has been modified from its
|
7 |
+
# original forms to accommodate minor architectural differences compared
|
8 |
+
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
9 |
+
#
|
10 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
11 |
+
# you may not use this file except in compliance with the License.
|
12 |
+
# You may obtain a copy of the License at
|
13 |
+
#
|
14 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
15 |
+
#
|
16 |
+
# Unless required by applicable law or agreed to in writing, software
|
17 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
18 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
19 |
+
# See the License for the specific language governing permissions and
|
20 |
+
# limitations under the License.
|
21 |
+
"""LG AI Research EXAONE Lab"""
|
22 |
+
|
23 |
+
import math
|
24 |
+
from typing import Optional, Tuple, Union
|
25 |
+
|
26 |
+
import torch
|
27 |
+
import torch.utils.checkpoint
|
28 |
+
from packaging import version
|
29 |
+
from torch import nn
|
30 |
+
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
|
31 |
+
|
32 |
+
from transformers.activations import ACT2FN
|
33 |
+
from transformers.cache_utils import Cache, DynamicCache, StaticCache
|
34 |
+
from transformers.generation import GenerationMixin
|
35 |
+
from transformers.modeling_attn_mask_utils import AttentionMaskConverter
|
36 |
+
from transformers.modeling_flash_attention_utils import _flash_attention_forward
|
37 |
+
from transformers.modeling_outputs import (
|
38 |
+
BaseModelOutputWithPast,
|
39 |
+
BaseModelOutputWithPastAndCrossAttentions,
|
40 |
+
CausalLMOutputWithPast,
|
41 |
+
QuestionAnsweringModelOutput,
|
42 |
+
SequenceClassifierOutputWithPast,
|
43 |
+
)
|
44 |
+
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
|
45 |
+
from transformers.modeling_utils import PreTrainedModel
|
46 |
+
from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
|
47 |
+
from transformers.utils import (
|
48 |
+
add_code_sample_docstrings,
|
49 |
+
add_start_docstrings,
|
50 |
+
add_start_docstrings_to_model_forward,
|
51 |
+
is_flash_attn_2_available,
|
52 |
+
logging,
|
53 |
+
)
|
54 |
+
from .configuration_exaone import ExaoneConfig
|
55 |
+
|
56 |
+
|
57 |
+
if is_flash_attn_2_available():
|
58 |
+
try:
|
59 |
+
import flash_attn
|
60 |
+
|
61 |
+
if version.parse(flash_attn.__version__) > version.parse("2.4.2"):
|
62 |
+
from flash_attn.ops.triton.layer_norm import rms_norm_fn
|
63 |
+
else:
|
64 |
+
from flash_attn.ops.triton.layernorm import rms_norm_fn
|
65 |
+
except ImportError:
|
66 |
+
pass
|
67 |
+
|
68 |
+
|
69 |
+
logger = logging.get_logger(__name__)
|
70 |
+
|
71 |
+
_CHECKPOINT_FOR_DOC = "exaone"
|
72 |
+
_CONFIG_FOR_DOC = "ExaoneConfig"
|
73 |
+
|
74 |
+
EXAONE_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
75 |
+
"exaone",
|
76 |
+
]
|
77 |
+
|
78 |
+
|
79 |
+
@torch.jit.script
|
80 |
+
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
81 |
+
"""
|
82 |
+
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
|
83 |
+
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
|
84 |
+
"""
|
85 |
+
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
|
86 |
+
if n_rep == 1:
|
87 |
+
return hidden_states
|
88 |
+
hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
|
89 |
+
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
|
90 |
+
|
91 |
+
|
92 |
+
def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
|
93 |
+
"""Applies Rotary Position Embedding to the query and key tensors.
|
94 |
+
|
95 |
+
Args:
|
96 |
+
q (`torch.Tensor`): The query tensor.
|
97 |
+
k (`torch.Tensor`): The key tensor.
|
98 |
+
cos (`torch.Tensor`): The cosine part of the rotary embedding.
|
99 |
+
sin (`torch.Tensor`): The sine part of the rotary embedding.
|
100 |
+
unsqueeze_dim (`int`, *optional*, defaults to 1):
|
101 |
+
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
|
102 |
+
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
|
103 |
+
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
|
104 |
+
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
|
105 |
+
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
|
106 |
+
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
|
107 |
+
Returns:
|
108 |
+
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
|
109 |
+
"""
|
110 |
+
cos = cos.unsqueeze(unsqueeze_dim)
|
111 |
+
sin = sin.unsqueeze(unsqueeze_dim)
|
112 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
113 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
114 |
+
return q_embed, k_embed
|
115 |
+
|
116 |
+
|
117 |
+
def rotate_half(x):
|
118 |
+
"""Rotates half the hidden dims of the input."""
|
119 |
+
x1 = x[..., : x.shape[-1] // 2]
|
120 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
121 |
+
return torch.cat((-x2, x1), dim=-1)
|
122 |
+
|
123 |
+
|
124 |
+
def _prepare_4d_causal_attention_mask_with_cache_position(
|
125 |
+
attention_mask: torch.Tensor,
|
126 |
+
sequence_length: int,
|
127 |
+
target_length: int,
|
128 |
+
dtype: torch.dtype,
|
129 |
+
device: torch.device,
|
130 |
+
min_dtype: float,
|
131 |
+
cache_position: torch.Tensor,
|
132 |
+
batch_size: int,
|
133 |
+
):
|
134 |
+
"""
|
135 |
+
Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
|
136 |
+
`(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
|
137 |
+
|
138 |
+
Args:
|
139 |
+
attention_mask (`torch.Tensor`):
|
140 |
+
A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
|
141 |
+
sequence_length (`int`):
|
142 |
+
The sequence length being processed.
|
143 |
+
target_length (`int`):
|
144 |
+
The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
|
145 |
+
dtype (`torch.dtype`):
|
146 |
+
The dtype to use for the 4D attention mask.
|
147 |
+
device (`torch.device`):
|
148 |
+
The device to plcae the 4D attention mask on.
|
149 |
+
min_dtype (`float`):
|
150 |
+
The minimum value representable with the dtype `dtype`.
|
151 |
+
cache_position (`torch.Tensor`):
|
152 |
+
Indices depicting the position of the input sequence tokens in the sequence.
|
153 |
+
batch_size (`torch.Tensor`):
|
154 |
+
Batch size.
|
155 |
+
"""
|
156 |
+
if attention_mask is not None and attention_mask.dim() == 4:
|
157 |
+
# In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
|
158 |
+
causal_mask = attention_mask
|
159 |
+
else:
|
160 |
+
causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device)
|
161 |
+
if sequence_length != 1:
|
162 |
+
causal_mask = torch.triu(causal_mask, diagonal=1)
|
163 |
+
causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1)
|
164 |
+
causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
|
165 |
+
if attention_mask is not None:
|
166 |
+
causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit
|
167 |
+
mask_length = attention_mask.shape[-1]
|
168 |
+
padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]
|
169 |
+
padding_mask = padding_mask == 0
|
170 |
+
causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(
|
171 |
+
padding_mask, min_dtype
|
172 |
+
)
|
173 |
+
|
174 |
+
return causal_mask
|
175 |
+
|
176 |
+
|
177 |
+
class ExaoneRMSNorm(torch.nn.Module):
|
178 |
+
def __init__(self, hidden_size, eps=1e-6):
|
179 |
+
super().__init__()
|
180 |
+
self.eps = eps
|
181 |
+
self.weight = torch.nn.Parameter(torch.ones(hidden_size))
|
182 |
+
|
183 |
+
def forward(self, hidden_states):
|
184 |
+
input_dtype = hidden_states.dtype
|
185 |
+
hidden_states = hidden_states.to(torch.float32)
|
186 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
187 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
|
188 |
+
return self.weight * hidden_states.to(input_dtype)
|
189 |
+
|
190 |
+
|
191 |
+
class ExaoneTritonRMSNorm(torch.nn.Module):
|
192 |
+
def __init__(
|
193 |
+
self,
|
194 |
+
hidden_size: int = 0,
|
195 |
+
eps: float = 1e-5,
|
196 |
+
):
|
197 |
+
super().__init__()
|
198 |
+
self.eps = eps
|
199 |
+
self.drop = None
|
200 |
+
self.weight = torch.nn.Parameter(torch.empty(hidden_size))
|
201 |
+
self.register_parameter("bias", None)
|
202 |
+
self.reset_parameters()
|
203 |
+
|
204 |
+
def reset_parameters(self):
|
205 |
+
torch.nn.init.ones_(self.weight)
|
206 |
+
|
207 |
+
def forward(self, x, residual=None, prenorm=False, residual_in_fp32=False):
|
208 |
+
return rms_norm_fn(
|
209 |
+
x,
|
210 |
+
self.weight,
|
211 |
+
self.bias,
|
212 |
+
residual=residual,
|
213 |
+
eps=self.eps,
|
214 |
+
dropout_p=self.drop.p if self.drop is not None and self.training else 0.0,
|
215 |
+
prenorm=prenorm,
|
216 |
+
residual_in_fp32=residual_in_fp32,
|
217 |
+
)
|
218 |
+
|
219 |
+
|
220 |
+
ALL_LAYERNORM_LAYERS.append(ExaoneRMSNorm)
|
221 |
+
ALL_LAYERNORM_LAYERS.append(ExaoneTritonRMSNorm)
|
222 |
+
|
223 |
+
|
224 |
+
class ExaoneRotaryEmbedding(nn.Module):
|
225 |
+
def __init__(self, config: ExaoneConfig, device=None):
|
226 |
+
super().__init__()
|
227 |
+
if config.rope_scaling is not None:
|
228 |
+
self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
|
229 |
+
else:
|
230 |
+
self.rope_type = "default"
|
231 |
+
self.rope_theta = config.rope_theta
|
232 |
+
self.max_seq_len = config.max_position_embeddings
|
233 |
+
self.original_max_seq_len = config.max_position_embeddings
|
234 |
+
|
235 |
+
self.config = config
|
236 |
+
if self.rope_type not in ROPE_INIT_FUNCTIONS:
|
237 |
+
raise KeyError(f"The EXAONE model does not support RoPE type: {self.rope_type}")
|
238 |
+
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
239 |
+
|
240 |
+
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
|
241 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
242 |
+
self.original_inv_freq = self.inv_freq
|
243 |
+
|
244 |
+
def _update_freq(self, position_ids, device):
|
245 |
+
"""
|
246 |
+
dynamic RoPE layers should recompute `inv_freq` in the following situations:
|
247 |
+
1 - growing beyond the cached sequence length (allow scaling)
|
248 |
+
2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
|
249 |
+
"""
|
250 |
+
seq_len = torch.max(position_ids) + 1
|
251 |
+
if seq_len > self.max_seq_len: # expand to seq_len
|
252 |
+
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, seq_len=seq_len)
|
253 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
254 |
+
self.max_seq_len = seq_len
|
255 |
+
|
256 |
+
if seq_len < self.original_max_seq_len and self.max_seq_len > self.original_max_seq_len: # reset to original
|
257 |
+
self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
|
258 |
+
self.max_seq_len = self.original_max_seq_len
|
259 |
+
|
260 |
+
@torch.no_grad()
|
261 |
+
def forward(self, x, position_ids):
|
262 |
+
if "dynamic" in self.rope_type:
|
263 |
+
self._update_freq(position_ids, device=x.device)
|
264 |
+
|
265 |
+
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
266 |
+
position_ids_expanded = position_ids[:, None, :].float()
|
267 |
+
|
268 |
+
device_type = x.device.type
|
269 |
+
device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
|
270 |
+
with torch.autocast(device_type=device_type, enabled=False):
|
271 |
+
freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2)
|
272 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
273 |
+
cos, sin = emb.cos(), emb.sin()
|
274 |
+
|
275 |
+
cos, sin = cos * self.attention_scaling, sin * self.attention_scaling
|
276 |
+
return cos.to(x.dtype), sin.to(x.dtype)
|
277 |
+
|
278 |
+
|
279 |
+
class ExaoneSelfAttention(nn.Module):
|
280 |
+
def __init__(self, config: ExaoneConfig, layer_idx: Optional[int] = None):
|
281 |
+
super().__init__()
|
282 |
+
self.config = config
|
283 |
+
self.layer_idx = layer_idx
|
284 |
+
self.embed_dim = config.hidden_size
|
285 |
+
self.num_heads = config.num_attention_heads
|
286 |
+
self.head_dim = self.embed_dim // self.num_heads
|
287 |
+
self.num_key_value_heads = config.num_key_value_heads
|
288 |
+
self.num_key_value_groups = self.num_heads // self.num_key_value_heads
|
289 |
+
self.attention_dropout_rate = config.attention_dropout
|
290 |
+
|
291 |
+
if self.head_dim * self.num_heads != self.embed_dim:
|
292 |
+
raise ValueError(
|
293 |
+
f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})."
|
294 |
+
)
|
295 |
+
|
296 |
+
self.rotary = ExaoneRotaryEmbedding(config)
|
297 |
+
|
298 |
+
self.k_proj = nn.Linear(self.embed_dim, self.num_key_value_heads * self.head_dim, bias=False)
|
299 |
+
self.v_proj = nn.Linear(self.embed_dim, self.num_key_value_heads * self.head_dim, bias=False)
|
300 |
+
self.q_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=False)
|
301 |
+
self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
|
302 |
+
|
303 |
+
def forward(
|
304 |
+
self,
|
305 |
+
hidden_states: torch.Tensor,
|
306 |
+
attention_mask: Optional[torch.Tensor] = None,
|
307 |
+
position_ids: Optional[torch.LongTensor] = None,
|
308 |
+
past_key_value: Optional[Cache] = None,
|
309 |
+
output_attentions: Optional[bool] = False,
|
310 |
+
use_cache: Optional[bool] = False,
|
311 |
+
cache_position: Optional[torch.LongTensor] = None,
|
312 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
313 |
+
**kwargs,
|
314 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
315 |
+
bsz, q_len, _ = hidden_states.size()
|
316 |
+
query_states = self.q_proj(hidden_states)
|
317 |
+
key_states = self.k_proj(hidden_states)
|
318 |
+
value_states = self.v_proj(hidden_states)
|
319 |
+
|
320 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
321 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
322 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
323 |
+
|
324 |
+
if position_embeddings is None:
|
325 |
+
cos, sin = self.rotary(value_states, position_ids=position_ids)
|
326 |
+
else:
|
327 |
+
cos, sin = position_embeddings
|
328 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
|
329 |
+
|
330 |
+
if past_key_value is not None:
|
331 |
+
# sin and cos are specific to RoPE models; cache_position needed for the static cache
|
332 |
+
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
|
333 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
334 |
+
|
335 |
+
key_states = repeat_kv(key_states, self.num_key_value_groups)
|
336 |
+
value_states = repeat_kv(value_states, self.num_key_value_groups)
|
337 |
+
|
338 |
+
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
|
339 |
+
|
340 |
+
if attention_mask is not None:
|
341 |
+
causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
|
342 |
+
attn_weights = attn_weights + causal_mask
|
343 |
+
|
344 |
+
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
|
345 |
+
attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout_rate, training=self.training)
|
346 |
+
attn_output = torch.matmul(attn_weights, value_states)
|
347 |
+
|
348 |
+
if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
|
349 |
+
raise ValueError(
|
350 |
+
f"Attention outputs should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
|
351 |
+
f" {attn_output.size()}"
|
352 |
+
)
|
353 |
+
|
354 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
355 |
+
attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
|
356 |
+
|
357 |
+
attn_output = self.out_proj(attn_output)
|
358 |
+
|
359 |
+
if not output_attentions:
|
360 |
+
attn_weights = None
|
361 |
+
|
362 |
+
return attn_output, attn_weights, past_key_value
|
363 |
+
|
364 |
+
|
365 |
+
class ExaoneFlashAttention(ExaoneSelfAttention):
|
366 |
+
def __init__(self, *args, **kwargs):
|
367 |
+
super().__init__(*args, **kwargs)
|
368 |
+
|
369 |
+
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
370 |
+
return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
|
371 |
+
|
372 |
+
def forward(
|
373 |
+
self,
|
374 |
+
hidden_states: torch.Tensor,
|
375 |
+
attention_mask: Optional[torch.Tensor] = None,
|
376 |
+
position_ids: Optional[torch.LongTensor] = None,
|
377 |
+
past_key_value: Optional[Cache] = None,
|
378 |
+
output_attentions: Optional[bool] = False,
|
379 |
+
use_cache: Optional[bool] = False,
|
380 |
+
cache_position: Optional[torch.LongTensor] = None,
|
381 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
382 |
+
**kwargs,
|
383 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
384 |
+
if isinstance(past_key_value, StaticCache):
|
385 |
+
raise ValueError(
|
386 |
+
"`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` "
|
387 |
+
"make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformers"
|
388 |
+
)
|
389 |
+
|
390 |
+
output_attentions = False
|
391 |
+
|
392 |
+
bsz, q_len, h_size = hidden_states.size()
|
393 |
+
|
394 |
+
query_states = self.q_proj(hidden_states)
|
395 |
+
key_states = self.k_proj(hidden_states)
|
396 |
+
value_states = self.v_proj(hidden_states)
|
397 |
+
|
398 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
399 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
400 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
401 |
+
|
402 |
+
if position_embeddings is None:
|
403 |
+
cos, sin = self.rotary(value_states, position_ids=position_ids)
|
404 |
+
else:
|
405 |
+
cos, sin = position_embeddings
|
406 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
|
407 |
+
|
408 |
+
if past_key_value is not None:
|
409 |
+
# sin and cos are specific to RoPE models; cache_position needed for the static cache
|
410 |
+
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
|
411 |
+
# Only update cache as shape of [bsz, n_head, q_len, head_dim]
|
412 |
+
# TODO: need to be fixed when transformers' KV cache layout is changed
|
413 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
414 |
+
|
415 |
+
query_states = query_states.transpose(1, 2)
|
416 |
+
key_states = key_states.transpose(1, 2)
|
417 |
+
value_states = value_states.transpose(1, 2)
|
418 |
+
|
419 |
+
# In PEFT, usually we cast the layer norms in float32 for training stability reasons
|
420 |
+
# therefore the input hidden states gets silently casted in float32. Hence, we need
|
421 |
+
# cast them back in the correct dtype just to be sure everything works as expected.
|
422 |
+
input_dtype = query_states.dtype
|
423 |
+
if input_dtype == torch.float32:
|
424 |
+
if torch.is_autocast_enabled():
|
425 |
+
target_dtype = torch.get_autocast_gpu_dtype()
|
426 |
+
# Handle the case where the model is quantized
|
427 |
+
elif hasattr(self.config, "_pre_quantization_dtype"):
|
428 |
+
target_dtype = self.config._pre_quantization_dtype
|
429 |
+
else:
|
430 |
+
target_dtype = self.q_proj.weight.dtype
|
431 |
+
|
432 |
+
logger.warning_once(
|
433 |
+
f"The input hidden states seems to be silently casted in float32, this might be related to"
|
434 |
+
f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
|
435 |
+
f" {target_dtype}."
|
436 |
+
)
|
437 |
+
|
438 |
+
query_states = query_states.to(target_dtype)
|
439 |
+
key_states = key_states.to(target_dtype)
|
440 |
+
value_states = value_states.to(target_dtype)
|
441 |
+
|
442 |
+
dropout_rate = self.attention_dropout_rate if self.training else 0.0
|
443 |
+
|
444 |
+
attn_output = _flash_attention_forward(
|
445 |
+
query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate, is_causal=True
|
446 |
+
)
|
447 |
+
|
448 |
+
attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
|
449 |
+
attn_output = self.out_proj(attn_output)
|
450 |
+
|
451 |
+
if not output_attentions:
|
452 |
+
attn_weights = None
|
453 |
+
|
454 |
+
return attn_output, attn_weights, past_key_value
|
455 |
+
|
456 |
+
|
457 |
+
class ExaoneSdpaAttention(ExaoneSelfAttention):
|
458 |
+
def __init__(self, *args, **kwargs):
|
459 |
+
super().__init__(*args, **kwargs)
|
460 |
+
|
461 |
+
def forward(
|
462 |
+
self,
|
463 |
+
hidden_states: torch.Tensor,
|
464 |
+
attention_mask: Optional[torch.Tensor] = None,
|
465 |
+
position_ids: Optional[torch.LongTensor] = None,
|
466 |
+
past_key_value: Optional[Cache] = None,
|
467 |
+
output_attentions: Optional[bool] = False,
|
468 |
+
use_cache: Optional[bool] = False,
|
469 |
+
cache_position: Optional[torch.LongTensor] = None,
|
470 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
471 |
+
**kwargs,
|
472 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
473 |
+
if output_attentions:
|
474 |
+
logger.warning_once(
|
475 |
+
"ExaoneModel is using ExaoneSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
|
476 |
+
'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
|
477 |
+
)
|
478 |
+
return super().forward(
|
479 |
+
hidden_states=hidden_states,
|
480 |
+
attention_mask=attention_mask,
|
481 |
+
position_ids=position_ids,
|
482 |
+
past_key_value=past_key_value,
|
483 |
+
output_attentions=output_attentions,
|
484 |
+
use_cache=use_cache,
|
485 |
+
cache_position=cache_position,
|
486 |
+
position_embeddings=position_embeddings,
|
487 |
+
**kwargs,
|
488 |
+
)
|
489 |
+
|
490 |
+
bsz, q_len, _ = hidden_states.size()
|
491 |
+
|
492 |
+
query_states = self.q_proj(hidden_states)
|
493 |
+
key_states = self.k_proj(hidden_states)
|
494 |
+
value_states = self.v_proj(hidden_states)
|
495 |
+
|
496 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
497 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
498 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
499 |
+
|
500 |
+
if position_embeddings is None:
|
501 |
+
cos, sin = self.rotary(value_states, position_ids=position_ids)
|
502 |
+
else:
|
503 |
+
cos, sin = position_embeddings
|
504 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
|
505 |
+
|
506 |
+
if past_key_value is not None:
|
507 |
+
# sin and cos are specific to RoPE models; cache_position needed for the static cache
|
508 |
+
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
|
509 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
510 |
+
|
511 |
+
key_states = repeat_kv(key_states, self.num_key_value_groups)
|
512 |
+
value_states = repeat_kv(value_states, self.num_key_value_groups)
|
513 |
+
|
514 |
+
causal_mask = attention_mask
|
515 |
+
if attention_mask is not None:
|
516 |
+
causal_mask = causal_mask[:, :, :, : key_states.shape[-2]]
|
517 |
+
|
518 |
+
# SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
|
519 |
+
# Reference: https://github.com/pytorch/pytorch/issues/112577.
|
520 |
+
if query_states.device.type == "cuda" and causal_mask is not None:
|
521 |
+
query_states = query_states.contiguous()
|
522 |
+
key_states = key_states.contiguous()
|
523 |
+
value_states = value_states.contiguous()
|
524 |
+
|
525 |
+
# We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment
|
526 |
+
# in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling.
|
527 |
+
is_causal = True if causal_mask is None and q_len > 1 else False
|
528 |
+
|
529 |
+
attn_output = torch.nn.functional.scaled_dot_product_attention(
|
530 |
+
query_states,
|
531 |
+
key_states,
|
532 |
+
value_states,
|
533 |
+
attn_mask=causal_mask,
|
534 |
+
dropout_p=self.attention_dropout_rate if self.training else 0.0,
|
535 |
+
is_causal=is_causal,
|
536 |
+
)
|
537 |
+
|
538 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
539 |
+
attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
|
540 |
+
|
541 |
+
attn_output = self.out_proj(attn_output)
|
542 |
+
|
543 |
+
return attn_output, None, past_key_value
|
544 |
+
|
545 |
+
|
546 |
+
class ExaoneAttention(nn.Module):
|
547 |
+
def __init__(self, config, layer_id=0):
|
548 |
+
super().__init__()
|
549 |
+
self.layer_id = layer_id
|
550 |
+
if "flash" in config._attn_implementation:
|
551 |
+
self.attention = ExaoneFlashAttention(config, self.layer_id)
|
552 |
+
elif "sdpa" in config._attn_implementation:
|
553 |
+
self.attention = ExaoneSdpaAttention(config, self.layer_id)
|
554 |
+
else:
|
555 |
+
self.attention = ExaoneSelfAttention(config, self.layer_id)
|
556 |
+
|
557 |
+
def forward(
|
558 |
+
self,
|
559 |
+
hidden_states: torch.Tensor,
|
560 |
+
attention_mask: Optional[torch.Tensor] = None,
|
561 |
+
position_ids: Optional[torch.LongTensor] = None,
|
562 |
+
past_key_value: Optional[Cache] = None,
|
563 |
+
output_attentions: Optional[bool] = False,
|
564 |
+
use_cache: Optional[bool] = False,
|
565 |
+
cache_position: Optional[torch.LongTensor] = None,
|
566 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
567 |
+
**kwargs,
|
568 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
569 |
+
return self.attention(
|
570 |
+
hidden_states=hidden_states,
|
571 |
+
attention_mask=attention_mask,
|
572 |
+
position_ids=position_ids,
|
573 |
+
past_key_value=past_key_value,
|
574 |
+
output_attentions=output_attentions,
|
575 |
+
use_cache=use_cache,
|
576 |
+
cache_position=cache_position,
|
577 |
+
position_embeddings=position_embeddings,
|
578 |
+
**kwargs,
|
579 |
+
)
|
580 |
+
|
581 |
+
|
582 |
+
class ExaoneGatedMLP(nn.Module):
|
583 |
+
def __init__(self, intermediate_size, config):
|
584 |
+
super().__init__()
|
585 |
+
self.config = config
|
586 |
+
embed_dim = config.hidden_size
|
587 |
+
self.c_fc_0 = nn.Linear(embed_dim, intermediate_size, bias=False)
|
588 |
+
self.c_fc_1 = nn.Linear(embed_dim, intermediate_size, bias=False)
|
589 |
+
self.c_proj = nn.Linear(intermediate_size, embed_dim, bias=False)
|
590 |
+
self.act = ACT2FN[config.activation_function]
|
591 |
+
|
592 |
+
def forward(self, hidden_states):
|
593 |
+
output_proj = self.c_proj(self.act(self.c_fc_0(hidden_states)) * self.c_fc_1(hidden_states))
|
594 |
+
return output_proj
|
595 |
+
|
596 |
+
|
597 |
+
class ExaoneBlock(nn.Module):
|
598 |
+
def __init__(self, config, layer_id):
|
599 |
+
super().__init__()
|
600 |
+
self.config = config
|
601 |
+
hidden_size = config.hidden_size
|
602 |
+
inner_dim = config.intermediate_size if config.intermediate_size is not None else 4 * hidden_size
|
603 |
+
self.ln_1 = ExaoneRMSNorm(hidden_size=hidden_size, eps=config.layer_norm_epsilon)
|
604 |
+
self.attn = ExaoneAttention(config, layer_id)
|
605 |
+
self.ln_2 = ExaoneRMSNorm(hidden_size=hidden_size, eps=config.layer_norm_epsilon)
|
606 |
+
self.mlp = ExaoneGatedMLP(inner_dim, config)
|
607 |
+
|
608 |
+
def forward(
|
609 |
+
self,
|
610 |
+
hidden_states: torch.Tensor,
|
611 |
+
attention_mask: Optional[torch.Tensor] = None,
|
612 |
+
position_ids: Optional[torch.LongTensor] = None,
|
613 |
+
past_key_value: Optional[Cache] = None,
|
614 |
+
output_attentions: Optional[bool] = False,
|
615 |
+
use_cache: Optional[bool] = False,
|
616 |
+
cache_position: Optional[torch.LongTensor] = None,
|
617 |
+
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
618 |
+
**kwargs,
|
619 |
+
) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
|
620 |
+
residual = hidden_states
|
621 |
+
hidden_states = self.ln_1(hidden_states)
|
622 |
+
|
623 |
+
hidden_states, self_attn_weights, present_key_value = self.attn(
|
624 |
+
hidden_states=hidden_states,
|
625 |
+
attention_mask=attention_mask,
|
626 |
+
position_ids=position_ids,
|
627 |
+
past_key_value=past_key_value,
|
628 |
+
output_attentions=output_attentions,
|
629 |
+
use_cache=use_cache,
|
630 |
+
cache_position=cache_position,
|
631 |
+
position_embeddings=position_embeddings,
|
632 |
+
**kwargs,
|
633 |
+
)
|
634 |
+
# residual connection
|
635 |
+
hidden_states = residual + hidden_states
|
636 |
+
|
637 |
+
residual = hidden_states
|
638 |
+
hidden_states = self.ln_2(hidden_states)
|
639 |
+
hidden_states = self.mlp(hidden_states)
|
640 |
+
|
641 |
+
hidden_states = residual + hidden_states
|
642 |
+
|
643 |
+
outputs = (hidden_states,)
|
644 |
+
|
645 |
+
if output_attentions:
|
646 |
+
outputs += (self_attn_weights,)
|
647 |
+
|
648 |
+
if use_cache:
|
649 |
+
outputs += (present_key_value,)
|
650 |
+
|
651 |
+
return outputs
|
652 |
+
|
653 |
+
|
654 |
+
class ExaonePreTrainedModel(PreTrainedModel):
|
655 |
+
"""
|
656 |
+
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
|
657 |
+
models.
|
658 |
+
"""
|
659 |
+
|
660 |
+
config_class = ExaoneConfig
|
661 |
+
base_model_prefix = "transformer"
|
662 |
+
supports_gradient_checkpointing = True
|
663 |
+
_no_split_modules = ["ExaoneBlock"]
|
664 |
+
_skip_keys_device_placement = "past_key_values"
|
665 |
+
_supports_flash_attn_2 = True
|
666 |
+
_supports_sdpa = True
|
667 |
+
_supports_cache_class = True
|
668 |
+
|
669 |
+
def __init__(self, *inputs, **kwargs):
|
670 |
+
super().__init__(*inputs, **kwargs)
|
671 |
+
|
672 |
+
def _init_weights(self, module):
|
673 |
+
"""Initialize the weights."""
|
674 |
+
if isinstance(module, (nn.Linear,)):
|
675 |
+
# Slightly different from the TF version which uses truncated_normal for initialization
|
676 |
+
# cf https://github.com/pytorch/pytorch/pull/5617
|
677 |
+
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
|
678 |
+
if module.bias is not None:
|
679 |
+
module.bias.data.zero_()
|
680 |
+
elif isinstance(module, nn.Embedding):
|
681 |
+
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
|
682 |
+
if module.padding_idx is not None:
|
683 |
+
module.weight.data[module.padding_idx].zero_()
|
684 |
+
elif isinstance(module, ExaoneRMSNorm):
|
685 |
+
module.weight.data.fill_(1.0)
|
686 |
+
|
687 |
+
|
688 |
+
EXAONE_START_DOCSTRING = r"""
|
689 |
+
|
690 |
+
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
691 |
+
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
692 |
+
etc.)
|
693 |
+
|
694 |
+
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
695 |
+
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
696 |
+
and behavior.
|
697 |
+
|
698 |
+
Parameters:
|
699 |
+
config ([`ExaoneConfig`]): Model configuration class with all the parameters of the model.
|
700 |
+
Initializing with a config file does not load the weights associated with the model, only the
|
701 |
+
configuration. Check out the `PreTrainedModel.from_pretrained` method to load the model weights.
|
702 |
+
"""
|
703 |
+
|
704 |
+
EXAONE_INPUTS_DOCSTRING = r"""
|
705 |
+
Args:
|
706 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
|
707 |
+
`input_ids_length` = `sequence_length` if `past_key_values` is `None` else
|
708 |
+
`past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
|
709 |
+
sequence tokens in the vocabulary.
|
710 |
+
|
711 |
+
If `past_key_values` is used, only `input_ids` that do not have their past calculated should be
|
712 |
+
passed as `input_ids`.
|
713 |
+
|
714 |
+
`What are input IDs? <../glossary.html#input-ids>`__
|
715 |
+
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
716 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
717 |
+
|
718 |
+
- 1 for tokens that are **not masked**,
|
719 |
+
- 0 for tokens that are **masked**.
|
720 |
+
|
721 |
+
`What are attention masks? <../glossary.html#attention-mask>`__
|
722 |
+
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
723 |
+
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
|
724 |
+
config.max_position_embeddings - 1]`.
|
725 |
+
|
726 |
+
`What are position IDs? <../glossary.html#position-ids>`_
|
727 |
+
past_key_values (`Cache`, *optional*):
|
728 |
+
Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
|
729 |
+
`past_key_values` output below). Can be used to speed up sequential decoding. This typically consists
|
730 |
+
in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or
|
731 |
+
`config.use_cache=True`.
|
732 |
+
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
|
733 |
+
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
|
734 |
+
This is useful if you want more control over how to convert `input_ids` indices into associated
|
735 |
+
vectors than the model's internal embedding lookup matrix.
|
736 |
+
|
737 |
+
If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
|
738 |
+
`past_key_values`).
|
739 |
+
use_cache (`bool`, *optional*):
|
740 |
+
If set to `True`, `past_key_values` key value states are returned and can be used to speed up
|
741 |
+
decoding (see `past_key_values`).
|
742 |
+
output_attentions (`bool`, *optional*):
|
743 |
+
Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
|
744 |
+
tensors for more detail.
|
745 |
+
output_hidden_states (`bool`, *optional*):
|
746 |
+
Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for
|
747 |
+
more detail.
|
748 |
+
return_dict (`bool`, *optional*):
|
749 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
750 |
+
cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
|
751 |
+
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
|
752 |
+
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
|
753 |
+
the complete sequence length.
|
754 |
+
"""
|
755 |
+
|
756 |
+
|
757 |
+
@add_start_docstrings(
|
758 |
+
"The bare EXAONE Model transformer outputting raw hidden-states without any specific head on top.",
|
759 |
+
EXAONE_START_DOCSTRING,
|
760 |
+
)
|
761 |
+
class ExaoneModel(ExaonePreTrainedModel):
|
762 |
+
def __init__(self, config):
|
763 |
+
super().__init__(config)
|
764 |
+
self.config = config
|
765 |
+
self.embed_dim = config.hidden_size
|
766 |
+
self.wte = nn.Embedding(config.vocab_size, self.embed_dim, self.config.pad_token_id)
|
767 |
+
self.drop = nn.Dropout(float(config.embed_dropout))
|
768 |
+
self.h = nn.ModuleList([ExaoneBlock(config, layer_id=i) for i in range(config.num_layers)])
|
769 |
+
self.ln_f = ExaoneRMSNorm(hidden_size=self.embed_dim, eps=config.layer_norm_epsilon)
|
770 |
+
self.rotary = ExaoneRotaryEmbedding(config)
|
771 |
+
self.gradient_checkpointing = False
|
772 |
+
# Initialize weights and apply final processing
|
773 |
+
self.post_init()
|
774 |
+
|
775 |
+
def get_input_embeddings(self):
|
776 |
+
return self.wte
|
777 |
+
|
778 |
+
def set_input_embeddings(self, new_embeddings):
|
779 |
+
self.wte = new_embeddings
|
780 |
+
|
781 |
+
@add_start_docstrings_to_model_forward(EXAONE_INPUTS_DOCSTRING)
|
782 |
+
@add_code_sample_docstrings(
|
783 |
+
checkpoint=_CHECKPOINT_FOR_DOC,
|
784 |
+
output_type=BaseModelOutputWithPastAndCrossAttentions,
|
785 |
+
config_class=_CONFIG_FOR_DOC,
|
786 |
+
)
|
787 |
+
def forward(
|
788 |
+
self,
|
789 |
+
input_ids: Optional[torch.Tensor] = None,
|
790 |
+
attention_mask: Optional[torch.Tensor] = None,
|
791 |
+
position_ids: Optional[torch.Tensor] = None,
|
792 |
+
past_key_values: Optional[Cache] = None,
|
793 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
794 |
+
use_cache: Optional[bool] = None,
|
795 |
+
output_attentions: Optional[bool] = None,
|
796 |
+
output_hidden_states: Optional[bool] = None,
|
797 |
+
return_dict: Optional[bool] = None,
|
798 |
+
cache_position: Optional[torch.LongTensor] = None,
|
799 |
+
) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPast]:
|
800 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
801 |
+
output_hidden_states = (
|
802 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
803 |
+
)
|
804 |
+
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
805 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
806 |
+
|
807 |
+
if self.gradient_checkpointing and self.training:
|
808 |
+
if use_cache:
|
809 |
+
logger.warning_once(
|
810 |
+
"`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
|
811 |
+
)
|
812 |
+
use_cache = False
|
813 |
+
|
814 |
+
if input_ids is not None and inputs_embeds is not None:
|
815 |
+
raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
|
816 |
+
elif input_ids is not None:
|
817 |
+
batch_size, seq_length = input_ids.shape[:2]
|
818 |
+
elif inputs_embeds is not None:
|
819 |
+
batch_size, seq_length = inputs_embeds.shape[:2]
|
820 |
+
else:
|
821 |
+
raise ValueError("You have to specify either input_ids or inputs_embeds")
|
822 |
+
|
823 |
+
return_legacy_cache = False
|
824 |
+
if (
|
825 |
+
use_cache and not isinstance(past_key_values, Cache) and not self.training
|
826 |
+
): # kept for BC (non `Cache` `past_key_values` inputs)
|
827 |
+
return_legacy_cache = True
|
828 |
+
past_key_values = DynamicCache.from_legacy_cache(past_key_values)
|
829 |
+
logger.warning_once(
|
830 |
+
"We detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. "
|
831 |
+
"Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)"
|
832 |
+
)
|
833 |
+
|
834 |
+
if inputs_embeds is None:
|
835 |
+
inputs_embeds = self.wte(input_ids)
|
836 |
+
|
837 |
+
if cache_position is None:
|
838 |
+
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
839 |
+
cache_position = torch.arange(
|
840 |
+
past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
|
841 |
+
)
|
842 |
+
if position_ids is None:
|
843 |
+
position_ids = cache_position.unsqueeze(0)
|
844 |
+
|
845 |
+
causal_mask = self._update_causal_mask(
|
846 |
+
attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
|
847 |
+
)
|
848 |
+
|
849 |
+
hidden_states = inputs_embeds
|
850 |
+
hidden_states = self.drop(hidden_states)
|
851 |
+
|
852 |
+
position_embeddings = self.rotary(hidden_states, position_ids)
|
853 |
+
|
854 |
+
all_hidden_states = () if output_hidden_states else None
|
855 |
+
all_self_attns = () if output_attentions else None
|
856 |
+
next_decoder_cache = None
|
857 |
+
|
858 |
+
for block in self.h:
|
859 |
+
if output_hidden_states:
|
860 |
+
all_hidden_states = all_hidden_states + (hidden_states,)
|
861 |
+
|
862 |
+
if self.gradient_checkpointing and self.training:
|
863 |
+
outputs = self._gradient_checkpointing_func(
|
864 |
+
block.__call__,
|
865 |
+
hidden_states,
|
866 |
+
causal_mask,
|
867 |
+
position_ids,
|
868 |
+
past_key_values,
|
869 |
+
output_attentions,
|
870 |
+
use_cache,
|
871 |
+
cache_position,
|
872 |
+
position_embeddings,
|
873 |
+
)
|
874 |
+
else:
|
875 |
+
outputs = block(
|
876 |
+
hidden_states,
|
877 |
+
attention_mask=causal_mask,
|
878 |
+
position_ids=position_ids,
|
879 |
+
past_key_value=past_key_values,
|
880 |
+
output_attentions=output_attentions,
|
881 |
+
use_cache=use_cache,
|
882 |
+
cache_position=cache_position,
|
883 |
+
position_embeddings=position_embeddings,
|
884 |
+
)
|
885 |
+
|
886 |
+
hidden_states = outputs[0]
|
887 |
+
if use_cache:
|
888 |
+
next_decoder_cache = outputs[2 if output_attentions else 1]
|
889 |
+
|
890 |
+
if output_attentions:
|
891 |
+
all_self_attns += (outputs[1],)
|
892 |
+
|
893 |
+
hidden_states = self.ln_f(hidden_states)
|
894 |
+
# Add last hidden state
|
895 |
+
if output_hidden_states:
|
896 |
+
all_hidden_states += (hidden_states,)
|
897 |
+
|
898 |
+
next_cache = None
|
899 |
+
if use_cache:
|
900 |
+
next_cache = next_decoder_cache.to_legacy_cache() if return_legacy_cache else next_decoder_cache
|
901 |
+
if not return_dict:
|
902 |
+
return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
|
903 |
+
|
904 |
+
return BaseModelOutputWithPast(
|
905 |
+
last_hidden_state=hidden_states,
|
906 |
+
past_key_values=next_cache,
|
907 |
+
hidden_states=all_hidden_states,
|
908 |
+
attentions=all_self_attns,
|
909 |
+
)
|
910 |
+
|
911 |
+
def _update_causal_mask(
|
912 |
+
self,
|
913 |
+
attention_mask: torch.Tensor,
|
914 |
+
input_tensor: torch.Tensor,
|
915 |
+
cache_position: torch.Tensor,
|
916 |
+
past_key_values: Cache,
|
917 |
+
output_attentions: bool,
|
918 |
+
):
|
919 |
+
# TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static
|
920 |
+
# KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes.
|
921 |
+
# (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using
|
922 |
+
# `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114
|
923 |
+
|
924 |
+
if self.config._attn_implementation == "flash_attention_2":
|
925 |
+
if attention_mask is not None and 0.0 in attention_mask:
|
926 |
+
return attention_mask
|
927 |
+
return None
|
928 |
+
|
929 |
+
# For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
|
930 |
+
# order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
|
931 |
+
# to infer the attention mask.
|
932 |
+
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
933 |
+
using_static_cache = isinstance(past_key_values, StaticCache)
|
934 |
+
|
935 |
+
# When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
|
936 |
+
if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions:
|
937 |
+
if AttentionMaskConverter._ignore_causal_mask_sdpa(
|
938 |
+
attention_mask,
|
939 |
+
inputs_embeds=input_tensor,
|
940 |
+
past_key_values_length=past_seen_tokens,
|
941 |
+
is_training=self.training,
|
942 |
+
):
|
943 |
+
return None
|
944 |
+
|
945 |
+
dtype, device = input_tensor.dtype, input_tensor.device
|
946 |
+
min_dtype = torch.finfo(dtype).min
|
947 |
+
sequence_length = input_tensor.shape[1]
|
948 |
+
if using_static_cache:
|
949 |
+
target_length = past_key_values.get_max_length()
|
950 |
+
else:
|
951 |
+
target_length = (
|
952 |
+
attention_mask.shape[-1]
|
953 |
+
if isinstance(attention_mask, torch.Tensor)
|
954 |
+
else past_seen_tokens + sequence_length + 1
|
955 |
+
)
|
956 |
+
|
957 |
+
# In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
|
958 |
+
causal_mask = _prepare_4d_causal_attention_mask_with_cache_position(
|
959 |
+
attention_mask,
|
960 |
+
sequence_length=sequence_length,
|
961 |
+
target_length=target_length,
|
962 |
+
dtype=dtype,
|
963 |
+
device=device,
|
964 |
+
min_dtype=min_dtype,
|
965 |
+
cache_position=cache_position,
|
966 |
+
batch_size=input_tensor.shape[0],
|
967 |
+
)
|
968 |
+
|
969 |
+
if (
|
970 |
+
self.config._attn_implementation == "sdpa"
|
971 |
+
and attention_mask is not None
|
972 |
+
and attention_mask.device.type == "cuda"
|
973 |
+
and not output_attentions
|
974 |
+
):
|
975 |
+
# Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
|
976 |
+
# using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
|
977 |
+
# Details: https://github.com/pytorch/pytorch/issues/110213
|
978 |
+
causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)
|
979 |
+
|
980 |
+
return causal_mask
|
981 |
+
|
982 |
+
|
983 |
+
@add_start_docstrings(
|
984 |
+
"""
|
985 |
+
The EXAONE Model transformer with a language modeling head on top (linear layer with weights tied to the input
|
986 |
+
embeddings).
|
987 |
+
""",
|
988 |
+
EXAONE_START_DOCSTRING,
|
989 |
+
)
|
990 |
+
class ExaoneForCausalLM(ExaonePreTrainedModel, GenerationMixin):
|
991 |
+
_tied_weights_keys = ["lm_head.weight"]
|
992 |
+
|
993 |
+
def __init__(self, config):
|
994 |
+
super().__init__(config)
|
995 |
+
self.transformer = ExaoneModel(config)
|
996 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
997 |
+
self.config = config
|
998 |
+
# Initialize weights and apply final processing
|
999 |
+
self.post_init()
|
1000 |
+
|
1001 |
+
def get_output_embeddings(self):
|
1002 |
+
return self.lm_head
|
1003 |
+
|
1004 |
+
def set_output_embeddings(self, new_embeddings):
|
1005 |
+
self.lm_head = new_embeddings
|
1006 |
+
|
1007 |
+
@add_start_docstrings_to_model_forward(EXAONE_INPUTS_DOCSTRING)
|
1008 |
+
@add_code_sample_docstrings(
|
1009 |
+
checkpoint=_CHECKPOINT_FOR_DOC,
|
1010 |
+
output_type=BaseModelOutputWithPast,
|
1011 |
+
config_class=_CONFIG_FOR_DOC,
|
1012 |
+
)
|
1013 |
+
def forward(
|
1014 |
+
self,
|
1015 |
+
input_ids: Optional[torch.Tensor] = None,
|
1016 |
+
attention_mask: Optional[torch.Tensor] = None,
|
1017 |
+
position_ids: Optional[torch.Tensor] = None,
|
1018 |
+
past_key_values: Optional[Cache] = None,
|
1019 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
1020 |
+
labels: Optional[torch.Tensor] = None,
|
1021 |
+
use_cache: Optional[bool] = None,
|
1022 |
+
output_attentions: Optional[bool] = None,
|
1023 |
+
output_hidden_states: Optional[bool] = None,
|
1024 |
+
return_dict: Optional[bool] = None,
|
1025 |
+
cache_position: Optional[torch.LongTensor] = None,
|
1026 |
+
) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPast]:
|
1027 |
+
r"""
|
1028 |
+
Args:
|
1029 |
+
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
1030 |
+
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
|
1031 |
+
`labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
|
1032 |
+
are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
|
1033 |
+
|
1034 |
+
Example:
|
1035 |
+
|
1036 |
+
```python
|
1037 |
+
>>> from transformers import AutoModelForCausalLM, AutoTokenizer
|
1038 |
+
|
1039 |
+
>>> model = AutoModelForCausalLM.from_pretrained("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct",
|
1040 |
+
trust_remote_code=True)
|
1041 |
+
>>> tokenizer = AutoTokenizer.from_pretrained("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct")
|
1042 |
+
|
1043 |
+
>>> prompt = "Explain how wonderful you are"
|
1044 |
+
>>> messages = [
|
1045 |
+
{"role": "system", "content": "You are a helpful assistant."},
|
1046 |
+
{"role": "user", "content": prompt}
|
1047 |
+
]
|
1048 |
+
>>> input_ids = tokenizer.apply_chat_template(
|
1049 |
+
messages,
|
1050 |
+
tokenize=True,
|
1051 |
+
add_generation_prompt=True,
|
1052 |
+
return_tensors="pt"
|
1053 |
+
)
|
1054 |
+
|
1055 |
+
>>> output = model.generate(input_ids, max_new_tokens=128)
|
1056 |
+
>>> tokenizer.decode(output[0], skip_special_tokens=True)
|
1057 |
+
"[|system|]You are a helpful assistant.\n[|user|]Explain how wonderful you are\n[|assistant|]Thank you for your kind words! I'm here to assist you with information, answer questions, and help you in any way I can. My goal is to provide accurate, helpful, and timely responses. Whether you need help with a specific task, want to learn something new, or just need someone to talk to, I'm here for you. How can I assist you today?"
|
1058 |
+
```
|
1059 |
+
"""
|
1060 |
+
|
1061 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
1062 |
+
output_hidden_states = (
|
1063 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
1064 |
+
)
|
1065 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
1066 |
+
transformer_outputs = self.transformer(
|
1067 |
+
input_ids,
|
1068 |
+
attention_mask=attention_mask,
|
1069 |
+
past_key_values=past_key_values,
|
1070 |
+
position_ids=position_ids,
|
1071 |
+
inputs_embeds=inputs_embeds,
|
1072 |
+
use_cache=use_cache,
|
1073 |
+
output_attentions=output_attentions,
|
1074 |
+
output_hidden_states=output_hidden_states,
|
1075 |
+
return_dict=return_dict,
|
1076 |
+
cache_position=cache_position,
|
1077 |
+
)
|
1078 |
+
hidden_states = transformer_outputs[0]
|
1079 |
+
lm_logits = self.lm_head(hidden_states)
|
1080 |
+
lm_logits = lm_logits.float()
|
1081 |
+
loss = None
|
1082 |
+
if labels is not None:
|
1083 |
+
lm_logits = lm_logits.to(torch.float32)
|
1084 |
+
|
1085 |
+
# Shift so that tokens < n predict n
|
1086 |
+
shift_logits = lm_logits[..., :-1, :].contiguous()
|
1087 |
+
shift_labels = labels[..., 1:].contiguous()
|
1088 |
+
# Flatten the tokens
|
1089 |
+
loss_fct = CrossEntropyLoss()
|
1090 |
+
loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
|
1091 |
+
|
1092 |
+
lm_logits = lm_logits.to(hidden_states.dtype)
|
1093 |
+
loss = loss.to(hidden_states.dtype)
|
1094 |
+
|
1095 |
+
if not return_dict:
|
1096 |
+
output = (lm_logits,) + transformer_outputs[1:]
|
1097 |
+
return ((loss,) + output) if loss is not None else output
|
1098 |
+
|
1099 |
+
return CausalLMOutputWithPast(
|
1100 |
+
loss=loss,
|
1101 |
+
logits=lm_logits,
|
1102 |
+
past_key_values=transformer_outputs.past_key_values,
|
1103 |
+
hidden_states=transformer_outputs.hidden_states,
|
1104 |
+
attentions=transformer_outputs.attentions,
|
1105 |
+
)
|
1106 |
+
|
1107 |
+
def prepare_inputs_for_generation(
|
1108 |
+
self,
|
1109 |
+
input_ids,
|
1110 |
+
past_key_values=None,
|
1111 |
+
attention_mask=None,
|
1112 |
+
inputs_embeds=None,
|
1113 |
+
cache_position=None,
|
1114 |
+
position_ids=None,
|
1115 |
+
use_cache=True,
|
1116 |
+
**kwargs,
|
1117 |
+
):
|
1118 |
+
# If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens
|
1119 |
+
# Exception 1: when passing input_embeds, input_ids may be missing entries
|
1120 |
+
# Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here
|
1121 |
+
if past_key_values is not None:
|
1122 |
+
if inputs_embeds is not None: # Exception 1
|
1123 |
+
input_ids = input_ids[:, -cache_position.shape[0] :]
|
1124 |
+
elif input_ids.shape[1] != cache_position.shape[0]: # Default case (the "else", a no op, is Exception 2)
|
1125 |
+
input_ids = input_ids[:, cache_position]
|
1126 |
+
|
1127 |
+
if attention_mask is not None and position_ids is None:
|
1128 |
+
# create position_ids on the fly for batch generation
|
1129 |
+
position_ids = attention_mask.long().cumsum(-1) - 1
|
1130 |
+
position_ids.masked_fill_(attention_mask == 0, 1)
|
1131 |
+
if past_key_values:
|
1132 |
+
position_ids = position_ids[:, -input_ids.shape[1] :]
|
1133 |
+
|
1134 |
+
# This `clone` call is needed to avoid recapturing cuda graphs with `torch.compile`'s `mode="reduce-overhead`, as otherwise the input `position_ids` would have various stride during the decoding. Here, simply using `.contiguous()` is not sufficient as in the batch size = 1 case, `position_ids` is already contiguous but with varying stride which retriggers a capture.
|
1135 |
+
position_ids = position_ids.clone(memory_format=torch.contiguous_format)
|
1136 |
+
|
1137 |
+
# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
|
1138 |
+
if inputs_embeds is not None and cache_position[0] == 0:
|
1139 |
+
model_inputs = {"inputs_embeds": inputs_embeds, "input_ids": None}
|
1140 |
+
else:
|
1141 |
+
model_inputs = {"input_ids": input_ids, "inputs_embeds": None}
|
1142 |
+
|
1143 |
+
if isinstance(past_key_values, StaticCache) and attention_mask.ndim == 2:
|
1144 |
+
if inputs_embeds is not None:
|
1145 |
+
batch_size, sequence_length, _ = inputs_embeds.shape
|
1146 |
+
device = inputs_embeds.device
|
1147 |
+
else:
|
1148 |
+
batch_size, sequence_length = input_ids.shape
|
1149 |
+
device = input_ids.device
|
1150 |
+
|
1151 |
+
dtype = self.lm_head.weight.dtype
|
1152 |
+
min_dtype = torch.finfo(dtype).min
|
1153 |
+
|
1154 |
+
attention_mask = _prepare_4d_causal_attention_mask_with_cache_position(
|
1155 |
+
attention_mask,
|
1156 |
+
sequence_length=sequence_length,
|
1157 |
+
target_length=past_key_values.get_max_length(),
|
1158 |
+
dtype=dtype,
|
1159 |
+
device=device,
|
1160 |
+
min_dtype=min_dtype,
|
1161 |
+
cache_position=cache_position,
|
1162 |
+
batch_size=batch_size,
|
1163 |
+
)
|
1164 |
+
|
1165 |
+
model_inputs.update(
|
1166 |
+
{
|
1167 |
+
"position_ids": position_ids,
|
1168 |
+
"cache_position": cache_position,
|
1169 |
+
"past_key_values": past_key_values,
|
1170 |
+
"use_cache": use_cache,
|
1171 |
+
"attention_mask": attention_mask,
|
1172 |
+
}
|
1173 |
+
)
|
1174 |
+
return model_inputs
|
1175 |
+
|
1176 |
+
|
1177 |
+
@add_start_docstrings(
|
1178 |
+
"""
|
1179 |
+
The EXAONE Model transformer with a sequence classification head on top (linear layer).
|
1180 |
+
|
1181 |
+
[`ExaoneForSequenceClassification`] uses the last token in order to do the classification, as
|
1182 |
+
other causal models (e.g. GPT-1) do.
|
1183 |
+
|
1184 |
+
Since it does classification on the last token, it requires to know the position of the last token. If a
|
1185 |
+
`pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
|
1186 |
+
row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
|
1187 |
+
guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
|
1188 |
+
the last value in each row of the batch).
|
1189 |
+
""",
|
1190 |
+
EXAONE_START_DOCSTRING,
|
1191 |
+
)
|
1192 |
+
class ExaoneForSequenceClassification(ExaonePreTrainedModel):
|
1193 |
+
def __init__(self, config):
|
1194 |
+
super().__init__(config)
|
1195 |
+
self.num_labels = config.num_labels
|
1196 |
+
self.transformer = ExaoneModel(config)
|
1197 |
+
self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
|
1198 |
+
|
1199 |
+
# Initialize weights and apply final processing
|
1200 |
+
self.post_init()
|
1201 |
+
|
1202 |
+
@add_start_docstrings_to_model_forward(EXAONE_INPUTS_DOCSTRING)
|
1203 |
+
@add_code_sample_docstrings(
|
1204 |
+
checkpoint=_CHECKPOINT_FOR_DOC,
|
1205 |
+
output_type=SequenceClassifierOutputWithPast,
|
1206 |
+
config_class=_CONFIG_FOR_DOC,
|
1207 |
+
)
|
1208 |
+
def forward(
|
1209 |
+
self,
|
1210 |
+
input_ids: Optional[torch.Tensor] = None,
|
1211 |
+
attention_mask: Optional[torch.Tensor] = None,
|
1212 |
+
position_ids: Optional[torch.Tensor] = None,
|
1213 |
+
past_key_values: Optional[Cache] = None,
|
1214 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
1215 |
+
labels: Optional[torch.Tensor] = None,
|
1216 |
+
use_cache: Optional[bool] = None,
|
1217 |
+
output_attentions: Optional[bool] = None,
|
1218 |
+
output_hidden_states: Optional[bool] = None,
|
1219 |
+
return_dict: Optional[bool] = None,
|
1220 |
+
) -> Union[Tuple[torch.Tensor], SequenceClassifierOutputWithPast]:
|
1221 |
+
r"""
|
1222 |
+
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
1223 |
+
Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
|
1224 |
+
config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
|
1225 |
+
`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
|
1226 |
+
"""
|
1227 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
1228 |
+
|
1229 |
+
transformer_outputs = self.transformer(
|
1230 |
+
input_ids,
|
1231 |
+
attention_mask=attention_mask,
|
1232 |
+
position_ids=position_ids,
|
1233 |
+
past_key_values=past_key_values,
|
1234 |
+
inputs_embeds=inputs_embeds,
|
1235 |
+
use_cache=use_cache,
|
1236 |
+
output_attentions=output_attentions,
|
1237 |
+
output_hidden_states=output_hidden_states,
|
1238 |
+
return_dict=return_dict,
|
1239 |
+
)
|
1240 |
+
hidden_states = transformer_outputs[0]
|
1241 |
+
logits = self.score(hidden_states)
|
1242 |
+
|
1243 |
+
if input_ids is not None:
|
1244 |
+
batch_size, sequence_length = input_ids.shape[:2]
|
1245 |
+
else:
|
1246 |
+
batch_size, sequence_length = inputs_embeds.shape[:2]
|
1247 |
+
|
1248 |
+
if self.config.pad_token_id is None and batch_size != 1:
|
1249 |
+
raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
|
1250 |
+
if self.config.pad_token_id is None:
|
1251 |
+
sequence_lengths = -1
|
1252 |
+
else:
|
1253 |
+
if input_ids is not None:
|
1254 |
+
# if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
|
1255 |
+
sequence_lengths = torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1
|
1256 |
+
sequence_lengths = sequence_lengths % input_ids.shape[-1]
|
1257 |
+
sequence_lengths = sequence_lengths.to(logits.device)
|
1258 |
+
else:
|
1259 |
+
sequence_lengths = -1
|
1260 |
+
logger.warning(
|
1261 |
+
f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be "
|
1262 |
+
"unexpected if using padding tokens in conjunction with `inputs_embeds.`"
|
1263 |
+
)
|
1264 |
+
|
1265 |
+
pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
|
1266 |
+
|
1267 |
+
loss = None
|
1268 |
+
if labels is not None:
|
1269 |
+
labels = labels.to(logits.device)
|
1270 |
+
if self.config.problem_type is None:
|
1271 |
+
if self.num_labels == 1:
|
1272 |
+
self.config.problem_type = "regression"
|
1273 |
+
elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
|
1274 |
+
self.config.problem_type = "single_label_classification"
|
1275 |
+
else:
|
1276 |
+
self.config.problem_type = "multi_label_classification"
|
1277 |
+
|
1278 |
+
if self.config.problem_type == "regression":
|
1279 |
+
loss_fct = MSELoss()
|
1280 |
+
if self.num_labels == 1:
|
1281 |
+
loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
|
1282 |
+
else:
|
1283 |
+
loss = loss_fct(pooled_logits, labels)
|
1284 |
+
elif self.config.problem_type == "single_label_classification":
|
1285 |
+
loss_fct = CrossEntropyLoss()
|
1286 |
+
loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
|
1287 |
+
elif self.config.problem_type == "multi_label_classification":
|
1288 |
+
loss_fct = BCEWithLogitsLoss()
|
1289 |
+
loss = loss_fct(pooled_logits, labels)
|
1290 |
+
if not return_dict:
|
1291 |
+
output = (pooled_logits,) + transformer_outputs[1:]
|
1292 |
+
return ((loss,) + output) if loss is not None else output
|
1293 |
+
|
1294 |
+
return SequenceClassifierOutputWithPast(
|
1295 |
+
loss=loss,
|
1296 |
+
logits=pooled_logits,
|
1297 |
+
past_key_values=transformer_outputs.past_key_values,
|
1298 |
+
hidden_states=transformer_outputs.hidden_states,
|
1299 |
+
attentions=transformer_outputs.attentions,
|
1300 |
+
)
|
1301 |
+
|
1302 |
+
|
1303 |
+
@add_start_docstrings(
|
1304 |
+
"""
|
1305 |
+
The EXAONE Model transformer with a span classification head on top for extractive question-answering tasks like
|
1306 |
+
SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
|
1307 |
+
""",
|
1308 |
+
EXAONE_START_DOCSTRING,
|
1309 |
+
)
|
1310 |
+
class ExaoneForQuestionAnswering(ExaonePreTrainedModel):
|
1311 |
+
def __init__(self, config):
|
1312 |
+
super().__init__(config)
|
1313 |
+
self.num_labels = config.num_labels
|
1314 |
+
self.transformer = ExaoneModel(config)
|
1315 |
+
self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
|
1316 |
+
|
1317 |
+
# Model parallel
|
1318 |
+
self.model_parallel = False
|
1319 |
+
self.device_map = None
|
1320 |
+
|
1321 |
+
# Initialize weights and apply final processing
|
1322 |
+
self.post_init()
|
1323 |
+
|
1324 |
+
def forward(
|
1325 |
+
self,
|
1326 |
+
input_ids: Optional[torch.LongTensor] = None,
|
1327 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
1328 |
+
position_ids: Optional[torch.LongTensor] = None,
|
1329 |
+
past_key_values: Optional[Cache] = None,
|
1330 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
1331 |
+
start_positions: Optional[torch.LongTensor] = None,
|
1332 |
+
end_positions: Optional[torch.LongTensor] = None,
|
1333 |
+
output_attentions: Optional[bool] = None,
|
1334 |
+
output_hidden_states: Optional[bool] = None,
|
1335 |
+
return_dict: Optional[bool] = None,
|
1336 |
+
) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
|
1337 |
+
r"""
|
1338 |
+
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
1339 |
+
Labels for position (index) of the start of the labelled span for computing the token classification loss.
|
1340 |
+
Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
|
1341 |
+
sequence are not taken into account for computing the loss.
|
1342 |
+
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
1343 |
+
Labels for position (index) of the end of the labelled span for computing the token classification loss.
|
1344 |
+
Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
|
1345 |
+
sequence are not taken into account for computing the loss.
|
1346 |
+
"""
|
1347 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
1348 |
+
|
1349 |
+
outputs = self.transformer(
|
1350 |
+
input_ids,
|
1351 |
+
attention_mask=attention_mask,
|
1352 |
+
position_ids=position_ids,
|
1353 |
+
past_key_values=past_key_values,
|
1354 |
+
inputs_embeds=inputs_embeds,
|
1355 |
+
output_attentions=output_attentions,
|
1356 |
+
output_hidden_states=output_hidden_states,
|
1357 |
+
return_dict=return_dict,
|
1358 |
+
)
|
1359 |
+
|
1360 |
+
sequence_output = outputs[0]
|
1361 |
+
|
1362 |
+
logits = self.qa_outputs(sequence_output)
|
1363 |
+
start_logits, end_logits = logits.split(1, dim=-1)
|
1364 |
+
start_logits = start_logits.squeeze(-1).contiguous()
|
1365 |
+
end_logits = end_logits.squeeze(-1).contiguous()
|
1366 |
+
|
1367 |
+
total_loss = None
|
1368 |
+
if start_positions is not None and end_positions is not None:
|
1369 |
+
# If we are on multi-GPU, split add a dimension
|
1370 |
+
if len(start_positions.size()) > 1:
|
1371 |
+
start_positions = start_positions.squeeze(-1).to(start_logits.device)
|
1372 |
+
if len(end_positions.size()) > 1:
|
1373 |
+
end_positions = end_positions.squeeze(-1).to(end_logits.device)
|
1374 |
+
# sometimes the start/end positions are outside our model inputs, we ignore these terms
|
1375 |
+
ignored_index = start_logits.size(1)
|
1376 |
+
start_positions = start_positions.clamp(0, ignored_index)
|
1377 |
+
end_positions = end_positions.clamp(0, ignored_index)
|
1378 |
+
|
1379 |
+
loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
|
1380 |
+
start_loss = loss_fct(start_logits, start_positions)
|
1381 |
+
end_loss = loss_fct(end_logits, end_positions)
|
1382 |
+
total_loss = (start_loss + end_loss) / 2
|
1383 |
+
|
1384 |
+
if not return_dict:
|
1385 |
+
output = (start_logits, end_logits) + outputs[2:]
|
1386 |
+
return ((total_loss,) + output) if total_loss is not None else output
|
1387 |
+
|
1388 |
+
return QuestionAnsweringModelOutput(
|
1389 |
+
loss=total_loss,
|
1390 |
+
start_logits=start_logits,
|
1391 |
+
end_logits=end_logits,
|
1392 |
+
hidden_states=outputs.hidden_states,
|
1393 |
+
attentions=outputs.attentions,
|
1394 |
+
)
|
special_tokens_map.json
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": {
|
3 |
+
"content": "[BOS]",
|
4 |
+
"lstrip": false,
|
5 |
+
"normalized": false,
|
6 |
+
"rstrip": false,
|
7 |
+
"single_word": false
|
8 |
+
},
|
9 |
+
"eos_token": {
|
10 |
+
"content": "[|endofturn|]",
|
11 |
+
"lstrip": false,
|
12 |
+
"normalized": false,
|
13 |
+
"rstrip": false,
|
14 |
+
"single_word": false
|
15 |
+
},
|
16 |
+
"pad_token": {
|
17 |
+
"content": "[PAD]",
|
18 |
+
"lstrip": false,
|
19 |
+
"normalized": false,
|
20 |
+
"rstrip": false,
|
21 |
+
"single_word": false
|
22 |
+
},
|
23 |
+
"unk_token": {
|
24 |
+
"content": "[UNK]",
|
25 |
+
"lstrip": false,
|
26 |
+
"normalized": false,
|
27 |
+
"rstrip": false,
|
28 |
+
"single_word": false
|
29 |
+
}
|
30 |
+
}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
ADDED
@@ -0,0 +1,3221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"added_tokens_decoder": {
|
4 |
+
"0": {
|
5 |
+
"content": "[PAD]",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": false,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false,
|
10 |
+
"special": true
|
11 |
+
},
|
12 |
+
"1": {
|
13 |
+
"content": "[BOS]",
|
14 |
+
"lstrip": false,
|
15 |
+
"normalized": false,
|
16 |
+
"rstrip": false,
|
17 |
+
"single_word": false,
|
18 |
+
"special": true
|
19 |
+
},
|
20 |
+
"2": {
|
21 |
+
"content": "[EOS]",
|
22 |
+
"lstrip": false,
|
23 |
+
"normalized": false,
|
24 |
+
"rstrip": false,
|
25 |
+
"single_word": false,
|
26 |
+
"special": true
|
27 |
+
},
|
28 |
+
"3": {
|
29 |
+
"content": "[UNK]",
|
30 |
+
"lstrip": false,
|
31 |
+
"normalized": false,
|
32 |
+
"rstrip": false,
|
33 |
+
"single_word": false,
|
34 |
+
"special": true
|
35 |
+
},
|
36 |
+
"4": {
|
37 |
+
"content": " ",
|
38 |
+
"lstrip": false,
|
39 |
+
"normalized": false,
|
40 |
+
"rstrip": false,
|
41 |
+
"single_word": false,
|
42 |
+
"special": false
|
43 |
+
},
|
44 |
+
"5": {
|
45 |
+
"content": " ",
|
46 |
+
"lstrip": false,
|
47 |
+
"normalized": false,
|
48 |
+
"rstrip": false,
|
49 |
+
"single_word": false,
|
50 |
+
"special": false
|
51 |
+
},
|
52 |
+
"6": {
|
53 |
+
"content": " ",
|
54 |
+
"lstrip": false,
|
55 |
+
"normalized": false,
|
56 |
+
"rstrip": false,
|
57 |
+
"single_word": false,
|
58 |
+
"special": false
|
59 |
+
},
|
60 |
+
"7": {
|
61 |
+
"content": " ",
|
62 |
+
"lstrip": false,
|
63 |
+
"normalized": false,
|
64 |
+
"rstrip": false,
|
65 |
+
"single_word": false,
|
66 |
+
"special": false
|
67 |
+
},
|
68 |
+
"8": {
|
69 |
+
"content": " ",
|
70 |
+
"lstrip": false,
|
71 |
+
"normalized": false,
|
72 |
+
"rstrip": false,
|
73 |
+
"single_word": false,
|
74 |
+
"special": false
|
75 |
+
},
|
76 |
+
"9": {
|
77 |
+
"content": " ",
|
78 |
+
"lstrip": false,
|
79 |
+
"normalized": false,
|
80 |
+
"rstrip": false,
|
81 |
+
"single_word": false,
|
82 |
+
"special": false
|
83 |
+
},
|
84 |
+
"10": {
|
85 |
+
"content": " ",
|
86 |
+
"lstrip": false,
|
87 |
+
"normalized": false,
|
88 |
+
"rstrip": false,
|
89 |
+
"single_word": false,
|
90 |
+
"special": false
|
91 |
+
},
|
92 |
+
"11": {
|
93 |
+
"content": " ",
|
94 |
+
"lstrip": false,
|
95 |
+
"normalized": false,
|
96 |
+
"rstrip": false,
|
97 |
+
"single_word": false,
|
98 |
+
"special": false
|
99 |
+
},
|
100 |
+
"12": {
|
101 |
+
"content": " ",
|
102 |
+
"lstrip": false,
|
103 |
+
"normalized": false,
|
104 |
+
"rstrip": false,
|
105 |
+
"single_word": false,
|
106 |
+
"special": false
|
107 |
+
},
|
108 |
+
"13": {
|
109 |
+
"content": " ",
|
110 |
+
"lstrip": false,
|
111 |
+
"normalized": false,
|
112 |
+
"rstrip": false,
|
113 |
+
"single_word": false,
|
114 |
+
"special": false
|
115 |
+
},
|
116 |
+
"14": {
|
117 |
+
"content": " ",
|
118 |
+
"lstrip": false,
|
119 |
+
"normalized": false,
|
120 |
+
"rstrip": false,
|
121 |
+
"single_word": false,
|
122 |
+
"special": false
|
123 |
+
},
|
124 |
+
"15": {
|
125 |
+
"content": " ",
|
126 |
+
"lstrip": false,
|
127 |
+
"normalized": false,
|
128 |
+
"rstrip": false,
|
129 |
+
"single_word": false,
|
130 |
+
"special": false
|
131 |
+
},
|
132 |
+
"16": {
|
133 |
+
"content": " ",
|
134 |
+
"lstrip": false,
|
135 |
+
"normalized": false,
|
136 |
+
"rstrip": false,
|
137 |
+
"single_word": false,
|
138 |
+
"special": false
|
139 |
+
},
|
140 |
+
"17": {
|
141 |
+
"content": " ",
|
142 |
+
"lstrip": false,
|
143 |
+
"normalized": false,
|
144 |
+
"rstrip": false,
|
145 |
+
"single_word": false,
|
146 |
+
"special": false
|
147 |
+
},
|
148 |
+
"18": {
|
149 |
+
"content": " ",
|
150 |
+
"lstrip": false,
|
151 |
+
"normalized": false,
|
152 |
+
"rstrip": false,
|
153 |
+
"single_word": false,
|
154 |
+
"special": false
|
155 |
+
},
|
156 |
+
"19": {
|
157 |
+
"content": " ",
|
158 |
+
"lstrip": false,
|
159 |
+
"normalized": false,
|
160 |
+
"rstrip": false,
|
161 |
+
"single_word": false,
|
162 |
+
"special": false
|
163 |
+
},
|
164 |
+
"20": {
|
165 |
+
"content": " ",
|
166 |
+
"lstrip": false,
|
167 |
+
"normalized": false,
|
168 |
+
"rstrip": false,
|
169 |
+
"single_word": false,
|
170 |
+
"special": false
|
171 |
+
},
|
172 |
+
"21": {
|
173 |
+
"content": " ",
|
174 |
+
"lstrip": false,
|
175 |
+
"normalized": false,
|
176 |
+
"rstrip": false,
|
177 |
+
"single_word": false,
|
178 |
+
"special": false
|
179 |
+
},
|
180 |
+
"22": {
|
181 |
+
"content": " ",
|
182 |
+
"lstrip": false,
|
183 |
+
"normalized": false,
|
184 |
+
"rstrip": false,
|
185 |
+
"single_word": false,
|
186 |
+
"special": false
|
187 |
+
},
|
188 |
+
"23": {
|
189 |
+
"content": " ",
|
190 |
+
"lstrip": false,
|
191 |
+
"normalized": false,
|
192 |
+
"rstrip": false,
|
193 |
+
"single_word": false,
|
194 |
+
"special": false
|
195 |
+
},
|
196 |
+
"24": {
|
197 |
+
"content": " ",
|
198 |
+
"lstrip": false,
|
199 |
+
"normalized": false,
|
200 |
+
"rstrip": false,
|
201 |
+
"single_word": false,
|
202 |
+
"special": false
|
203 |
+
},
|
204 |
+
"25": {
|
205 |
+
"content": " ",
|
206 |
+
"lstrip": false,
|
207 |
+
"normalized": false,
|
208 |
+
"rstrip": false,
|
209 |
+
"single_word": false,
|
210 |
+
"special": false
|
211 |
+
},
|
212 |
+
"26": {
|
213 |
+
"content": " ",
|
214 |
+
"lstrip": false,
|
215 |
+
"normalized": false,
|
216 |
+
"rstrip": false,
|
217 |
+
"single_word": false,
|
218 |
+
"special": false
|
219 |
+
},
|
220 |
+
"27": {
|
221 |
+
"content": " ",
|
222 |
+
"lstrip": false,
|
223 |
+
"normalized": false,
|
224 |
+
"rstrip": false,
|
225 |
+
"single_word": false,
|
226 |
+
"special": false
|
227 |
+
},
|
228 |
+
"28": {
|
229 |
+
"content": " ",
|
230 |
+
"lstrip": false,
|
231 |
+
"normalized": false,
|
232 |
+
"rstrip": false,
|
233 |
+
"single_word": false,
|
234 |
+
"special": false
|
235 |
+
},
|
236 |
+
"29": {
|
237 |
+
"content": " ",
|
238 |
+
"lstrip": false,
|
239 |
+
"normalized": false,
|
240 |
+
"rstrip": false,
|
241 |
+
"single_word": false,
|
242 |
+
"special": false
|
243 |
+
},
|
244 |
+
"30": {
|
245 |
+
"content": " ",
|
246 |
+
"lstrip": false,
|
247 |
+
"normalized": false,
|
248 |
+
"rstrip": false,
|
249 |
+
"single_word": false,
|
250 |
+
"special": false
|
251 |
+
},
|
252 |
+
"31": {
|
253 |
+
"content": " ",
|
254 |
+
"lstrip": false,
|
255 |
+
"normalized": false,
|
256 |
+
"rstrip": false,
|
257 |
+
"single_word": false,
|
258 |
+
"special": false
|
259 |
+
},
|
260 |
+
"32": {
|
261 |
+
"content": " ",
|
262 |
+
"lstrip": false,
|
263 |
+
"normalized": false,
|
264 |
+
"rstrip": false,
|
265 |
+
"single_word": false,
|
266 |
+
"special": false
|
267 |
+
},
|
268 |
+
"33": {
|
269 |
+
"content": " ",
|
270 |
+
"lstrip": false,
|
271 |
+
"normalized": false,
|
272 |
+
"rstrip": false,
|
273 |
+
"single_word": false,
|
274 |
+
"special": false
|
275 |
+
},
|
276 |
+
"34": {
|
277 |
+
"content": "\t\t\t\t\t\t\t\t\t",
|
278 |
+
"lstrip": false,
|
279 |
+
"normalized": false,
|
280 |
+
"rstrip": false,
|
281 |
+
"single_word": false,
|
282 |
+
"special": false
|
283 |
+
},
|
284 |
+
"35": {
|
285 |
+
"content": "\t\t\t\t\t\t\t\t",
|
286 |
+
"lstrip": false,
|
287 |
+
"normalized": false,
|
288 |
+
"rstrip": false,
|
289 |
+
"single_word": false,
|
290 |
+
"special": false
|
291 |
+
},
|
292 |
+
"36": {
|
293 |
+
"content": "\t\t\t\t\t\t\t",
|
294 |
+
"lstrip": false,
|
295 |
+
"normalized": false,
|
296 |
+
"rstrip": false,
|
297 |
+
"single_word": false,
|
298 |
+
"special": false
|
299 |
+
},
|
300 |
+
"37": {
|
301 |
+
"content": "\t\t\t\t\t\t",
|
302 |
+
"lstrip": false,
|
303 |
+
"normalized": false,
|
304 |
+
"rstrip": false,
|
305 |
+
"single_word": false,
|
306 |
+
"special": false
|
307 |
+
},
|
308 |
+
"38": {
|
309 |
+
"content": "\t\t\t\t\t",
|
310 |
+
"lstrip": false,
|
311 |
+
"normalized": false,
|
312 |
+
"rstrip": false,
|
313 |
+
"single_word": false,
|
314 |
+
"special": false
|
315 |
+
},
|
316 |
+
"39": {
|
317 |
+
"content": "\t\t\t\t",
|
318 |
+
"lstrip": false,
|
319 |
+
"normalized": false,
|
320 |
+
"rstrip": false,
|
321 |
+
"single_word": false,
|
322 |
+
"special": false
|
323 |
+
},
|
324 |
+
"40": {
|
325 |
+
"content": "\t\t\t",
|
326 |
+
"lstrip": false,
|
327 |
+
"normalized": false,
|
328 |
+
"rstrip": false,
|
329 |
+
"single_word": false,
|
330 |
+
"special": false
|
331 |
+
},
|
332 |
+
"41": {
|
333 |
+
"content": "\t\t",
|
334 |
+
"lstrip": false,
|
335 |
+
"normalized": false,
|
336 |
+
"rstrip": false,
|
337 |
+
"single_word": false,
|
338 |
+
"special": false
|
339 |
+
},
|
340 |
+
"42": {
|
341 |
+
"content": "<|endoftext|>",
|
342 |
+
"lstrip": false,
|
343 |
+
"normalized": false,
|
344 |
+
"rstrip": false,
|
345 |
+
"single_word": false,
|
346 |
+
"special": true
|
347 |
+
},
|
348 |
+
"43": {
|
349 |
+
"content": "<|c|>",
|
350 |
+
"lstrip": false,
|
351 |
+
"normalized": false,
|
352 |
+
"rstrip": false,
|
353 |
+
"single_word": false,
|
354 |
+
"special": true
|
355 |
+
},
|
356 |
+
"44": {
|
357 |
+
"content": "<|c++|>",
|
358 |
+
"lstrip": false,
|
359 |
+
"normalized": false,
|
360 |
+
"rstrip": false,
|
361 |
+
"single_word": false,
|
362 |
+
"special": true
|
363 |
+
},
|
364 |
+
"45": {
|
365 |
+
"content": "<|python|>",
|
366 |
+
"lstrip": false,
|
367 |
+
"normalized": false,
|
368 |
+
"rstrip": false,
|
369 |
+
"single_word": false,
|
370 |
+
"special": true
|
371 |
+
},
|
372 |
+
"46": {
|
373 |
+
"content": "<|javascript|>",
|
374 |
+
"lstrip": false,
|
375 |
+
"normalized": false,
|
376 |
+
"rstrip": false,
|
377 |
+
"single_word": false,
|
378 |
+
"special": true
|
379 |
+
},
|
380 |
+
"47": {
|
381 |
+
"content": "<|markdown|>",
|
382 |
+
"lstrip": false,
|
383 |
+
"normalized": false,
|
384 |
+
"rstrip": false,
|
385 |
+
"single_word": false,
|
386 |
+
"special": true
|
387 |
+
},
|
388 |
+
"48": {
|
389 |
+
"content": "<|html|>",
|
390 |
+
"lstrip": false,
|
391 |
+
"normalized": false,
|
392 |
+
"rstrip": false,
|
393 |
+
"single_word": false,
|
394 |
+
"special": true
|
395 |
+
},
|
396 |
+
"49": {
|
397 |
+
"content": "<|css|>",
|
398 |
+
"lstrip": false,
|
399 |
+
"normalized": false,
|
400 |
+
"rstrip": false,
|
401 |
+
"single_word": false,
|
402 |
+
"special": true
|
403 |
+
},
|
404 |
+
"50": {
|
405 |
+
"content": "<|vue|>",
|
406 |
+
"lstrip": false,
|
407 |
+
"normalized": false,
|
408 |
+
"rstrip": false,
|
409 |
+
"single_word": false,
|
410 |
+
"special": true
|
411 |
+
},
|
412 |
+
"51": {
|
413 |
+
"content": "<|java|>",
|
414 |
+
"lstrip": false,
|
415 |
+
"normalized": false,
|
416 |
+
"rstrip": false,
|
417 |
+
"single_word": false,
|
418 |
+
"special": true
|
419 |
+
},
|
420 |
+
"52": {
|
421 |
+
"content": "PI:URL",
|
422 |
+
"lstrip": false,
|
423 |
+
"normalized": false,
|
424 |
+
"rstrip": false,
|
425 |
+
"single_word": false,
|
426 |
+
"special": true
|
427 |
+
},
|
428 |
+
"53": {
|
429 |
+
"content": "PI:EMAIL",
|
430 |
+
"lstrip": false,
|
431 |
+
"normalized": false,
|
432 |
+
"rstrip": false,
|
433 |
+
"single_word": false,
|
434 |
+
"special": true
|
435 |
+
},
|
436 |
+
"54": {
|
437 |
+
"content": "PI:ACCOUNT_NUM",
|
438 |
+
"lstrip": false,
|
439 |
+
"normalized": false,
|
440 |
+
"rstrip": false,
|
441 |
+
"single_word": false,
|
442 |
+
"special": true
|
443 |
+
},
|
444 |
+
"55": {
|
445 |
+
"content": "PI:PHONE_NUM",
|
446 |
+
"lstrip": false,
|
447 |
+
"normalized": false,
|
448 |
+
"rstrip": false,
|
449 |
+
"single_word": false,
|
450 |
+
"special": true
|
451 |
+
},
|
452 |
+
"56": {
|
453 |
+
"content": "PI:BUSINESS_NUM",
|
454 |
+
"lstrip": false,
|
455 |
+
"normalized": false,
|
456 |
+
"rstrip": false,
|
457 |
+
"single_word": false,
|
458 |
+
"special": true
|
459 |
+
},
|
460 |
+
"57": {
|
461 |
+
"content": "PI:ANNON",
|
462 |
+
"lstrip": false,
|
463 |
+
"normalized": false,
|
464 |
+
"rstrip": false,
|
465 |
+
"single_word": false,
|
466 |
+
"special": true
|
467 |
+
},
|
468 |
+
"58": {
|
469 |
+
"content": "PI:KEY",
|
470 |
+
"lstrip": false,
|
471 |
+
"normalized": false,
|
472 |
+
"rstrip": false,
|
473 |
+
"single_word": false,
|
474 |
+
"special": true
|
475 |
+
},
|
476 |
+
"59": {
|
477 |
+
"content": "PI:ID",
|
478 |
+
"lstrip": false,
|
479 |
+
"normalized": false,
|
480 |
+
"rstrip": false,
|
481 |
+
"single_word": false,
|
482 |
+
"special": true
|
483 |
+
},
|
484 |
+
"60": {
|
485 |
+
"content": "PI:IP_ADDRESS",
|
486 |
+
"lstrip": false,
|
487 |
+
"normalized": false,
|
488 |
+
"rstrip": false,
|
489 |
+
"single_word": false,
|
490 |
+
"special": true
|
491 |
+
},
|
492 |
+
"61": {
|
493 |
+
"content": "PI:USER",
|
494 |
+
"lstrip": false,
|
495 |
+
"normalized": false,
|
496 |
+
"rstrip": false,
|
497 |
+
"single_word": false,
|
498 |
+
"special": true
|
499 |
+
},
|
500 |
+
"62": {
|
501 |
+
"content": "[unused0]",
|
502 |
+
"lstrip": false,
|
503 |
+
"normalized": false,
|
504 |
+
"rstrip": false,
|
505 |
+
"single_word": false,
|
506 |
+
"special": true
|
507 |
+
},
|
508 |
+
"63": {
|
509 |
+
"content": "[unused1]",
|
510 |
+
"lstrip": false,
|
511 |
+
"normalized": false,
|
512 |
+
"rstrip": false,
|
513 |
+
"single_word": false,
|
514 |
+
"special": true
|
515 |
+
},
|
516 |
+
"64": {
|
517 |
+
"content": "[unused2]",
|
518 |
+
"lstrip": false,
|
519 |
+
"normalized": false,
|
520 |
+
"rstrip": false,
|
521 |
+
"single_word": false,
|
522 |
+
"special": true
|
523 |
+
},
|
524 |
+
"65": {
|
525 |
+
"content": "[unused3]",
|
526 |
+
"lstrip": false,
|
527 |
+
"normalized": false,
|
528 |
+
"rstrip": false,
|
529 |
+
"single_word": false,
|
530 |
+
"special": true
|
531 |
+
},
|
532 |
+
"66": {
|
533 |
+
"content": "[unused4]",
|
534 |
+
"lstrip": false,
|
535 |
+
"normalized": false,
|
536 |
+
"rstrip": false,
|
537 |
+
"single_word": false,
|
538 |
+
"special": true
|
539 |
+
},
|
540 |
+
"67": {
|
541 |
+
"content": "[unused5]",
|
542 |
+
"lstrip": false,
|
543 |
+
"normalized": false,
|
544 |
+
"rstrip": false,
|
545 |
+
"single_word": false,
|
546 |
+
"special": true
|
547 |
+
},
|
548 |
+
"68": {
|
549 |
+
"content": "[unused6]",
|
550 |
+
"lstrip": false,
|
551 |
+
"normalized": false,
|
552 |
+
"rstrip": false,
|
553 |
+
"single_word": false,
|
554 |
+
"special": true
|
555 |
+
},
|
556 |
+
"69": {
|
557 |
+
"content": "[unused7]",
|
558 |
+
"lstrip": false,
|
559 |
+
"normalized": false,
|
560 |
+
"rstrip": false,
|
561 |
+
"single_word": false,
|
562 |
+
"special": true
|
563 |
+
},
|
564 |
+
"70": {
|
565 |
+
"content": "[unused8]",
|
566 |
+
"lstrip": false,
|
567 |
+
"normalized": false,
|
568 |
+
"rstrip": false,
|
569 |
+
"single_word": false,
|
570 |
+
"special": true
|
571 |
+
},
|
572 |
+
"71": {
|
573 |
+
"content": "[unused9]",
|
574 |
+
"lstrip": false,
|
575 |
+
"normalized": false,
|
576 |
+
"rstrip": false,
|
577 |
+
"single_word": false,
|
578 |
+
"special": true
|
579 |
+
},
|
580 |
+
"72": {
|
581 |
+
"content": "[unused10]",
|
582 |
+
"lstrip": false,
|
583 |
+
"normalized": false,
|
584 |
+
"rstrip": false,
|
585 |
+
"single_word": false,
|
586 |
+
"special": true
|
587 |
+
},
|
588 |
+
"73": {
|
589 |
+
"content": "[unused11]",
|
590 |
+
"lstrip": false,
|
591 |
+
"normalized": false,
|
592 |
+
"rstrip": false,
|
593 |
+
"single_word": false,
|
594 |
+
"special": true
|
595 |
+
},
|
596 |
+
"74": {
|
597 |
+
"content": "[unused12]",
|
598 |
+
"lstrip": false,
|
599 |
+
"normalized": false,
|
600 |
+
"rstrip": false,
|
601 |
+
"single_word": false,
|
602 |
+
"special": true
|
603 |
+
},
|
604 |
+
"75": {
|
605 |
+
"content": "[unused13]",
|
606 |
+
"lstrip": false,
|
607 |
+
"normalized": false,
|
608 |
+
"rstrip": false,
|
609 |
+
"single_word": false,
|
610 |
+
"special": true
|
611 |
+
},
|
612 |
+
"76": {
|
613 |
+
"content": "[unused14]",
|
614 |
+
"lstrip": false,
|
615 |
+
"normalized": false,
|
616 |
+
"rstrip": false,
|
617 |
+
"single_word": false,
|
618 |
+
"special": true
|
619 |
+
},
|
620 |
+
"77": {
|
621 |
+
"content": "[unused15]",
|
622 |
+
"lstrip": false,
|
623 |
+
"normalized": false,
|
624 |
+
"rstrip": false,
|
625 |
+
"single_word": false,
|
626 |
+
"special": true
|
627 |
+
},
|
628 |
+
"78": {
|
629 |
+
"content": "[unused16]",
|
630 |
+
"lstrip": false,
|
631 |
+
"normalized": false,
|
632 |
+
"rstrip": false,
|
633 |
+
"single_word": false,
|
634 |
+
"special": true
|
635 |
+
},
|
636 |
+
"79": {
|
637 |
+
"content": "[unused17]",
|
638 |
+
"lstrip": false,
|
639 |
+
"normalized": false,
|
640 |
+
"rstrip": false,
|
641 |
+
"single_word": false,
|
642 |
+
"special": true
|
643 |
+
},
|
644 |
+
"80": {
|
645 |
+
"content": "[unused18]",
|
646 |
+
"lstrip": false,
|
647 |
+
"normalized": false,
|
648 |
+
"rstrip": false,
|
649 |
+
"single_word": false,
|
650 |
+
"special": true
|
651 |
+
},
|
652 |
+
"81": {
|
653 |
+
"content": "[unused19]",
|
654 |
+
"lstrip": false,
|
655 |
+
"normalized": false,
|
656 |
+
"rstrip": false,
|
657 |
+
"single_word": false,
|
658 |
+
"special": true
|
659 |
+
},
|
660 |
+
"82": {
|
661 |
+
"content": "[unused20]",
|
662 |
+
"lstrip": false,
|
663 |
+
"normalized": false,
|
664 |
+
"rstrip": false,
|
665 |
+
"single_word": false,
|
666 |
+
"special": true
|
667 |
+
},
|
668 |
+
"83": {
|
669 |
+
"content": "[unused21]",
|
670 |
+
"lstrip": false,
|
671 |
+
"normalized": false,
|
672 |
+
"rstrip": false,
|
673 |
+
"single_word": false,
|
674 |
+
"special": true
|
675 |
+
},
|
676 |
+
"84": {
|
677 |
+
"content": "[unused22]",
|
678 |
+
"lstrip": false,
|
679 |
+
"normalized": false,
|
680 |
+
"rstrip": false,
|
681 |
+
"single_word": false,
|
682 |
+
"special": true
|
683 |
+
},
|
684 |
+
"85": {
|
685 |
+
"content": "[unused23]",
|
686 |
+
"lstrip": false,
|
687 |
+
"normalized": false,
|
688 |
+
"rstrip": false,
|
689 |
+
"single_word": false,
|
690 |
+
"special": true
|
691 |
+
},
|
692 |
+
"86": {
|
693 |
+
"content": "[unused24]",
|
694 |
+
"lstrip": false,
|
695 |
+
"normalized": false,
|
696 |
+
"rstrip": false,
|
697 |
+
"single_word": false,
|
698 |
+
"special": true
|
699 |
+
},
|
700 |
+
"87": {
|
701 |
+
"content": "[unused25]",
|
702 |
+
"lstrip": false,
|
703 |
+
"normalized": false,
|
704 |
+
"rstrip": false,
|
705 |
+
"single_word": false,
|
706 |
+
"special": true
|
707 |
+
},
|
708 |
+
"88": {
|
709 |
+
"content": "[unused26]",
|
710 |
+
"lstrip": false,
|
711 |
+
"normalized": false,
|
712 |
+
"rstrip": false,
|
713 |
+
"single_word": false,
|
714 |
+
"special": true
|
715 |
+
},
|
716 |
+
"89": {
|
717 |
+
"content": "[unused27]",
|
718 |
+
"lstrip": false,
|
719 |
+
"normalized": false,
|
720 |
+
"rstrip": false,
|
721 |
+
"single_word": false,
|
722 |
+
"special": true
|
723 |
+
},
|
724 |
+
"90": {
|
725 |
+
"content": "[unused28]",
|
726 |
+
"lstrip": false,
|
727 |
+
"normalized": false,
|
728 |
+
"rstrip": false,
|
729 |
+
"single_word": false,
|
730 |
+
"special": true
|
731 |
+
},
|
732 |
+
"91": {
|
733 |
+
"content": "[unused29]",
|
734 |
+
"lstrip": false,
|
735 |
+
"normalized": false,
|
736 |
+
"rstrip": false,
|
737 |
+
"single_word": false,
|
738 |
+
"special": true
|
739 |
+
},
|
740 |
+
"92": {
|
741 |
+
"content": "[unused30]",
|
742 |
+
"lstrip": false,
|
743 |
+
"normalized": false,
|
744 |
+
"rstrip": false,
|
745 |
+
"single_word": false,
|
746 |
+
"special": true
|
747 |
+
},
|
748 |
+
"93": {
|
749 |
+
"content": "[unused31]",
|
750 |
+
"lstrip": false,
|
751 |
+
"normalized": false,
|
752 |
+
"rstrip": false,
|
753 |
+
"single_word": false,
|
754 |
+
"special": true
|
755 |
+
},
|
756 |
+
"94": {
|
757 |
+
"content": "[unused32]",
|
758 |
+
"lstrip": false,
|
759 |
+
"normalized": false,
|
760 |
+
"rstrip": false,
|
761 |
+
"single_word": false,
|
762 |
+
"special": true
|
763 |
+
},
|
764 |
+
"95": {
|
765 |
+
"content": "[unused33]",
|
766 |
+
"lstrip": false,
|
767 |
+
"normalized": false,
|
768 |
+
"rstrip": false,
|
769 |
+
"single_word": false,
|
770 |
+
"special": true
|
771 |
+
},
|
772 |
+
"96": {
|
773 |
+
"content": "[unused34]",
|
774 |
+
"lstrip": false,
|
775 |
+
"normalized": false,
|
776 |
+
"rstrip": false,
|
777 |
+
"single_word": false,
|
778 |
+
"special": true
|
779 |
+
},
|
780 |
+
"97": {
|
781 |
+
"content": "[unused35]",
|
782 |
+
"lstrip": false,
|
783 |
+
"normalized": false,
|
784 |
+
"rstrip": false,
|
785 |
+
"single_word": false,
|
786 |
+
"special": true
|
787 |
+
},
|
788 |
+
"98": {
|
789 |
+
"content": "[unused36]",
|
790 |
+
"lstrip": false,
|
791 |
+
"normalized": false,
|
792 |
+
"rstrip": false,
|
793 |
+
"single_word": false,
|
794 |
+
"special": true
|
795 |
+
},
|
796 |
+
"99": {
|
797 |
+
"content": "[unused37]",
|
798 |
+
"lstrip": false,
|
799 |
+
"normalized": false,
|
800 |
+
"rstrip": false,
|
801 |
+
"single_word": false,
|
802 |
+
"special": true
|
803 |
+
},
|
804 |
+
"100": {
|
805 |
+
"content": "[unused38]",
|
806 |
+
"lstrip": false,
|
807 |
+
"normalized": false,
|
808 |
+
"rstrip": false,
|
809 |
+
"single_word": false,
|
810 |
+
"special": true
|
811 |
+
},
|
812 |
+
"101": {
|
813 |
+
"content": "[unused39]",
|
814 |
+
"lstrip": false,
|
815 |
+
"normalized": false,
|
816 |
+
"rstrip": false,
|
817 |
+
"single_word": false,
|
818 |
+
"special": true
|
819 |
+
},
|
820 |
+
"102": {
|
821 |
+
"content": "[unused40]",
|
822 |
+
"lstrip": false,
|
823 |
+
"normalized": false,
|
824 |
+
"rstrip": false,
|
825 |
+
"single_word": false,
|
826 |
+
"special": true
|
827 |
+
},
|
828 |
+
"103": {
|
829 |
+
"content": "[unused41]",
|
830 |
+
"lstrip": false,
|
831 |
+
"normalized": false,
|
832 |
+
"rstrip": false,
|
833 |
+
"single_word": false,
|
834 |
+
"special": true
|
835 |
+
},
|
836 |
+
"104": {
|
837 |
+
"content": "[unused42]",
|
838 |
+
"lstrip": false,
|
839 |
+
"normalized": false,
|
840 |
+
"rstrip": false,
|
841 |
+
"single_word": false,
|
842 |
+
"special": true
|
843 |
+
},
|
844 |
+
"105": {
|
845 |
+
"content": "[unused43]",
|
846 |
+
"lstrip": false,
|
847 |
+
"normalized": false,
|
848 |
+
"rstrip": false,
|
849 |
+
"single_word": false,
|
850 |
+
"special": true
|
851 |
+
},
|
852 |
+
"106": {
|
853 |
+
"content": "[unused44]",
|
854 |
+
"lstrip": false,
|
855 |
+
"normalized": false,
|
856 |
+
"rstrip": false,
|
857 |
+
"single_word": false,
|
858 |
+
"special": true
|
859 |
+
},
|
860 |
+
"107": {
|
861 |
+
"content": "[unused45]",
|
862 |
+
"lstrip": false,
|
863 |
+
"normalized": false,
|
864 |
+
"rstrip": false,
|
865 |
+
"single_word": false,
|
866 |
+
"special": true
|
867 |
+
},
|
868 |
+
"108": {
|
869 |
+
"content": "[unused46]",
|
870 |
+
"lstrip": false,
|
871 |
+
"normalized": false,
|
872 |
+
"rstrip": false,
|
873 |
+
"single_word": false,
|
874 |
+
"special": true
|
875 |
+
},
|
876 |
+
"109": {
|
877 |
+
"content": "[unused47]",
|
878 |
+
"lstrip": false,
|
879 |
+
"normalized": false,
|
880 |
+
"rstrip": false,
|
881 |
+
"single_word": false,
|
882 |
+
"special": true
|
883 |
+
},
|
884 |
+
"110": {
|
885 |
+
"content": "[unused48]",
|
886 |
+
"lstrip": false,
|
887 |
+
"normalized": false,
|
888 |
+
"rstrip": false,
|
889 |
+
"single_word": false,
|
890 |
+
"special": true
|
891 |
+
},
|
892 |
+
"111": {
|
893 |
+
"content": "[unused49]",
|
894 |
+
"lstrip": false,
|
895 |
+
"normalized": false,
|
896 |
+
"rstrip": false,
|
897 |
+
"single_word": false,
|
898 |
+
"special": true
|
899 |
+
},
|
900 |
+
"112": {
|
901 |
+
"content": "[unused50]",
|
902 |
+
"lstrip": false,
|
903 |
+
"normalized": false,
|
904 |
+
"rstrip": false,
|
905 |
+
"single_word": false,
|
906 |
+
"special": true
|
907 |
+
},
|
908 |
+
"113": {
|
909 |
+
"content": "[unused51]",
|
910 |
+
"lstrip": false,
|
911 |
+
"normalized": false,
|
912 |
+
"rstrip": false,
|
913 |
+
"single_word": false,
|
914 |
+
"special": true
|
915 |
+
},
|
916 |
+
"114": {
|
917 |
+
"content": "[unused52]",
|
918 |
+
"lstrip": false,
|
919 |
+
"normalized": false,
|
920 |
+
"rstrip": false,
|
921 |
+
"single_word": false,
|
922 |
+
"special": true
|
923 |
+
},
|
924 |
+
"115": {
|
925 |
+
"content": "[unused53]",
|
926 |
+
"lstrip": false,
|
927 |
+
"normalized": false,
|
928 |
+
"rstrip": false,
|
929 |
+
"single_word": false,
|
930 |
+
"special": true
|
931 |
+
},
|
932 |
+
"116": {
|
933 |
+
"content": "[unused54]",
|
934 |
+
"lstrip": false,
|
935 |
+
"normalized": false,
|
936 |
+
"rstrip": false,
|
937 |
+
"single_word": false,
|
938 |
+
"special": true
|
939 |
+
},
|
940 |
+
"117": {
|
941 |
+
"content": "[unused55]",
|
942 |
+
"lstrip": false,
|
943 |
+
"normalized": false,
|
944 |
+
"rstrip": false,
|
945 |
+
"single_word": false,
|
946 |
+
"special": true
|
947 |
+
},
|
948 |
+
"118": {
|
949 |
+
"content": "[unused56]",
|
950 |
+
"lstrip": false,
|
951 |
+
"normalized": false,
|
952 |
+
"rstrip": false,
|
953 |
+
"single_word": false,
|
954 |
+
"special": true
|
955 |
+
},
|
956 |
+
"119": {
|
957 |
+
"content": "[unused57]",
|
958 |
+
"lstrip": false,
|
959 |
+
"normalized": false,
|
960 |
+
"rstrip": false,
|
961 |
+
"single_word": false,
|
962 |
+
"special": true
|
963 |
+
},
|
964 |
+
"120": {
|
965 |
+
"content": "[unused58]",
|
966 |
+
"lstrip": false,
|
967 |
+
"normalized": false,
|
968 |
+
"rstrip": false,
|
969 |
+
"single_word": false,
|
970 |
+
"special": true
|
971 |
+
},
|
972 |
+
"121": {
|
973 |
+
"content": "[unused59]",
|
974 |
+
"lstrip": false,
|
975 |
+
"normalized": false,
|
976 |
+
"rstrip": false,
|
977 |
+
"single_word": false,
|
978 |
+
"special": true
|
979 |
+
},
|
980 |
+
"122": {
|
981 |
+
"content": "[unused60]",
|
982 |
+
"lstrip": false,
|
983 |
+
"normalized": false,
|
984 |
+
"rstrip": false,
|
985 |
+
"single_word": false,
|
986 |
+
"special": true
|
987 |
+
},
|
988 |
+
"123": {
|
989 |
+
"content": "[unused61]",
|
990 |
+
"lstrip": false,
|
991 |
+
"normalized": false,
|
992 |
+
"rstrip": false,
|
993 |
+
"single_word": false,
|
994 |
+
"special": true
|
995 |
+
},
|
996 |
+
"124": {
|
997 |
+
"content": "[unused62]",
|
998 |
+
"lstrip": false,
|
999 |
+
"normalized": false,
|
1000 |
+
"rstrip": false,
|
1001 |
+
"single_word": false,
|
1002 |
+
"special": true
|
1003 |
+
},
|
1004 |
+
"125": {
|
1005 |
+
"content": "[unused63]",
|
1006 |
+
"lstrip": false,
|
1007 |
+
"normalized": false,
|
1008 |
+
"rstrip": false,
|
1009 |
+
"single_word": false,
|
1010 |
+
"special": true
|
1011 |
+
},
|
1012 |
+
"126": {
|
1013 |
+
"content": "[unused64]",
|
1014 |
+
"lstrip": false,
|
1015 |
+
"normalized": false,
|
1016 |
+
"rstrip": false,
|
1017 |
+
"single_word": false,
|
1018 |
+
"special": true
|
1019 |
+
},
|
1020 |
+
"127": {
|
1021 |
+
"content": "[unused65]",
|
1022 |
+
"lstrip": false,
|
1023 |
+
"normalized": false,
|
1024 |
+
"rstrip": false,
|
1025 |
+
"single_word": false,
|
1026 |
+
"special": true
|
1027 |
+
},
|
1028 |
+
"128": {
|
1029 |
+
"content": "[unused66]",
|
1030 |
+
"lstrip": false,
|
1031 |
+
"normalized": false,
|
1032 |
+
"rstrip": false,
|
1033 |
+
"single_word": false,
|
1034 |
+
"special": true
|
1035 |
+
},
|
1036 |
+
"129": {
|
1037 |
+
"content": "[unused67]",
|
1038 |
+
"lstrip": false,
|
1039 |
+
"normalized": false,
|
1040 |
+
"rstrip": false,
|
1041 |
+
"single_word": false,
|
1042 |
+
"special": true
|
1043 |
+
},
|
1044 |
+
"130": {
|
1045 |
+
"content": "[unused68]",
|
1046 |
+
"lstrip": false,
|
1047 |
+
"normalized": false,
|
1048 |
+
"rstrip": false,
|
1049 |
+
"single_word": false,
|
1050 |
+
"special": true
|
1051 |
+
},
|
1052 |
+
"131": {
|
1053 |
+
"content": "[unused69]",
|
1054 |
+
"lstrip": false,
|
1055 |
+
"normalized": false,
|
1056 |
+
"rstrip": false,
|
1057 |
+
"single_word": false,
|
1058 |
+
"special": true
|
1059 |
+
},
|
1060 |
+
"132": {
|
1061 |
+
"content": "[unused70]",
|
1062 |
+
"lstrip": false,
|
1063 |
+
"normalized": false,
|
1064 |
+
"rstrip": false,
|
1065 |
+
"single_word": false,
|
1066 |
+
"special": true
|
1067 |
+
},
|
1068 |
+
"133": {
|
1069 |
+
"content": "[unused71]",
|
1070 |
+
"lstrip": false,
|
1071 |
+
"normalized": false,
|
1072 |
+
"rstrip": false,
|
1073 |
+
"single_word": false,
|
1074 |
+
"special": true
|
1075 |
+
},
|
1076 |
+
"134": {
|
1077 |
+
"content": "[unused72]",
|
1078 |
+
"lstrip": false,
|
1079 |
+
"normalized": false,
|
1080 |
+
"rstrip": false,
|
1081 |
+
"single_word": false,
|
1082 |
+
"special": true
|
1083 |
+
},
|
1084 |
+
"135": {
|
1085 |
+
"content": "[unused73]",
|
1086 |
+
"lstrip": false,
|
1087 |
+
"normalized": false,
|
1088 |
+
"rstrip": false,
|
1089 |
+
"single_word": false,
|
1090 |
+
"special": true
|
1091 |
+
},
|
1092 |
+
"136": {
|
1093 |
+
"content": "[unused74]",
|
1094 |
+
"lstrip": false,
|
1095 |
+
"normalized": false,
|
1096 |
+
"rstrip": false,
|
1097 |
+
"single_word": false,
|
1098 |
+
"special": true
|
1099 |
+
},
|
1100 |
+
"137": {
|
1101 |
+
"content": "[unused75]",
|
1102 |
+
"lstrip": false,
|
1103 |
+
"normalized": false,
|
1104 |
+
"rstrip": false,
|
1105 |
+
"single_word": false,
|
1106 |
+
"special": true
|
1107 |
+
},
|
1108 |
+
"138": {
|
1109 |
+
"content": "[unused76]",
|
1110 |
+
"lstrip": false,
|
1111 |
+
"normalized": false,
|
1112 |
+
"rstrip": false,
|
1113 |
+
"single_word": false,
|
1114 |
+
"special": true
|
1115 |
+
},
|
1116 |
+
"139": {
|
1117 |
+
"content": "[unused77]",
|
1118 |
+
"lstrip": false,
|
1119 |
+
"normalized": false,
|
1120 |
+
"rstrip": false,
|
1121 |
+
"single_word": false,
|
1122 |
+
"special": true
|
1123 |
+
},
|
1124 |
+
"140": {
|
1125 |
+
"content": "[unused78]",
|
1126 |
+
"lstrip": false,
|
1127 |
+
"normalized": false,
|
1128 |
+
"rstrip": false,
|
1129 |
+
"single_word": false,
|
1130 |
+
"special": true
|
1131 |
+
},
|
1132 |
+
"141": {
|
1133 |
+
"content": "[unused79]",
|
1134 |
+
"lstrip": false,
|
1135 |
+
"normalized": false,
|
1136 |
+
"rstrip": false,
|
1137 |
+
"single_word": false,
|
1138 |
+
"special": true
|
1139 |
+
},
|
1140 |
+
"142": {
|
1141 |
+
"content": "[unused80]",
|
1142 |
+
"lstrip": false,
|
1143 |
+
"normalized": false,
|
1144 |
+
"rstrip": false,
|
1145 |
+
"single_word": false,
|
1146 |
+
"special": true
|
1147 |
+
},
|
1148 |
+
"143": {
|
1149 |
+
"content": "[unused81]",
|
1150 |
+
"lstrip": false,
|
1151 |
+
"normalized": false,
|
1152 |
+
"rstrip": false,
|
1153 |
+
"single_word": false,
|
1154 |
+
"special": true
|
1155 |
+
},
|
1156 |
+
"144": {
|
1157 |
+
"content": "[unused82]",
|
1158 |
+
"lstrip": false,
|
1159 |
+
"normalized": false,
|
1160 |
+
"rstrip": false,
|
1161 |
+
"single_word": false,
|
1162 |
+
"special": true
|
1163 |
+
},
|
1164 |
+
"145": {
|
1165 |
+
"content": "[unused83]",
|
1166 |
+
"lstrip": false,
|
1167 |
+
"normalized": false,
|
1168 |
+
"rstrip": false,
|
1169 |
+
"single_word": false,
|
1170 |
+
"special": true
|
1171 |
+
},
|
1172 |
+
"146": {
|
1173 |
+
"content": "[unused84]",
|
1174 |
+
"lstrip": false,
|
1175 |
+
"normalized": false,
|
1176 |
+
"rstrip": false,
|
1177 |
+
"single_word": false,
|
1178 |
+
"special": true
|
1179 |
+
},
|
1180 |
+
"147": {
|
1181 |
+
"content": "[unused85]",
|
1182 |
+
"lstrip": false,
|
1183 |
+
"normalized": false,
|
1184 |
+
"rstrip": false,
|
1185 |
+
"single_word": false,
|
1186 |
+
"special": true
|
1187 |
+
},
|
1188 |
+
"148": {
|
1189 |
+
"content": "[unused86]",
|
1190 |
+
"lstrip": false,
|
1191 |
+
"normalized": false,
|
1192 |
+
"rstrip": false,
|
1193 |
+
"single_word": false,
|
1194 |
+
"special": true
|
1195 |
+
},
|
1196 |
+
"149": {
|
1197 |
+
"content": "[unused87]",
|
1198 |
+
"lstrip": false,
|
1199 |
+
"normalized": false,
|
1200 |
+
"rstrip": false,
|
1201 |
+
"single_word": false,
|
1202 |
+
"special": true
|
1203 |
+
},
|
1204 |
+
"150": {
|
1205 |
+
"content": "[unused88]",
|
1206 |
+
"lstrip": false,
|
1207 |
+
"normalized": false,
|
1208 |
+
"rstrip": false,
|
1209 |
+
"single_word": false,
|
1210 |
+
"special": true
|
1211 |
+
},
|
1212 |
+
"151": {
|
1213 |
+
"content": "[unused89]",
|
1214 |
+
"lstrip": false,
|
1215 |
+
"normalized": false,
|
1216 |
+
"rstrip": false,
|
1217 |
+
"single_word": false,
|
1218 |
+
"special": true
|
1219 |
+
},
|
1220 |
+
"152": {
|
1221 |
+
"content": "[unused90]",
|
1222 |
+
"lstrip": false,
|
1223 |
+
"normalized": false,
|
1224 |
+
"rstrip": false,
|
1225 |
+
"single_word": false,
|
1226 |
+
"special": true
|
1227 |
+
},
|
1228 |
+
"153": {
|
1229 |
+
"content": "[unused91]",
|
1230 |
+
"lstrip": false,
|
1231 |
+
"normalized": false,
|
1232 |
+
"rstrip": false,
|
1233 |
+
"single_word": false,
|
1234 |
+
"special": true
|
1235 |
+
},
|
1236 |
+
"154": {
|
1237 |
+
"content": "[unused92]",
|
1238 |
+
"lstrip": false,
|
1239 |
+
"normalized": false,
|
1240 |
+
"rstrip": false,
|
1241 |
+
"single_word": false,
|
1242 |
+
"special": true
|
1243 |
+
},
|
1244 |
+
"155": {
|
1245 |
+
"content": "[unused93]",
|
1246 |
+
"lstrip": false,
|
1247 |
+
"normalized": false,
|
1248 |
+
"rstrip": false,
|
1249 |
+
"single_word": false,
|
1250 |
+
"special": true
|
1251 |
+
},
|
1252 |
+
"156": {
|
1253 |
+
"content": "[unused94]",
|
1254 |
+
"lstrip": false,
|
1255 |
+
"normalized": false,
|
1256 |
+
"rstrip": false,
|
1257 |
+
"single_word": false,
|
1258 |
+
"special": true
|
1259 |
+
},
|
1260 |
+
"157": {
|
1261 |
+
"content": "[unused95]",
|
1262 |
+
"lstrip": false,
|
1263 |
+
"normalized": false,
|
1264 |
+
"rstrip": false,
|
1265 |
+
"single_word": false,
|
1266 |
+
"special": true
|
1267 |
+
},
|
1268 |
+
"158": {
|
1269 |
+
"content": "[unused96]",
|
1270 |
+
"lstrip": false,
|
1271 |
+
"normalized": false,
|
1272 |
+
"rstrip": false,
|
1273 |
+
"single_word": false,
|
1274 |
+
"special": true
|
1275 |
+
},
|
1276 |
+
"159": {
|
1277 |
+
"content": "[unused97]",
|
1278 |
+
"lstrip": false,
|
1279 |
+
"normalized": false,
|
1280 |
+
"rstrip": false,
|
1281 |
+
"single_word": false,
|
1282 |
+
"special": true
|
1283 |
+
},
|
1284 |
+
"160": {
|
1285 |
+
"content": "[unused98]",
|
1286 |
+
"lstrip": false,
|
1287 |
+
"normalized": false,
|
1288 |
+
"rstrip": false,
|
1289 |
+
"single_word": false,
|
1290 |
+
"special": true
|
1291 |
+
},
|
1292 |
+
"161": {
|
1293 |
+
"content": "[unused99]",
|
1294 |
+
"lstrip": false,
|
1295 |
+
"normalized": false,
|
1296 |
+
"rstrip": false,
|
1297 |
+
"single_word": false,
|
1298 |
+
"special": true
|
1299 |
+
},
|
1300 |
+
"162": {
|
1301 |
+
"content": "[extra_id_0]",
|
1302 |
+
"lstrip": false,
|
1303 |
+
"normalized": false,
|
1304 |
+
"rstrip": false,
|
1305 |
+
"single_word": false,
|
1306 |
+
"special": true
|
1307 |
+
},
|
1308 |
+
"163": {
|
1309 |
+
"content": "[extra_id_1]",
|
1310 |
+
"lstrip": false,
|
1311 |
+
"normalized": false,
|
1312 |
+
"rstrip": false,
|
1313 |
+
"single_word": false,
|
1314 |
+
"special": true
|
1315 |
+
},
|
1316 |
+
"164": {
|
1317 |
+
"content": "[extra_id_2]",
|
1318 |
+
"lstrip": false,
|
1319 |
+
"normalized": false,
|
1320 |
+
"rstrip": false,
|
1321 |
+
"single_word": false,
|
1322 |
+
"special": true
|
1323 |
+
},
|
1324 |
+
"165": {
|
1325 |
+
"content": "[extra_id_3]",
|
1326 |
+
"lstrip": false,
|
1327 |
+
"normalized": false,
|
1328 |
+
"rstrip": false,
|
1329 |
+
"single_word": false,
|
1330 |
+
"special": true
|
1331 |
+
},
|
1332 |
+
"166": {
|
1333 |
+
"content": "[extra_id_4]",
|
1334 |
+
"lstrip": false,
|
1335 |
+
"normalized": false,
|
1336 |
+
"rstrip": false,
|
1337 |
+
"single_word": false,
|
1338 |
+
"special": true
|
1339 |
+
},
|
1340 |
+
"167": {
|
1341 |
+
"content": "[extra_id_5]",
|
1342 |
+
"lstrip": false,
|
1343 |
+
"normalized": false,
|
1344 |
+
"rstrip": false,
|
1345 |
+
"single_word": false,
|
1346 |
+
"special": true
|
1347 |
+
},
|
1348 |
+
"168": {
|
1349 |
+
"content": "[extra_id_6]",
|
1350 |
+
"lstrip": false,
|
1351 |
+
"normalized": false,
|
1352 |
+
"rstrip": false,
|
1353 |
+
"single_word": false,
|
1354 |
+
"special": true
|
1355 |
+
},
|
1356 |
+
"169": {
|
1357 |
+
"content": "[extra_id_7]",
|
1358 |
+
"lstrip": false,
|
1359 |
+
"normalized": false,
|
1360 |
+
"rstrip": false,
|
1361 |
+
"single_word": false,
|
1362 |
+
"special": true
|
1363 |
+
},
|
1364 |
+
"170": {
|
1365 |
+
"content": "[extra_id_8]",
|
1366 |
+
"lstrip": false,
|
1367 |
+
"normalized": false,
|
1368 |
+
"rstrip": false,
|
1369 |
+
"single_word": false,
|
1370 |
+
"special": true
|
1371 |
+
},
|
1372 |
+
"171": {
|
1373 |
+
"content": "[extra_id_9]",
|
1374 |
+
"lstrip": false,
|
1375 |
+
"normalized": false,
|
1376 |
+
"rstrip": false,
|
1377 |
+
"single_word": false,
|
1378 |
+
"special": true
|
1379 |
+
},
|
1380 |
+
"172": {
|
1381 |
+
"content": "[extra_id_10]",
|
1382 |
+
"lstrip": false,
|
1383 |
+
"normalized": false,
|
1384 |
+
"rstrip": false,
|
1385 |
+
"single_word": false,
|
1386 |
+
"special": true
|
1387 |
+
},
|
1388 |
+
"173": {
|
1389 |
+
"content": "[extra_id_11]",
|
1390 |
+
"lstrip": false,
|
1391 |
+
"normalized": false,
|
1392 |
+
"rstrip": false,
|
1393 |
+
"single_word": false,
|
1394 |
+
"special": true
|
1395 |
+
},
|
1396 |
+
"174": {
|
1397 |
+
"content": "[extra_id_12]",
|
1398 |
+
"lstrip": false,
|
1399 |
+
"normalized": false,
|
1400 |
+
"rstrip": false,
|
1401 |
+
"single_word": false,
|
1402 |
+
"special": true
|
1403 |
+
},
|
1404 |
+
"175": {
|
1405 |
+
"content": "[extra_id_13]",
|
1406 |
+
"lstrip": false,
|
1407 |
+
"normalized": false,
|
1408 |
+
"rstrip": false,
|
1409 |
+
"single_word": false,
|
1410 |
+
"special": true
|
1411 |
+
},
|
1412 |
+
"176": {
|
1413 |
+
"content": "[extra_id_14]",
|
1414 |
+
"lstrip": false,
|
1415 |
+
"normalized": false,
|
1416 |
+
"rstrip": false,
|
1417 |
+
"single_word": false,
|
1418 |
+
"special": true
|
1419 |
+
},
|
1420 |
+
"177": {
|
1421 |
+
"content": "[extra_id_15]",
|
1422 |
+
"lstrip": false,
|
1423 |
+
"normalized": false,
|
1424 |
+
"rstrip": false,
|
1425 |
+
"single_word": false,
|
1426 |
+
"special": true
|
1427 |
+
},
|
1428 |
+
"178": {
|
1429 |
+
"content": "[extra_id_16]",
|
1430 |
+
"lstrip": false,
|
1431 |
+
"normalized": false,
|
1432 |
+
"rstrip": false,
|
1433 |
+
"single_word": false,
|
1434 |
+
"special": true
|
1435 |
+
},
|
1436 |
+
"179": {
|
1437 |
+
"content": "[extra_id_17]",
|
1438 |
+
"lstrip": false,
|
1439 |
+
"normalized": false,
|
1440 |
+
"rstrip": false,
|
1441 |
+
"single_word": false,
|
1442 |
+
"special": true
|
1443 |
+
},
|
1444 |
+
"180": {
|
1445 |
+
"content": "[extra_id_18]",
|
1446 |
+
"lstrip": false,
|
1447 |
+
"normalized": false,
|
1448 |
+
"rstrip": false,
|
1449 |
+
"single_word": false,
|
1450 |
+
"special": true
|
1451 |
+
},
|
1452 |
+
"181": {
|
1453 |
+
"content": "[extra_id_19]",
|
1454 |
+
"lstrip": false,
|
1455 |
+
"normalized": false,
|
1456 |
+
"rstrip": false,
|
1457 |
+
"single_word": false,
|
1458 |
+
"special": true
|
1459 |
+
},
|
1460 |
+
"182": {
|
1461 |
+
"content": "[extra_id_20]",
|
1462 |
+
"lstrip": false,
|
1463 |
+
"normalized": false,
|
1464 |
+
"rstrip": false,
|
1465 |
+
"single_word": false,
|
1466 |
+
"special": true
|
1467 |
+
},
|
1468 |
+
"183": {
|
1469 |
+
"content": "[extra_id_21]",
|
1470 |
+
"lstrip": false,
|
1471 |
+
"normalized": false,
|
1472 |
+
"rstrip": false,
|
1473 |
+
"single_word": false,
|
1474 |
+
"special": true
|
1475 |
+
},
|
1476 |
+
"184": {
|
1477 |
+
"content": "[extra_id_22]",
|
1478 |
+
"lstrip": false,
|
1479 |
+
"normalized": false,
|
1480 |
+
"rstrip": false,
|
1481 |
+
"single_word": false,
|
1482 |
+
"special": true
|
1483 |
+
},
|
1484 |
+
"185": {
|
1485 |
+
"content": "[extra_id_23]",
|
1486 |
+
"lstrip": false,
|
1487 |
+
"normalized": false,
|
1488 |
+
"rstrip": false,
|
1489 |
+
"single_word": false,
|
1490 |
+
"special": true
|
1491 |
+
},
|
1492 |
+
"186": {
|
1493 |
+
"content": "[extra_id_24]",
|
1494 |
+
"lstrip": false,
|
1495 |
+
"normalized": false,
|
1496 |
+
"rstrip": false,
|
1497 |
+
"single_word": false,
|
1498 |
+
"special": true
|
1499 |
+
},
|
1500 |
+
"187": {
|
1501 |
+
"content": "[extra_id_25]",
|
1502 |
+
"lstrip": false,
|
1503 |
+
"normalized": false,
|
1504 |
+
"rstrip": false,
|
1505 |
+
"single_word": false,
|
1506 |
+
"special": true
|
1507 |
+
},
|
1508 |
+
"188": {
|
1509 |
+
"content": "[extra_id_26]",
|
1510 |
+
"lstrip": false,
|
1511 |
+
"normalized": false,
|
1512 |
+
"rstrip": false,
|
1513 |
+
"single_word": false,
|
1514 |
+
"special": true
|
1515 |
+
},
|
1516 |
+
"189": {
|
1517 |
+
"content": "[extra_id_27]",
|
1518 |
+
"lstrip": false,
|
1519 |
+
"normalized": false,
|
1520 |
+
"rstrip": false,
|
1521 |
+
"single_word": false,
|
1522 |
+
"special": true
|
1523 |
+
},
|
1524 |
+
"190": {
|
1525 |
+
"content": "[extra_id_28]",
|
1526 |
+
"lstrip": false,
|
1527 |
+
"normalized": false,
|
1528 |
+
"rstrip": false,
|
1529 |
+
"single_word": false,
|
1530 |
+
"special": true
|
1531 |
+
},
|
1532 |
+
"191": {
|
1533 |
+
"content": "[extra_id_29]",
|
1534 |
+
"lstrip": false,
|
1535 |
+
"normalized": false,
|
1536 |
+
"rstrip": false,
|
1537 |
+
"single_word": false,
|
1538 |
+
"special": true
|
1539 |
+
},
|
1540 |
+
"192": {
|
1541 |
+
"content": "[extra_id_30]",
|
1542 |
+
"lstrip": false,
|
1543 |
+
"normalized": false,
|
1544 |
+
"rstrip": false,
|
1545 |
+
"single_word": false,
|
1546 |
+
"special": true
|
1547 |
+
},
|
1548 |
+
"193": {
|
1549 |
+
"content": "[extra_id_31]",
|
1550 |
+
"lstrip": false,
|
1551 |
+
"normalized": false,
|
1552 |
+
"rstrip": false,
|
1553 |
+
"single_word": false,
|
1554 |
+
"special": true
|
1555 |
+
},
|
1556 |
+
"194": {
|
1557 |
+
"content": "[extra_id_32]",
|
1558 |
+
"lstrip": false,
|
1559 |
+
"normalized": false,
|
1560 |
+
"rstrip": false,
|
1561 |
+
"single_word": false,
|
1562 |
+
"special": true
|
1563 |
+
},
|
1564 |
+
"195": {
|
1565 |
+
"content": "[extra_id_33]",
|
1566 |
+
"lstrip": false,
|
1567 |
+
"normalized": false,
|
1568 |
+
"rstrip": false,
|
1569 |
+
"single_word": false,
|
1570 |
+
"special": true
|
1571 |
+
},
|
1572 |
+
"196": {
|
1573 |
+
"content": "[extra_id_34]",
|
1574 |
+
"lstrip": false,
|
1575 |
+
"normalized": false,
|
1576 |
+
"rstrip": false,
|
1577 |
+
"single_word": false,
|
1578 |
+
"special": true
|
1579 |
+
},
|
1580 |
+
"197": {
|
1581 |
+
"content": "[extra_id_35]",
|
1582 |
+
"lstrip": false,
|
1583 |
+
"normalized": false,
|
1584 |
+
"rstrip": false,
|
1585 |
+
"single_word": false,
|
1586 |
+
"special": true
|
1587 |
+
},
|
1588 |
+
"198": {
|
1589 |
+
"content": "[extra_id_36]",
|
1590 |
+
"lstrip": false,
|
1591 |
+
"normalized": false,
|
1592 |
+
"rstrip": false,
|
1593 |
+
"single_word": false,
|
1594 |
+
"special": true
|
1595 |
+
},
|
1596 |
+
"199": {
|
1597 |
+
"content": "[extra_id_37]",
|
1598 |
+
"lstrip": false,
|
1599 |
+
"normalized": false,
|
1600 |
+
"rstrip": false,
|
1601 |
+
"single_word": false,
|
1602 |
+
"special": true
|
1603 |
+
},
|
1604 |
+
"200": {
|
1605 |
+
"content": "[extra_id_38]",
|
1606 |
+
"lstrip": false,
|
1607 |
+
"normalized": false,
|
1608 |
+
"rstrip": false,
|
1609 |
+
"single_word": false,
|
1610 |
+
"special": true
|
1611 |
+
},
|
1612 |
+
"201": {
|
1613 |
+
"content": "[extra_id_39]",
|
1614 |
+
"lstrip": false,
|
1615 |
+
"normalized": false,
|
1616 |
+
"rstrip": false,
|
1617 |
+
"single_word": false,
|
1618 |
+
"special": true
|
1619 |
+
},
|
1620 |
+
"202": {
|
1621 |
+
"content": "[extra_id_40]",
|
1622 |
+
"lstrip": false,
|
1623 |
+
"normalized": false,
|
1624 |
+
"rstrip": false,
|
1625 |
+
"single_word": false,
|
1626 |
+
"special": true
|
1627 |
+
},
|
1628 |
+
"203": {
|
1629 |
+
"content": "[extra_id_41]",
|
1630 |
+
"lstrip": false,
|
1631 |
+
"normalized": false,
|
1632 |
+
"rstrip": false,
|
1633 |
+
"single_word": false,
|
1634 |
+
"special": true
|
1635 |
+
},
|
1636 |
+
"204": {
|
1637 |
+
"content": "[extra_id_42]",
|
1638 |
+
"lstrip": false,
|
1639 |
+
"normalized": false,
|
1640 |
+
"rstrip": false,
|
1641 |
+
"single_word": false,
|
1642 |
+
"special": true
|
1643 |
+
},
|
1644 |
+
"205": {
|
1645 |
+
"content": "[extra_id_43]",
|
1646 |
+
"lstrip": false,
|
1647 |
+
"normalized": false,
|
1648 |
+
"rstrip": false,
|
1649 |
+
"single_word": false,
|
1650 |
+
"special": true
|
1651 |
+
},
|
1652 |
+
"206": {
|
1653 |
+
"content": "[extra_id_44]",
|
1654 |
+
"lstrip": false,
|
1655 |
+
"normalized": false,
|
1656 |
+
"rstrip": false,
|
1657 |
+
"single_word": false,
|
1658 |
+
"special": true
|
1659 |
+
},
|
1660 |
+
"207": {
|
1661 |
+
"content": "[extra_id_45]",
|
1662 |
+
"lstrip": false,
|
1663 |
+
"normalized": false,
|
1664 |
+
"rstrip": false,
|
1665 |
+
"single_word": false,
|
1666 |
+
"special": true
|
1667 |
+
},
|
1668 |
+
"208": {
|
1669 |
+
"content": "[extra_id_46]",
|
1670 |
+
"lstrip": false,
|
1671 |
+
"normalized": false,
|
1672 |
+
"rstrip": false,
|
1673 |
+
"single_word": false,
|
1674 |
+
"special": true
|
1675 |
+
},
|
1676 |
+
"209": {
|
1677 |
+
"content": "[extra_id_47]",
|
1678 |
+
"lstrip": false,
|
1679 |
+
"normalized": false,
|
1680 |
+
"rstrip": false,
|
1681 |
+
"single_word": false,
|
1682 |
+
"special": true
|
1683 |
+
},
|
1684 |
+
"210": {
|
1685 |
+
"content": "[extra_id_48]",
|
1686 |
+
"lstrip": false,
|
1687 |
+
"normalized": false,
|
1688 |
+
"rstrip": false,
|
1689 |
+
"single_word": false,
|
1690 |
+
"special": true
|
1691 |
+
},
|
1692 |
+
"211": {
|
1693 |
+
"content": "[extra_id_49]",
|
1694 |
+
"lstrip": false,
|
1695 |
+
"normalized": false,
|
1696 |
+
"rstrip": false,
|
1697 |
+
"single_word": false,
|
1698 |
+
"special": true
|
1699 |
+
},
|
1700 |
+
"212": {
|
1701 |
+
"content": "[extra_id_50]",
|
1702 |
+
"lstrip": false,
|
1703 |
+
"normalized": false,
|
1704 |
+
"rstrip": false,
|
1705 |
+
"single_word": false,
|
1706 |
+
"special": true
|
1707 |
+
},
|
1708 |
+
"213": {
|
1709 |
+
"content": "[extra_id_51]",
|
1710 |
+
"lstrip": false,
|
1711 |
+
"normalized": false,
|
1712 |
+
"rstrip": false,
|
1713 |
+
"single_word": false,
|
1714 |
+
"special": true
|
1715 |
+
},
|
1716 |
+
"214": {
|
1717 |
+
"content": "[extra_id_52]",
|
1718 |
+
"lstrip": false,
|
1719 |
+
"normalized": false,
|
1720 |
+
"rstrip": false,
|
1721 |
+
"single_word": false,
|
1722 |
+
"special": true
|
1723 |
+
},
|
1724 |
+
"215": {
|
1725 |
+
"content": "[extra_id_53]",
|
1726 |
+
"lstrip": false,
|
1727 |
+
"normalized": false,
|
1728 |
+
"rstrip": false,
|
1729 |
+
"single_word": false,
|
1730 |
+
"special": true
|
1731 |
+
},
|
1732 |
+
"216": {
|
1733 |
+
"content": "[extra_id_54]",
|
1734 |
+
"lstrip": false,
|
1735 |
+
"normalized": false,
|
1736 |
+
"rstrip": false,
|
1737 |
+
"single_word": false,
|
1738 |
+
"special": true
|
1739 |
+
},
|
1740 |
+
"217": {
|
1741 |
+
"content": "[extra_id_55]",
|
1742 |
+
"lstrip": false,
|
1743 |
+
"normalized": false,
|
1744 |
+
"rstrip": false,
|
1745 |
+
"single_word": false,
|
1746 |
+
"special": true
|
1747 |
+
},
|
1748 |
+
"218": {
|
1749 |
+
"content": "[extra_id_56]",
|
1750 |
+
"lstrip": false,
|
1751 |
+
"normalized": false,
|
1752 |
+
"rstrip": false,
|
1753 |
+
"single_word": false,
|
1754 |
+
"special": true
|
1755 |
+
},
|
1756 |
+
"219": {
|
1757 |
+
"content": "[extra_id_57]",
|
1758 |
+
"lstrip": false,
|
1759 |
+
"normalized": false,
|
1760 |
+
"rstrip": false,
|
1761 |
+
"single_word": false,
|
1762 |
+
"special": true
|
1763 |
+
},
|
1764 |
+
"220": {
|
1765 |
+
"content": "[extra_id_58]",
|
1766 |
+
"lstrip": false,
|
1767 |
+
"normalized": false,
|
1768 |
+
"rstrip": false,
|
1769 |
+
"single_word": false,
|
1770 |
+
"special": true
|
1771 |
+
},
|
1772 |
+
"221": {
|
1773 |
+
"content": "[extra_id_59]",
|
1774 |
+
"lstrip": false,
|
1775 |
+
"normalized": false,
|
1776 |
+
"rstrip": false,
|
1777 |
+
"single_word": false,
|
1778 |
+
"special": true
|
1779 |
+
},
|
1780 |
+
"222": {
|
1781 |
+
"content": "[extra_id_60]",
|
1782 |
+
"lstrip": false,
|
1783 |
+
"normalized": false,
|
1784 |
+
"rstrip": false,
|
1785 |
+
"single_word": false,
|
1786 |
+
"special": true
|
1787 |
+
},
|
1788 |
+
"223": {
|
1789 |
+
"content": "[extra_id_61]",
|
1790 |
+
"lstrip": false,
|
1791 |
+
"normalized": false,
|
1792 |
+
"rstrip": false,
|
1793 |
+
"single_word": false,
|
1794 |
+
"special": true
|
1795 |
+
},
|
1796 |
+
"224": {
|
1797 |
+
"content": "[extra_id_62]",
|
1798 |
+
"lstrip": false,
|
1799 |
+
"normalized": false,
|
1800 |
+
"rstrip": false,
|
1801 |
+
"single_word": false,
|
1802 |
+
"special": true
|
1803 |
+
},
|
1804 |
+
"225": {
|
1805 |
+
"content": "[extra_id_63]",
|
1806 |
+
"lstrip": false,
|
1807 |
+
"normalized": false,
|
1808 |
+
"rstrip": false,
|
1809 |
+
"single_word": false,
|
1810 |
+
"special": true
|
1811 |
+
},
|
1812 |
+
"226": {
|
1813 |
+
"content": "[extra_id_64]",
|
1814 |
+
"lstrip": false,
|
1815 |
+
"normalized": false,
|
1816 |
+
"rstrip": false,
|
1817 |
+
"single_word": false,
|
1818 |
+
"special": true
|
1819 |
+
},
|
1820 |
+
"227": {
|
1821 |
+
"content": "[extra_id_65]",
|
1822 |
+
"lstrip": false,
|
1823 |
+
"normalized": false,
|
1824 |
+
"rstrip": false,
|
1825 |
+
"single_word": false,
|
1826 |
+
"special": true
|
1827 |
+
},
|
1828 |
+
"228": {
|
1829 |
+
"content": "[extra_id_66]",
|
1830 |
+
"lstrip": false,
|
1831 |
+
"normalized": false,
|
1832 |
+
"rstrip": false,
|
1833 |
+
"single_word": false,
|
1834 |
+
"special": true
|
1835 |
+
},
|
1836 |
+
"229": {
|
1837 |
+
"content": "[extra_id_67]",
|
1838 |
+
"lstrip": false,
|
1839 |
+
"normalized": false,
|
1840 |
+
"rstrip": false,
|
1841 |
+
"single_word": false,
|
1842 |
+
"special": true
|
1843 |
+
},
|
1844 |
+
"230": {
|
1845 |
+
"content": "[extra_id_68]",
|
1846 |
+
"lstrip": false,
|
1847 |
+
"normalized": false,
|
1848 |
+
"rstrip": false,
|
1849 |
+
"single_word": false,
|
1850 |
+
"special": true
|
1851 |
+
},
|
1852 |
+
"231": {
|
1853 |
+
"content": "[extra_id_69]",
|
1854 |
+
"lstrip": false,
|
1855 |
+
"normalized": false,
|
1856 |
+
"rstrip": false,
|
1857 |
+
"single_word": false,
|
1858 |
+
"special": true
|
1859 |
+
},
|
1860 |
+
"232": {
|
1861 |
+
"content": "[extra_id_70]",
|
1862 |
+
"lstrip": false,
|
1863 |
+
"normalized": false,
|
1864 |
+
"rstrip": false,
|
1865 |
+
"single_word": false,
|
1866 |
+
"special": true
|
1867 |
+
},
|
1868 |
+
"233": {
|
1869 |
+
"content": "[extra_id_71]",
|
1870 |
+
"lstrip": false,
|
1871 |
+
"normalized": false,
|
1872 |
+
"rstrip": false,
|
1873 |
+
"single_word": false,
|
1874 |
+
"special": true
|
1875 |
+
},
|
1876 |
+
"234": {
|
1877 |
+
"content": "[extra_id_72]",
|
1878 |
+
"lstrip": false,
|
1879 |
+
"normalized": false,
|
1880 |
+
"rstrip": false,
|
1881 |
+
"single_word": false,
|
1882 |
+
"special": true
|
1883 |
+
},
|
1884 |
+
"235": {
|
1885 |
+
"content": "[extra_id_73]",
|
1886 |
+
"lstrip": false,
|
1887 |
+
"normalized": false,
|
1888 |
+
"rstrip": false,
|
1889 |
+
"single_word": false,
|
1890 |
+
"special": true
|
1891 |
+
},
|
1892 |
+
"236": {
|
1893 |
+
"content": "[extra_id_74]",
|
1894 |
+
"lstrip": false,
|
1895 |
+
"normalized": false,
|
1896 |
+
"rstrip": false,
|
1897 |
+
"single_word": false,
|
1898 |
+
"special": true
|
1899 |
+
},
|
1900 |
+
"237": {
|
1901 |
+
"content": "[extra_id_75]",
|
1902 |
+
"lstrip": false,
|
1903 |
+
"normalized": false,
|
1904 |
+
"rstrip": false,
|
1905 |
+
"single_word": false,
|
1906 |
+
"special": true
|
1907 |
+
},
|
1908 |
+
"238": {
|
1909 |
+
"content": "[extra_id_76]",
|
1910 |
+
"lstrip": false,
|
1911 |
+
"normalized": false,
|
1912 |
+
"rstrip": false,
|
1913 |
+
"single_word": false,
|
1914 |
+
"special": true
|
1915 |
+
},
|
1916 |
+
"239": {
|
1917 |
+
"content": "[extra_id_77]",
|
1918 |
+
"lstrip": false,
|
1919 |
+
"normalized": false,
|
1920 |
+
"rstrip": false,
|
1921 |
+
"single_word": false,
|
1922 |
+
"special": true
|
1923 |
+
},
|
1924 |
+
"240": {
|
1925 |
+
"content": "[extra_id_78]",
|
1926 |
+
"lstrip": false,
|
1927 |
+
"normalized": false,
|
1928 |
+
"rstrip": false,
|
1929 |
+
"single_word": false,
|
1930 |
+
"special": true
|
1931 |
+
},
|
1932 |
+
"241": {
|
1933 |
+
"content": "[extra_id_79]",
|
1934 |
+
"lstrip": false,
|
1935 |
+
"normalized": false,
|
1936 |
+
"rstrip": false,
|
1937 |
+
"single_word": false,
|
1938 |
+
"special": true
|
1939 |
+
},
|
1940 |
+
"242": {
|
1941 |
+
"content": "[extra_id_80]",
|
1942 |
+
"lstrip": false,
|
1943 |
+
"normalized": false,
|
1944 |
+
"rstrip": false,
|
1945 |
+
"single_word": false,
|
1946 |
+
"special": true
|
1947 |
+
},
|
1948 |
+
"243": {
|
1949 |
+
"content": "[extra_id_81]",
|
1950 |
+
"lstrip": false,
|
1951 |
+
"normalized": false,
|
1952 |
+
"rstrip": false,
|
1953 |
+
"single_word": false,
|
1954 |
+
"special": true
|
1955 |
+
},
|
1956 |
+
"244": {
|
1957 |
+
"content": "[extra_id_82]",
|
1958 |
+
"lstrip": false,
|
1959 |
+
"normalized": false,
|
1960 |
+
"rstrip": false,
|
1961 |
+
"single_word": false,
|
1962 |
+
"special": true
|
1963 |
+
},
|
1964 |
+
"245": {
|
1965 |
+
"content": "[extra_id_83]",
|
1966 |
+
"lstrip": false,
|
1967 |
+
"normalized": false,
|
1968 |
+
"rstrip": false,
|
1969 |
+
"single_word": false,
|
1970 |
+
"special": true
|
1971 |
+
},
|
1972 |
+
"246": {
|
1973 |
+
"content": "[extra_id_84]",
|
1974 |
+
"lstrip": false,
|
1975 |
+
"normalized": false,
|
1976 |
+
"rstrip": false,
|
1977 |
+
"single_word": false,
|
1978 |
+
"special": true
|
1979 |
+
},
|
1980 |
+
"247": {
|
1981 |
+
"content": "[extra_id_85]",
|
1982 |
+
"lstrip": false,
|
1983 |
+
"normalized": false,
|
1984 |
+
"rstrip": false,
|
1985 |
+
"single_word": false,
|
1986 |
+
"special": true
|
1987 |
+
},
|
1988 |
+
"248": {
|
1989 |
+
"content": "[extra_id_86]",
|
1990 |
+
"lstrip": false,
|
1991 |
+
"normalized": false,
|
1992 |
+
"rstrip": false,
|
1993 |
+
"single_word": false,
|
1994 |
+
"special": true
|
1995 |
+
},
|
1996 |
+
"249": {
|
1997 |
+
"content": "[extra_id_87]",
|
1998 |
+
"lstrip": false,
|
1999 |
+
"normalized": false,
|
2000 |
+
"rstrip": false,
|
2001 |
+
"single_word": false,
|
2002 |
+
"special": true
|
2003 |
+
},
|
2004 |
+
"250": {
|
2005 |
+
"content": "[extra_id_88]",
|
2006 |
+
"lstrip": false,
|
2007 |
+
"normalized": false,
|
2008 |
+
"rstrip": false,
|
2009 |
+
"single_word": false,
|
2010 |
+
"special": true
|
2011 |
+
},
|
2012 |
+
"251": {
|
2013 |
+
"content": "[extra_id_89]",
|
2014 |
+
"lstrip": false,
|
2015 |
+
"normalized": false,
|
2016 |
+
"rstrip": false,
|
2017 |
+
"single_word": false,
|
2018 |
+
"special": true
|
2019 |
+
},
|
2020 |
+
"252": {
|
2021 |
+
"content": "[extra_id_90]",
|
2022 |
+
"lstrip": false,
|
2023 |
+
"normalized": false,
|
2024 |
+
"rstrip": false,
|
2025 |
+
"single_word": false,
|
2026 |
+
"special": true
|
2027 |
+
},
|
2028 |
+
"253": {
|
2029 |
+
"content": "[extra_id_91]",
|
2030 |
+
"lstrip": false,
|
2031 |
+
"normalized": false,
|
2032 |
+
"rstrip": false,
|
2033 |
+
"single_word": false,
|
2034 |
+
"special": true
|
2035 |
+
},
|
2036 |
+
"254": {
|
2037 |
+
"content": "[extra_id_92]",
|
2038 |
+
"lstrip": false,
|
2039 |
+
"normalized": false,
|
2040 |
+
"rstrip": false,
|
2041 |
+
"single_word": false,
|
2042 |
+
"special": true
|
2043 |
+
},
|
2044 |
+
"255": {
|
2045 |
+
"content": "[extra_id_93]",
|
2046 |
+
"lstrip": false,
|
2047 |
+
"normalized": false,
|
2048 |
+
"rstrip": false,
|
2049 |
+
"single_word": false,
|
2050 |
+
"special": true
|
2051 |
+
},
|
2052 |
+
"256": {
|
2053 |
+
"content": "[extra_id_94]",
|
2054 |
+
"lstrip": false,
|
2055 |
+
"normalized": false,
|
2056 |
+
"rstrip": false,
|
2057 |
+
"single_word": false,
|
2058 |
+
"special": true
|
2059 |
+
},
|
2060 |
+
"257": {
|
2061 |
+
"content": "[extra_id_95]",
|
2062 |
+
"lstrip": false,
|
2063 |
+
"normalized": false,
|
2064 |
+
"rstrip": false,
|
2065 |
+
"single_word": false,
|
2066 |
+
"special": true
|
2067 |
+
},
|
2068 |
+
"258": {
|
2069 |
+
"content": "[extra_id_96]",
|
2070 |
+
"lstrip": false,
|
2071 |
+
"normalized": false,
|
2072 |
+
"rstrip": false,
|
2073 |
+
"single_word": false,
|
2074 |
+
"special": true
|
2075 |
+
},
|
2076 |
+
"259": {
|
2077 |
+
"content": "[extra_id_97]",
|
2078 |
+
"lstrip": false,
|
2079 |
+
"normalized": false,
|
2080 |
+
"rstrip": false,
|
2081 |
+
"single_word": false,
|
2082 |
+
"special": true
|
2083 |
+
},
|
2084 |
+
"260": {
|
2085 |
+
"content": "[extra_id_98]",
|
2086 |
+
"lstrip": false,
|
2087 |
+
"normalized": false,
|
2088 |
+
"rstrip": false,
|
2089 |
+
"single_word": false,
|
2090 |
+
"special": true
|
2091 |
+
},
|
2092 |
+
"261": {
|
2093 |
+
"content": "[extra_id_99]",
|
2094 |
+
"lstrip": false,
|
2095 |
+
"normalized": false,
|
2096 |
+
"rstrip": false,
|
2097 |
+
"single_word": false,
|
2098 |
+
"special": true
|
2099 |
+
},
|
2100 |
+
"262": {
|
2101 |
+
"content": "[extra_id_100]",
|
2102 |
+
"lstrip": false,
|
2103 |
+
"normalized": false,
|
2104 |
+
"rstrip": false,
|
2105 |
+
"single_word": false,
|
2106 |
+
"special": true
|
2107 |
+
},
|
2108 |
+
"263": {
|
2109 |
+
"content": "[extra_id_101]",
|
2110 |
+
"lstrip": false,
|
2111 |
+
"normalized": false,
|
2112 |
+
"rstrip": false,
|
2113 |
+
"single_word": false,
|
2114 |
+
"special": true
|
2115 |
+
},
|
2116 |
+
"264": {
|
2117 |
+
"content": "[extra_id_102]",
|
2118 |
+
"lstrip": false,
|
2119 |
+
"normalized": false,
|
2120 |
+
"rstrip": false,
|
2121 |
+
"single_word": false,
|
2122 |
+
"special": true
|
2123 |
+
},
|
2124 |
+
"265": {
|
2125 |
+
"content": "[extra_id_103]",
|
2126 |
+
"lstrip": false,
|
2127 |
+
"normalized": false,
|
2128 |
+
"rstrip": false,
|
2129 |
+
"single_word": false,
|
2130 |
+
"special": true
|
2131 |
+
},
|
2132 |
+
"266": {
|
2133 |
+
"content": "[extra_id_104]",
|
2134 |
+
"lstrip": false,
|
2135 |
+
"normalized": false,
|
2136 |
+
"rstrip": false,
|
2137 |
+
"single_word": false,
|
2138 |
+
"special": true
|
2139 |
+
},
|
2140 |
+
"267": {
|
2141 |
+
"content": "[extra_id_105]",
|
2142 |
+
"lstrip": false,
|
2143 |
+
"normalized": false,
|
2144 |
+
"rstrip": false,
|
2145 |
+
"single_word": false,
|
2146 |
+
"special": true
|
2147 |
+
},
|
2148 |
+
"268": {
|
2149 |
+
"content": "[extra_id_106]",
|
2150 |
+
"lstrip": false,
|
2151 |
+
"normalized": false,
|
2152 |
+
"rstrip": false,
|
2153 |
+
"single_word": false,
|
2154 |
+
"special": true
|
2155 |
+
},
|
2156 |
+
"269": {
|
2157 |
+
"content": "[extra_id_107]",
|
2158 |
+
"lstrip": false,
|
2159 |
+
"normalized": false,
|
2160 |
+
"rstrip": false,
|
2161 |
+
"single_word": false,
|
2162 |
+
"special": true
|
2163 |
+
},
|
2164 |
+
"270": {
|
2165 |
+
"content": "[extra_id_108]",
|
2166 |
+
"lstrip": false,
|
2167 |
+
"normalized": false,
|
2168 |
+
"rstrip": false,
|
2169 |
+
"single_word": false,
|
2170 |
+
"special": true
|
2171 |
+
},
|
2172 |
+
"271": {
|
2173 |
+
"content": "[extra_id_109]",
|
2174 |
+
"lstrip": false,
|
2175 |
+
"normalized": false,
|
2176 |
+
"rstrip": false,
|
2177 |
+
"single_word": false,
|
2178 |
+
"special": true
|
2179 |
+
},
|
2180 |
+
"272": {
|
2181 |
+
"content": "[extra_id_110]",
|
2182 |
+
"lstrip": false,
|
2183 |
+
"normalized": false,
|
2184 |
+
"rstrip": false,
|
2185 |
+
"single_word": false,
|
2186 |
+
"special": true
|
2187 |
+
},
|
2188 |
+
"273": {
|
2189 |
+
"content": "[extra_id_111]",
|
2190 |
+
"lstrip": false,
|
2191 |
+
"normalized": false,
|
2192 |
+
"rstrip": false,
|
2193 |
+
"single_word": false,
|
2194 |
+
"special": true
|
2195 |
+
},
|
2196 |
+
"274": {
|
2197 |
+
"content": "[extra_id_112]",
|
2198 |
+
"lstrip": false,
|
2199 |
+
"normalized": false,
|
2200 |
+
"rstrip": false,
|
2201 |
+
"single_word": false,
|
2202 |
+
"special": true
|
2203 |
+
},
|
2204 |
+
"275": {
|
2205 |
+
"content": "[extra_id_113]",
|
2206 |
+
"lstrip": false,
|
2207 |
+
"normalized": false,
|
2208 |
+
"rstrip": false,
|
2209 |
+
"single_word": false,
|
2210 |
+
"special": true
|
2211 |
+
},
|
2212 |
+
"276": {
|
2213 |
+
"content": "[extra_id_114]",
|
2214 |
+
"lstrip": false,
|
2215 |
+
"normalized": false,
|
2216 |
+
"rstrip": false,
|
2217 |
+
"single_word": false,
|
2218 |
+
"special": true
|
2219 |
+
},
|
2220 |
+
"277": {
|
2221 |
+
"content": "[extra_id_115]",
|
2222 |
+
"lstrip": false,
|
2223 |
+
"normalized": false,
|
2224 |
+
"rstrip": false,
|
2225 |
+
"single_word": false,
|
2226 |
+
"special": true
|
2227 |
+
},
|
2228 |
+
"278": {
|
2229 |
+
"content": "[extra_id_116]",
|
2230 |
+
"lstrip": false,
|
2231 |
+
"normalized": false,
|
2232 |
+
"rstrip": false,
|
2233 |
+
"single_word": false,
|
2234 |
+
"special": true
|
2235 |
+
},
|
2236 |
+
"279": {
|
2237 |
+
"content": "[extra_id_117]",
|
2238 |
+
"lstrip": false,
|
2239 |
+
"normalized": false,
|
2240 |
+
"rstrip": false,
|
2241 |
+
"single_word": false,
|
2242 |
+
"special": true
|
2243 |
+
},
|
2244 |
+
"280": {
|
2245 |
+
"content": "[extra_id_118]",
|
2246 |
+
"lstrip": false,
|
2247 |
+
"normalized": false,
|
2248 |
+
"rstrip": false,
|
2249 |
+
"single_word": false,
|
2250 |
+
"special": true
|
2251 |
+
},
|
2252 |
+
"281": {
|
2253 |
+
"content": "[extra_id_119]",
|
2254 |
+
"lstrip": false,
|
2255 |
+
"normalized": false,
|
2256 |
+
"rstrip": false,
|
2257 |
+
"single_word": false,
|
2258 |
+
"special": true
|
2259 |
+
},
|
2260 |
+
"282": {
|
2261 |
+
"content": "[extra_id_120]",
|
2262 |
+
"lstrip": false,
|
2263 |
+
"normalized": false,
|
2264 |
+
"rstrip": false,
|
2265 |
+
"single_word": false,
|
2266 |
+
"special": true
|
2267 |
+
},
|
2268 |
+
"283": {
|
2269 |
+
"content": "[extra_id_121]",
|
2270 |
+
"lstrip": false,
|
2271 |
+
"normalized": false,
|
2272 |
+
"rstrip": false,
|
2273 |
+
"single_word": false,
|
2274 |
+
"special": true
|
2275 |
+
},
|
2276 |
+
"284": {
|
2277 |
+
"content": "[extra_id_122]",
|
2278 |
+
"lstrip": false,
|
2279 |
+
"normalized": false,
|
2280 |
+
"rstrip": false,
|
2281 |
+
"single_word": false,
|
2282 |
+
"special": true
|
2283 |
+
},
|
2284 |
+
"285": {
|
2285 |
+
"content": "[extra_id_123]",
|
2286 |
+
"lstrip": false,
|
2287 |
+
"normalized": false,
|
2288 |
+
"rstrip": false,
|
2289 |
+
"single_word": false,
|
2290 |
+
"special": true
|
2291 |
+
},
|
2292 |
+
"286": {
|
2293 |
+
"content": "[extra_id_124]",
|
2294 |
+
"lstrip": false,
|
2295 |
+
"normalized": false,
|
2296 |
+
"rstrip": false,
|
2297 |
+
"single_word": false,
|
2298 |
+
"special": true
|
2299 |
+
},
|
2300 |
+
"287": {
|
2301 |
+
"content": "[extra_id_125]",
|
2302 |
+
"lstrip": false,
|
2303 |
+
"normalized": false,
|
2304 |
+
"rstrip": false,
|
2305 |
+
"single_word": false,
|
2306 |
+
"special": true
|
2307 |
+
},
|
2308 |
+
"288": {
|
2309 |
+
"content": "[extra_id_126]",
|
2310 |
+
"lstrip": false,
|
2311 |
+
"normalized": false,
|
2312 |
+
"rstrip": false,
|
2313 |
+
"single_word": false,
|
2314 |
+
"special": true
|
2315 |
+
},
|
2316 |
+
"289": {
|
2317 |
+
"content": "[extra_id_127]",
|
2318 |
+
"lstrip": false,
|
2319 |
+
"normalized": false,
|
2320 |
+
"rstrip": false,
|
2321 |
+
"single_word": false,
|
2322 |
+
"special": true
|
2323 |
+
},
|
2324 |
+
"290": {
|
2325 |
+
"content": "[extra_id_128]",
|
2326 |
+
"lstrip": false,
|
2327 |
+
"normalized": false,
|
2328 |
+
"rstrip": false,
|
2329 |
+
"single_word": false,
|
2330 |
+
"special": true
|
2331 |
+
},
|
2332 |
+
"291": {
|
2333 |
+
"content": "[extra_id_129]",
|
2334 |
+
"lstrip": false,
|
2335 |
+
"normalized": false,
|
2336 |
+
"rstrip": false,
|
2337 |
+
"single_word": false,
|
2338 |
+
"special": true
|
2339 |
+
},
|
2340 |
+
"292": {
|
2341 |
+
"content": "[extra_id_130]",
|
2342 |
+
"lstrip": false,
|
2343 |
+
"normalized": false,
|
2344 |
+
"rstrip": false,
|
2345 |
+
"single_word": false,
|
2346 |
+
"special": true
|
2347 |
+
},
|
2348 |
+
"293": {
|
2349 |
+
"content": "[extra_id_131]",
|
2350 |
+
"lstrip": false,
|
2351 |
+
"normalized": false,
|
2352 |
+
"rstrip": false,
|
2353 |
+
"single_word": false,
|
2354 |
+
"special": true
|
2355 |
+
},
|
2356 |
+
"294": {
|
2357 |
+
"content": "[extra_id_132]",
|
2358 |
+
"lstrip": false,
|
2359 |
+
"normalized": false,
|
2360 |
+
"rstrip": false,
|
2361 |
+
"single_word": false,
|
2362 |
+
"special": true
|
2363 |
+
},
|
2364 |
+
"295": {
|
2365 |
+
"content": "[extra_id_133]",
|
2366 |
+
"lstrip": false,
|
2367 |
+
"normalized": false,
|
2368 |
+
"rstrip": false,
|
2369 |
+
"single_word": false,
|
2370 |
+
"special": true
|
2371 |
+
},
|
2372 |
+
"296": {
|
2373 |
+
"content": "[extra_id_134]",
|
2374 |
+
"lstrip": false,
|
2375 |
+
"normalized": false,
|
2376 |
+
"rstrip": false,
|
2377 |
+
"single_word": false,
|
2378 |
+
"special": true
|
2379 |
+
},
|
2380 |
+
"297": {
|
2381 |
+
"content": "[extra_id_135]",
|
2382 |
+
"lstrip": false,
|
2383 |
+
"normalized": false,
|
2384 |
+
"rstrip": false,
|
2385 |
+
"single_word": false,
|
2386 |
+
"special": true
|
2387 |
+
},
|
2388 |
+
"298": {
|
2389 |
+
"content": "[extra_id_136]",
|
2390 |
+
"lstrip": false,
|
2391 |
+
"normalized": false,
|
2392 |
+
"rstrip": false,
|
2393 |
+
"single_word": false,
|
2394 |
+
"special": true
|
2395 |
+
},
|
2396 |
+
"299": {
|
2397 |
+
"content": "[extra_id_137]",
|
2398 |
+
"lstrip": false,
|
2399 |
+
"normalized": false,
|
2400 |
+
"rstrip": false,
|
2401 |
+
"single_word": false,
|
2402 |
+
"special": true
|
2403 |
+
},
|
2404 |
+
"300": {
|
2405 |
+
"content": "[extra_id_138]",
|
2406 |
+
"lstrip": false,
|
2407 |
+
"normalized": false,
|
2408 |
+
"rstrip": false,
|
2409 |
+
"single_word": false,
|
2410 |
+
"special": true
|
2411 |
+
},
|
2412 |
+
"301": {
|
2413 |
+
"content": "[extra_id_139]",
|
2414 |
+
"lstrip": false,
|
2415 |
+
"normalized": false,
|
2416 |
+
"rstrip": false,
|
2417 |
+
"single_word": false,
|
2418 |
+
"special": true
|
2419 |
+
},
|
2420 |
+
"302": {
|
2421 |
+
"content": "[extra_id_140]",
|
2422 |
+
"lstrip": false,
|
2423 |
+
"normalized": false,
|
2424 |
+
"rstrip": false,
|
2425 |
+
"single_word": false,
|
2426 |
+
"special": true
|
2427 |
+
},
|
2428 |
+
"303": {
|
2429 |
+
"content": "[extra_id_141]",
|
2430 |
+
"lstrip": false,
|
2431 |
+
"normalized": false,
|
2432 |
+
"rstrip": false,
|
2433 |
+
"single_word": false,
|
2434 |
+
"special": true
|
2435 |
+
},
|
2436 |
+
"304": {
|
2437 |
+
"content": "[extra_id_142]",
|
2438 |
+
"lstrip": false,
|
2439 |
+
"normalized": false,
|
2440 |
+
"rstrip": false,
|
2441 |
+
"single_word": false,
|
2442 |
+
"special": true
|
2443 |
+
},
|
2444 |
+
"305": {
|
2445 |
+
"content": "[extra_id_143]",
|
2446 |
+
"lstrip": false,
|
2447 |
+
"normalized": false,
|
2448 |
+
"rstrip": false,
|
2449 |
+
"single_word": false,
|
2450 |
+
"special": true
|
2451 |
+
},
|
2452 |
+
"306": {
|
2453 |
+
"content": "[extra_id_144]",
|
2454 |
+
"lstrip": false,
|
2455 |
+
"normalized": false,
|
2456 |
+
"rstrip": false,
|
2457 |
+
"single_word": false,
|
2458 |
+
"special": true
|
2459 |
+
},
|
2460 |
+
"307": {
|
2461 |
+
"content": "[extra_id_145]",
|
2462 |
+
"lstrip": false,
|
2463 |
+
"normalized": false,
|
2464 |
+
"rstrip": false,
|
2465 |
+
"single_word": false,
|
2466 |
+
"special": true
|
2467 |
+
},
|
2468 |
+
"308": {
|
2469 |
+
"content": "[extra_id_146]",
|
2470 |
+
"lstrip": false,
|
2471 |
+
"normalized": false,
|
2472 |
+
"rstrip": false,
|
2473 |
+
"single_word": false,
|
2474 |
+
"special": true
|
2475 |
+
},
|
2476 |
+
"309": {
|
2477 |
+
"content": "[extra_id_147]",
|
2478 |
+
"lstrip": false,
|
2479 |
+
"normalized": false,
|
2480 |
+
"rstrip": false,
|
2481 |
+
"single_word": false,
|
2482 |
+
"special": true
|
2483 |
+
},
|
2484 |
+
"310": {
|
2485 |
+
"content": "[extra_id_148]",
|
2486 |
+
"lstrip": false,
|
2487 |
+
"normalized": false,
|
2488 |
+
"rstrip": false,
|
2489 |
+
"single_word": false,
|
2490 |
+
"special": true
|
2491 |
+
},
|
2492 |
+
"311": {
|
2493 |
+
"content": "[extra_id_149]",
|
2494 |
+
"lstrip": false,
|
2495 |
+
"normalized": false,
|
2496 |
+
"rstrip": false,
|
2497 |
+
"single_word": false,
|
2498 |
+
"special": true
|
2499 |
+
},
|
2500 |
+
"312": {
|
2501 |
+
"content": "[extra_id_150]",
|
2502 |
+
"lstrip": false,
|
2503 |
+
"normalized": false,
|
2504 |
+
"rstrip": false,
|
2505 |
+
"single_word": false,
|
2506 |
+
"special": true
|
2507 |
+
},
|
2508 |
+
"313": {
|
2509 |
+
"content": "[extra_id_151]",
|
2510 |
+
"lstrip": false,
|
2511 |
+
"normalized": false,
|
2512 |
+
"rstrip": false,
|
2513 |
+
"single_word": false,
|
2514 |
+
"special": true
|
2515 |
+
},
|
2516 |
+
"314": {
|
2517 |
+
"content": "[extra_id_152]",
|
2518 |
+
"lstrip": false,
|
2519 |
+
"normalized": false,
|
2520 |
+
"rstrip": false,
|
2521 |
+
"single_word": false,
|
2522 |
+
"special": true
|
2523 |
+
},
|
2524 |
+
"315": {
|
2525 |
+
"content": "[extra_id_153]",
|
2526 |
+
"lstrip": false,
|
2527 |
+
"normalized": false,
|
2528 |
+
"rstrip": false,
|
2529 |
+
"single_word": false,
|
2530 |
+
"special": true
|
2531 |
+
},
|
2532 |
+
"316": {
|
2533 |
+
"content": "[extra_id_154]",
|
2534 |
+
"lstrip": false,
|
2535 |
+
"normalized": false,
|
2536 |
+
"rstrip": false,
|
2537 |
+
"single_word": false,
|
2538 |
+
"special": true
|
2539 |
+
},
|
2540 |
+
"317": {
|
2541 |
+
"content": "[extra_id_155]",
|
2542 |
+
"lstrip": false,
|
2543 |
+
"normalized": false,
|
2544 |
+
"rstrip": false,
|
2545 |
+
"single_word": false,
|
2546 |
+
"special": true
|
2547 |
+
},
|
2548 |
+
"318": {
|
2549 |
+
"content": "[extra_id_156]",
|
2550 |
+
"lstrip": false,
|
2551 |
+
"normalized": false,
|
2552 |
+
"rstrip": false,
|
2553 |
+
"single_word": false,
|
2554 |
+
"special": true
|
2555 |
+
},
|
2556 |
+
"319": {
|
2557 |
+
"content": "[extra_id_157]",
|
2558 |
+
"lstrip": false,
|
2559 |
+
"normalized": false,
|
2560 |
+
"rstrip": false,
|
2561 |
+
"single_word": false,
|
2562 |
+
"special": true
|
2563 |
+
},
|
2564 |
+
"320": {
|
2565 |
+
"content": "[extra_id_158]",
|
2566 |
+
"lstrip": false,
|
2567 |
+
"normalized": false,
|
2568 |
+
"rstrip": false,
|
2569 |
+
"single_word": false,
|
2570 |
+
"special": true
|
2571 |
+
},
|
2572 |
+
"321": {
|
2573 |
+
"content": "[extra_id_159]",
|
2574 |
+
"lstrip": false,
|
2575 |
+
"normalized": false,
|
2576 |
+
"rstrip": false,
|
2577 |
+
"single_word": false,
|
2578 |
+
"special": true
|
2579 |
+
},
|
2580 |
+
"322": {
|
2581 |
+
"content": "[extra_id_160]",
|
2582 |
+
"lstrip": false,
|
2583 |
+
"normalized": false,
|
2584 |
+
"rstrip": false,
|
2585 |
+
"single_word": false,
|
2586 |
+
"special": true
|
2587 |
+
},
|
2588 |
+
"323": {
|
2589 |
+
"content": "[extra_id_161]",
|
2590 |
+
"lstrip": false,
|
2591 |
+
"normalized": false,
|
2592 |
+
"rstrip": false,
|
2593 |
+
"single_word": false,
|
2594 |
+
"special": true
|
2595 |
+
},
|
2596 |
+
"324": {
|
2597 |
+
"content": "[extra_id_162]",
|
2598 |
+
"lstrip": false,
|
2599 |
+
"normalized": false,
|
2600 |
+
"rstrip": false,
|
2601 |
+
"single_word": false,
|
2602 |
+
"special": true
|
2603 |
+
},
|
2604 |
+
"325": {
|
2605 |
+
"content": "[extra_id_163]",
|
2606 |
+
"lstrip": false,
|
2607 |
+
"normalized": false,
|
2608 |
+
"rstrip": false,
|
2609 |
+
"single_word": false,
|
2610 |
+
"special": true
|
2611 |
+
},
|
2612 |
+
"326": {
|
2613 |
+
"content": "[extra_id_164]",
|
2614 |
+
"lstrip": false,
|
2615 |
+
"normalized": false,
|
2616 |
+
"rstrip": false,
|
2617 |
+
"single_word": false,
|
2618 |
+
"special": true
|
2619 |
+
},
|
2620 |
+
"327": {
|
2621 |
+
"content": "[extra_id_165]",
|
2622 |
+
"lstrip": false,
|
2623 |
+
"normalized": false,
|
2624 |
+
"rstrip": false,
|
2625 |
+
"single_word": false,
|
2626 |
+
"special": true
|
2627 |
+
},
|
2628 |
+
"328": {
|
2629 |
+
"content": "[extra_id_166]",
|
2630 |
+
"lstrip": false,
|
2631 |
+
"normalized": false,
|
2632 |
+
"rstrip": false,
|
2633 |
+
"single_word": false,
|
2634 |
+
"special": true
|
2635 |
+
},
|
2636 |
+
"329": {
|
2637 |
+
"content": "[extra_id_167]",
|
2638 |
+
"lstrip": false,
|
2639 |
+
"normalized": false,
|
2640 |
+
"rstrip": false,
|
2641 |
+
"single_word": false,
|
2642 |
+
"special": true
|
2643 |
+
},
|
2644 |
+
"330": {
|
2645 |
+
"content": "[extra_id_168]",
|
2646 |
+
"lstrip": false,
|
2647 |
+
"normalized": false,
|
2648 |
+
"rstrip": false,
|
2649 |
+
"single_word": false,
|
2650 |
+
"special": true
|
2651 |
+
},
|
2652 |
+
"331": {
|
2653 |
+
"content": "[extra_id_169]",
|
2654 |
+
"lstrip": false,
|
2655 |
+
"normalized": false,
|
2656 |
+
"rstrip": false,
|
2657 |
+
"single_word": false,
|
2658 |
+
"special": true
|
2659 |
+
},
|
2660 |
+
"332": {
|
2661 |
+
"content": "[extra_id_170]",
|
2662 |
+
"lstrip": false,
|
2663 |
+
"normalized": false,
|
2664 |
+
"rstrip": false,
|
2665 |
+
"single_word": false,
|
2666 |
+
"special": true
|
2667 |
+
},
|
2668 |
+
"333": {
|
2669 |
+
"content": "[extra_id_171]",
|
2670 |
+
"lstrip": false,
|
2671 |
+
"normalized": false,
|
2672 |
+
"rstrip": false,
|
2673 |
+
"single_word": false,
|
2674 |
+
"special": true
|
2675 |
+
},
|
2676 |
+
"334": {
|
2677 |
+
"content": "[extra_id_172]",
|
2678 |
+
"lstrip": false,
|
2679 |
+
"normalized": false,
|
2680 |
+
"rstrip": false,
|
2681 |
+
"single_word": false,
|
2682 |
+
"special": true
|
2683 |
+
},
|
2684 |
+
"335": {
|
2685 |
+
"content": "[extra_id_173]",
|
2686 |
+
"lstrip": false,
|
2687 |
+
"normalized": false,
|
2688 |
+
"rstrip": false,
|
2689 |
+
"single_word": false,
|
2690 |
+
"special": true
|
2691 |
+
},
|
2692 |
+
"336": {
|
2693 |
+
"content": "[extra_id_174]",
|
2694 |
+
"lstrip": false,
|
2695 |
+
"normalized": false,
|
2696 |
+
"rstrip": false,
|
2697 |
+
"single_word": false,
|
2698 |
+
"special": true
|
2699 |
+
},
|
2700 |
+
"337": {
|
2701 |
+
"content": "[extra_id_175]",
|
2702 |
+
"lstrip": false,
|
2703 |
+
"normalized": false,
|
2704 |
+
"rstrip": false,
|
2705 |
+
"single_word": false,
|
2706 |
+
"special": true
|
2707 |
+
},
|
2708 |
+
"338": {
|
2709 |
+
"content": "[extra_id_176]",
|
2710 |
+
"lstrip": false,
|
2711 |
+
"normalized": false,
|
2712 |
+
"rstrip": false,
|
2713 |
+
"single_word": false,
|
2714 |
+
"special": true
|
2715 |
+
},
|
2716 |
+
"339": {
|
2717 |
+
"content": "[extra_id_177]",
|
2718 |
+
"lstrip": false,
|
2719 |
+
"normalized": false,
|
2720 |
+
"rstrip": false,
|
2721 |
+
"single_word": false,
|
2722 |
+
"special": true
|
2723 |
+
},
|
2724 |
+
"340": {
|
2725 |
+
"content": "[extra_id_178]",
|
2726 |
+
"lstrip": false,
|
2727 |
+
"normalized": false,
|
2728 |
+
"rstrip": false,
|
2729 |
+
"single_word": false,
|
2730 |
+
"special": true
|
2731 |
+
},
|
2732 |
+
"341": {
|
2733 |
+
"content": "[extra_id_179]",
|
2734 |
+
"lstrip": false,
|
2735 |
+
"normalized": false,
|
2736 |
+
"rstrip": false,
|
2737 |
+
"single_word": false,
|
2738 |
+
"special": true
|
2739 |
+
},
|
2740 |
+
"342": {
|
2741 |
+
"content": "[extra_id_180]",
|
2742 |
+
"lstrip": false,
|
2743 |
+
"normalized": false,
|
2744 |
+
"rstrip": false,
|
2745 |
+
"single_word": false,
|
2746 |
+
"special": true
|
2747 |
+
},
|
2748 |
+
"343": {
|
2749 |
+
"content": "[extra_id_181]",
|
2750 |
+
"lstrip": false,
|
2751 |
+
"normalized": false,
|
2752 |
+
"rstrip": false,
|
2753 |
+
"single_word": false,
|
2754 |
+
"special": true
|
2755 |
+
},
|
2756 |
+
"344": {
|
2757 |
+
"content": "[extra_id_182]",
|
2758 |
+
"lstrip": false,
|
2759 |
+
"normalized": false,
|
2760 |
+
"rstrip": false,
|
2761 |
+
"single_word": false,
|
2762 |
+
"special": true
|
2763 |
+
},
|
2764 |
+
"345": {
|
2765 |
+
"content": "[extra_id_183]",
|
2766 |
+
"lstrip": false,
|
2767 |
+
"normalized": false,
|
2768 |
+
"rstrip": false,
|
2769 |
+
"single_word": false,
|
2770 |
+
"special": true
|
2771 |
+
},
|
2772 |
+
"346": {
|
2773 |
+
"content": "[extra_id_184]",
|
2774 |
+
"lstrip": false,
|
2775 |
+
"normalized": false,
|
2776 |
+
"rstrip": false,
|
2777 |
+
"single_word": false,
|
2778 |
+
"special": true
|
2779 |
+
},
|
2780 |
+
"347": {
|
2781 |
+
"content": "[extra_id_185]",
|
2782 |
+
"lstrip": false,
|
2783 |
+
"normalized": false,
|
2784 |
+
"rstrip": false,
|
2785 |
+
"single_word": false,
|
2786 |
+
"special": true
|
2787 |
+
},
|
2788 |
+
"348": {
|
2789 |
+
"content": "[extra_id_186]",
|
2790 |
+
"lstrip": false,
|
2791 |
+
"normalized": false,
|
2792 |
+
"rstrip": false,
|
2793 |
+
"single_word": false,
|
2794 |
+
"special": true
|
2795 |
+
},
|
2796 |
+
"349": {
|
2797 |
+
"content": "[extra_id_187]",
|
2798 |
+
"lstrip": false,
|
2799 |
+
"normalized": false,
|
2800 |
+
"rstrip": false,
|
2801 |
+
"single_word": false,
|
2802 |
+
"special": true
|
2803 |
+
},
|
2804 |
+
"350": {
|
2805 |
+
"content": "[extra_id_188]",
|
2806 |
+
"lstrip": false,
|
2807 |
+
"normalized": false,
|
2808 |
+
"rstrip": false,
|
2809 |
+
"single_word": false,
|
2810 |
+
"special": true
|
2811 |
+
},
|
2812 |
+
"351": {
|
2813 |
+
"content": "[extra_id_189]",
|
2814 |
+
"lstrip": false,
|
2815 |
+
"normalized": false,
|
2816 |
+
"rstrip": false,
|
2817 |
+
"single_word": false,
|
2818 |
+
"special": true
|
2819 |
+
},
|
2820 |
+
"352": {
|
2821 |
+
"content": "[extra_id_190]",
|
2822 |
+
"lstrip": false,
|
2823 |
+
"normalized": false,
|
2824 |
+
"rstrip": false,
|
2825 |
+
"single_word": false,
|
2826 |
+
"special": true
|
2827 |
+
},
|
2828 |
+
"353": {
|
2829 |
+
"content": "[extra_id_191]",
|
2830 |
+
"lstrip": false,
|
2831 |
+
"normalized": false,
|
2832 |
+
"rstrip": false,
|
2833 |
+
"single_word": false,
|
2834 |
+
"special": true
|
2835 |
+
},
|
2836 |
+
"354": {
|
2837 |
+
"content": "[extra_id_192]",
|
2838 |
+
"lstrip": false,
|
2839 |
+
"normalized": false,
|
2840 |
+
"rstrip": false,
|
2841 |
+
"single_word": false,
|
2842 |
+
"special": true
|
2843 |
+
},
|
2844 |
+
"355": {
|
2845 |
+
"content": "[extra_id_193]",
|
2846 |
+
"lstrip": false,
|
2847 |
+
"normalized": false,
|
2848 |
+
"rstrip": false,
|
2849 |
+
"single_word": false,
|
2850 |
+
"special": true
|
2851 |
+
},
|
2852 |
+
"356": {
|
2853 |
+
"content": "[extra_id_194]",
|
2854 |
+
"lstrip": false,
|
2855 |
+
"normalized": false,
|
2856 |
+
"rstrip": false,
|
2857 |
+
"single_word": false,
|
2858 |
+
"special": true
|
2859 |
+
},
|
2860 |
+
"357": {
|
2861 |
+
"content": "[extra_id_195]",
|
2862 |
+
"lstrip": false,
|
2863 |
+
"normalized": false,
|
2864 |
+
"rstrip": false,
|
2865 |
+
"single_word": false,
|
2866 |
+
"special": true
|
2867 |
+
},
|
2868 |
+
"358": {
|
2869 |
+
"content": "[extra_id_196]",
|
2870 |
+
"lstrip": false,
|
2871 |
+
"normalized": false,
|
2872 |
+
"rstrip": false,
|
2873 |
+
"single_word": false,
|
2874 |
+
"special": true
|
2875 |
+
},
|
2876 |
+
"359": {
|
2877 |
+
"content": "[extra_id_197]",
|
2878 |
+
"lstrip": false,
|
2879 |
+
"normalized": false,
|
2880 |
+
"rstrip": false,
|
2881 |
+
"single_word": false,
|
2882 |
+
"special": true
|
2883 |
+
},
|
2884 |
+
"360": {
|
2885 |
+
"content": "[extra_id_198]",
|
2886 |
+
"lstrip": false,
|
2887 |
+
"normalized": false,
|
2888 |
+
"rstrip": false,
|
2889 |
+
"single_word": false,
|
2890 |
+
"special": true
|
2891 |
+
},
|
2892 |
+
"361": {
|
2893 |
+
"content": "[|endofturn|]",
|
2894 |
+
"lstrip": false,
|
2895 |
+
"normalized": false,
|
2896 |
+
"rstrip": false,
|
2897 |
+
"single_word": false,
|
2898 |
+
"special": true
|
2899 |
+
}
|
2900 |
+
},
|
2901 |
+
"additional_special_token": [
|
2902 |
+
"[unused0]",
|
2903 |
+
"[unused1]",
|
2904 |
+
"[unused2]",
|
2905 |
+
"[unused3]",
|
2906 |
+
"[unused4]",
|
2907 |
+
"[unused5]",
|
2908 |
+
"[unused6]",
|
2909 |
+
"[unused7]",
|
2910 |
+
"[unused8]",
|
2911 |
+
"[unused9]",
|
2912 |
+
"[unused10]",
|
2913 |
+
"[unused11]",
|
2914 |
+
"[unused12]",
|
2915 |
+
"[unused13]",
|
2916 |
+
"[unused14]",
|
2917 |
+
"[unused15]",
|
2918 |
+
"[unused16]",
|
2919 |
+
"[unused17]",
|
2920 |
+
"[unused18]",
|
2921 |
+
"[unused19]",
|
2922 |
+
"[unused20]",
|
2923 |
+
"[unused21]",
|
2924 |
+
"[unused22]",
|
2925 |
+
"[unused23]",
|
2926 |
+
"[unused24]",
|
2927 |
+
"[unused25]",
|
2928 |
+
"[unused26]",
|
2929 |
+
"[unused27]",
|
2930 |
+
"[unused28]",
|
2931 |
+
"[unused29]",
|
2932 |
+
"[unused30]",
|
2933 |
+
"[unused31]",
|
2934 |
+
"[unused32]",
|
2935 |
+
"[unused33]",
|
2936 |
+
"[unused34]",
|
2937 |
+
"[unused35]",
|
2938 |
+
"[unused36]",
|
2939 |
+
"[unused37]",
|
2940 |
+
"[unused38]",
|
2941 |
+
"[unused39]",
|
2942 |
+
"[unused40]",
|
2943 |
+
"[unused41]",
|
2944 |
+
"[unused42]",
|
2945 |
+
"[unused43]",
|
2946 |
+
"[unused44]",
|
2947 |
+
"[unused45]",
|
2948 |
+
"[unused46]",
|
2949 |
+
"[unused47]",
|
2950 |
+
"[unused48]",
|
2951 |
+
"[unused49]",
|
2952 |
+
"[unused50]",
|
2953 |
+
"[unused51]",
|
2954 |
+
"[unused52]",
|
2955 |
+
"[unused53]",
|
2956 |
+
"[unused54]",
|
2957 |
+
"[unused55]",
|
2958 |
+
"[unused56]",
|
2959 |
+
"[unused57]",
|
2960 |
+
"[unused58]",
|
2961 |
+
"[unused59]",
|
2962 |
+
"[unused60]",
|
2963 |
+
"[unused61]",
|
2964 |
+
"[unused62]",
|
2965 |
+
"[unused63]",
|
2966 |
+
"[unused64]",
|
2967 |
+
"[unused65]",
|
2968 |
+
"[unused66]",
|
2969 |
+
"[unused67]",
|
2970 |
+
"[unused68]",
|
2971 |
+
"[unused69]",
|
2972 |
+
"[unused70]",
|
2973 |
+
"[unused71]",
|
2974 |
+
"[unused72]",
|
2975 |
+
"[unused73]",
|
2976 |
+
"[unused74]",
|
2977 |
+
"[unused75]",
|
2978 |
+
"[unused76]",
|
2979 |
+
"[unused77]",
|
2980 |
+
"[unused78]",
|
2981 |
+
"[unused79]",
|
2982 |
+
"[unused80]",
|
2983 |
+
"[unused81]",
|
2984 |
+
"[unused82]",
|
2985 |
+
"[unused83]",
|
2986 |
+
"[unused84]",
|
2987 |
+
"[unused85]",
|
2988 |
+
"[unused86]",
|
2989 |
+
"[unused87]",
|
2990 |
+
"[unused88]",
|
2991 |
+
"[unused89]",
|
2992 |
+
"[unused90]",
|
2993 |
+
"[unused91]",
|
2994 |
+
"[unused92]",
|
2995 |
+
"[unused93]",
|
2996 |
+
"[unused94]",
|
2997 |
+
"[unused95]",
|
2998 |
+
"[unused96]",
|
2999 |
+
"[unused97]",
|
3000 |
+
"[unused98]",
|
3001 |
+
"[unused99]",
|
3002 |
+
"[extra_id_0]",
|
3003 |
+
"[extra_id_1]",
|
3004 |
+
"[extra_id_2]",
|
3005 |
+
"[extra_id_3]",
|
3006 |
+
"[extra_id_4]",
|
3007 |
+
"[extra_id_5]",
|
3008 |
+
"[extra_id_6]",
|
3009 |
+
"[extra_id_7]",
|
3010 |
+
"[extra_id_8]",
|
3011 |
+
"[extra_id_9]",
|
3012 |
+
"[extra_id_10]",
|
3013 |
+
"[extra_id_11]",
|
3014 |
+
"[extra_id_12]",
|
3015 |
+
"[extra_id_13]",
|
3016 |
+
"[extra_id_14]",
|
3017 |
+
"[extra_id_15]",
|
3018 |
+
"[extra_id_16]",
|
3019 |
+
"[extra_id_17]",
|
3020 |
+
"[extra_id_18]",
|
3021 |
+
"[extra_id_19]",
|
3022 |
+
"[extra_id_20]",
|
3023 |
+
"[extra_id_21]",
|
3024 |
+
"[extra_id_22]",
|
3025 |
+
"[extra_id_23]",
|
3026 |
+
"[extra_id_24]",
|
3027 |
+
"[extra_id_25]",
|
3028 |
+
"[extra_id_26]",
|
3029 |
+
"[extra_id_27]",
|
3030 |
+
"[extra_id_28]",
|
3031 |
+
"[extra_id_29]",
|
3032 |
+
"[extra_id_30]",
|
3033 |
+
"[extra_id_31]",
|
3034 |
+
"[extra_id_32]",
|
3035 |
+
"[extra_id_33]",
|
3036 |
+
"[extra_id_34]",
|
3037 |
+
"[extra_id_35]",
|
3038 |
+
"[extra_id_36]",
|
3039 |
+
"[extra_id_37]",
|
3040 |
+
"[extra_id_38]",
|
3041 |
+
"[extra_id_39]",
|
3042 |
+
"[extra_id_40]",
|
3043 |
+
"[extra_id_41]",
|
3044 |
+
"[extra_id_42]",
|
3045 |
+
"[extra_id_43]",
|
3046 |
+
"[extra_id_44]",
|
3047 |
+
"[extra_id_45]",
|
3048 |
+
"[extra_id_46]",
|
3049 |
+
"[extra_id_47]",
|
3050 |
+
"[extra_id_48]",
|
3051 |
+
"[extra_id_49]",
|
3052 |
+
"[extra_id_50]",
|
3053 |
+
"[extra_id_51]",
|
3054 |
+
"[extra_id_52]",
|
3055 |
+
"[extra_id_53]",
|
3056 |
+
"[extra_id_54]",
|
3057 |
+
"[extra_id_55]",
|
3058 |
+
"[extra_id_56]",
|
3059 |
+
"[extra_id_57]",
|
3060 |
+
"[extra_id_58]",
|
3061 |
+
"[extra_id_59]",
|
3062 |
+
"[extra_id_60]",
|
3063 |
+
"[extra_id_61]",
|
3064 |
+
"[extra_id_62]",
|
3065 |
+
"[extra_id_63]",
|
3066 |
+
"[extra_id_64]",
|
3067 |
+
"[extra_id_65]",
|
3068 |
+
"[extra_id_66]",
|
3069 |
+
"[extra_id_67]",
|
3070 |
+
"[extra_id_68]",
|
3071 |
+
"[extra_id_69]",
|
3072 |
+
"[extra_id_70]",
|
3073 |
+
"[extra_id_71]",
|
3074 |
+
"[extra_id_72]",
|
3075 |
+
"[extra_id_73]",
|
3076 |
+
"[extra_id_74]",
|
3077 |
+
"[extra_id_75]",
|
3078 |
+
"[extra_id_76]",
|
3079 |
+
"[extra_id_77]",
|
3080 |
+
"[extra_id_78]",
|
3081 |
+
"[extra_id_79]",
|
3082 |
+
"[extra_id_80]",
|
3083 |
+
"[extra_id_81]",
|
3084 |
+
"[extra_id_82]",
|
3085 |
+
"[extra_id_83]",
|
3086 |
+
"[extra_id_84]",
|
3087 |
+
"[extra_id_85]",
|
3088 |
+
"[extra_id_86]",
|
3089 |
+
"[extra_id_87]",
|
3090 |
+
"[extra_id_88]",
|
3091 |
+
"[extra_id_89]",
|
3092 |
+
"[extra_id_90]",
|
3093 |
+
"[extra_id_91]",
|
3094 |
+
"[extra_id_92]",
|
3095 |
+
"[extra_id_93]",
|
3096 |
+
"[extra_id_94]",
|
3097 |
+
"[extra_id_95]",
|
3098 |
+
"[extra_id_96]",
|
3099 |
+
"[extra_id_97]",
|
3100 |
+
"[extra_id_98]",
|
3101 |
+
"[extra_id_99]",
|
3102 |
+
"[extra_id_100]",
|
3103 |
+
"[extra_id_101]",
|
3104 |
+
"[extra_id_102]",
|
3105 |
+
"[extra_id_103]",
|
3106 |
+
"[extra_id_104]",
|
3107 |
+
"[extra_id_105]",
|
3108 |
+
"[extra_id_106]",
|
3109 |
+
"[extra_id_107]",
|
3110 |
+
"[extra_id_108]",
|
3111 |
+
"[extra_id_109]",
|
3112 |
+
"[extra_id_110]",
|
3113 |
+
"[extra_id_111]",
|
3114 |
+
"[extra_id_112]",
|
3115 |
+
"[extra_id_113]",
|
3116 |
+
"[extra_id_114]",
|
3117 |
+
"[extra_id_115]",
|
3118 |
+
"[extra_id_116]",
|
3119 |
+
"[extra_id_117]",
|
3120 |
+
"[extra_id_118]",
|
3121 |
+
"[extra_id_119]",
|
3122 |
+
"[extra_id_120]",
|
3123 |
+
"[extra_id_121]",
|
3124 |
+
"[extra_id_122]",
|
3125 |
+
"[extra_id_123]",
|
3126 |
+
"[extra_id_124]",
|
3127 |
+
"[extra_id_125]",
|
3128 |
+
"[extra_id_126]",
|
3129 |
+
"[extra_id_127]",
|
3130 |
+
"[extra_id_128]",
|
3131 |
+
"[extra_id_129]",
|
3132 |
+
"[extra_id_130]",
|
3133 |
+
"[extra_id_131]",
|
3134 |
+
"[extra_id_132]",
|
3135 |
+
"[extra_id_133]",
|
3136 |
+
"[extra_id_134]",
|
3137 |
+
"[extra_id_135]",
|
3138 |
+
"[extra_id_136]",
|
3139 |
+
"[extra_id_137]",
|
3140 |
+
"[extra_id_138]",
|
3141 |
+
"[extra_id_139]",
|
3142 |
+
"[extra_id_140]",
|
3143 |
+
"[extra_id_141]",
|
3144 |
+
"[extra_id_142]",
|
3145 |
+
"[extra_id_143]",
|
3146 |
+
"[extra_id_144]",
|
3147 |
+
"[extra_id_145]",
|
3148 |
+
"[extra_id_146]",
|
3149 |
+
"[extra_id_147]",
|
3150 |
+
"[extra_id_148]",
|
3151 |
+
"[extra_id_149]",
|
3152 |
+
"[extra_id_150]",
|
3153 |
+
"[extra_id_151]",
|
3154 |
+
"[extra_id_152]",
|
3155 |
+
"[extra_id_153]",
|
3156 |
+
"[extra_id_154]",
|
3157 |
+
"[extra_id_155]",
|
3158 |
+
"[extra_id_156]",
|
3159 |
+
"[extra_id_157]",
|
3160 |
+
"[extra_id_158]",
|
3161 |
+
"[extra_id_159]",
|
3162 |
+
"[extra_id_160]",
|
3163 |
+
"[extra_id_161]",
|
3164 |
+
"[extra_id_162]",
|
3165 |
+
"[extra_id_163]",
|
3166 |
+
"[extra_id_164]",
|
3167 |
+
"[extra_id_165]",
|
3168 |
+
"[extra_id_166]",
|
3169 |
+
"[extra_id_167]",
|
3170 |
+
"[extra_id_168]",
|
3171 |
+
"[extra_id_169]",
|
3172 |
+
"[extra_id_170]",
|
3173 |
+
"[extra_id_171]",
|
3174 |
+
"[extra_id_172]",
|
3175 |
+
"[extra_id_173]",
|
3176 |
+
"[extra_id_174]",
|
3177 |
+
"[extra_id_175]",
|
3178 |
+
"[extra_id_176]",
|
3179 |
+
"[extra_id_177]",
|
3180 |
+
"[extra_id_178]",
|
3181 |
+
"[extra_id_179]",
|
3182 |
+
"[extra_id_180]",
|
3183 |
+
"[extra_id_181]",
|
3184 |
+
"[extra_id_182]",
|
3185 |
+
"[extra_id_183]",
|
3186 |
+
"[extra_id_184]",
|
3187 |
+
"[extra_id_185]",
|
3188 |
+
"[extra_id_186]",
|
3189 |
+
"[extra_id_187]",
|
3190 |
+
"[extra_id_188]",
|
3191 |
+
"[extra_id_189]",
|
3192 |
+
"[extra_id_190]",
|
3193 |
+
"[extra_id_191]",
|
3194 |
+
"[extra_id_192]",
|
3195 |
+
"[extra_id_193]",
|
3196 |
+
"[extra_id_194]",
|
3197 |
+
"[extra_id_195]",
|
3198 |
+
"[extra_id_196]",
|
3199 |
+
"[extra_id_197]",
|
3200 |
+
"[extra_id_198]",
|
3201 |
+
"[|endofturn|]",
|
3202 |
+
"PI:URL",
|
3203 |
+
"PI:EMAIL",
|
3204 |
+
"PI:ACCOUNT_NUM",
|
3205 |
+
"PI:PHONE_NUM",
|
3206 |
+
"PI:BUSINESS_NUM",
|
3207 |
+
"PI:ANNON",
|
3208 |
+
"PI:KEY",
|
3209 |
+
"PI:ID",
|
3210 |
+
"PI:IP_ADDRESS",
|
3211 |
+
"PI:USER"
|
3212 |
+
],
|
3213 |
+
"bos_token": "[BOS]",
|
3214 |
+
"chat_template": "{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{{ '[|system|]You are EXAONE model from LG AI Research, a helpful assistant.[|endofturn|]\n' }}{% endif %}{{ '[|' + message['role'] + '|]' + message['content'] }}{% if message['role'] == 'user' %}{{ '\n' }}{% else %}{{ '[|endofturn|]\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '[|assistant|]' }}{% endif %}",
|
3215 |
+
"clean_up_tokenization_spaces": true,
|
3216 |
+
"eos_token": "[|endofturn|]",
|
3217 |
+
"model_max_length": 1000000000000000019884624838656,
|
3218 |
+
"pad_token": "[PAD]",
|
3219 |
+
"tokenizer_class": "GPT2Tokenizer",
|
3220 |
+
"unk_token": "[UNK]"
|
3221 |
+
}
|
vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|