|
from dataclasses import dataclass |
|
from typing import Optional, Tuple |
|
|
|
import torch |
|
import torch.nn as nn |
|
from transformers import ErnieModel, ErniePreTrainedModel, PretrainedConfig |
|
from transformers.file_utils import ModelOutput |
|
|
|
from .decode_utils import UIEDecoder |
|
|
|
|
|
@dataclass |
|
class UIEModelOutput(ModelOutput): |
|
""" |
|
Output class for outputs of UIE. |
|
losses (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): |
|
Total spn extraction losses is the sum of a Cross-Entropy for the start and end positions. |
|
start_prob (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): |
|
Span-start scores (after Sigmoid). |
|
end_prob (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): |
|
Span-end scores (after Sigmoid). |
|
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): |
|
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layers, + |
|
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. |
|
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. |
|
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): |
|
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, |
|
sequence_length)`. |
|
Attention weights after the attention softmax, used to compute the weighted average in the self-attention |
|
heads. |
|
""" |
|
loss: Optional[torch.FloatTensor] = None |
|
start_prob: torch.FloatTensor = None |
|
end_prob: torch.FloatTensor = None |
|
start_positions: torch.FloatTensor = None |
|
end_positions: torch.FloatTensor = None |
|
hidden_states: Optional[Tuple[torch.FloatTensor]] = None |
|
attentions: Optional[Tuple[torch.FloatTensor]] = None |
|
|
|
|
|
class UIEModel(ErniePreTrainedModel, UIEDecoder): |
|
""" |
|
UIE model based on Bert model. |
|
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the |
|
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads |
|
etc.) |
|
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. |
|
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage |
|
and behavior. |
|
Parameters: |
|
config ([`PretrainedConfig`]): Model configuration class with all the parameters of the model. |
|
Initializing with a config file does not load the weights associated with the model, only the |
|
configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. |
|
""" |
|
|
|
def __init__(self, config: PretrainedConfig): |
|
super(UIEModel, self).__init__(config) |
|
self.encoder = ErnieModel(config) |
|
self.config = config |
|
hidden_size = self.config.hidden_size |
|
|
|
self.linear_start = nn.Linear(hidden_size, 1) |
|
self.linear_end = nn.Linear(hidden_size, 1) |
|
self.sigmoid = nn.Sigmoid() |
|
|
|
self.post_init() |
|
|
|
def forward( |
|
self, |
|
input_ids: Optional[torch.Tensor] = None, |
|
token_type_ids: Optional[torch.Tensor] = None, |
|
position_ids: Optional[torch.Tensor] = None, |
|
attention_mask: Optional[torch.Tensor] = None, |
|
head_mask: Optional[torch.Tensor] = None, |
|
inputs_embeds: Optional[torch.Tensor] = None, |
|
start_positions: Optional[torch.Tensor] = None, |
|
end_positions: Optional[torch.Tensor] = None, |
|
output_attentions: Optional[bool] = None, |
|
output_hidden_states: Optional[bool] = None, |
|
) -> UIEModelOutput: |
|
""" |
|
Args: |
|
input_ids (`torch.LongTensor` of shape `({0})`): |
|
Indices of input sequence tokens in the vocabulary. |
|
Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and |
|
[`PreTrainedTokenizer.__call__`] for details. |
|
[What are input IDs?](../glossary#input-ids) |
|
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): |
|
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: |
|
- 1 for tokens that are **not masked**, |
|
- 0 for tokens that are **masked**. |
|
[What are attention masks?](../glossary#attention-mask) |
|
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): |
|
Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, |
|
1]`: |
|
- 0 corresponds to a *sentence A* token, |
|
- 1 corresponds to a *sentence B* token. |
|
[What are token type IDs?](../glossary#token-type-ids) |
|
position_ids (`torch.LongTensor` of shape `({0})`, *optional*): |
|
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, |
|
config.max_position_embeddings - 1]`. |
|
[What are position IDs?](../glossary#position-ids) |
|
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): |
|
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: |
|
- 1 indicates the head is **not masked**, |
|
- 0 indicates the head is **masked**. |
|
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): |
|
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This |
|
is useful if you want more control over how to convert `input_ids` indices into associated vectors than the |
|
model's internal embedding lookup matrix. |
|
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): |
|
Labels for position (index) of the start of the labelled span for computing the token classification loss. |
|
Positions are clamped to the length of the sequence (`sequence_length`). Position outsides of the sequence |
|
are not taken into account for computing the loss. |
|
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): |
|
Labels for position (index) of the end of the labelled span for computing the token classification loss. |
|
Positions are clamped to the length of the sequence (`sequence_length`). Position outsides of the sequence |
|
are not taken into account for computing the loss. |
|
output_attentions (`bool`, *optional*): |
|
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned |
|
tensors for more detail. |
|
output_hidden_states (`bool`, *optional*): |
|
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for |
|
more detail. |
|
return_dict (`bool`, *optional*): |
|
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. |
|
""" |
|
outputs = self.encoder( |
|
input_ids=input_ids, |
|
token_type_ids=token_type_ids, |
|
position_ids=position_ids, |
|
attention_mask=attention_mask, |
|
head_mask=head_mask, |
|
inputs_embeds=inputs_embeds, |
|
output_attentions=output_attentions, |
|
output_hidden_states=output_hidden_states, |
|
) |
|
sequence_output = outputs[0] |
|
|
|
start_logits = self.linear_start(sequence_output) |
|
start_logits = torch.squeeze(start_logits, -1) |
|
start_prob = self.sigmoid(start_logits) |
|
|
|
end_logits = self.linear_end(sequence_output) |
|
end_logits = torch.squeeze(end_logits, -1) |
|
end_prob = self.sigmoid(end_logits) |
|
|
|
total_loss = None |
|
if start_positions is not None and end_positions is not None: |
|
loss_fct = nn.BCELoss() |
|
start_loss = loss_fct(start_prob, start_positions) |
|
end_loss = loss_fct(end_prob, end_positions) |
|
|
|
total_loss = (start_loss + end_loss) / 2.0 |
|
|
|
return UIEModelOutput( |
|
loss=total_loss, |
|
start_prob=start_prob, |
|
end_prob=end_prob, |
|
hidden_states=outputs.hidden_states, |
|
attentions=outputs.attentions, |
|
) |
|
|
