handler.py

from typing import  Dict, List, Any
from transformers import AutoModel, AutoTokenizer
import torch


class EndpointHandler():
    def __init__(self, path=""):
        # load the optimized model
        self.model = AutoModel.from_pretrained(path, trust_remote_code=True)
        self.model.eval()
        self.tokenizer = AutoTokenizer.from_pretrained('allenai/led-base-16384')
        
        # create inference pipeline
        #self.pipeline = pipeline("token-classification", model=model, tokenizer=tokenizer)


    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
        """
        Args:
            data (:obj:):
                includes the input data and the parameters for the inference.
        Return:
            A :obj:`list`:. The object returned should be a list of one list like [[{"label": 0.9939950108528137}]] containing :
                - "label": A string representing what the label/class is. There can be multiple labels.
                - "score": A score between 0 and 1 describing how confident the model is for this label/class.
        """
        text = data['inputs'].pop("text", "")
        label_tolerance = data['inputs'].pop("label_tolerance", 0)
        backup_tolerance = data['inputs'].pop("backup_tolerance", None)
        
        # Return labeled results and backup results based on tolerances
        inputs = self.preprocess_text(text)
        outputs = self.model(input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask'])

        # Extract labeled results
        predictions = self.extract_results(input_ids=inputs['input_ids'][0].tolist(), offset_mapping=inputs['offset_mapping'], logits=outputs['logits'], 
            label_tolerance=label_tolerance, backup_tolerance=backup_tolerance)
        
        return predictions
    
    def preprocess_text(self, text):
        
        inputs = self.tokenizer(text, return_offsets_mapping=True)
        input_ids = torch.tensor([inputs["input_ids"]])#, dtype=torch.fp32)
        attention_mask = torch.tensor([inputs["attention_mask"]])#, dtype=torch.fp32)
        
        return {"input_ids": input_ids, "attention_mask": attention_mask, "offset_mapping": inputs["offset_mapping"]}

    def extract_results(self, input_ids, offset_mapping, logits, label_tolerance=0, backup_tolerance=None):

        def convert_indices_to_result_obj(indices_array):
            result_array = []
            if (indices_array):
                for result_indices in indices_array:
                    text = self.tokenizer.decode(input_ids[result_indices[0]:result_indices[-1]])
                    indices = [offset_mapping[result_indices[0]-1][1], offset_mapping[result_indices[-2]][1]]
                    if text != "" and not text.isspace():
                        while True:
                            if text[0] == " ":
                                text = text[1:]
                                indices[0] += 1
                            else:
                                break
                        result_array.append({'text': text, 'indices': indices})
            return result_array
                
        
        # Extract labeled results first
        labeled_result_indices = []
        result_indices = []
        for index, token_logits in enumerate(logits.tolist()[0]):

            if (len(result_indices) > 0):
                if token_logits[2] > label_tolerance:
                    result_indices.append(index)
                else:
                    result_indices.append(index)
                    labeled_result_indices.append(result_indices)
                    result_indices = []
                    
            elif (token_logits[1] > label_tolerance):
                result_indices.append(index)

        if (len(result_indices) > 0):
            labeled_result_indices.append(result_indices)

        
        # Extract backup results, avoiding overlapping with labeled results
        backup_result_indices = []
        result_indices = []
        if (backup_tolerance):
            for index, token_logits in enumerate(logits.tolist()[0]):

                if (len(result_indices) > 0):
                    if token_logits[2] > backup_tolerance:
                        result_indices.append(index)
                    else:
                        # Check if backup result overlaps at all with any labeled result. If it does just ignore it
                        result_indices.append(index)
                        overlaps_labeled_result = False
                        if (len(labeled_result_indices) > 0):
                            for index in result_indices:
                                for group in labeled_result_indices:
                                    for labeled_index in group:
                                        if (index == labeled_index):
                                            overlaps_labeled_result = True
                        if (not overlaps_labeled_result):         
                            backup_result_indices.append(result_indices)
                        
                        result_indices = []
                        
                elif (token_logits[1] > backup_tolerance):
                    result_indices.append(index)

        # Convert both labeled results and backup results to {name: "", indices: []}
        labeled_results = convert_indices_to_result_obj(labeled_result_indices)
        backup_results = convert_indices_to_result_obj(backup_result_indices)
            
            
        return {'labeled_results': labeled_results, 'backup_results': backup_results}