Upload 12 files

cv APP/README.md

@@ -0,0 +1,12 @@
+---
+title: Demo
+emoji: 🌖
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

cv APP/all_datasets.py

@@ -0,0 +1,129 @@
+from imports import *
+from utils import normalize, replace_all
+
+class NerFeatures(object):
+    def __init__(self, input_ids, token_type_ids, attention_mask, valid_ids, labels, label_masks):
+        self.input_ids = torch.as_tensor(input_ids, dtype=torch.long)
+        self.labels = torch.as_tensor(labels, dtype=torch.long)
+        self.token_type_ids = torch.as_tensor(token_type_ids, dtype=torch.long)
+        self.attention_mask = torch.as_tensor(attention_mask, dtype=torch.long)
+        self.valid_ids = torch.as_tensor(valid_ids, dtype=torch.long)
+        self.label_masks = torch.as_tensor(label_masks, dtype=torch.long)
+
+class NerOutput(OrderedDict):
+    loss: Optional[torch.FloatTensor] = torch.FloatTensor([0.0])
+    tags: Optional[List[int]] = []
+    cls_metrics: Optional[List[int]] = []
+    def __getitem__(self, k):
+        if isinstance(k, str):
+            inner_dict = {k: v for (k, v) in self.items()}
+            return inner_dict[k]
+        else:
+            return self.to_tuple()[k]
+    def __setattr__(self, name, value):
+        if name in self.keys() and value is not None:
+            super().__setitem__(name, value)
+        super().__setattr__(name, value)
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        super().__setattr__(key, value)
+    def to_tuple(self) -> Tuple[Any]:
+        return tuple(self[k] for k in self.keys())
+
+class NerDataset(Dataset):
+    def __init__(self, features: List[NerFeatures], device: str = 'cpu'):
+        self.examples = features
+        self.device = device
+
+    def __len__(self):
+        return len(self.examples)
+
+    def __getitem__(self, index):
+        return {key: val.to(self.device) for key, val in self.examples[index].__dict__.items()}     
+
+# return sentiment dataset at tensor type
+def sentiment_dataset(path_folder, train_file_name, test_file_name):
+    def extract(path):
+        data = pd.read_csv(os.path.join(path), encoding="utf-8").dropna() 
+        label = [np.argmax(i) for i in data[["negative", "positive", "neutral"]].values.astype(float)]
+        # text = data["text"].apply(lambda x: x.replace("_"," "))
+        text = data["text"]#.apply(lambda x: normalize(x))
+        return text, label
+    x_train, y_train = extract(os.path.join(path_folder, train_file_name))
+    x_test, y_test = extract(os.path.join(path_folder, test_file_name))
+    train_set = datasets.Dataset.from_pandas(pd.DataFrame(data=zip(x_train,y_train), columns=['text','label']))
+    test_set = datasets.Dataset.from_pandas(pd.DataFrame(data=zip(x_test,y_test), columns=['text','label']))
+    custom_dt = datasets.DatasetDict({'train': train_set, 'test': test_set})
+    tokenizer = AutoTokenizer.from_pretrained('wonrax/phobert-base-vietnamese-sentiment', use_fast=False)
+    def tokenize(batch):
+        return tokenizer(list(batch['text']), padding=True, truncation=True)
+    custom_tokenized = custom_dt.map(tokenize, batched=True, batch_size=None)
+    custom_tokenized.set_format('torch',columns=["input_ids", 'token_type_ids', "attention_mask", "label"]) 
+    return custom_tokenized
+
+# get feature for ner task
+def feature_for_phobert(data, tokenizer, max_seq_len: int=256, use_crf: bool = False) -> List[NerFeatures]:
+    features = []
+    tokens = []
+    tag_ids = []
+
+    idx2tag = {0: 'B-chỗ để xe', 1: 'B-con người', 2: 'B-công việc', 3: 'B-cơ sở vật chất', 4: 'B-dự án', 5: 'B-lương', 6: 'B-môi trường làm việc', 7: 'B-ot/thời gian', 8: 'B-văn phòng', 9: 'B-đãi ngộ', 10: 'I-chỗ để xe', 11: 'I-con người', 12: 'I-công việc', 13: 'I-cơ sở vật chất', 14: 'I-dự án', 15: 'I-lương', 16: 'I-môi trường làm việc', 17: 'I-ot/thời gian', 18: 'I-văn phòng', 19: 'I-đãi ngộ', 20: 'O'}
+    tag2idx = {v: k for k, v in idx2tag.items()}
+    for id, tokens in enumerate(data):
+        if tokens == []:
+            continue
+        tag_ids = [tag2idx[i[1]] for i in tokens]
+        seq_len = len(tokens)
+        sentence = ' '.join([tok[0] for tok in tokens])
+        encoding = tokenizer(sentence, padding='max_length', truncation=True, max_length=max_seq_len)
+        subwords = tokenizer.tokenize(sentence)
+        valid_ids = np.zeros(len(encoding.input_ids), dtype=int)
+        label_marks = np.zeros(len(encoding.input_ids), dtype=int)
+        valid_labels = np.ones(len(encoding.input_ids), dtype=int) * -100
+        i = 1
+        for idx, subword in enumerate(subwords): # subwords[:max_seq_len-2]
+            if idx != 0 and subwords[idx-1].endswith("@@"):
+                continue
+            if use_crf:
+                valid_ids[i-1] = idx + 1
+            else:
+                valid_ids[idx+1] = 1
+            valid_labels[idx+1] = tag_ids[i-1]
+            i += 1
+        if max_seq_len >= seq_len:
+            label_padding_size = (max_seq_len - seq_len)
+            label_marks[:seq_len] = [1] * seq_len
+            tag_ids.extend([0] * label_padding_size)
+        else:
+            tag_ids = tag_ids[:max_seq_len]
+            label_marks[:-2] = [1] * (max_seq_len - 2)
+            tag_ids[-2:] = [0] * 2
+        if use_crf and label_marks[0] == 0:
+            try:
+                raise f"{sentence} - {tag_ids} have mark == 0 at index 0!"
+            except:
+                print(f"{sentence} - {tag_ids} have mark == 0 at index 0!")
+                break 
+        items = {key: val for key, val in encoding.items()}
+        items['labels'] = tag_ids if use_crf else valid_labels
+        items['valid_ids'] = valid_ids
+        items['label_masks'] = label_marks if use_crf else valid_ids
+        features.append(NerFeatures(**items))
+        for k, v in items.items():
+            assert len(v) == max_seq_len, f"Expected length of {k} is {max_seq_len} but got {len(v)}"
+        tokens = []
+        tag_ids = []
+    return features 
+
+# create ner dataset
+def topic_dataset(path_folder, file_name, tokenizer, use_crf=True):
+    data = read_csv_to_ner_data(os.path.join(path_folder, file_name))
+    train_data, test_data = train_test_split(data, test_size=0.2, random_state=42)
+    # token2idx, idx2token = get_dict_map(train_data+test_data, 'token')
+    #tag2idx, idx2tag = get_dict_map(data, 'tag')
+
+    train_set = NerDataset(feature_for_phobert(train_data, tokenizer=tokenizer, use_crf=use_crf))
+    test_set = NerDataset(feature_for_phobert(test_data, tokenizer=tokenizer, use_crf=use_crf))
+    return train_set, test_set
+
+

cv APP/android-studio-2023.1.1.28-linux.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:139d0dbb4909353b68fbf55c09b6d31a34512044a9d4f02ce0f1a9accca128f9
+size 1162099040

cv APP/app.py

@@ -0,0 +1,198 @@
+import gradio as gr
+from imports import *
+from parse_info import *
+#os.system("apt-get install poppler-utils")
+token = os.environ.get("HF_TOKEN")
+login(token=token)
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+dict_ = {
+    0: "negative",
+    1: "positive",
+    2: "neutral"}
+tokenizer_sent = AutoTokenizer.from_pretrained("nam194/sentiment", use_fast=False)
+model_sent = AutoModelForSequenceClassification.from_pretrained("nam194/sentiment", num_labels=3, use_auth_token=True).to(device)
+def cvt2cls(data):
+    data = list(set(data))
+    try: 
+        data.remove(20)
+    except:
+        pass
+    for i, num in enumerate(data):
+        if num == 20:
+            continue
+        if num>=10:
+            data[i] -= 10
+    return data
+ner_tags = {0: 'B-chỗ để xe', 1: 'B-con người', 2: 'B-công việc', 3: 'B-cơ sở vật chất', 4: 'B-dự án', 5: 'B-lương', 6: 'B-môi trường làm việc', 7: 'B-ot/thời gian', 8: 'B-văn phòng', 9: 'B-đãi ngộ', 10: 'I-chỗ để xe', 11: 'I-con người', 12: 'I-công việc', 13: 'I-cơ sở vật chất', 14: 'I-dự án', 15: 'I-lương', 16: 'I-môi trường làm việc', 17: 'I-ot/thời gian', 18: 'I-văn phòng', 19: 'I-đãi ngộ', 20: 'O'}
+topic_tags = {0: 'chỗ để xe', 1: 'con người', 2: 'công việc', 3: 'cơ sở vật chất', 4: 'dự án', 5: 'lương', 6: 'môi trường làm việc', 7: 'ot/thời gian', 8: 'văn phòng', 9: 'đãi ngộ'}
+config = RobertaConfig.from_pretrained("nam194/ner", num_labels=21)
+tokenizer_topic = AutoTokenizer.from_pretrained("nam194/ner", use_fast=False) 
+model_topic = PhoBertLstmCrf.from_pretrained("nam194/ner", config=config, from_tf=False).to(device)
+model_topic.resize_token_embeddings(len(tokenizer_topic)) 
+
+
+def sentiment(sent: str):
+    print("\n--------------------------------------------------------------------------------------------------------------------------\n")
+    print("New review inference at: ", datetime.utcnow())
+    print("review: ", sent)
+    print("\n--------------------------------------------------------------------------------------------------------------------------\n")
+    sent_ = normalize(text=sent) 
+    input_sent = torch.tensor([tokenizer_sent.encode(sent_)]).to(device)
+    with torch.no_grad():
+        out_sent = model_sent(input_sent)
+        logits_sent = out_sent.logits.softmax(dim=-1).tolist()[0]
+        pred_sent = dict_[np.argmax(logits_sent)]
+
+    sent = replace_all(text=sent) 
+    sent_segment = sent.split(".")
+    for i, s in enumerate(sent_segment):
+        s = s.strip()
+        sent_segment[i] = underthesea.word_tokenize(s, format="text").split()
+    dump = [[i, 'O'] for s in sent_segment for i in s]
+    dump_set = NerDataset(feature_for_phobert([dump], tokenizer=tokenizer_topic, use_crf=True))    
+    dump_iter = DataLoader(dump_set, batch_size=1)
+    with torch.no_grad():
+        for idx, batch in enumerate(dump_iter):
+            batch = { k:v.to(device) for k, v in batch.items() }        
+            outputs = model_topic(**batch)
+    pred_topic = list(set([topic_tags[i] for i in cvt2cls(outputs["tags"][0])]))
+    return "Sentiment: " + pred_sent + "\n" + "Topic in sentence: " + ". ".join([i.capitalize() for i in pred_topic]) # str({"sentiment": pred_sent, "topic": pred_topic})
+    
+
+processor = transformers.AutoProcessor.from_pretrained("nam194/resume_parsing_layoutlmv3_large_custom_label", use_auth_token=True, apply_ocr=False) 
+model = transformers.LayoutLMv3ForTokenClassification.from_pretrained("nam194/resume_parsing_layoutlmv3_large_custom_label").to(device)
+# model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8).to(device)
+label_list = ['person_name', 'dob_key', 'dob_value', 'gender_key', 'gender_value', 'phonenumber_key', 'phonenumber_value', 'email_key', 'email_value',
+              'address_key', 'address_value', 'socical_address_value', 'education', 'education_name', 'education_time', 'experience', 'experience_name',
+              'experience_time', 'information', 'undefined', 'designation_key', 'designation_value', 'degree_key', 'degree_value', 'skill_key', 'skill_value']
+id2label = {0: 'person_name', 1: 'dob_key', 2: 'dob_value', 3: 'gender_key', 4: 'gender_value', 5: 'phonenumber_key', 6: 'phonenumber_value',
+            7: 'email_key', 8: 'email_value', 9: 'address_key', 10: 'address_value', 11: 'socical_address_value', 12: 'education', 13: 'education_name',
+            14: 'education_time', 15: 'experience', 16: 'experience_name', 17: 'experience_time', 18: 'information', 19: 'undefined', 20: 'designation_key',
+            21: 'designation_value', 22: 'degree_key', 23: 'degree_value', 24: 'skill_key', 25: 'skill_value'}
+key_list = ["person_name","dob_value","gender_value","phonenumber_value","email_value","address_value",
+                "socical_address_value","education_name","education_time","experience_name","experience_time",
+                "designation_value","degree_value","skill_value"]
+label2id = {v: k for k, v in id2label.items()}
+def pred_resume(pdf_path) -> dict:
+    global key_list, device
+    result = {}
+    for i in key_list:
+        result[i] = []
+    DPI = 200/77
+    global label_list, id2label, label2id
+
+    # read pdf, convert to img
+    doc = fitz.open(pdf_path.name)
+    num_pages = len(doc)
+    images = pdf2image.convert_from_path(pdf_path.name)
+    block_dict = {} 
+
+    # get all data in pdf
+    page_num = 1
+    for page in doc: 
+        file_dict = page.get_text('dict') 
+        block = file_dict['blocks'] 
+        block_dict[page_num] = block 
+        page_num += 1
+
+    # predict each page in pdf
+    for page_num, blocks in block_dict.items():
+        bboxes, words = [], [] # store bounding boxes, text in a page
+        image = images[page_num-1]
+        for block in blocks:
+            if block['type'] == 0:
+                for line in block['lines']:
+                    for span in line['spans']:
+                        xmin, ymin, xmax, ymax = [int(i)*DPI for i in list(span['bbox'])]
+                        text = span['text'].strip()
+                        if text.replace(" ","") !=  "":
+                            bboxes.append(normalize_bbox([xmin, ymin, xmax, ymax], image.size))
+                            words.append(decontracted(text))
+        text_reverse = {str(bboxes[i]): words[i] for i,_ in enumerate(words)}
+        fake_label = ["O"] * len(words)
+        encoding = processor(image, words, boxes=bboxes, word_labels=fake_label, truncation=True, stride=256,
+                      padding="max_length", max_length=512, return_overflowing_tokens=True, return_offsets_mapping=True)
+        labels = encoding["labels"]
+        key_box = encoding["bbox"]
+        offset_mapping = encoding.pop('offset_mapping')
+        overflow_to_sample_mapping = encoding.pop('overflow_to_sample_mapping')
+        encoding = {k: torch.tensor(v) for k,v in encoding.items() if k != "labels"}
+        x = []
+        for i in range(0, len(encoding['pixel_values'])):
+            x.append(encoding['pixel_values'][i])
+        x = torch.stack(x)
+        encoding['pixel_values'] = x
+
+        # forawrd to model
+        with torch.no_grad():
+            outputs = model(**{k: v.to(device) for k,v in encoding.items() if k != "labels"})
+        
+        # process output
+        predictions = outputs["logits"].argmax(-1).squeeze().tolist()
+        if outputs["logits"].shape[0] > 1:
+            for i, label in enumerate(labels):
+                if i>0:
+                    labels[i] = labels[i][256:]
+                    predictions[i] = predictions[i][256:]
+                    key_box[i] = key_box[i][256:]
+            predictions = [j for i in predictions for j in i]
+        key_box = [j for i in key_box for j in i]
+        labels = [j for i in labels for j in i]
+        true_predictions = [id2label[pred] for pred, label in zip(predictions, labels) if label != -100]
+        key_box = [box for box, label in zip(key_box, labels) if label != -100]
+        for box, pred in zip(key_box, true_predictions):
+            if pred in key_list:
+                result[pred].append(text_reverse[str(box)])
+    result = {k: list(set(v)) for k, v in result.items()}
+    print("\n--------------------------------------------------------------------------------------------------------------------------\n")
+    print("New resume inference at: ", datetime.utcnow())
+    print("Pdf name: ", pdf_path.name)
+    print("Result: ", result)
+    print("\n--------------------------------------------------------------------------------------------------------------------------\n")    
+    return result
+def norm(result: dict) -> str:
+    result = ast.literal_eval(result)
+    result["person_name"] = " ".join([parse_string(i).capitalize() for i in " ".join(result["person_name"]).split()])
+    result["email_value"] = parse_email(result["email_value"])
+    result["phonenumber_value"] = "".join([i for i in "".join(result["phonenumber_value"]) if i.isdigit()])
+    result["address_value"] = parse_address(result["address_value"])
+    result["designation_value"] = parse_designation(result["designation_value"])
+    result["experience_time"] = parse_time(result["experience_time"])
+    result["gender_value"] = parse_gender(result["gender_value"])
+    result["skill_value"] = parse_skill(result["skill_value"])
+    result["education_name"] = parse_designation(result["education_name"])
+    result["experience_name"] = parse_designation(result["experience_name"])
+    for k, v in result.items():
+        if isinstance(v, list):
+            result[k] = ". ".join([i for i in result[k]])
+        if isinstance(v, int) or isinstance(v, float):
+            result[k] = str(result[k])
+    return "Tên: "+result["person_name"]+"\n"+"Ngày sinh: "+result["dob_value"]+"\n"+"Giới tính: "+result["gender_value"]+"\n"+"Chức danh: "+result["designation_value"]+"\n"+"Số điện thoại: "+result["phonenumber_value"]+"\n"+"Email: "+result["email_value"]+"\n"+"Địa chỉ: "+result["address_value"]+"\n"+"Tên công ty/công việc: "+result["experience_name"]+"\n"+"Tên trường học: "+result["education_name"]+"\n"+"Kỹ năng: "+result["skill_value"]+"\n"+"Năm kinh nghiệm: "+result["experience_time"]
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("DEMO PROJECTS: REVIEW ANALYSIS AND EXTRACT INFOMATION FROM RESUME")
+    with gr.Tab("Review analysis"):
+        text_input = gr.Textbox(label="Input sentence (ex: Sếp tốt, bảo hiểm đóng full lương bảo hiểm cho nhân viên. Hàng năm tăng lương ổn OT không trả thêm tiền, chỉ cho ngày nghỉ và hỗ trợ ăn tối.):", placeholder="input here...")
+        text_output = gr.Textbox(label="Result:")
+        text_button = gr.Button("Predict")
+    with gr.Tab("Extract infomation from resume"):
+        with gr.Column():
+            file_input = gr.File(label="Upload pdf", file_types=[".pdf"])
+        with gr.Column():
+            cv_output = gr.Textbox(label="Information fields")
+            resume_button = gr.Button("Extract")
+        with gr.Column():
+            normalize_output = gr.Textbox(label="Normalize by rule-based:")
+            normalize_button = gr.Button("Normailze")
+
+    # with gr.Accordion("Open for More!"):
+    #     gr.Markdown("Look at me...")
+
+    text_button.click(sentiment, inputs=text_input, outputs=text_output)
+    resume_button.click(pred_resume, inputs=file_input, outputs=cv_output)
+    normalize_button.click(norm, inputs=cv_output, outputs=normalize_output)
+
+demo.launch()

cv APP/gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+vncorenlp_segmenter/VnCoreNLP-1.1.1.jar filter=lfs diff=lfs merge=lfs -text

cv APP/imports.py

@@ -0,0 +1,31 @@
+import numpy as np
+import pandas as pd
+import seaborn as sns
+from typing import Optional, List, Tuple, Any
+from collections import OrderedDict
+import os, ast, re, string, torch, transformers, datasets, chardet, gdown
+from sklearn.preprocessing import MultiLabelBinarizer, LabelEncoder
+from torch.utils.data import Dataset, DataLoader
+from sklearn.model_selection import train_test_split
+from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification, Trainer, TrainingArguments, logging, RobertaForTokenClassification, RobertaConfig, AutoConfig
+from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
+from torchcrf import CRF
+from accelerate import Accelerator
+import torch.nn as nn
+import torch.nn.functional as F
+import underthesea 
+from utils import *
+from all_datasets import *
+from model import *
+
+from huggingface_hub import login
+import PIL, fitz, pdf2image, re, unicodedata
+from transformers import AutoProcessor, LayoutLMv3ForTokenClassification
+from unidecode import unidecode
+
+from pathlib import Path
+from nltk import everygrams
+from collections import Counter
+from typing import List, Optional
+from datetime import datetime
+from dateutil import parser, relativedelta

cv APP/model.py

@@ -0,0 +1,54 @@
+from imports import *
+from all_datasets import *
+
+class PhoBertLstmCrf(RobertaForTokenClassification):
+    def __init__(self, config):
+        super(PhoBertLstmCrf, self).__init__(config=config)
+        self.num_labels = config.num_labels
+        self.lstm = nn.LSTM(input_size=config.hidden_size,
+                            hidden_size=config.hidden_size // 2,
+                            num_layers=1,
+                            batch_first=True,
+                            bidirectional=True)
+        self.crf = CRF(config.num_labels, batch_first=True)
+
+    @staticmethod
+    def sort_batch(src_tensor, lengths):
+        """
+        Sort a minibatch by the length of the sequences with the longest sequences first
+        return the sorted batch targes and sequence lengths.
+        This way the output can be used by pack_padd ed_sequences(...)
+        """
+        seq_lengths, perm_idx = lengths.sort(0, descending=True)
+        seq_tensor = src_tensor[perm_idx]
+        _, reversed_idx = perm_idx.sort(0, descending=False)
+        return seq_tensor, seq_lengths, reversed_idx
+
+    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, valid_ids=None,
+                label_masks=None):
+        seq_outputs = self.roberta(input_ids=input_ids,
+                                   token_type_ids=token_type_ids,
+                                   attention_mask=attention_mask,
+                                   head_mask=None)[0]
+
+        batch_size, max_len, feat_dim = seq_outputs.shape
+        seq_lens = torch.sum(label_masks, dim=-1)
+        range_vector = torch.arange(0, batch_size, dtype=torch.long, device=seq_outputs.device).unsqueeze(1)
+        seq_outputs = seq_outputs[range_vector, valid_ids]
+
+        sorted_seq_outputs, sorted_seq_lens, reversed_idx = self.sort_batch(src_tensor=seq_outputs,
+                                                                            lengths=seq_lens)
+        packed_words = pack_padded_sequence(sorted_seq_outputs, sorted_seq_lens.cpu(), True)
+        lstm_outs, _ = self.lstm(packed_words)
+        lstm_outs, _ = pad_packed_sequence(lstm_outs, batch_first=True, total_length=max_len)
+        seq_outputs = lstm_outs[reversed_idx]
+
+        seq_outputs = self.dropout(seq_outputs)
+        logits = self.classifier(seq_outputs)
+        seq_tags = self.crf.decode(logits, mask=label_masks != 0)
+
+        if labels is not None:
+            log_likelihood = self.crf(logits, labels, mask=label_masks.type(torch.uint8))
+            return NerOutput(loss=-1.0 * log_likelihood, tags=seq_tags, cls_metrics=seq_tags)
+        else:
+            return NerOutput(tags=seq_tags)

cv APP/packages.txt

@@ -0,0 +1 @@
+poppler-utils

cv APP/parse_info.py

@@ -0,0 +1,112 @@
+from imports import *
+
+punc = list(string.punctuation)
+def parse_string(inp: str, rep=" ", punc=punc, excp=[]) -> str:
+    try:
+        for i in excp:
+            punc.remove(i)
+    except:
+        pass
+    inp = inp.lower()
+    inp = re.sub(r"won\'t", "will not", inp)
+    inp = re.sub(r"can\'t", "can not", inp)
+    inp = re.sub(r"\'re", " are", inp)
+    inp = re.sub(r"\'s", " of", inp)
+    inp = re.sub(r"\'d", " would", inp)
+    inp = re.sub(r"\'ll", " will", inp)
+    inp = re.sub(r"\'t", " not", inp)
+    inp = re.sub(r"\'ve", " have", inp)
+    inp = re.sub(r"\'m", " am", inp)  
+    for i in punc:
+        inp = inp.replace(i,rep)
+    return " ".join(inp.split())
+
+def parse_time(inp: List):
+    duration = 0
+    for i, _ in enumerate(inp):
+        inp[i] = inp[i].lower()
+        now = datetime.utcnow().strftime("%d/%m/%Y")
+        _ = ["đến", " to ", "–"] # list that split 2 time point word
+        __ = ["now", "hiện tại", " nay", " đến nay", "present"] # end time point
+        for j in _:
+            inp[i] = inp[i].replace(j," - ")
+        for j in __:
+            inp[i] = inp[i].replace(j,now)
+        for j in inp[i]:
+            if j.isalpha():
+                inp[i] = inp[i].replace(j,"").strip()
+        inp[i] = parse_string(" ".join(inp[i].split(" ")), rep="", excp=["/","-"])
+
+        time_point = inp[i].split("-") # split to 2 time point
+        if len(time_point) != 2: # must be splitted to 2 time point
+            continue
+        try:
+            d1 = parser.parse(time_point[0]).strftime("%d-%m-%Y")
+            d2 = parser.parse(time_point[1]).strftime("%d-%m-%Y")
+            duration += (datetime.strptime(d2, "%d-%m-%Y") - datetime.strptime(d1, "%d-%m-%Y")).days
+        except:
+            continue
+    return "{:.1f} năm".format(np.abs(duration/365))
+
+filename = "./skills.csv"
+detected = chardet.detect(Path(filename).read_bytes()) # "ISO-8859-1"
+skill_list = pd.read_csv(filename, encoding=detected["encoding"])
+skill_list = [i.replace("\n","") for i in skill_list["Skill"].to_list()]
+def parse_skill(inp: List) -> list:
+    res = []
+    for i, _ in enumerate(inp):
+        if "," in _:
+            _ = [j.strip() for j in _.split(",")]
+            inp.extend(_)
+    inp = [parse_string(i) for i in inp]
+    for ngram in Counter(map(' '.join, everygrams(" ".join(inp).split(), 1, 3))).keys():
+        if ngram in skill_list:
+            res.append(ngram)
+    return ". ".join([i.capitalize() for i in list(set(res))])
+
+def parse_gender(inp: List) -> str:
+    inp = " ".join([parse_string(i) for i in inp])
+    gender = ["nam", "nữ", "female", "male", "bisexual", "asexual", "heterosexual", "homosexual", "lgbt"]
+    for gen in gender:
+        if gen in inp:
+            return gen
+    return ""
+
+def parse_address(inp: List) -> str:
+    inp = [parse_string(i, excp=",") for i in inp]
+    for i, _ in enumerate(inp):
+        inp[i] = " ".join([j.capitalize() for j in inp[i].split()])
+    return ". ".join(inp)
+
+def parse_designation(inp: List) -> str:
+    inp = list(set([parse_string(i) for i in inp]))
+    for i, _ in enumerate(inp):
+        inp[i] = " ".join([j.capitalize() for j in inp[i].split()])
+    return ". ".join(inp)    
+
+def parse_email(inp: List) -> str:
+    inp = list(set([parse_string(i, rep="", excp=["@","."]) for i in inp]))
+    return " ".join(inp)
+
+def decontracted(phrase) -> str:
+    phrase = re.sub(r"â€|™|“|”|;|ü|\xad|\xa0|\u200b|·|∙|�|●|�|§|•|!|▪|©|\?|\]|\[|\)|\(", "", phrase)
+    phrase = phrase.strip()
+    phrase = unicodedata.normalize("NFC", phrase)
+    if "   " in phrase or "  " in phrase: # check space character 
+        phrase = phrase.replace("   ","_").replace("  ","_").replace(" ","").replace("_"," ")
+    tmp = phrase.split(" ")
+    check_parse = True
+    for i in tmp:
+        if len(i) > 1:
+            check_parse = False
+            break
+    if check_parse:
+        phrase = phrase.replace(" ","")
+    # phrase = phrase.replace("   "," ").replace("  "," ")
+    return phrase.replace("\n"," ")
+
+def normalize_bbox(bbox, size): # must normalize bbox to [0;1000]
+    return [int(1000 * bbox[0] / size[0]),
+            int(1000 * bbox[1] / size[1]),
+            int(1000 * bbox[2] / size[0]),
+            int(1000 * bbox[3] / size[1])]

cv APP/requirements.txt

@@ -0,0 +1,19 @@
+torch
+transformers
+huggingface_hub
+gdown
+pymupdf
+unidecode
+pdf2image
+chardet
+python-dateutil
+datasets
+underthesea
+accelerate
+pytorch-crf==0.7.2
+sklearn-crfsuite
+scikit-learn
+numpy
+pandas
+install-jdk
+seaborn

cv APP/utils.py

@@ -0,0 +1,113 @@
+from imports import *
+import unicodedata
+dict_map = {
+    "òa": "oà",
+    "Òa": "Oà",
+    "ÒA": "OÀ",
+    "óa": "oá",
+    "Óa": "Oá",
+    "ÓA": "OÁ",
+    "ỏa": "oả",
+    "Ỏa": "Oả",
+    "ỎA": "OẢ",
+    "õa": "oã",
+    "Õa": "Oã",
+    "ÕA": "OÃ",
+    "ọa": "oạ",
+    "Ọa": "Oạ",
+    "ỌA": "OẠ",
+    "òe": "oè",
+    "Òe": "Oè",
+    "ÒE": "OÈ",
+    "óe": "oé",
+    "Óe": "Oé",
+    "ÓE": "OÉ",
+    "ỏe": "oẻ",
+    "Ỏe": "Oẻ",
+    "ỎE": "OẺ",
+    "õe": "oẽ",
+    "Õe": "Oẽ",
+    "ÕE": "OẼ",
+    "ọe": "oẹ",
+    "Ọe": "Oẹ",
+    "ỌE": "OẸ",
+    "ùy": "uỳ",
+    "Ùy": "Uỳ",
+    "ÙY": "UỲ",
+    "úy": "uý",
+    "Úy": "Uý",
+    "ÚY": "UÝ",
+    "ủy": "uỷ",
+    "Ủy": "Uỷ",
+    "ỦY": "UỶ",
+    "ũy": "uỹ",
+    "Ũy": "Uỹ",
+    "ŨY": "UỸ",
+    "ụy": "uỵ",
+    "Ụy": "Uỵ",
+    "ỤY": "UỴ",
+    }
+
+### Normalize functions ###
+def replace_all(text, dict_map=dict_map):
+    for i, j in dict_map.items():
+        text = unicodedata.normalize('NFC',str(text)).replace(i, j)
+    return text
+def normalize(text, segment=True):
+    text = replace_all(text, dict_map)
+    if segment:
+        text = text.split(".")
+        text = ". ".join([underthesea.word_tokenize(i, format="text") for i in text])
+    return text
+def text_preprocess(document):
+    punc = [i for i in ["\"", "-", ".", ":"]]#string.punctuation.replace(",","")]
+    stopword = [" thì ", " được ", " có ", " là "]
+    acronyms = {" wfh": " làm việc tại nhà ", " ot": " làm tăng ca ", " team": " nhóm ", " pm": " quản lý dự án ", " flexible": " linh động ",
+                " office": " văn phòng ", " feedback": " phản hồi ", " cty": " công ty ", " hr": " tuyển dụng ", " effective": " hiệu quả ",
+                " suggest": " gợi ý ", " hong": " không ", " ko": " không ", " vp": " văn phòng ", " plan ": " kế hoạch ", " planning": " lên kế hoạch ",
+                " family": " gia đình ", " leaders": " trưởng nhóm ", " leader": " trưởng nhóm ", ",": " , "}
+
+    document = re.sub(r"\n"," . ", document)
+    document = re.sub(r"\t"," ", document)
+    document = re.sub(r"\r","", document)
+    for p in punc:
+        document = document.replace(p," ")
+    for acr in acronyms:
+        tmp = [acr, acr.upper(), acr[0].upper()+acr[1:]]
+        for j in tmp:
+            document = re.sub(j, acronyms[acr], document)
+            #document = re.sub(j, acr.upper(), document)
+    for sw in stopword:
+        document = re.sub(sw, " ", document)
+
+    document = re.sub("   ", " ", document)
+    document = re.sub("  ", " ", document)
+    try:
+        document = document.split(".")
+        document = ". ".join([underthesea.word_tokenize(i, format="text") for i in document])
+    except:
+        pass
+    return document.lower()
+
+### Compute metrics for multiclass classification problem
+def compute_metrics(pred):
+    labels = pred.label_ids
+    preds = pred.predictions.argmax(-1)
+    f1 = f1_score(labels, preds, average="weighted")
+    acc = accuracy_score(labels, preds)
+    return {"accuracy": acc, "f1": f1}
+
+### Make multilabel result from Ner result
+# mb and cls_class just a dictionary map id to class name, see train.py
+def convert2cls(data, mb, cls_class):
+    data = list(set(data))
+    try:
+        data.remove(20)
+    except:
+        pass
+    for i, num in enumerate(data):
+        if num>=10:
+            data[i] -= 10
+        data[i] = cls_class[data[i]]
+    data = mb.transform([data])[0]
+    return list(data)