adding JointBERT IDSF checkpoint folder, load_model.py and utils.py from IDSF; modified app.py and requirements.txt

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 jdk-8u361-linux-aarch64.rpm filter=lfs diff=lfs merge=lfs -text
 VnCoreNLP-1.2.jar filter=lfs diff=lfs merge=lfs -text
 models/postagger/vi-tagger filter=lfs diff=lfs merge=lfs -text
+JointBERT-CRF_PhoBERTencoder/ filter=lfs diff=lfs merge=lfs -text

JointBERT-CRF_PhoBERTencoder/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "vinai/phobert-base",
+  "architectures": [
+    "JointPhoBERT"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "finetuning_task": "word-level",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 258,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "PhobertTokenizer",
+  "type_vocab_size": 1,
+  "vocab_size": 64001
+}

JointBERT-CRF_PhoBERTencoder/eval_dev_results.txt

@@ -0,0 +1,8 @@
+***** Eval results *****
+ intent_acc = 0.984
+ loss = 0.7875628210604191
+ mean_intent_slot = 0.9723264354415622
+ semantic_frame_acc = 0.874
+ slot_f1 = 0.9606528708831245
+ slot_precision = 0.959254947613504
+ slot_recall = 0.9620548744892002

JointBERT-CRF_PhoBERTencoder/eval_test_results.txt

@@ -0,0 +1,8 @@
+***** Eval results *****
+ intent_acc = 0.973124300111982
+ loss = 0.9325816652604512
+ mean_intent_slot = 0.9598045425009019
+ semantic_frame_acc = 0.8533034714445689
+ slot_f1 = 0.9464847848898217
+ slot_precision = 0.9445026178010472
+ slot_recall = 0.9484752891692955

JointBERT-CRF_PhoBERTencoder/events.out.tfevents.1617863943.d86fb58144ae.20305.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbe6f738dce99ab0769bc25063ad5cd2017725eb5789e7dbf61081166cf81c32
+size 17078

JointBERT-CRF_PhoBERTencoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e6947fac4a325becd62e5932ca6d2f7d15014d486ac30308e56bfd9b0e7d451
+size 540968940

JointBERT-CRF_PhoBERTencoder/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a16e0479297d9c9fe69f4fa4041819e22092ff6145f8e97d5eb56c74180e07b
+size 1583

app.py

@@ -51,6 +51,48 @@ my_classifier = pipeline(
   "token-classification", model=model_checkpoint, aggregation_strategy="simple", pipeline_class=MyPipeline)
 
 
+#################### IDSF #######################
+from utils import get_intent_labels, get_slot_labels, load_tokenizer
+import argparse
+import load_model as lm
+
+parser = argparse.ArgumentParser()
+
+# parser.add_argument("--input_file", default="sample_pred_in.txt", type=str, help="Input file for prediction")
+# parser.add_argument("--output_file", default="sample_pred_out.txt", type=str, help="Output file for prediction")
+parser.add_argument("--model_dir", default="./JointBERT-CRF_PhoBERTencoder", type=str, help="Path to save, load model")
+
+parser.add_argument("--batch_size", default=32, type=int, help="Batch size for prediction")
+parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
+
+pred_config = parser.parse_args()    
+
+# load model and args
+args = lm.get_args(pred_config)
+device = lm.get_device(pred_config)
+model = lm.load_model(pred_config, args, device)
+
+intent_label_lst = get_intent_labels(args)
+slot_label_lst = get_slot_labels(args)
+
+# Convert input file to TensorDataset
+pad_token_label_id = args.ignore_index
+tokenizer = load_tokenizer(args) 
+
+
+
+
+#################### END IDSF #######################
+
+def remove_disfluency(example, prediction):
+  characters = list(example)
+
+  for entity in reversed(prediction):
+    characters[entity['start']:entity['end']] = ''
+  
+  return " ".join("".join(characters).split())
+
+
 import gradio as gr
 
 def ner(text):
@@ -64,23 +106,40 @@ def ner(text):
     entity['entity'] = entity.pop('entity_group')
 
   # Remove Disfluency-entities to return a sentence with "Fluency" version
-  list_str = list(text)
+  fluency_sentence = remove_disfluency(text, output)
+
+
+  #################### IDSF #######################
+  words, slot_preds, intent_pred = lm.predict(fluency_sentence)[0][0],  lm.predict(fluency_sentence)[1][0],  lm.predict(fluency_sentence)[2][0]
+
+  slot_tokens = []    
+
+  for word, pred in zip(words, slot_preds):
+      if pred == 'O':
+          slot_tokens.extend([(word, None), (" ", None)])
+      elif pred[0] == 'I':
+          added_tokens = list(slot_tokens[-2]) 
+          added_tokens[0] += f' {word}'
+          slot_tokens[-2] = tuple(added_tokens)
+      else:
+          slot_tokens.extend([(word, pred[2:]), (" ", None)])
+  
+  intent_label = intent_label_lst[intent_pred]
+
+  #################### END IDSF ####################### 
 
-  for entity in output[::-1]: # if we use default order of output list, we will shorten the length of the sentence, so the words later are not in the correct start and end index
-    start = max(0, entity['start'] - 1)
-    end = min(len(list_str), entity['end'] + 1)
 
-    list_str[start:end] = ' '
 
-  fluency_sentence = "".join(list_str).strip() # use strip() in case we need to remove entity at the beginning or the end of sentence
-                                               # (without strip(): "Giá vé khứ hồi à nhầm giá vé một chiều ..." -> " giá vé một chiều ...")
   fluency_sentence = fluency_sentence[0].upper() + fluency_sentence[1:] # since capitalize() just lowercase whole sentence first then uppercase the first letter
 
   # Replace words like "Đà_Nẵng" to "Đà Nẵng"  
   text = text.replace("_", " ")
   fluency_sentence = fluency_sentence.replace("_", " ")
     
-  return {'text': text, 'entities': output}, fluency_sentence
+  return {'text': text, 'entities': output}, fluency_sentence, slot_tokens, intent_label 
+
+
+################################### Gradio Demo ####################################
 
 examples = ['Tôi cần thuê à tôi muốn bay một chuyến khứ hồi từ Đà Nẵng đến Đà Lạt', 
             'Giá vé một chiều à không khứ hồi từ Đà Nẵng đến Vinh dưới 2 triệu đồng giá vé khứ hồi từ Quy Nhơn đến Vinh dưới 3 triệu đồng giá vé khứ hồi từ Buôn Ma Thuột đến Quy Nhơn à đến Vinh dưới 4 triệu rưỡi', 
@@ -91,7 +150,8 @@ examples = ['Tôi cần thuê à tôi muốn bay một chuyến khứ hồi từ
 
 demo = gr.Interface(ner, 
                     gr.Textbox(label='Sentence', placeholder="Enter your sentence here..."),
-                    outputs=[gr.HighlightedText(label='Highlighted Output'), gr.Textbox(label='"Fluency" version')],
+                    outputs=[gr.HighlightedText(label='Disfluency Highlighted'), gr.Textbox(label='"Fluency" version'),
+                             gr.HighlightedText(label='Slot Filling Highlighted'), gr.Textbox(label='Intent Label')],
                     examples=examples,
                     title="Disfluency Detection",
                     description="This is an easy-to-use built in Gradio for desmontrating a NER System that identifies disfluency-entities in \

load_model.py

@@ -0,0 +1,250 @@
+import gradio as gr
+
+import argparse
+import logging
+import os
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, SequentialSampler, TensorDataset
+from tqdm import tqdm
+from utils import MODEL_CLASSES, get_intent_labels, get_slot_labels, init_logger, load_tokenizer
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_device(pred_config):
+    return "cuda" if torch.cuda.is_available() and not pred_config.no_cuda else "cpu"
+
+
+def get_args(pred_config):
+    args = torch.load(os.path.join(pred_config.model_dir, "training_args.bin"))
+
+    args.model_dir = pred_config.model_dir
+    args.data_dir = 'PhoATIS'
+
+    return args
+
+
+def load_model(pred_config, args, device):
+    # Check whether model exists
+    if not os.path.exists(pred_config.model_dir):
+        raise Exception("Model doesn't exists! Train first!")
+
+    try:
+        model = MODEL_CLASSES[args.model_type][1].from_pretrained(
+            args.model_dir, args=args, intent_label_lst=get_intent_labels(args), slot_label_lst=get_slot_labels(args)
+        )
+        model.to(device)
+        model.eval()
+        logger.info("***** Model Loaded *****")
+    except Exception:
+        raise Exception("Some model files might be missing...")
+
+    return model
+
+def convert_input_file_to_tensor_dataset(
+    lines,
+    pred_config,
+    args,
+    tokenizer,
+    pad_token_label_id,
+    cls_token_segment_id=0,
+    pad_token_segment_id=0,
+    sequence_a_segment_id=0,
+    mask_padding_with_zero=True,
+):
+    # Setting based on the current model type
+    cls_token = tokenizer.cls_token
+    sep_token = tokenizer.sep_token
+    unk_token = tokenizer.unk_token
+    pad_token_id = tokenizer.pad_token_id
+
+    all_input_ids = []
+    all_attention_mask = []
+    all_token_type_ids = []
+    all_slot_label_mask = []
+
+    for words in lines:
+        tokens = []
+        slot_label_mask = []
+        for word in words:
+            word_tokens = tokenizer.tokenize(word)
+            if not word_tokens:
+                word_tokens = [unk_token]  # For handling the bad-encoded word
+            tokens.extend(word_tokens)
+            # Use the real label id for the first token of the word, and padding ids for the remaining tokens
+            slot_label_mask.extend([pad_token_label_id + 1] + [pad_token_label_id] * (len(word_tokens) - 1))
+
+        # Account for [CLS] and [SEP]
+        special_tokens_count = 2
+        if len(tokens) > args.max_seq_len - special_tokens_count:
+            tokens = tokens[: (args.max_seq_len - special_tokens_count)]
+            slot_label_mask = slot_label_mask[: (args.max_seq_len - special_tokens_count)]
+
+        # Add [SEP] token
+        tokens += [sep_token]
+        token_type_ids = [sequence_a_segment_id] * len(tokens)
+        slot_label_mask += [pad_token_label_id]
+
+        # Add [CLS] token
+        tokens = [cls_token] + tokens
+        token_type_ids = [cls_token_segment_id] + token_type_ids
+        slot_label_mask = [pad_token_label_id] + slot_label_mask
+
+        input_ids = tokenizer.convert_tokens_to_ids(tokens)
+
+        # The mask has 1 for real tokens and 0 for padding tokens. Only real tokens are attended to.
+        attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
+
+        # Zero-pad up to the sequence length.
+        padding_length = args.max_seq_len - len(input_ids)
+        input_ids = input_ids + ([pad_token_id] * padding_length)
+        attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)
+        token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length)
+        slot_label_mask = slot_label_mask + ([pad_token_label_id] * padding_length)
+
+        all_input_ids.append(input_ids)
+        all_attention_mask.append(attention_mask)
+        all_token_type_ids.append(token_type_ids)
+        all_slot_label_mask.append(slot_label_mask)
+
+    # Change to Tensor
+    all_input_ids = torch.tensor(all_input_ids, dtype=torch.long)
+    all_attention_mask = torch.tensor(all_attention_mask, dtype=torch.long)
+    all_token_type_ids = torch.tensor(all_token_type_ids, dtype=torch.long)
+    all_slot_label_mask = torch.tensor(all_slot_label_mask, dtype=torch.long)
+
+    dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_slot_label_mask)
+
+    return dataset
+
+def predict(text):
+
+    lines = text
+    dataset = convert_input_file_to_tensor_dataset(lines, pred_config, args, tokenizer, pad_token_label_id)
+
+    # Predict
+    sampler = SequentialSampler(dataset)
+    data_loader = DataLoader(dataset, sampler=sampler, batch_size=pred_config.batch_size)
+
+    all_slot_label_mask = None
+    intent_preds = None
+    slot_preds = None
+
+    for batch in tqdm(data_loader, desc="Predicting"):
+        batch = tuple(t.to(device) for t in batch)
+        with torch.no_grad():
+            inputs = {
+                "input_ids": batch[0],
+                "attention_mask": batch[1],
+                "intent_label_ids": None,
+                "slot_labels_ids": None,
+            }
+            if args.model_type != "distilbert":
+                inputs["token_type_ids"] = batch[2]
+            outputs = model(**inputs)
+            _, (intent_logits, slot_logits) = outputs[:2]
+
+            # Intent Prediction
+            if intent_preds is None:
+                intent_preds = intent_logits.detach().cpu().numpy()
+            else:
+                intent_preds = np.append(intent_preds, intent_logits.detach().cpu().numpy(), axis=0)
+
+            # Slot prediction
+            if slot_preds is None:
+                if args.use_crf:
+                    # decode() in `torchcrf` returns list with best index directly
+                    slot_preds = np.array(model.crf.decode(slot_logits))
+                else:
+                    slot_preds = slot_logits.detach().cpu().numpy()
+                all_slot_label_mask = batch[3].detach().cpu().numpy()
+            else:
+                if args.use_crf:
+                    slot_preds = np.append(slot_preds, np.array(model.crf.decode(slot_logits)), axis=0)
+                else:
+                    slot_preds = np.append(slot_preds, slot_logits.detach().cpu().numpy(), axis=0)
+                all_slot_label_mask = np.append(all_slot_label_mask, batch[3].detach().cpu().numpy(), axis=0)
+
+    intent_preds = np.argmax(intent_preds, axis=1)
+
+    if not args.use_crf:
+        slot_preds = np.argmax(slot_preds, axis=2)
+
+    slot_label_map = {i: label for i, label in enumerate(slot_label_lst)}
+    slot_preds_list = [[] for _ in range(slot_preds.shape[0])]
+
+    for i in range(slot_preds.shape[0]):
+        for j in range(slot_preds.shape[1]):
+            if all_slot_label_mask[i, j] != pad_token_label_id:
+                slot_preds_list[i].append(slot_label_map[slot_preds[i][j]])
+
+    return (lines, slot_preds_list, intent_preds)
+
+
+def text_analysis(text):
+    text = [text.strip().split()]
+
+    words, slot_preds, intent_pred = predict(text)[0][0],  predict(text)[1][0],  predict(text)[2][0]
+
+    slot_tokens = []    
+
+    for word, pred in zip(words, slot_preds):
+        if pred == 'O':
+            slot_tokens.extend([(word, None), (" ", None)])
+        elif pred[0] == 'I':
+            added_tokens = list(slot_tokens[-2]) 
+            added_tokens[0] += f' {word}'
+            slot_tokens[-2] = tuple(added_tokens)
+        else:
+            slot_tokens.extend([(word, pred[2:]), (" ", None)])
+    
+    intent_label = intent_label_lst[intent_pred]
+
+    return slot_tokens, intent_label
+
+    
+
+if __name__ == "__main__":
+    init_logger()
+    parser = argparse.ArgumentParser()
+
+    # parser.add_argument("--input_file", default="sample_pred_in.txt", type=str, help="Input file for prediction")
+    # parser.add_argument("--output_file", default="sample_pred_out.txt", type=str, help="Output file for prediction")
+    parser.add_argument("--model_dir", default="./JointBERT-CRF_PhoBERTencoder", type=str, help="Path to save, load model")
+
+    parser.add_argument("--batch_size", default=32, type=int, help="Batch size for prediction")
+    parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
+
+    pred_config = parser.parse_args()    
+
+    # load model and args
+    args = get_args(pred_config)
+    device = get_device(pred_config)
+    model = load_model(pred_config, args, device)
+    logger.info(args)
+
+    intent_label_lst = get_intent_labels(args)
+    slot_label_lst = get_slot_labels(args)
+
+    # Convert input file to TensorDataset
+    pad_token_label_id = args.ignore_index
+    tokenizer = load_tokenizer(args)   
+
+
+    examples = ["tôi muốn bay một chuyến khứ_hồi từ đà_nẵng đến đà_lạt",
+                ("giá vé khứ_hồi từ đà_nẵng đến vinh dưới 2 triệu đồng giá vé khứ_hồi từ quy nhơn đến vinh dưới 3 triệu đồng giá vé khứ_hồi từ" 
+                " buôn_ma_thuột đến vinh dưới 4 triệu rưỡi"),
+                "cho tôi biết các chuyến bay đến đà_nẵng vào ngày 14 tháng sáu",
+                "những chuyến bay nào khởi_hành từ thành_phố hồ_chí_minh bay đến frankfurt mà nối chuyến ở singapore và hạ_cánh trước 9 giờ tối"]
+    
+    demo = gr.Interface(
+        text_analysis,
+        gr.Textbox(placeholder="Enter sentence here...", label="Input"),
+        [gr.HighlightedText(label='Highlighted Output'), gr.Textbox(label='Intent Label')],
+        examples=examples,
+    )
+
+    demo.launch() 

requirements.txt

@@ -1,3 +1,5 @@
 transformers
 torch
-py_vncorenlp
+py_vncorenlp
+numpy
+tqdm

utils.py

@@ -0,0 +1,115 @@
+import logging
+import os
+import random
+
+import numpy as np
+import torch
+from model import JointPhoBERT, JointXLMR
+from seqeval.metrics import f1_score, precision_score, recall_score
+from transformers import (
+    AutoTokenizer,
+    RobertaConfig,
+    XLMRobertaConfig,
+    XLMRobertaTokenizer,
+)
+
+
+MODEL_CLASSES = {
+    "xlmr": (XLMRobertaConfig, JointXLMR, XLMRobertaTokenizer),
+    "phobert": (RobertaConfig, JointPhoBERT, AutoTokenizer),
+}
+
+MODEL_PATH_MAP = {
+    "xlmr": "xlm-roberta-base",
+    "phobert": "vinai/phobert-base",
+}
+
+
+def get_intent_labels(args):
+    return [
+        label.strip()
+        for label in open(os.path.join(args.data_dir, args.token_level, args.intent_label_file), "r", encoding="utf-8")
+    ]
+
+
+def get_slot_labels(args):
+    return [
+        label.strip()
+        for label in open(os.path.join(args.data_dir, args.token_level, args.slot_label_file), "r", encoding="utf-8")
+    ]
+
+
+def load_tokenizer(args):
+    return MODEL_CLASSES[args.model_type][2].from_pretrained(args.model_name_or_path)
+
+
+def init_logger():
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO,
+    )
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if not args.no_cuda and torch.cuda.is_available():
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def compute_metrics(intent_preds, intent_labels, slot_preds, slot_labels):
+    assert len(intent_preds) == len(intent_labels) == len(slot_preds) == len(slot_labels)
+    results = {}
+    intent_result = get_intent_acc(intent_preds, intent_labels)
+    slot_result = get_slot_metrics(slot_preds, slot_labels)
+    sementic_result = get_sentence_frame_acc(intent_preds, intent_labels, slot_preds, slot_labels)
+
+    mean_intent_slot = (intent_result["intent_acc"] + slot_result["slot_f1"]) / 2
+
+    results.update(intent_result)
+    results.update(slot_result)
+    results.update(sementic_result)
+    results["mean_intent_slot"] = mean_intent_slot
+
+    return results
+
+
+def get_slot_metrics(preds, labels):
+    assert len(preds) == len(labels)
+    return {
+        "slot_precision": precision_score(labels, preds),
+        "slot_recall": recall_score(labels, preds),
+        "slot_f1": f1_score(labels, preds),
+    }
+
+
+def get_intent_acc(preds, labels):
+    acc = (preds == labels).mean()
+    return {"intent_acc": acc}
+
+
+def read_prediction_text(args):
+    return [text.strip() for text in open(os.path.join(args.pred_dir, args.pred_input_file), "r", encoding="utf-8")]
+
+
+def get_sentence_frame_acc(intent_preds, intent_labels, slot_preds, slot_labels):
+    """For the cases that intent and all the slots are correct (in one sentence)"""
+    # Get the intent comparison result
+    intent_result = intent_preds == intent_labels
+
+    # Get the slot comparision result
+    slot_result = []
+    for preds, labels in zip(slot_preds, slot_labels):
+        assert len(preds) == len(labels)
+        one_sent_result = True
+        for p, l in zip(preds, labels):
+            if p != l:
+                one_sent_result = False
+                break
+        slot_result.append(one_sent_result)
+    slot_result = np.array(slot_result)
+
+    semantic_acc = np.multiply(intent_result, slot_result).mean()
+    return {"semantic_frame_acc": semantic_acc}