all

README.md

@@ -0,0 +1,13 @@
+---
+title: FeinbergQuizNotes
+emoji: 💻
+colorFrom: gray
+colorTo: green
+sdk: gradio
+sdk_version: 3.0.26
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,50 @@
+import os
+import gradio as gr
+import pandas as pd
+from question_generation.pipelines import pipeline
+import docx2txt
+
+qa_list = []
+def process_file(Notes):
+
+  os.system("pip install -U transformers==3.0.0")
+  os.system("python -m nltk.downloader punkt")
+  nlp = pipeline("question-generation", model="valhalla/t5-small-qg-prepend", qg_format="prepend")
+  
+  target_word_doc = Notes.name
+  raw_word_file = docx2txt.process(target_word_doc)
+
+  #remove empty lines
+  preprocessed_sentence_list = [i for i in raw_word_file.splitlines() if i != ""]
+
+  #grab content
+  processed_sentence_list = []
+  content = False
+  for i in preprocessed_sentence_list:
+    if "Outline" in i:
+      content = True
+      continue
+    if "Summary Learning Points" in i:
+      content = False
+      continue
+    if "Learning Activity" in i:
+      content = False
+      continue
+    if content == True:
+      processed_sentence_list.append(i.lstrip())
+
+  qa_list.extend(nlp(" ".join(processed_sentence_list)))
+  formatted_questions = "\n".join([str(idx+1) + ". " + i["question"] for idx, i in enumerate(qa_list)])
+  formatted_answers = "\n".join([str(idx+1) + ". " + i["answer"] for idx, i in enumerate(qa_list)])
+  return [formatted_questions, formatted_answers]
+
+def reveal_answer():
+  global qa_list
+  
+  qa_list = []
+  return formatted_answers
+
+io = gr.Interface(process_file, "file", outputs=
+                  [gr.Textbox(lines=1, label="Questions"), 
+                   gr.Textbox(lines=1, label="Answers")])
+io.launch()

question_generation/README.md

@@ -0,0 +1,352 @@
+# Question Generation using 🤗transformers
+
+- [Question Generation using 🤗transformers](#question-generation-using-transformers)
+  - [Project Details](#project-details)
+  - [Initial experiments](#initial-experiments)
+    - [answer aware question generation](#answer-aware-question-generation)
+    - [answer extraction models](#answer-extraction-models)
+    - [Multitask QA-QG](#multitask-qa-qg)
+    - [End-to-End question generation (answer agnostic)](#end-to-end-question-generation-answer-agnostic)
+  - [Results](#results)
+  - [Requirements](#requirements)
+  - [Usage](#usage)
+      - [Question Generation](#question-generation)
+      - [Multitask QA-QG](#multitask-qa-qg-1)
+      - [End-to-end question generation (without answer supervision)](#end-to-end-question-generation-without-answer-supervision)
+  - [Fine-tuning](#fine-tuning)
+    - [Data processing](#data-processing)
+    - [training](#training)
+    - [Evaluation](#evaluation)
+  - [Applications 🚀](#applications-)
+  - [Relevant papers](#relevant-papers)
+
+
+## Project Details
+Question generation is the task of automatically generating questions from a text paragraph. The most straight-forward way for this is answer aware question generation. In answer aware question generation the model is presented with the answer and the passage and asked to generate a question for that answer by considering the passage context. While there are many papers available for QG task, it's still not as mainstream as QA. One of the reasons is most of the earlier papers use complicated models/processing pipelines and have no pre-trained models available. Few recent papers, specifically UniLM and ProphetNet have SOTA pre-trained weights availble for QG but the usage seems quite complicated. 
+
+This project is aimed as an open source study on question generation with pre-trained transformers (specifically seq-2-seq models) using straight-forward end-to-end methods without much complicated pipelines. The goal is to provide simplified data processing and training scripts and easy to use pipelines for inference.
+ 
+
+## Initial experiments
+Initial experiments are conducted using the SQuADv1 dataset and T5 model with different input processing formats as described below.
+
+### answer aware question generation
+
+For answer aware models the input text can be processed in two ways.
+
+**1. prepend format:**
+
+ Here the answer is simply added before the context and seperated by sep token. For example
+
+ `42 [SEP] 42 is the answer to life, the universe and everything.`
+
+ for T5 model the input is processed like this
+
+ `answer: 42  context: 42 is the answer to life, the universe and everything.`
+
+**2. highlight format**
+
+Here the answer span is highlighted within the text with special highlight tokens.
+
+`<hl> 42 <hl> is the answer to life, the universe and everything.`
+
+This idea is proposed in the "A Recurrent BERT-based Model for Question Generation" [paper](https://www.aclweb.org/anthology/D19-5821.pdf). See section 4.3 
+
+### answer extraction models
+
+As the answer aware models need answers for generating question, we need something which can extract answer like spans from the text. This can be done using various methods like NER, noun-phrase extarction etc. But here a model is trained to extract answer like spans, to see how it'll work. With T5, answer extarction is done using the text-to-format. 
+
+As the highlight format will need to know the position of extracted answer spans the input for answer extraction is processed as follows
+
+  1. split the text into senteces. 
+  2. for each sentence that has answers, highlight the sentence with `<hl>` tokens.
+  3. for the target text join the answers in that sentence with `<sep>` tokens.
+
+For example for this text 
+
+`Python is a programming language. Created by Guido van Rossum and first released in 1991.` 
+
+following examples will be created
+
+Input text:
+`<hl> Python is a programming language. <hl> Created by Guido van Rossum and first released in 1991.`
+
+target text:
+`Python <sep>`
+
+and 
+
+Input text:
+`Python is a programming language. <hl> Created by Guido van Rossum and first released in 1991 <hl>.`
+
+target text:
+`Guido van Rossum <sep> 1991 <sep>`
+
+At inference time the text is split into sentences and each sentence is highlighted.
+
+### Multitask QA-QG
+
+For answer aware question generation we usually need 3 models, first which will extract answer like spans, second model will generate question on that answer and third will be a QA model which will take the question and produce an answer,
+then we can compare the two answers to see if the generated question is correct or not.
+
+Having 3 models for single task is lot of complexity, so goal is to create a multi-task model which can do all of these 3 tasks
+
+1. extract answer like spans
+2. generate question based on the answer
+3. QA
+
+T5 model is fine-tuned in multi-task way using task prefixes as described in the paper.
+
+<p align="center">
+  <img width="80%", src="https://i.ibb.co/TBS3nsr/t5-ss-2.png">
+</p>
+
+### End-to-End question generation (answer agnostic)
+
+In end-to-end question generation the model is aksed to generate questions without providing the answers. [This](https://arxiv.org/pdf/2005.01107v1.pdf) paper discusses these ideas in more detail. Here the T5 model is trained to generate multiple questions simultaneously by just providing the context. The questions are seperated by the `<sep>` token. Here's how the examples are processed
+
+input text: `Python is a programming language. Created by Guido van Rossum and first released in 1991.`
+
+target text: `Who created Python ? <sep> When was python released ? <sep>`
+
+**All the training details can be found in [this](https://app.wandb.ai/psuraj/question-generation) wandb project**
+
+## Results
+
+Results on the SQuAD1.0 dev set using above approaches. For decoding, beam search with num_beams 4 is used with max decoding length set to 32. 
+
+For multitask qa-qg models the EM and F1 scores are privded as QA-EM and QA-F1.
+
+The [nlg-eval](https://github.com/Maluuba/nlg-eval) package is used for calculating the metrics.
+
+
+| Name                                                                       | BLEU-4  | METEOR  | ROUGE-L | QA-EM  | QA-F1  | QG-FORMAT |
+|----------------------------------------------------------------------------|---------|---------|---------|--------|--------|-----------|
+| [t5-base-qg-hl](https://huggingface.co/valhalla/t5-base-qg-hl)             | 21.3226 | 27.0854 | 43.5962 | -      | -      | highlight |
+| [t5-base-qa-qg-hl](https://huggingface.co/valhalla/t5-base-qa-qg-hl)       | 21.0141 | 26.9113 | 43.2484 | 82.46  | 90.272 | highlight |
+| [t5-small-qa-qg-hl](https://huggingface.co/valhalla/t5-small-qa-qg-hl)     | 18.9872 | 25.2217 | 40.7893 | 76.121 | 84.904 | highlight |
+| [t5-small-qg-hl](https://huggingface.co/valhalla/t5-small-qg-hl)           | 18.5921 | 24.9915 | 40.1886 | -      | -      | highlight |
+| [t5-small-qg-prepend](https://huggingface.co/valhalla/t5-small-qg-prepend) | 18.2791 | 24.6722 | 39.958  | -      | -      | prepend   |
+
+
+## Requirements
+```
+transformers==3.0.0
+nltk
+nlp==0.2.0 # only if you want to fine-tune.
+```
+
+after installing `nltk` do
+```bash
+python -m nltk.downloader punkt
+```
+
+## Usage
+Use the pipeline whch mimics 🤗transformers pipeline for easy inference.
+
+The pipeline is divided into 3 tasks
+1. `question-generation`: for single task question generation models.
+2. `multitask-qa-qg`: for multi-task qa,qg models.
+3. `e2e-qg`: for end-to-end question generation.
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+#### Question Generation
+
+```python3
+from pipelines import pipeline
+
+nlp = pipeline("question-generation")
+nlp("42 is the answer to life, the universe and everything.")
+=> [{'answer': '42', 'question': 'What is the answer to life, the universe and everything?'}]
+```
+
+**prepend format**
+```python3
+nlp = pipeline("question-generation", model="valhalla/t5-small-qg-prepend", qg_format="prepend")
+nlp("42 is the answer to life, the universe and everything.")
+=> [{'answer': '42 ', 'question': 'What is the answer to life, the universe, and everything?'}]
+```
+
+#### Multitask QA-QG
+```python3
+nlp = pipeline("multitask-qa-qg")
+
+# to generate questions simply pass the text
+nlp("42 is the answer to life, the universe and everything.")
+=> [{'answer': '42', 'question': 'What is the answer to life, the universe and everything?'}]
+
+# for qa pass a dict with "question" and "context"
+nlp({
+    "question": "What is 42 ?",
+    "context": "42 is the answer to life, the universe and everything."
+})
+=> 'the answer to life, the universe and everything'
+```
+
+#### End-to-end question generation (without answer supervision)
+```python3
+nlp = pipeline("e2e-qg")
+nlp("Python is a programming language. Created by Guido van Rossum and first released in 1991.")
+=> [
+    'What is a programming language?',
+    'Who created Python?',
+    'When was Python first released?'
+]
+```
+
+By default both pipelines will use the t5-small* models, to use the other models pass the path through `model` paramter.
+
+By default the `question-generation` pipeline will download the [valhalla/t5-small-qg-hl](https://huggingface.co/valhalla/t5-small-qg-hl) model with `highlight` qg format. If you want to use prepend format then provide the path to the prepend model and set `qg_format` to `"prepend"`. For extracting answer like spans it uses [valhalla/t5-small-qa-qg-hl](https://huggingface.co/valhalla/t5-small-qa-qg-hl) model, you can provide a different model through `ans_model` parameter.
+
+The `multitask-qa-qg` model is for multitask models which can extract answer like spans, do qg and qa, so it won't need seperate `ans_model`. By default [valhalla/t5-small-qa-qg-hl](https://huggingface.co/valhalla/t5-small-qa-qg-hl) model is used with `highlight` format. If you want to use prepend format then provide the path to the prepend model and set `qg_format` to `"prepend"`
+
+The `e2e-qg` pipeline is for end-to-end question generation. These models can generate multiple questions simultaneously without answer supervision. By default it uses [valhalla/t5-small-e2e-qg](https://huggingface.co/valhalla/t5-small-e2e-qg)
+
+## Fine-tuning
+
+### Data processing 
+
+To support different data formats the trainer expects pre-processed cached dataset, so you can process the data the way you want.
+The cached dataset should be saved using `torch.save` and it should return a `dict` with `source_ids`, `target_ids`, `attention_mask` keys from `__getitem__`.
+
+- `source_ids`: encoded source text
+- `target_ids`: encoded target text
+- `attention_mask`: attention mask for the `source_ids`
+  
+The `T2TDataCollator` takes care of preparing right `input_ids` and `labels`. It also trims the batches dynamically to remove excessive padding tokens, to speed up the training.
+
+The `data/squad_multitask` containes the modifed SQuAD dataset for answer aware question generation (using both prepend and highlight formats), question answering (text-to-text), answer extraction and end-to-end question generation. This dataset can be loaded using the awesome 🤗`nlp` library, this makes processing very easy.
+
+To process and cache the dataset use `prepare_data.py` script. It will load the correct tokenizer depending on the `model_type` argument. It adds two new tokens `<sep>` and `<hl>` to the tokenizer and saves it at `{model_type}_qg_tokenizer` path. You should pass this tokenizer to the fine-tuning script.
+
+The datasets will be saved in `data/` directory. You should provide filenames using `train_file_name` and `valid_file_name` arguments.
+
+**process data for single task question generation with highlight_qg_format**
+```bash
+python prepare_data.py \
+    --task qg \
+    --model_type t5 \
+    --dataset_path data/squad_multitask/ \
+    --qg_format highlight_qg_format \
+    --max_source_length 512 \
+    --max_target_length 32 \
+    --train_file_name train_data_qg_hl_t5.pt \
+    --valid_file_name valid_data_qg_hl_t5.pt \
+```
+
+**process data for multi-task qa-qg with highlight_qg_format**
+
+`valid_for_qg_only` argument is used to decide if the validation set should only contain data for qg task. For my multi-task experiments I used validation data with only qg task so that the eval loss curve can be easly compared with other single task models
+
+```bash
+python prepare_data.py \
+    --task multi \
+    --valid_for_qg_only \ 
+    --model_type t5 \
+    --dataset_path data/squad_multitask/ \
+    --qg_format highlight_qg_format \
+    --max_source_length 512 \
+    --max_target_length 32 \
+    --train_file_name train_data_qa_qg_hl_t5.pt \
+    --valid_file_name valid_data_qg_hl_t5.pt \
+```
+
+**process dataset for end-to-end question generation**
+```bash
+python prepare_data.py \
+    --task e2e_qg \
+    --valid_for_qg_only \ 
+    --model_type t5 \
+    --dataset_path data/squad_multitask/ \
+    --qg_format highlight_qg_format \
+    --max_source_length 512 \
+    --max_target_length 32 \
+    --train_file_name train_data_e2e_qg_t5.pt \
+    --valid_file_name valid_data_e2e_qg_t5.pt \
+```
+
+### training
+Use the `run_qg.py` script to  start training. It uses transformers `Trainer` class for training the models.
+
+
+```bash
+python run_qg.py \
+    --model_name_or_path t5-small \
+    --model_type t5 \
+    --tokenizer_name_or_path t5_qg_tokenizer \
+    --output_dir t5-small-qg-hl \
+    --train_file_path data/train_data_qg_hl_t5.pt \
+    --valid_file_path data/valid_data_qg_hl_t5.pt \
+    --per_device_train_batch_size 32 \
+    --per_device_eval_batch_size 32 \
+    --gradient_accumulation_steps 8 \
+    --learning_rate 1e-4 \
+    --num_train_epochs 10 \
+    --seed 42 \
+    --do_train \
+    --do_eval \
+    --evaluate_during_training \
+    --logging_steps 100
+```
+
+or if you want to train it from script or notebook then
+
+```python3
+from run_qg import run_qg
+
+args_dict = {
+    "model_name_or_path": "t5-small",
+    "model_type": "t5",
+    "tokenizer_name_or_path": "t5_qg_tokenizer",
+    "output_dir": "t5-small-qg-hl",
+    "train_file_path": "data/train_data_qg_hl_t5.pt",
+    "valid_file_path": "data/valid_data_qg_hl_t5.pt",
+    "per_device_train_batch_size": 32,
+    "per_device_eval_batch_size": 32,
+    "gradient_accumulation_steps": 8,
+    "learning_rate": 1e-4,
+    "num_train_epochs": 10,
+    "seed": 42,
+    "do_train": True,
+    "do_eval": True,
+    "evaluate_during_training": True,
+    "logging_steps": 100
+}
+
+# start training
+run_qg(args_dict)
+```
+
+### Evaluation
+
+Use the `eval.py` script for evaluting the model. 
+
+```bash
+python eval.py \
+    --model_name_or_path t5-base-qg-hl \
+    --valid_file_path valid_data_qg_hl_t5.pt \
+    --model_type t5 \
+    --num_beams 4 \
+    --max_decoding_length 32 \
+    --output_path hypothesis_t5-base-qg-hl.txt
+```
+
+This will save the output at {output_path} file.
+
+To calculate the metrics install the [nlg-eval](https://github.com/Maluuba/nlg-eval) package and run
+
+```bash
+nlg-eval --hypothesis=hypothesis_t5-base-qg-hl.txt --references=data/references.txt --no-skipthoughts --no-glove 
+```
+
+## Applications 🚀
+
+1. A simple Trivia Quiz on topics of your choice - <br/>
+   [Medium article](https://medium.com/@nvarshney97/using-the-latest-nlp-techniques-for-fun-98f31ce7b556) and its [Colab Notebook](https://colab.research.google.com/gist/nrjvarshney/39ed6c80e2fe293b9e7eca5bc3a45b7d/quiz.ipynb)
+2. [Autocards, Accelerating learning through machine-generated flashcards](https://paulbricman.com/docs/tools/autocards/)
+
+## Relevant papers
+- https://arxiv.org/abs/1906.05416
+- https://www.aclweb.org/anthology/D19-5821/
+- https://arxiv.org/abs/2005.01107v1

question_generation/data/squad_multitask/dataset_infos.json

@@ -0,0 +1 @@
+{"prepend_qg_format": {"description": "Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.\n", "citation": "@article{2016arXiv160605250R,\n       author = {{Rajpurkar}, Pranav and {Zhang}, Jian and {Lopyrev},\n                 Konstantin and {Liang}, Percy},\n        title = \"{SQuAD: 100,000+ Questions for Machine Comprehension of Text}\",\n      journal = {arXiv e-prints},\n         year = 2016,\n          eid = {arXiv:1606.05250},\n        pages = {arXiv:1606.05250},\narchivePrefix = {arXiv},\n       eprint = {1606.05250},\n}\n", "homepage": "https://rajpurkar.github.io/SQuAD-explorer/", "license": "", "features": {"source_text": {"dtype": "string", "id": null, "_type": "Value"}, "target_text": {"dtype": "string", "id": null, "_type": "Value"}, "task": {"dtype": "string", "id": null, "_type": "Value"}}, "supervised_keys": null, "builder_name": "squad_multitask", "config_name": "prepend_qg_format", "version": {"version_str": "1.0.0", "description": "New split API (https://tensorflow.org/datasets/splits)", "nlp_version_to_prepare": null, "major": 1, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 225952922, "num_examples": 253276, "dataset_name": "squad_multitask"}, "validation": {"name": "validation", "num_bytes": 27650081, "num_examples": 30020, "dataset_name": "squad_multitask"}}, "download_checksums": {"https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json": {"num_bytes": 30288272, "checksum": "3527663986b8295af4f7fcdff1ba1ff3f72d07d61a20f487cb238a6ef92fd955"}, "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json": {"num_bytes": 4854279, "checksum": "95aa6a52d5d6a735563366753ca50492a658031da74f301ac5238b03966972c9"}}, "download_size": 35142551, "dataset_size": 253603003, "size_in_bytes": 288745554}, "highlight_qg_format": {"description": "Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.\n", "citation": "@article{2016arXiv160605250R,\n       author = {{Rajpurkar}, Pranav and {Zhang}, Jian and {Lopyrev},\n                 Konstantin and {Liang}, Percy},\n        title = \"{SQuAD: 100,000+ Questions for Machine Comprehension of Text}\",\n      journal = {arXiv e-prints},\n         year = 2016,\n          eid = {arXiv:1606.05250},\n        pages = {arXiv:1606.05250},\narchivePrefix = {arXiv},\n       eprint = {1606.05250},\n}\n", "homepage": "https://rajpurkar.github.io/SQuAD-explorer/", "license": "", "features": {"source_text": {"dtype": "string", "id": null, "_type": "Value"}, "target_text": {"dtype": "string", "id": null, "_type": "Value"}, "task": {"dtype": "string", "id": null, "_type": "Value"}}, "supervised_keys": null, "builder_name": "squad_multitask", "config_name": "highlight_qg_format", "version": {"version_str": "1.0.0", "description": "New split API (https://tensorflow.org/datasets/splits)", "nlp_version_to_prepare": null, "major": 1, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 226286197, "num_examples": 253276, "dataset_name": "squad_multitask"}, "validation": {"name": "validation", "num_bytes": 27698388, "num_examples": 30020, "dataset_name": "squad_multitask"}}, "download_checksums": {"https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json": {"num_bytes": 30288272, "checksum": "3527663986b8295af4f7fcdff1ba1ff3f72d07d61a20f487cb238a6ef92fd955"}, "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json": {"num_bytes": 4854279, "checksum": "95aa6a52d5d6a735563366753ca50492a658031da74f301ac5238b03966972c9"}}, "download_size": 35142551, "dataset_size": 253984585, "size_in_bytes": 289127136}, "prepend_highlight_qg_format": {"description": "Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.\n", "citation": "@article{2016arXiv160605250R,\n       author = {{Rajpurkar}, Pranav and {Zhang}, Jian and {Lopyrev},\n                 Konstantin and {Liang}, Percy},\n        title = \"{SQuAD: 100,000+ Questions for Machine Comprehension of Text}\",\n      journal = {arXiv e-prints},\n         year = 2016,\n          eid = {arXiv:1606.05250},\n        pages = {arXiv:1606.05250},\narchivePrefix = {arXiv},\n       eprint = {1606.05250},\n}\n", "homepage": "https://rajpurkar.github.io/SQuAD-explorer/", "license": "", "features": {"source_text": {"dtype": "string", "id": null, "_type": "Value"}, "target_text": {"dtype": "string", "id": null, "_type": "Value"}, "task": {"dtype": "string", "id": null, "_type": "Value"}}, "supervised_keys": null, "builder_name": "squad_multitask", "config_name": "prepend_highlight_qg_format", "version": {"version_str": "1.0.0", "description": "New split API (https://tensorflow.org/datasets/splits)", "nlp_version_to_prepare": null, "major": 1, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 227967699, "num_examples": 253276, "dataset_name": "squad_multitask"}, "validation": {"name": "validation", "num_bytes": 27893191, "num_examples": 30020, "dataset_name": "squad_multitask"}}, "download_checksums": {"https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json": {"num_bytes": 30288272, "checksum": "3527663986b8295af4f7fcdff1ba1ff3f72d07d61a20f487cb238a6ef92fd955"}, "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json": {"num_bytes": 4854279, "checksum": "95aa6a52d5d6a735563366753ca50492a658031da74f301ac5238b03966972c9"}}, "download_size": 35142551, "dataset_size": 255860890, "size_in_bytes": 291003441}}

question_generation/data/squad_multitask/dummy/plain_text/1.0.0/dummy_data/dev

@@ -0,0 +1,25 @@
+{
+	"data": [	
+		{ "title": "dev test",
+			"paragraphs": [
+				{ "context": "This is a test context.",
+					"qas": [
+						{ "question": "Is this a test?",
+							"id": "2",
+							"answers": [
+								{ "answer_start": 6,
+								  "text": "This is a test text"
+								}
+							]
+						}
+					]
+		   	}
+			]
+		}
+	]
+}
+
+
+
+
+

question_generation/data/squad_multitask/dummy/plain_text/1.0.0/dummy_data/train

@@ -0,0 +1,25 @@
+{
+	"data": [	
+		{ "title": "train test",
+			"paragraphs": [
+				{ "context": "This is a test context.",
+					"qas": [
+						{ "question": "Is this a test?",
+							"id": "1",
+							"answers": [
+								{ "answer_start": 1,
+								  "text": "This is a test text"
+								}
+							]
+						}
+					]
+		   	}
+			]
+		}
+	]
+}
+
+
+
+
+

question_generation/data/squad_multitask/squad_multitask.py

@@ -0,0 +1,239 @@
+# coding=utf-8
+# Copyright 2020 The TensorFlow Datasets Authors and the HuggingFace NLP Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""SQUAD: The Stanford Question Answering Dataset."""
+
+from __future__ import absolute_import, division, print_function
+
+import json
+import logging
+import os
+
+import nltk
+nltk.download('punkt')
+
+import nlp
+
+
+_CITATION = """\
+@article{2016arXiv160605250R,
+       author = {{Rajpurkar}, Pranav and {Zhang}, Jian and {Lopyrev},
+                 Konstantin and {Liang}, Percy},
+        title = "{SQuAD: 100,000+ Questions for Machine Comprehension of Text}",
+      journal = {arXiv e-prints},
+         year = 2016,
+          eid = {arXiv:1606.05250},
+        pages = {arXiv:1606.05250},
+archivePrefix = {arXiv},
+       eprint = {1606.05250},
+}
+"""
+
+_DESCRIPTION = """\
+Stanford Question Answering Dataset (SQuAD) is a reading comprehension \
+dataset, consisting of questions posed by crowdworkers on a set of Wikipedia \
+articles, where the answer to every question is a segment of text, or span, \
+from the corresponding reading passage, or the question might be unanswerable.
+"""
+
+QG_FORMATS = [
+    "prepend",
+    "highlight",
+    "prepend_highlight",
+]
+
+
+class SquadMultitaskConfig(nlp.BuilderConfig):
+    """BuilderConfig for SQUAD."""
+
+    def __init__(self, qg_format="highlight", **kwargs):
+        """BuilderConfig for SQUAD.
+
+    Args:
+      **kwargs: keyword arguments forwarded to super.
+    """
+        super(SquadMultitaskConfig, self).__init__(**kwargs)
+        self.qg_format = qg_format
+
+
+class SquadMultitask(nlp.GeneratorBasedBuilder):
+    """SQUAD: The Stanford Question Answering Dataset. Version 1.1."""
+
+    _URL = "https://rajpurkar.github.io/SQuAD-explorer/dataset/"
+    _DEV_FILE = "dev-v1.1.json"
+    _TRAINING_FILE = "train-v1.1.json"
+
+    BUILDER_CONFIGS = [
+        SquadMultitaskConfig(
+            name=f"{format_}_qg_format",
+            version=nlp.Version("1.0.0", "New split API (https://tensorflow.org/datasets/splits)"),
+            description="Plain text",
+            qg_format=format_
+        )
+        for format_ in QG_FORMATS
+    ]
+
+    def _info(self):
+        return nlp.DatasetInfo(
+            description=_DESCRIPTION,
+            features=nlp.Features(
+                {
+                    "source_text": nlp.Value("string"),
+                    "target_text": nlp.Value("string"),
+                    "task": nlp.Value("string"),
+                }
+            ),
+            # No default supervised_keys (as we have to pass both question
+            # and context as input).
+            supervised_keys=None,
+            homepage="https://rajpurkar.github.io/SQuAD-explorer/",
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        urls_to_download = {
+            "train": os.path.join(self._URL, self._TRAINING_FILE),
+            "dev": os.path.join(self._URL, self._DEV_FILE),
+        }
+        downloaded_files = dl_manager.download_and_extract(urls_to_download)
+
+        return [
+            nlp.SplitGenerator(name=nlp.Split.TRAIN, gen_kwargs={"filepath": downloaded_files["train"]}),
+            nlp.SplitGenerator(name=nlp.Split.VALIDATION, gen_kwargs={"filepath": downloaded_files["dev"]}),
+        ]
+    
+    def _get_correct_alignement(self, context, answer):
+        """ Some original examples in SQuAD have indices wrong by 1 or 2 character. We test and fix this here. """
+        gold_text = answer['text']
+        start_idx = answer['answer_start']
+        end_idx = start_idx + len(gold_text)
+        if context[start_idx:end_idx] == gold_text:
+            return start_idx, end_idx       # When the gold label position is good
+        elif context[start_idx-1:end_idx-1] == gold_text:
+            return start_idx-1, end_idx-1   # When the gold label is off by one character
+        elif context[start_idx-2:end_idx-2] == gold_text:
+            return start_idx-2, end_idx-2   # When the gold label is off by two character
+        else:
+            raise ValueError()
+    
+    def process_qa_text(self, context, question, answer):
+        ans_gen_input = f"question: {question}  context: {context}"
+        ans_gen_target = f"{answer}"
+        return {"source_text": ans_gen_input, "target_text": ans_gen_target, "task": "qa"}
+
+    def process_qg_text(self, context, question, answer):
+        answer_text = answer['text'].strip()
+        
+        if self.config.qg_format == "prepend":
+            que_gen_input = f"answer: {answer_text}  context: {context}"
+        elif self.config.qg_format == "highlight":
+            start_pos, end_pos = self._get_correct_alignement(context, answer)
+            que_gen_input = f"generate question: {context[:start_pos]} {{hl_token}} {answer_text} {{hl_token}} {context[end_pos:]}"
+        else:
+            start_pos, end_pos = self._get_correct_alignement(context, answer)
+            que_gen_input = f"answer: {answer_text} context: {context[:start_pos]} {{hl_token}} {answer_text} {{hl_token}} {context[end_pos:]}"
+        
+        que_gen_target = f"{question}"
+        return {"source_text": que_gen_input, "target_text": que_gen_target, "task": "qg"}
+    
+    def process_e2e_qg(self, paragraph):
+        source_text = f"generate questions: {paragraph['context'].strip()}"
+        questions = [qas['question'].strip() for qas in paragraph['qas']]
+        target_text = " {sep_token} ".join(questions)
+        target_text = f"{target_text} {{sep_token}}"
+        return {"source_text": source_text, "target_text": target_text, "task": "e2e_qg"}
+
+    def process_ans_ext(self, paragraph):
+        context = paragraph['context'].strip()
+    
+        # split into sentences
+        sents = nltk.sent_tokenize(context)
+
+        # get positions of the sentences
+        positions = []
+        for i, sent in enumerate(sents):
+            if i == 0:
+                start, end = 0, len(sent)
+            else:
+                start, end = (prev_end + 1), (prev_end + len(sent) + 1)
+            prev_end = end
+            positions.append({'start': start, 'end': end})
+        
+        # get answers
+        answers = [qa['answers'][0] for qa in paragraph['qas']]
+
+        # get list of answers for each sentence
+        sent_answers = []
+        for pos, sent in zip(positions, sents):
+            target_answers = []
+            for ans in answers:
+                if ans['answer_start'] in range(pos['start'], pos['end']):
+                    target_answers.append(ans['text'].strip())
+            sent_answers.append(target_answers)
+
+        # build inputs and targets
+        examples = []
+        for i, ans in enumerate(sent_answers):
+            context = "extract answers:"
+            if len(ans) == 0: continue
+            ans = list(set(ans))
+            for j, sent in enumerate(sents):
+                if i == j:
+                    sent = "{hl_token} %s {hl_token}" % sent
+                context = "%s %s" % (context, sent)
+                context = context.strip()
+            input_text = context
+            target_text = " {sep_token} ".join(ans) + " {sep_token}"
+
+            examples.append({'source_text': input_text, "target_text": target_text, "task": "ans_ext"})
+        
+        return examples
+
+    def _generate_examples(self, filepath):
+        """This function returns the examples in the raw (text) form."""
+        logging.info("generating examples from = %s", filepath)
+        count = 0
+        tasks = ['qa', 'qg', 'ans_ext', 'e2e_qg']
+        with open(filepath) as f:
+            squad = json.load(f)
+            for article in squad["data"]:
+                title = article.get("title", "").strip()
+                for paragraph in article["paragraphs"]:
+                    context = paragraph["context"].strip()
+                    
+                    if 'ans_ext' in tasks:
+                        ans_ext_examples = self.process_ans_ext(paragraph)
+                        for example in ans_ext_examples:
+                                yield count, example
+                                count += 1
+                    
+                    if 'e2e_qg' in tasks:
+                        yield count, self.process_e2e_qg(paragraph)
+                        count += 1
+                    
+                    for qa in paragraph["qas"]:
+                        question = qa["question"].strip()
+                        id_ = qa["id"]
+
+                        answers = [answer["text"].strip() for answer in qa["answers"]]
+                        for task in tasks:
+                            if task == 'qa':
+                                yield count, self.process_qa_text(context, question, answers[0])
+                                count += 1
+                            
+                            if task == 'qg':
+                                yield count, self.process_qg_text(context, question, qa["answers"][0])
+                                count += 1

question_generation/data_collator.py

@@ -0,0 +1,83 @@
+from typing import Dict, List, Optional
+
+import torch
+
+
+def trim_batch(
+    input_ids, pad_token_id, attention_mask=None,
+):
+    """Remove columns that are populated exclusively by pad_token_id"""
+    keep_column_mask = input_ids.ne(pad_token_id).any(dim=0)
+    if attention_mask is None:
+        return input_ids[:, keep_column_mask]
+    else:
+        return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])
+
+
+# prepares lm_labels from target_ids, returns examples with keys as expected by the forward method
+# this is necessacry because the trainer directly passes this dict as arguments to the model
+# so make sure the keys match the parameter names of the forward method
+class T2TDataCollator():
+    def __init__(self, tokenizer, model_type="t5", mode='training', using_tpu=False):
+        self.tokenizer = tokenizer
+        self.model_type = model_type
+        self.mode = mode
+        self.using_tpu = using_tpu
+
+    def __call__(self, batch: List) -> Dict[str, torch.Tensor]:
+        """
+        Take a list of samples from a Dataset and collate them into a batch.
+        Returns:
+            A dictionary of tensors
+        """
+        input_ids = torch.stack([example['source_ids'] for example in batch])
+        target_ids = torch.stack([example['target_ids'] for example in batch])
+        attention_mask = torch.stack([example['attention_mask'] for example in batch])
+
+        pad_token_id = self.tokenizer.pad_token_id
+        
+        # don't trim on tpu, for some reason trimming leads to slower training on TPU
+        if not self.using_tpu:
+            input_ids, attention_mask = trim_batch(input_ids, pad_token_id, attention_mask=attention_mask)
+            target_ids = trim_batch(target_ids, pad_token_id)
+        
+        if self.model_type == "t5":
+            lm_labels = target_ids.clone()
+            decoder_input_ids = self._shift_right_t5(lm_labels)
+            if self.mode == 'training':
+                lm_labels[lm_labels[:, :] == pad_token_id] = -100
+        else:
+            decoder_input_ids = target_ids[:, :-1].contiguous()
+            lm_labels = target_ids[:, 1:].clone()
+            if self.mode == 'training':
+                lm_labels[target_ids[:, 1:] == pad_token_id] = -100
+
+        params =  {
+            "input_ids": input_ids, 
+            "attention_mask": attention_mask,
+            "labels": lm_labels,
+            "decoder_input_ids": decoder_input_ids
+        }
+        
+        return params
+    
+    def _shift_right_t5(self, input_ids):
+        decoder_start_token_id = self.tokenizer.pad_token_id
+        pad_token_id = self.tokenizer.pad_token_id
+
+        assert (
+            decoder_start_token_id is not None
+        ), "self.model.config.decoder_start_token_id has to be defined. In T5 it is usually set to the pad_token_id. See T5 docs for more information"
+
+        # shift inputs to the right
+        shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+        shifted_input_ids[..., 1:] = input_ids[..., :-1].clone()
+        shifted_input_ids[..., 0] = decoder_start_token_id
+
+        assert pad_token_id is not None, "self.model.config.pad_token_id has to be defined."
+        # replace possible -100 values in labels by `pad_token_id`
+        shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+        assert torch.all(shifted_input_ids >= 0).item(), "Verify that `labels` has only positive values and -100"
+
+        return shifted_input_ids

question_generation/eval.py

@@ -0,0 +1,92 @@
+import logging
+from dataclasses import dataclass, field
+from typing import Optional
+
+import torch
+from tqdm.auto import tqdm
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, HfArgumentParser
+
+from data_collator import T2TDataCollator
+
+device = 'cuda' if torch.cuda.is_available else 'cpu'
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class EvalArguments:
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    valid_file_path: str = field(
+        metadata={"help": "Path for cached valid dataset"}
+    )
+    model_type: str = field(metadata={"help": "One of 't5', 'bart'"})
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    num_beams: Optional[int] = field(
+        default=4,
+        metadata={"help": "num_beams to use for decoding"}
+    )
+    max_decoding_length: Optional[int] = field(
+        default=32,
+        metadata={"help": "maximum length for decoding"}
+    )
+    output_path: Optional[str] = field(
+        default="hypothesis.txt",
+        metadata={"help": "path to save the generated questions."}
+    )
+
+def get_predictions(model, tokenizer, data_loader, num_beams=4, max_length=32, length_penalty=1):
+    model.to(device)
+    
+    predictions = []
+    model.eval()
+    with torch.no_grad():
+        for batch in tqdm(data_loader):
+            outs = model.generate(
+                input_ids=batch['input_ids'].to(device), 
+                attention_mask=batch['attention_mask'].to(device),
+                num_beams=num_beams,
+                max_length=max_length,
+                length_penalty=length_penalty,
+            )
+
+            prediction = [tokenizer.decode(ids, skip_special_tokens=True) for ids in outs]
+            predictions.extend(prediction)
+
+    return predictions
+
+def main():
+    parser = HfArgumentParser((EvalArguments,))
+    args = parser.parse_args_into_dataclasses()[0]
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.tokenizer_name_or_path if args.tokenizer_name_or_path else args.model_name_or_path,
+    )
+    model = AutoModelForSeq2SeqLM.from_pretrained(args.model_name_or_path)
+
+    valid_dataset = torch.load(args.valid_file_path)
+    collator = T2TDataCollator(
+        tokenizer=tokenizer,
+        model_type=args.model_type,
+        mode="inference"
+    )
+    loader = torch.utils.data.DataLoader(valid_dataset, batch_size=32, collate_fn=collator)
+
+    predictions = get_predictions(
+        model=model,
+        tokenizer=tokenizer,
+        data_loader=loader,
+        num_beams=args.num_beams,
+        max_length=args.max_decoding_length
+    )
+
+    with open(args.output_path, 'w') as f:
+        f.write("\n".join(predictions))
+    
+    logging.info(f"Output saved at {args.output_path}")
+
+
+if __name__ == "__main__":
+    main()

question_generation/pipelines.py

@@ -0,0 +1,386 @@
+import itertools
+import logging
+from typing import Optional, Dict, Union
+
+from nltk import sent_tokenize
+
+import torch
+from transformers import(
+    AutoModelForSeq2SeqLM, 
+    AutoTokenizer,
+    PreTrainedModel,
+    PreTrainedTokenizer,
+)
+
+logger = logging.getLogger(__name__)
+
+class QGPipeline:
+    """Poor man's QG pipeline"""
+    def __init__(
+        self,
+        model: PreTrainedModel,
+        tokenizer: PreTrainedTokenizer,
+        ans_model: PreTrainedModel,
+        ans_tokenizer: PreTrainedTokenizer,
+        qg_format: str,
+        use_cuda: bool
+    ):
+        self.model = model
+        self.tokenizer = tokenizer
+
+        self.ans_model = ans_model
+        self.ans_tokenizer = ans_tokenizer
+
+        self.qg_format = qg_format
+
+        self.device = "cuda" if torch.cuda.is_available() and use_cuda else "cpu"
+        self.model.to(self.device)
+
+        if self.ans_model is not self.model:
+            self.ans_model.to(self.device)
+
+        assert self.model.__class__.__name__ in ["T5ForConditionalGeneration", "BartForConditionalGeneration"]
+        
+        if "T5ForConditionalGeneration" in self.model.__class__.__name__:
+            self.model_type = "t5"
+        else:
+            self.model_type = "bart"
+
+    def __call__(self, inputs: str):
+        inputs = " ".join(inputs.split())
+        sents, answers = self._extract_answers(inputs)
+        flat_answers = list(itertools.chain(*answers))
+        
+        if len(flat_answers) == 0:
+          return []
+
+        if self.qg_format == "prepend":
+            qg_examples = self._prepare_inputs_for_qg_from_answers_prepend(inputs, answers)
+        else:
+            qg_examples = self._prepare_inputs_for_qg_from_answers_hl(sents, answers)
+        
+        qg_inputs = [example['source_text'] for example in qg_examples]
+        questions = self._generate_questions(qg_inputs)
+        output = [{'answer': example['answer'], 'question': que} for example, que in zip(qg_examples, questions)]
+        return output
+    
+    def _generate_questions(self, inputs):
+        inputs = self._tokenize(inputs, padding=True, truncation=True)
+        
+        outs = self.model.generate(
+            input_ids=inputs['input_ids'].to(self.device), 
+            attention_mask=inputs['attention_mask'].to(self.device), 
+            max_length=32,
+            num_beams=4,
+        )
+        
+        questions = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in outs]
+        return questions
+    
+    def _extract_answers(self, context):
+        sents, inputs = self._prepare_inputs_for_ans_extraction(context)
+        inputs = self._tokenize(inputs, padding=True, truncation=True)
+
+        outs = self.ans_model.generate(
+            input_ids=inputs['input_ids'].to(self.device), 
+            attention_mask=inputs['attention_mask'].to(self.device), 
+            max_length=32,
+        )
+        
+        dec = [self.ans_tokenizer.decode(ids, skip_special_tokens=False) for ids in outs]
+        answers = [item.split('<sep>') for item in dec]
+        answers = [i[:-1] for i in answers]
+        
+        return sents, answers
+    
+    def _tokenize(self,
+        inputs,
+        padding=True,
+        truncation=True,
+        add_special_tokens=True,
+        max_length=512
+    ):
+        inputs = self.tokenizer.batch_encode_plus(
+            inputs, 
+            max_length=max_length,
+            add_special_tokens=add_special_tokens,
+            truncation=truncation,
+            padding="max_length" if padding else False,
+            pad_to_max_length=padding,
+            return_tensors="pt"
+        )
+        return inputs
+    
+    def _prepare_inputs_for_ans_extraction(self, text):
+        sents = sent_tokenize(text)
+
+        inputs = []
+        for i in range(len(sents)):
+            source_text = "extract answers:"
+            for j, sent in enumerate(sents):
+                if i == j:
+                    sent = "<hl> %s <hl>" % sent
+                source_text = "%s %s" % (source_text, sent)
+                source_text = source_text.strip()
+            
+            if self.model_type == "t5":
+                source_text = source_text + " </s>"
+            inputs.append(source_text)
+
+        return sents, inputs
+    
+    def _prepare_inputs_for_qg_from_answers_hl(self, sents, answers):
+        inputs = []
+        for i, answer in enumerate(answers):
+            if len(answer) == 0: continue
+            for answer_text in answer:
+                sent = sents[i]
+                sents_copy = sents[:]
+                
+                answer_text = answer_text.strip()
+                
+                ans_start_idx = sent.index(answer_text)
+                
+                sent = f"{sent[:ans_start_idx]} <hl> {answer_text} <hl> {sent[ans_start_idx + len(answer_text): ]}"
+                sents_copy[i] = sent
+                
+                source_text = " ".join(sents_copy)
+                source_text = f"generate question: {source_text}" 
+                if self.model_type == "t5":
+                    source_text = source_text + " </s>"
+                
+                inputs.append({"answer": answer_text, "source_text": source_text})
+        
+        return inputs
+    
+    def _prepare_inputs_for_qg_from_answers_prepend(self, context, answers):
+        flat_answers = list(itertools.chain(*answers))
+        examples = []
+        for answer in flat_answers:
+            source_text = f"answer: {answer} context: {context}"
+            if self.model_type == "t5":
+                source_text = source_text + " </s>"
+            
+            examples.append({"answer": answer, "source_text": source_text})
+        return examples
+
+    
+class MultiTaskQAQGPipeline(QGPipeline):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+    
+    def __call__(self, inputs: Union[Dict, str]):
+        if type(inputs) is str:
+            # do qg
+            return super().__call__(inputs)
+        else:
+            # do qa
+            return self._extract_answer(inputs["question"], inputs["context"])
+    
+    def _prepare_inputs_for_qa(self, question, context):
+        source_text = f"question: {question}  context: {context}"
+        if self.model_type == "t5":
+            source_text = source_text + " </s>"
+        return  source_text
+    
+    def _extract_answer(self, question, context):
+        source_text = self._prepare_inputs_for_qa(question, context)
+        inputs = self._tokenize([source_text], padding=False)
+    
+        outs = self.model.generate(
+            input_ids=inputs['input_ids'].to(self.device), 
+            attention_mask=inputs['attention_mask'].to(self.device), 
+            max_length=16,
+        )
+
+        answer = self.tokenizer.decode(outs[0], skip_special_tokens=True)
+        return answer
+
+
+class E2EQGPipeline:
+    def __init__(
+        self,
+        model: PreTrainedModel,
+        tokenizer: PreTrainedTokenizer,
+        use_cuda: bool
+    ) :
+
+        self.model = model
+        self.tokenizer = tokenizer
+
+        self.device = "cuda" if torch.cuda.is_available() and use_cuda else "cpu"
+        self.model.to(self.device)
+
+        assert self.model.__class__.__name__ in ["T5ForConditionalGeneration", "BartForConditionalGeneration"]
+        
+        if "T5ForConditionalGeneration" in self.model.__class__.__name__:
+            self.model_type = "t5"
+        else:
+            self.model_type = "bart"
+        
+        self.default_generate_kwargs = {
+            "max_length": 256,
+            "num_beams": 4,
+            "length_penalty": 1.5,
+            "no_repeat_ngram_size": 3,
+            "early_stopping": True,
+        }
+    
+    def __call__(self, context: str, **generate_kwargs):
+        inputs = self._prepare_inputs_for_e2e_qg(context)
+
+        # TODO: when overrding default_generate_kwargs all other arguments need to be passsed
+        # find a better way to do this
+        if not generate_kwargs:
+            generate_kwargs = self.default_generate_kwargs
+        
+        input_length = inputs["input_ids"].shape[-1]
+        
+        # max_length = generate_kwargs.get("max_length", 256)
+        # if input_length < max_length:
+        #     logger.warning(
+        #         "Your max_length is set to {}, but you input_length is only {}. You might consider decreasing max_length manually, e.g. summarizer('...', max_length=50)".format(
+        #             max_length, input_length
+        #         )
+        #     )
+
+        outs = self.model.generate(
+            input_ids=inputs['input_ids'].to(self.device), 
+            attention_mask=inputs['attention_mask'].to(self.device),
+            **generate_kwargs
+        )
+
+        prediction = self.tokenizer.decode(outs[0], skip_special_tokens=True)
+        questions = prediction.split("<sep>")
+        questions = [question.strip() for question in questions[:-1]]
+        return questions
+    
+    def _prepare_inputs_for_e2e_qg(self, context):
+        source_text = f"generate questions: {context}"
+        if self.model_type == "t5":
+            source_text = source_text + " </s>"
+        
+        inputs = self._tokenize([source_text], padding=False)
+        return inputs
+    
+    def _tokenize(
+        self,
+        inputs,
+        padding=True,
+        truncation=True,
+        add_special_tokens=True,
+        max_length=512
+    ):
+        inputs = self.tokenizer.batch_encode_plus(
+            inputs, 
+            max_length=max_length,
+            add_special_tokens=add_special_tokens,
+            truncation=truncation,
+            padding="max_length" if padding else False,
+            pad_to_max_length=padding,
+            return_tensors="pt"
+        )
+        return inputs
+
+
+SUPPORTED_TASKS = {
+    "question-generation": {
+        "impl": QGPipeline,
+        "default": {
+            "model": "valhalla/t5-small-qg-hl",
+            "ans_model": "valhalla/t5-small-qa-qg-hl",
+        }
+    },
+    "multitask-qa-qg": {
+        "impl": MultiTaskQAQGPipeline,
+        "default": {
+            "model": "valhalla/t5-small-qa-qg-hl",
+        }
+    },
+    "e2e-qg": {
+        "impl": E2EQGPipeline,
+        "default": {
+            "model": "valhalla/t5-small-e2e-qg",
+        }
+    }
+}
+
+def pipeline(
+    task: str,
+    model: Optional = None,
+    tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None,
+    qg_format: Optional[str] = "highlight",
+    ans_model: Optional = None,
+    ans_tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None,
+    use_cuda: Optional[bool] = True,
+    **kwargs,
+):
+    # Retrieve the task
+    if task not in SUPPORTED_TASKS:
+        raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys())))
+
+    targeted_task = SUPPORTED_TASKS[task]
+    task_class = targeted_task["impl"]
+
+    # Use default model/config/tokenizer for the task if no model is provided
+    if model is None:
+        model = targeted_task["default"]["model"]
+    
+    # Try to infer tokenizer from model or config name (if provided as str)
+    if tokenizer is None:
+        if isinstance(model, str):
+            tokenizer = model
+        else:
+            # Impossible to guest what is the right tokenizer here
+            raise Exception(
+                "Impossible to guess which tokenizer to use. "
+                "Please provided a PretrainedTokenizer class or a path/identifier to a pretrained tokenizer."
+            )
+    
+    # Instantiate tokenizer if needed
+    if isinstance(tokenizer, (str, tuple)):
+        if isinstance(tokenizer, tuple):
+            # For tuple we have (tokenizer name, {kwargs})
+            tokenizer = AutoTokenizer.from_pretrained(tokenizer[0], **tokenizer[1])
+        else:
+            tokenizer = AutoTokenizer.from_pretrained(tokenizer)
+    
+    # Instantiate model if needed
+    if isinstance(model, str):
+        model = AutoModelForSeq2SeqLM.from_pretrained(model)
+    
+    if task == "question-generation":
+        if ans_model is None:
+            # load default ans model
+            ans_model = targeted_task["default"]["ans_model"]
+            ans_tokenizer = AutoTokenizer.from_pretrained(ans_model)
+            ans_model = AutoModelForSeq2SeqLM.from_pretrained(ans_model)
+        else:
+            # Try to infer tokenizer from model or config name (if provided as str)
+            if ans_tokenizer is None:
+                if isinstance(ans_model, str):
+                    ans_tokenizer = ans_model
+                else:
+                    # Impossible to guest what is the right tokenizer here
+                    raise Exception(
+                        "Impossible to guess which tokenizer to use. "
+                        "Please provided a PretrainedTokenizer class or a path/identifier to a pretrained tokenizer."
+                    )
+            
+            # Instantiate tokenizer if needed
+            if isinstance(ans_tokenizer, (str, tuple)):
+                if isinstance(ans_tokenizer, tuple):
+                    # For tuple we have (tokenizer name, {kwargs})
+                    ans_tokenizer = AutoTokenizer.from_pretrained(ans_tokenizer[0], **ans_tokenizer[1])
+                else:
+                    ans_tokenizer = AutoTokenizer.from_pretrained(ans_tokenizer)
+
+            if isinstance(ans_model, str):
+                ans_model = AutoModelForSeq2SeqLM.from_pretrained(ans_model)
+    
+    if task == "e2e-qg":
+        return task_class(model=model, tokenizer=tokenizer, use_cuda=use_cuda)
+    elif task == "question-generation":
+        return task_class(model=model, tokenizer=tokenizer, ans_model=ans_model, ans_tokenizer=ans_tokenizer, qg_format=qg_format, use_cuda=use_cuda)
+    else:
+        return task_class(model=model, tokenizer=tokenizer, ans_model=model, ans_tokenizer=tokenizer, qg_format=qg_format, use_cuda=use_cuda)

question_generation/prepare_data.py

@@ -0,0 +1,204 @@
+import os
+import logging
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+import torch
+import nlp
+from transformers import T5Tokenizer, BartTokenizer, HfArgumentParser
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+    task: str = field(
+        metadata={"help": "Which task 'qa', 'qg', 'e2e_qg', 'ans_ext', 'multi'. 'multi' means 'qa', 'qg', 'ans_ext' tasks"}, 
+    )
+    model_type: str = field(metadata={"help": "One of 't5', 'bart'"})
+    dataset_path: Optional[str] = field(
+        default="data/squad_multitask",
+        metadata={"help": "Path for dataset directory"}, 
+    )
+    train_file_name: Optional[str] = field(
+        default=None,
+        metadata={"help": "name for cached train dataset"},
+    )
+    valid_file_name: Optional[str] = field(
+        default=None,
+        metadata={"help": "name for cached valid dataset"},
+    )
+    valid_for_qg_only: bool = field(
+        default=False,
+        metadata={"help": "For multitask dataset valid split should contain only qg task or all tasks."}
+    )
+    qg_format: Optional[str] = field(
+        default='highlight_qg_format',
+        metadata={"help": "How to format inputs for que generation, 'highlight_qg_format' or 'prepend_qg_format'"}, 
+    )
+    max_source_length: Optional[int] = field(
+        default=512,
+        metadata={"help": "Max input length for the source text"},
+    )
+    max_target_length: Optional[int] = field(
+        default=32,
+        metadata={"help": "Max input length for the target text"},
+    )
+
+class DataProcessor:
+    def __init__(self, tokenizer, model_type="t5", max_source_length=512, max_target_length=32):
+        self.tokenizer = tokenizer
+        self.max_source_length = max_source_length
+        self.max_target_length = max_target_length
+        self.model_type = model_type
+        self.hl_token = "<hl>"
+        
+        if model_type == "t5":
+            self.sep_token = "<sep>"
+        elif model_type == "bart":
+            self.sep_token = "<sep>"
+        else:
+            self.sep_token = "[SEP]"
+  
+    def process(self, dataset):
+        if self.model_type == "t5":
+            dataset = dataset.map(self._add_eos_examples)
+        
+        dataset = dataset.map(self._add_special_tokens)
+        dataset = dataset.map(self._convert_to_features, batched=True)
+        
+        return dataset
+  
+    def _add_eos_examples(self, example):
+        example['source_text'] = example['source_text'] + " </s>"
+        example['target_text'] = example['target_text'] + " </s>"
+        return example
+  
+    def _add_special_tokens(self, example):
+        example['source_text'] = example['source_text'].replace("{hl_token}", self.hl_token)    
+        example['target_text'] = example['target_text'].replace("{sep_token}", self.sep_token)
+        return example
+  
+    # tokenize the examples
+    def _convert_to_features(self, example_batch):
+        source_encoding = self.tokenizer.batch_encode_plus(
+            example_batch['source_text'],
+            max_length=self.max_source_length,
+            padding='max_length',
+            pad_to_max_length=True,
+            truncation=True, 
+        )
+        target_encoding = self.tokenizer.batch_encode_plus(
+            example_batch['target_text'],
+            max_length=self.max_target_length,
+            padding='max_length',
+            pad_to_max_length=True,
+            truncation=True, 
+        )
+
+        encodings = {
+            'source_ids': source_encoding['input_ids'], 
+            'target_ids': target_encoding['input_ids'],
+            'attention_mask': source_encoding['attention_mask'],
+        }
+
+        return encodings
+
+
+def filter_qa(example):
+    return example['task'] == 'qa'
+
+def filter_qg(example):
+    return example['task'] == 'qg'
+
+def filter_e2e_qg(example):
+    return example['task'] == 'e2e_qg'
+
+def filter_ans_ext(example):
+    return example['task'] == 'ans_ext'
+
+def filter_multi(example):
+    return example['task'] != 'e2e_qg'
+
+
+TASK_TO_FILTER_FN = {
+    'qa': filter_qa,
+    'qg': filter_qg,
+    'e2e_qg': filter_e2e_qg,
+    'ans_ext': filter_ans_ext,
+    'multi': filter_multi
+}
+
+
+def main():
+    parser = HfArgumentParser((DataTrainingArguments,))
+
+    data_args = parser.parse_args_into_dataclasses()[0]
+
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO
+    )
+
+    if data_args.model_type == 't5':
+        tokenizer = T5Tokenizer.from_pretrained("t5-base")
+    else:
+        tokenizer = BartTokenizer.from_pretrained("facebook/bart-base")
+    
+    tokenizer.add_tokens(['<sep>', '<hl>'])
+    
+    train_dataset = nlp.load_dataset(data_args.dataset_path, name=data_args.qg_format, split=nlp.Split.TRAIN)
+    valid_dataset = nlp.load_dataset(data_args.dataset_path, name=data_args.qg_format, split=nlp.Split.VALIDATION)
+
+    processor = DataProcessor(
+        tokenizer,
+        model_type=data_args.model_type,
+        max_source_length=data_args.max_source_length,
+        max_target_length=data_args.max_target_length
+    )
+
+    train_dataset = train_dataset.filter(TASK_TO_FILTER_FN[data_args.task])
+    if data_args.task == 'multi' and data_args.valid_for_qg_only:
+        logger.info("processing valid data only for qg task")
+        valid_dataset = valid_dataset.filter(filter_qg)
+    else:
+        valid_dataset = valid_dataset.filter(TASK_TO_FILTER_FN[data_args.task])
+
+    
+    train_dataset = processor.process(train_dataset)
+    valid_dataset = processor.process(valid_dataset)
+
+    columns = ["source_ids", "target_ids", "attention_mask"]
+    train_dataset.set_format(type='torch', columns=columns)
+    valid_dataset.set_format(type='torch', columns=columns)
+
+    if data_args.train_file_name is None:
+        train_file_name = f"train_data_{data_args.task}_{data_args.qg_format}_{data_args.model_type}.pt"
+        train_path = os.path.join("data", train_file_name)
+
+        valid_file_name = f"valid_data_{data_args.task}_{data_args.qg_format}_{data_args.model_type}.pt"
+        valid_path = os.path.join("data", valid_file_name)
+    else:
+        train_path = os.path.join("data", data_args.train_file_name)
+        valid_path = os.path.join("data", data_args.valid_file_name)
+    
+    torch.save(train_dataset, train_path)
+    logger.info(f"saved train dataset at {train_path}")
+    
+    torch.save(valid_dataset, valid_path)
+    logger.info(f"saved validation dataset at {valid_path}")
+    
+    tokenizer_path = f"{data_args.model_type}_qg_tokenizer"
+    if not os.path.exists(tokenizer_path):
+        os.mkdir(tokenizer_path)
+    tokenizer.save_pretrained(tokenizer_path)
+    logger.info(f"saved tokenizer at {tokenizer_path}")
+
+
+if __name__ == "__main__":
+    main()

question_generation/run_qg.py

@@ -0,0 +1,236 @@
+import dataclasses
+import json
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+import numpy as np
+import torch
+
+from transformers import (
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    T5Tokenizer,
+    BartTokenizer,
+    HfArgumentParser,
+    DataCollator,
+    TrainingArguments,
+    set_seed,
+)
+
+from trainer import Trainer
+from data_collator import T2TDataCollator
+from utils import freeze_embeds, assert_not_all_frozen
+
+MODEL_TYPE_TO_TOKENIZER = {
+    "t5": T5Tokenizer,
+    "bart": BartTokenizer,
+}
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    model_type: str = field(metadata={"help": "One of 't5', 'bart'"})
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
+    )
+    label_smoothing: Optional[float] = field(
+        default=0,
+        metadata={"help": "label smoothing rate, set to > 0 if you want to enable lable smoothing"}
+    )
+    freeze_embeds: bool = field(
+        default=False,
+        metadata={"help": "Freeze token embeddings and positional embeddings for bart, just token embeddings for t5."}
+    )
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+    train_file_path: str = field(
+        metadata={"help": "Path for cached train dataset"},
+    )
+    valid_file_path: str = field(
+        metadata={"help": "Path for cached valid dataset"},
+    )
+    data_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Path for data files"}, 
+    )
+    task: Optional[str] = field(
+        default=None,
+        metadata={"help": "Which task 'qa', 'qg', 'e2e_qg', 'ans_ext', 'multi'. 'multi' means 'qa', 'qg', 'ans_ext' tasks"}, 
+    )
+    qg_format: Optional[str] = field(
+        default='prepend_qg_format',
+        metadata={"help": "How to format inputs for que generation, 'highlight_qg_format' or 'prepend_qg_format'"}, 
+    )
+    max_source_length: Optional[int] = field(
+        default=512,
+        metadata={"help": "Max input length for the source text"},
+    )
+    max_target_length: Optional[int] = field(
+        default=32,
+        metadata={"help": "Max input length for the target text"},
+    )
+
+
+def main(args_file=None):
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+
+    if (len(sys.argv) == 2 and sys.argv[1].endswith(".json")) or args_file is not None:
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        args_file_path = os.path.abspath(sys.argv[1]) if args_file is None else args_file
+        model_args, data_args, training_args = parser.parse_json_file(json_file=args_file_path)
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    assert model_args.model_type in list(MODEL_TYPE_TO_TOKENIZER.keys()), "model type should be 't5' or 'bart'"
+
+    if (
+        os.path.exists(training_args.output_dir)
+        and os.listdir(training_args.output_dir)
+        and training_args.do_train
+        and not training_args.overwrite_output_dir
+    ):
+        raise ValueError(
+            f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
+        )
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        training_args.local_rank,
+        training_args.device,
+        training_args.n_gpu,
+        bool(training_args.local_rank != -1),
+        training_args.fp16,
+    )
+    logger.info("Training/evaluation parameters %s", training_args)
+
+    # Set seed
+    set_seed(training_args.seed)
+
+    # Set project name
+    os.environ["WANDB_PROJECT"] = "question-generation"
+
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+    tokenizer_cls = MODEL_TYPE_TO_TOKENIZER[model_args.model_type]
+    tokenizer = tokenizer_cls.from_pretrained(
+        model_args.tokenizer_name_or_path if model_args.tokenizer_name_or_path else model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+    )
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+    )
+
+    model.resize_token_embeddings(len(tokenizer))
+
+    if model_args.freeze_embeds:
+        logger.info("freezing embeddings of the model")
+        freeze_embeds(model)
+        assert_not_all_frozen(model)
+
+    # Get datasets
+    logger.info('loading dataset')
+    
+    train_dataset = torch.load(data_args.train_file_path) if training_args.do_train else None
+    valid_dataset = torch.load(data_args.valid_file_path) if training_args.do_eval else None
+    
+    logger.info('finished loading dataset')
+
+    # Initialize data_collator
+    data_collator = T2TDataCollator(
+        tokenizer=tokenizer,
+        model_type=model_args.model_type,
+        mode="training",
+        using_tpu=training_args.tpu_num_cores is not None
+    )
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=valid_dataset,
+        data_collator=data_collator,
+        prediction_loss_only=True,
+        label_smoothing=model_args.label_smoothing
+    )
+
+    # disable wandb console logs
+    logging.getLogger('wandb.run_manager').setLevel(logging.WARNING)
+
+    # Training
+    if training_args.do_train:
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        )
+        trainer.save_model()
+        # For convenience, we also re-save the tokenizer to the same directory,
+        # so that you can share your model easily on huggingface.co/models =)
+        if trainer.is_world_master():
+            tokenizer.save_pretrained(training_args.output_dir)
+
+    # Evaluation
+    results = {}
+    if training_args.do_eval and training_args.local_rank in [-1, 0]:
+        logger.info("*** Evaluate ***")
+
+        eval_output = trainer.evaluate()
+
+        output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
+        with open(output_eval_file, "w") as writer:
+            logger.info("***** Eval results *****")
+            for key in sorted(eval_output.keys()):
+                logger.info("  %s = %s", key, str(eval_output[key]))
+                writer.write("%s = %s\n" % (key, str(eval_output[key])))
+    
+        results.update(eval_output)
+    
+    return results
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+def run_qg(args_dict):
+    with open("args.json", 'w') as f:
+        json.dump(args_dict, f)
+    
+    main(args_file="args.json")
+
+if __name__ == "__main__":
+    main()

question_generation/trainer.py

@@ -0,0 +1,56 @@
+from typing import Any, Dict, Union
+
+import torch
+from torch import nn
+
+from transformers import Trainer as HFTrainer
+from transformers.file_utils import is_apex_available
+
+if is_apex_available():
+    from apex import amp
+
+from utils import label_smoothed_nll_loss
+
+class Trainer(HFTrainer):
+    def __init__(self, label_smoothing: float = 0, **kwargs):
+        super().__init__(**kwargs)
+        self.label_smoothing = label_smoothing
+    
+    # override to support label smoothing
+    def _training_step(
+        self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], optimizer: torch.optim.Optimizer
+    ) -> float:
+        model.train()
+        for k, v in inputs.items():
+            if isinstance(v, torch.Tensor):
+                inputs[k] = v.to(self.args.device)
+
+
+        # Our model outputs do not work with DataParallel, so forcing return tuple.
+        if isinstance(model, nn.DataParallel):
+            inputs["return_tuple"] = True
+
+        if self.label_smoothing == 0:
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
+        else:
+            labels = inputs.pop("labels")
+            labels[labels == -100] = model.config.pad_token_id
+            outputs = model(**inputs)
+            lprobs = torch.nn.functional.log_softmax(outputs[0], dim=-1)
+            loss, nll_loss = label_smoothed_nll_loss(
+                lprobs, labels, self.label_smoothing, ignore_index=model.config.pad_token_id
+            )
+
+        if self.args.n_gpu > 1:
+            loss = loss.mean()  # mean() to average on multi-gpu parallel training
+        if self.args.gradient_accumulation_steps > 1:
+            loss = loss / self.args.gradient_accumulation_steps
+
+        if self.args.fp16:
+            with amp.scale_loss(loss, optimizer) as scaled_loss:
+                scaled_loss.backward()
+        else:
+            loss.backward()
+
+        return loss.item()

question_generation/utils.py

@@ -0,0 +1,49 @@
+from typing import Callable, Dict, Iterable, List
+from torch import nn
+
+# these functions are taken from transformers repo
+def grad_status(model: nn.Module) -> Iterable:
+    return (par.requires_grad for par in model.parameters())
+
+def freeze_params(model: nn.Module):
+    for par in model.parameters():
+        par.requires_grad = False
+
+def freeze_embeds(model: nn.Module):
+    """Freeze token embeddings and positional embeddings for bart, just token embeddings for t5."""
+    try:
+        freeze_params(model.model.shared)
+        for d in [model.model.encoder, model.model.decoder]:
+            freeze_params(d.embed_positions)
+            freeze_params(d.embed_tokens)
+    except AttributeError:
+        freeze_params(model.shared)
+        for d in [model.encoder, model.decoder]:
+            freeze_params(d.embed_tokens)
+
+def assert_not_all_frozen(model):
+    model_grads: List[bool] = list(grad_status(model))
+    npars = len(model_grads)
+    assert any(model_grads), f"none of {npars} weights require grad"
+
+def label_smoothed_nll_loss(lprobs, target, epsilon, ignore_index=-100):
+    """From fairseq"""
+    if target.dim() == lprobs.dim() - 1:
+        target = target.unsqueeze(-1)
+    nll_loss = -lprobs.gather(dim=-1, index=target)
+    smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
+    if ignore_index is not None:
+        pad_mask = target.eq(ignore_index)
+        nll_loss.masked_fill_(pad_mask, 0.0)
+        smooth_loss.masked_fill_(pad_mask, 0.0)
+        bs = pad_mask.long().sum()
+    else:
+        nll_loss = nll_loss.squeeze(-1)
+        smooth_loss = smooth_loss.squeeze(-1)
+        bs = lprobs.shape[0]
+
+    nll_loss = nll_loss.sum()  # mean()? Scared to break other math.
+    smooth_loss = smooth_loss.sum()
+    eps_i = epsilon / lprobs.size(-1)
+    loss = (1.0 - epsilon) * nll_loss + eps_i * smooth_loss
+    return loss / bs, nll_loss / bs