online trainer

.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+.vscode
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

app.py

@@ -1,7 +1,96 @@
 import streamlit as st
-# To make things easier later, we're also importing numpy and pandas for
-# working with sample data.
-import numpy as np
-import pandas as pd
+from utils.load_dataset import load_datasets
+from utils.load_tasks import load_tasks
+from utils.load_models import load_models
+from trainer import train_estimtator
+from datetime import datetime
+import logging
 
-st.title('My first app')
+logger = logging.getLogger(__name__)
+
+
+def main():
+    parameter = st.experimental_get_query_params()
+    parameter["model_name_or_path"] = parameter.get("model_name_or_path", ["none"])
+    parameter["dataset"] = parameter.get("dataset", ["none"])
+    parameter["task"] = parameter.get("task", ["none"])
+    ### hyperparameter
+    parameter["epochs"] = parameter.get("epochs", [3])
+    parameter["learning_rate"] = parameter.get("learning_rate", [5e-5])
+    parameter["per_device_train_batch_size"] = parameter.get("per_device_train_batch_size", [8])
+    parameter["per_device_eval_batch_size"] = parameter.get("per_device_eval_batch_size", [8])
+    st.experimental_set_query_params(**parameter)
+
+    dataset_list = load_datasets()
+    task_list = load_tasks()
+    model_list = load_models()
+
+    st.header("Hugging Face model & dataset")
+    col1, col2 = st.beta_columns(2)
+    parameter["model_name_or_path"] = col1.selectbox("Model ID:", parameter["model_name_or_path"] + model_list)
+    st.experimental_set_query_params(**parameter)
+
+    parameter["dataset"] = col2.selectbox("Dataset:", parameter["dataset"] + dataset_list)
+    st.experimental_set_query_params(**parameter)
+
+    parameter["task"] = col1.selectbox("Task:", parameter["task"] + task_list)
+    st.experimental_set_query_params(**parameter)
+
+    use_auth_token = col2.text_input("HF auth token to upload your model:", help="api_xxxxx")
+
+    my_expander = st.beta_expander("Hyperparameters")
+    col1, col2 = my_expander.beta_columns(2)
+    parameter["epochs"] = col1.number_input("Epoch", 3)
+    st.experimental_set_query_params(**parameter)
+
+    parameter["learning_rate"] = col2.text_input("Learning Rate", 5e-5)
+    st.experimental_set_query_params(**parameter)
+
+    parameter["per_device_train_batch_size"] = col1.number_input("Training Batch Size", 8)
+    st.experimental_set_query_params(**parameter)
+
+    parameter["per_device_eval_batch_size"] = col2.number_input("Eval Batch Size", 8)
+    st.experimental_set_query_params(**parameter)
+    st.markdown("---")
+
+    st.header("Amazon Sagemaker configuration")
+
+    config = {}
+
+    config["job_name"] = st.text_input(
+        "model name",
+        f"{parameter['model_name_or_path'][0] if isinstance(parameter['model_name_or_path'],list)else parameter['model_name_or_path']}-job-{str(datetime.today()).split()[0]}",
+    )
+    col1, col2 = st.beta_columns(2)
+
+    config["aws_sagemaker_role"] = col1.text_input("AWS IAM role for sagemaker job")
+    config["instance_type"] = col2.selectbox(
+        "Instance type",
+        [
+            "single-gpu | ml.p3.2xlarge",
+            "multi-gpu | ml.p3.16xlarge",
+        ],
+    )
+    config["region"] = col1.selectbox(
+        "AWS Region",
+        ["eu-central-1", "eu-west-1", "us-east-1", "us-east-1", "us-west-1", "us-west-2"],
+    )
+    config["instance_count"] = col2.number_input("Instance count", 1)
+    config["use_spot"] = col1.selectbox("use spot instances", [False, True])
+    st.markdown("---")
+
+    st.header("Credentials")
+    # sagemaker config
+    col1, col2 = st.beta_columns(2)
+    config["aws_access_key_id"] = col1.text_input("Aws Secret Key ID")
+    config["aws_secret_accesskey"] = col2.text_input("Aws Secret Access Key")
+
+    if use_auth_token:
+        parameter["use_auth_token"] = use_auth_token
+
+    if st.button("Start training on SageMaker"):
+        train_estimtator(parameter, config)
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,3 @@
+sagemaker
+transformers
+datasets

trainer.py

@@ -0,0 +1,139 @@
+from sagemaker.huggingface import HuggingFace
+import logging
+import sys
+from contextlib import contextmanager
+from io import StringIO
+from streamlit.report_thread import REPORT_CONTEXT_ATTR_NAME
+from threading import current_thread
+import streamlit as st
+import sys
+import sagemaker
+import boto3
+
+
+@contextmanager
+def st_redirect(src, dst):
+    placeholder = st.empty()
+    output_func = getattr(placeholder, dst)
+
+    with StringIO() as buffer:
+        old_write = src.write
+
+        def new_write(b):
+            if getattr(current_thread(), REPORT_CONTEXT_ATTR_NAME, None):
+                buffer.write(b)
+                output_func(buffer.getvalue())
+            else:
+                old_write(b)
+
+        try:
+            src.write = new_write
+            yield
+        finally:
+            src.write = old_write
+
+
+@contextmanager
+def st_stdout(dst):
+    with st_redirect(sys.stdout, dst):
+        yield
+
+
+@contextmanager
+def st_stderr(dst):
+    with st_redirect(sys.stderr, dst):
+        yield
+
+
+task2script = {
+    "text-classification": {
+        "entry_point": "run_glue.py",
+        "source_dir": "examples/text-classification",
+    },
+    "token-classification": {
+        "entry_point": "run_ner.py",
+        "source_dir": "examples/token-classification",
+    },
+    "question-answering": {
+        "entry_point": "run_qa.py",
+        "source_dir": "examples/question-answering",
+    },
+    "summarization": {
+        "entry_point": "run_summarization.py",
+        "source_dir": "examples/seq2seq",
+    },
+    "translation": {
+        "entry_point": "run_translation.py",
+        "source_dir": "examples/seq2seq",
+    },
+    "causal-language-modeling": {
+        "entry_point": "run_clm.py",
+        "source_dir": "examples/language-modeling",
+    },
+    "masked-language-modeling": {
+        "entry_point": "run_mlm.py",
+        "source_dir": "examples/language-modeling",
+    },
+}
+
+
+def train_estimtator(parameter, config):
+    with st_stdout("code"):
+        logger = logging.getLogger(__name__)
+
+        logging.basicConfig(
+            level=logging.getLevelName("INFO"),
+            handlers=[logging.StreamHandler(sys.stdout)],
+            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        )
+        logger.info = print
+
+        # git configuration to download our fine-tuning script
+        git_config = {"repo": "https://github.com/huggingface/transformers.git", "branch": "v4.4.2"}
+
+        # creating fine-tuning script
+        entry_point = task2script[parameter["task"]]["entry_point"]
+        source_dir = task2script[parameter["task"]]["source_dir"]
+        # create train file
+        # iam configuration
+        session = boto3.session.Session(
+            aws_access_key_id=config["aws_access_key_id"],
+            aws_secret_access_key=config["aws_secret_accesskey"],
+            region_name=config["region"],
+        )
+        sess = sagemaker.Session(boto_session=session)
+
+        iam = session.client(
+            "iam", aws_access_key_id=config["aws_access_key_id"], aws_secret_access_key=config["aws_secret_accesskey"]
+        )
+        role = iam.get_role(RoleName=config["aws_sagemaker_role"])["Role"]["Arn"]
+
+        logger.info(f"role: {role}")
+        instance_type = config["instance_type"].split("|")[1].split("|")[0].strip()
+        logger.info(f"instance_type: {instance_type}")
+
+        hyperparameters = {
+            "output_dir": "/opt/ml/model",
+            "do_train": True,
+            "do_eval": True,
+            "do_predict": True,
+            **parameter,
+        }
+        del hyperparameters["task"]
+        # create estimator
+        huggingface_estimator = HuggingFace(
+            entry_point=entry_point,
+            source_dir=source_dir,
+            git_config=git_config,
+            base_job_name=config["job_name"],
+            instance_type=instance_type,
+            sagemaker_session=sess,
+            instance_count=config["instance_count"],
+            role=role,
+            transformers_version="4.4",
+            pytorch_version="1.6",
+            py_version="py36",
+            hyperparameters=hyperparameters,
+        )
+        # train
+        huggingface_estimator.fit()

utils/load_dataset.py

@@ -0,0 +1,7 @@
+import streamlit as st
+import datasets as ds
+
+
+@st.cache
+def load_datasets():
+    return ds.list_datasets(with_community_datasets=False)

utils/load_models.py

@@ -0,0 +1,8 @@
+import requests
+import streamlit as st
+
+
+@st.cache
+def load_models():
+    res = requests.get("https://huggingface.co/api/models").json()
+    return [model["modelId"] for model in res]

utils/load_tasks.py

@@ -0,0 +1,15 @@
+import streamlit as st
+import datasets as ds
+
+
+@st.cache
+def load_tasks():
+    return [
+  'causal-language-modeling',
+  'masked-language-modeling',
+  'question-answering',
+  'summarization',
+  'text-classification',
+  'token-classification',
+  'translation',
+    ]