fastchat/data/prepare_all.py

"""Prepare all datasets."""

import argparse
import os

from fastchat.utils import run_cmd


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--prefix", type=str, default="~/datasets/sharegpt_20230521")
    parser.add_argument(
        "--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf"
    )
    parser.add_argument("--seq-len", type=int, default=4096)
    args = parser.parse_args()

    in_prefix = args.prefix
    model_path = args.model_name_or_path
    seq_len = args.seq_len
    prefix = (
        f"{in_prefix}_{seq_len}".replace("4096", "4k")
        .replace("8192", "8k")
        .replace("16384", "16k")
    )

    cmd_list = [
        f"python3 -m fastchat.data.clean_sharegpt --in {in_prefix}_html.json --out {prefix}_clean.json",
        f"python3 -m fastchat.data.optional_clean --in {prefix}_clean.json --out {prefix}_clean_lang.json --skip-lang ko",
        f"python3 -m fastchat.data.split_long_conversation --in {prefix}_clean_lang.json --out {prefix}_clean_lang_split.json --model-name {model_path} --max-length {seq_len}",
        f"python3 -m fastchat.data.filter_wrong_format --in {prefix}_clean_lang_split.json --out {prefix}_clean_lang_split.json",
        f"python3 -m fastchat.data.split_train_test --in {prefix}_clean_lang_split.json --ratio 0.99",
        f"python3 -m fastchat.data.hardcoded_questions",
        f"python3 -m fastchat.data.merge --in {prefix}_clean_lang_split_train.json hardcoded.json --out {prefix}_clean_lang_split_identity.json",
        f"python3 -m fastchat.data.extract_gpt4_only --in {prefix}_clean_lang_split_identity.json",
        f"python3 -m fastchat.data.extract_single_round --in {prefix}_clean_lang_split_identity.json",
    ]

    for cmd in cmd_list:
        ret = run_cmd(cmd)
        if ret != 0:
            exit(ret)