[update]add code

.gitignore

@@ -0,0 +1,6 @@
+
+.git/
+.idea/
+
+**/flagged/
+**/__pycache__/

examples/exercises/chinese_modern_poetry/1.prepare_data.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+
+from datasets import load_dataset
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset_path", default="Iess/chinese_modern_poetry", type=str)
+    parser.add_argument("--dataset_name", default=None, type=str)
+    parser.add_argument("--dataset_split", default=None, type=str)
+    parser.add_argument(
+        "--dataset_cache_dir",
+        default=(project_path / "hub_datasets").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    dataset = load_dataset(
+        path=args.dataset_path,
+        name=args.dataset_name,
+        split=args.dataset_split,
+        cache_dir=args.dataset_cache_dir
+    )
+    print(dataset)
+
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/chinese_modern_poetry/2.train_model.py

@@ -0,0 +1,327 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from collections import defaultdict
+from dataclasses import dataclass, field
+import os
+import platform
+import sys
+from typing import Optional
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, '../../../'))
+
+import bitsandbytes as bnb
+from datasets import Dataset, DatasetDict, load_dataset
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+import torch
+from transformers.data.data_collator import DataCollatorForLanguageModeling
+from transformers.trainer import Trainer
+from transformers.training_args import TrainingArguments
+from transformers.models.auto import AutoModelForCausalLM, AutoTokenizer
+from transformers.utils.quantization_config import BitsAndBytesConfig
+
+from project_settings import project_path
+from toolbox.transformers.data.dataset.dataset import SFTDataset, ChatGLM2SFTDataset
+from toolbox.transformers.data.data_collator import SFTDataCollator
+from toolbox.transformers.modules.loss import TargetLMLoss
+from toolbox.transformers.trainer import LoRATrainer
+
+
+def get_args():
+    """
+    python3 2.train_model.py --pretrained_model_name_or_path /data/tianxing/PycharmProjects/Transformers/pretrained_models/huggingface/YeungNLP/firefly-chatglm2-6b
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset_path", default="Iess/chinese_modern_poetry", type=str)
+    parser.add_argument("--dataset_name", default=None, type=str)
+    parser.add_argument("--dataset_split", default=None, type=str)
+    parser.add_argument(
+        "--dataset_cache_dir",
+        default=(project_path / "hub_datasets").as_posix(),
+        type=str
+    )
+
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        default="Qwen/Qwen-7B",
+        type=str
+    )
+    parser.add_argument("--cache_dir", default="cache_dir", type=str)
+
+    # train
+    parser.add_argument("--output_dir", default="serialization_dir", type=str)
+    parser.add_argument("--overwrite_output_dir", action="store_true")
+    parser.add_argument("--evaluation_strategy", default="no", choices=["no", "steps", "epoch"], type=str)
+    parser.add_argument("--per_device_train_batch_size", default=4, type=int)
+    parser.add_argument("--gradient_accumulation_steps", default=4, type=int)
+    parser.add_argument("--learning_rate", default=2e-4, type=float)
+    parser.add_argument("--weight_decay", default=0, type=float)
+    parser.add_argument("--max_grad_norm", default=0.3, type=float)
+    parser.add_argument("--num_train_epochs", default=1.0, type=float)
+    parser.add_argument("--max_steps", default=-1, type=int)
+    parser.add_argument("--lr_scheduler_type", default="constant_with_warmup", type=str)
+    parser.add_argument("--warmup_ratio", default=0.0, type=float)
+    parser.add_argument("--warmup_steps", default=3000, type=int)
+    parser.add_argument("--logging_steps", default=300, type=int)
+    parser.add_argument("--save_strategy", default="steps", type=str)
+    parser.add_argument("--save_steps", default=500, type=int)
+    parser.add_argument("--save_total_limit", default=2, type=int)
+    parser.add_argument("--no_cuda", action="store_true")
+    parser.add_argument("--seed", default=3407, type=str, help="https://arxiv.org/abs/2109.08203")
+    # parser.add_argument("--fp16", action="store_true")
+    parser.add_argument("--fp16", action="store_false")
+    parser.add_argument("--half_precision_backend", default="auto", type=str)
+    parser.add_argument("--dataloader_num_workers", default=0, type=int)
+    parser.add_argument("--disable_tqdm", action="store_true")
+    # parser.add_argument("--disable_tqdm", action="store_false")
+    parser.add_argument("--remove_unused_columns", action="store_true")
+    # parser.add_argument("--remove_unused_columns", action="store_false")
+    # parser.add_argument("--deepspeed", default="ds_z3_config.json", type=str)
+    parser.add_argument("--deepspeed", default=None, type=str)
+    parser.add_argument("--optim", default="paged_adamw_32bit", type=str)
+    parser.add_argument("--report_to", default="tensorboard", type=str)
+    parser.add_argument("--resume_from_checkpoint", default="file_dir/serialization_dir/checkpoint-103000", type=str)
+    # parser.add_argument("--gradient_checkpointing", action="store_true")
+    parser.add_argument("--gradient_checkpointing", action="store_false")
+
+    # dataset process
+    parser.add_argument("--truncate_longer_samples", action="store_true")
+    parser.add_argument("--max_seq_length", default=1024, type=int)
+
+    # lora
+    parser.add_argument("--lora_rank", default=64, type=int)
+    parser.add_argument("--lora_alpha", default=16, type=int)
+    parser.add_argument("--lora_dropout", default=0.05, type=int)
+
+    args = parser.parse_args()
+    return args
+
+
+def verify_model_dtype(model):
+    """
+    查看模型种各种类型的参数的情况
+    """
+    dtype2param_num = defaultdict(int)   # 每种数据类型的参数量
+    dtype2param_name = defaultdict(list)   # 每种数据类型的参数名称
+    dtype2trainable_param_num = defaultdict(int)   # 每种数据类型参与训练的参数量
+    dtype2trainable_param_name = defaultdict(list)   # 每种数据类型参与训练的参数名称
+    for name, p in model.named_parameters():
+        dtype = p.dtype
+        dtype2param_num[dtype] += p.numel()
+        dtype2param_name[dtype].append(name)
+        if p.requires_grad:
+            dtype2trainable_param_num[dtype] += p.numel()
+            dtype2trainable_param_name[dtype].append(name)
+
+    # 统计全部参数中, 各种类型参数分布.
+    total = 0
+    print('verify all params of the model')
+    for k, v in dtype2param_num.items():
+        total += v
+    for k, v in dtype2param_num.items():
+        print(k, v, v / total)
+    for k, v in dtype2trainable_param_name.items():
+        print(k, v)
+
+    print()
+    # 统计可训练参数中, 各种类型参数分布.
+    print('verify trainable params the model')
+    total_trainable = 0
+    for k, v in dtype2trainable_param_num.items():
+        total_trainable += v
+    for k, v in dtype2trainable_param_num.items():
+        print(k, v, v / total_trainable)
+    for k, v in dtype2trainable_param_num.items():
+        print(k, v)
+
+
+def find_all_linear_names(model):
+    """
+    找出所有全连接层，为所有全连接添加adapter
+    """
+    cls = bnb.nn.Linear4bit
+    lora_module_names = set()
+    for name, module in model.named_modules():
+        if isinstance(module, cls):
+            names = name.split('.')
+            lora_module_names.add(names[0] if len(names) == 1 else names[-1])
+
+    if 'lm_head' in lora_module_names:  # needed for 16-bit
+        lora_module_names.remove('lm_head')
+    return list(lora_module_names)
+
+
+def main():
+    args = get_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    os.makedirs(args.cache_dir, exist_ok=True)
+
+    # dataset
+    dataset_dict = load_dataset(
+        path=args.dataset_path,
+        name=args.dataset_name,
+        split=args.dataset_split,
+        cache_dir=args.dataset_cache_dir
+    )
+    train_dataset = dataset_dict["train"]
+    print(train_dataset)
+
+    # training_args
+    training_args = TrainingArguments(
+        output_dir=args.output_dir,
+        overwrite_output_dir=args.overwrite_output_dir,
+        evaluation_strategy=args.evaluation_strategy,
+        per_device_train_batch_size=args.per_device_train_batch_size,
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        learning_rate=args.learning_rate,
+        weight_decay=args.weight_decay,
+        max_grad_norm=args.max_grad_norm,
+        num_train_epochs=args.num_train_epochs,
+        max_steps=args.max_steps,
+        lr_scheduler_type=args.lr_scheduler_type,
+        warmup_steps=args.warmup_steps,
+        logging_steps=args.logging_steps,
+        save_strategy=args.save_strategy,
+        save_steps=args.save_steps,
+        save_total_limit=args.save_total_limit,
+        no_cuda=args.no_cuda,
+        fp16=args.fp16,
+        half_precision_backend=args.half_precision_backend,
+        dataloader_num_workers=args.dataloader_num_workers,
+        disable_tqdm=args.disable_tqdm,
+        remove_unused_columns=args.remove_unused_columns,
+        # deepspeed=args.deepspeed,
+        optim=args.optim,
+        report_to=args.report_to,
+        resume_from_checkpoint=args.resume_from_checkpoint,
+        gradient_checkpointing=args.gradient_checkpointing,
+    )
+
+    # pretrained model
+    model = AutoModelForCausalLM.from_pretrained(
+        args.pretrained_model_name_or_path,
+        device_map={"": 0},
+        load_in_4bit=True,
+        torch_dtype=torch.float16,
+        trust_remote_code=True,
+        quantization_config=BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type="nf4",
+            llm_int8_threshold=6.0,
+            llm_int8_has_fp16_weight=False,
+        ),
+    )
+    # tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.pretrained_model_name_or_path,
+        trust_remote_code=True,
+        use_fast=False if model.config.model_type == "llama" else True
+    )
+    # QWenTokenizer比较特殊, pad_token_id, bos_token_id, eos_token_id 均 为None. eod_id对应的token为<|endoftext|>
+    if tokenizer.__class__.__name__ == "QWenTokenizer":
+        tokenizer.pad_token_id = tokenizer.eod_id
+        tokenizer.bos_token_id = tokenizer.eod_id
+        tokenizer.eos_token_id = tokenizer.eod_id
+
+    # model
+    # casts all the non int8 modules to full precision (fp32) for stability
+    model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=args.gradient_checkpointing)
+    print(f"memory footprint of model: {model.get_memory_footprint() / (1024*1024*1024)} GB")
+
+    # 找到所有需要插入adapter的全连接层
+    target_modules = find_all_linear_names(model)
+    config = LoraConfig(
+        r=args.lora_rank,
+        lora_alpha=args.lora_alpha,
+        target_modules=target_modules,
+        lora_dropout=args.lora_dropout,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+    model = get_peft_model(model, config)
+    model.print_trainable_parameters()
+    model.config.torch_dtype = torch.float32
+
+    # 查看模型种各种类型的参数的情况
+    verify_model_dtype(model)
+
+    # 初始化损失函数
+    loss_func = TargetLMLoss(ignore_index=-100)
+
+    data_collator = SFTDataCollator(tokenizer, args.max_seq_length)
+
+    # dataset
+    def encode_with_truncation(examples):
+        prompt_ = examples.pop('prompt')
+        response_ = examples.pop('response')
+        utterances = [
+            prompt_,
+            response_
+        ]
+
+        utterances_ids = tokenizer(utterances, add_special_tokens=False).input_ids
+
+        input_ids = [tokenizer.bos_token_id]
+        target_mask = [0]
+        for i, utterances_id in enumerate(utterances_ids):
+            input_ids += (utterances_id + [tokenizer.eos_token_id])
+
+            if i % 2 == 0:
+                target_mask += [0] * (len(utterances_id) + 1)
+            else:
+                target_mask += [1] * (len(utterances_id) + 1)
+
+        assert len(input_ids) == len(target_mask)
+
+        input_ids = input_ids[:args.max_seq_length]
+        target_mask = target_mask[:args.max_seq_length]
+        attention_mask = [1] * len(input_ids)
+
+        assert len(input_ids) == len(target_mask) == len(attention_mask)
+
+        inputs = {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "target_mask": target_mask
+        }
+        return inputs
+
+    train_dataset = train_dataset.map(
+        encode_with_truncation,
+        batched=False,
+        keep_in_memory=False,
+        num_proc=None if platform.system() == "Windows" else os.cpu_count(),
+        cache_file_name=os.path.join(args.cache_dir, "train.cache")
+    )
+    train_dataset.set_format(type=None, columns=["input_ids", "attention_mask", "target_mask"])
+    print("Train Dataset Examples Batch Number: {}".format(len(train_dataset)))
+
+    # 初始化 Trainer
+    trainer = LoRATrainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        # tokenizer=tokenizer,
+        data_collator=data_collator,
+        compute_loss=loss_func
+    )
+    train_result = trainer.train()
+
+    # 保存最好的 checkpoint
+    final_save_path = os.path.join(training_args.output_dir, "final")
+    trainer.save_model(final_save_path)  # Saves the tokenizer too
+    # 保存训练指标
+    metrics = train_result.metrics
+    trainer.log_metrics("train", metrics)
+    trainer.save_metrics("train", metrics)
+    trainer.save_state()
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/chinese_modern_poetry/3.merge_lora.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
+import torch
+"""
+使用该脚本，将lora的权重合并大base model中
+"""
+
+
+def get_args():
+    """
+    python3 3.merge_lora.py \
+    --pretrained_model_name_or_path /data/tianxing/PycharmProjects/Transformers/pretrained_models/huggingface/Qwen/Qwen-7B \
+    --adapter_name_or_path /data/tianxing/PycharmProjects/Transformers/examples/exercises/chinese_modern_poetry/file_dir/serialization_dir/checkpoint-27000 \
+    --save_directory /data/tianxing/PycharmProjects/Transformers/trained_models/qwen_7b_modern_poetry
+
+    """
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        default="YeungNLP/firefly-chatglm2-6b",
+        type=str
+    )
+    parser.add_argument(
+        "--adapter_name_or_path",
+        default="YeungNLP/firefly-baichuan-7b-qlora-sft",
+        type=str
+    )
+    parser.add_argument("--save_directory", default="save_directory", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    config = AutoConfig.from_pretrained(
+        args.pretrained_model_name_or_path,
+        trust_remote_code=True,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.pretrained_model_name_or_path,
+        trust_remote_code=True,
+        # llama不支持fast
+        use_fast=False if config.model_type == 'llama' else True
+    )
+
+    model = AutoModelForCausalLM.from_pretrained(
+        args.pretrained_model_name_or_path,
+        trust_remote_code=True,
+        low_cpu_mem_usage=True,
+        torch_dtype=torch.float16,
+        # device_map='auto',
+        device_map={"": "cpu"}
+    )
+    model = PeftModel.from_pretrained(model, args.adapter_name_or_path, device_map={"": "cpu"})
+    model = model.merge_and_unload()
+
+    tokenizer.save_pretrained(args.save_directory)
+    model.save_pretrained(args.save_directory)
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/chinese_modern_poetry/4.test_model.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, '../../../'))
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+from project_settings import project_path
+"""
+单轮对话，不具有对话历史的记忆功能
+"""
+
+
+def get_args():
+    """
+    python3 4.test_model.py --pretrained_model_name_or_path /data/tianxing/PycharmProjects/Transformers/trained_models/qwen_7b_chinese_modern_poetry
+    python3 4.test_model.py --pretrained_model_name_or_path /data/tianxing/PycharmProjects/Transformers/trained_models/qwen_7b_modern_poetry
+
+python3 4.test_model.py --pretrained_model_name_or_path /data/tianxing/PycharmProjects/Transformers/pretrained_models/huggingface/Qwen/Qwen-7B
+
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        # default="YeungNLP/firefly-chatglm2-6b",
+        default=(project_path / "trained_models/firefly_chatglm2_6b_intent").as_posix(),
+        type=str
+    )
+    parser.add_argument("--max_new_tokens", default=512, type=int)
+    parser.add_argument("--top_p", default=0.9, type=float)
+    parser.add_argument("--temperature", default=0.35, type=float)
+    parser.add_argument("--repetition_penalty", default=1.0, type=float)
+    parser.add_argument('--device', default="cuda" if torch.cuda.is_available() else "cpu", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    model = AutoModelForCausalLM.from_pretrained(
+        args.pretrained_model_name_or_path,
+        trust_remote_code=True,
+        # low_cpu_mem_usage=True,
+        torch_dtype=torch.float16,
+        # device_map="auto",
+        device_map={"": 0},
+        # offload_folder="./offload",
+    ).to(args.device).eval()
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.pretrained_model_name_or_path,
+        trust_remote_code=True,
+        # llama不支持fast
+        use_fast=False if model.config.model_type == "llama" else True,
+        padding_side="left"
+
+    )
+
+    # QWenTokenizer比较特殊, pad_token_id, bos_token_id, eos_token_id 均 为None. eod_id对应的token为<|endoftext|>
+    if tokenizer.__class__.__name__ == "QWenTokenizer":
+        tokenizer.pad_token_id = tokenizer.eod_id
+        tokenizer.bos_token_id = tokenizer.eod_id
+        tokenizer.eos_token_id = tokenizer.eod_id
+
+    text = input("User: ")
+    while True:
+        text = text.strip()
+        # chatglm使用官方的数据组织格式
+        if model.config.model_type == "chatglm":
+            text = "[Round 1]\n\n问：{}\n\n答：".format(text)
+            input_ids = tokenizer(text, return_tensors="pt", add_special_tokens=False).input_ids.to(args.device)
+        # 为了兼容qwen-7b，因为其对eos_token进行tokenize，无法得到对应的eos_token_id
+        else:
+            input_ids = tokenizer(
+                text,
+                return_tensors="pt",
+                add_special_tokens=False,
+            ).input_ids.to(args.device)
+            bos_token_id = torch.tensor([[tokenizer.bos_token_id]], dtype=torch.long).to(args.device)
+            eos_token_id = torch.tensor([[tokenizer.eos_token_id]], dtype=torch.long).to(args.device)
+            input_ids = torch.concat([bos_token_id, input_ids, eos_token_id], dim=1)
+        with torch.no_grad():
+            outputs = model.generate(
+                input_ids=input_ids, max_new_tokens=args.max_new_tokens, do_sample=True,
+                top_p=args.top_p, temperature=args.temperature, repetition_penalty=args.repetition_penalty,
+                eos_token_id=tokenizer.eos_token_id
+            )
+        outputs = outputs.tolist()[0][len(input_ids[0]):]
+        response = tokenizer.decode(outputs)
+        response = response.strip().replace(tokenizer.eos_token, "").strip()
+        print("LLM: {}".format(response))
+        text = input('User: ')
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/chinese_modern_poetry/run.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+
+# sh run.sh --stage 0 --stop_stage 0 --system_version centos
+# sh run.sh --stage 1 --stop_stage 1 --system_version centos
+# sh run.sh --stage 2 --stop_stage 2 --system_version centos
+# sh run.sh --stage 4 --stop_stage 4 --system_version centos --final_model_name qwen_7b_modern_poetry
+
+# bitsandbytes
+export LD_LIBRARY_PATH="/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+# params
+system_version="windows";
+verbose=true;
+stage=0 # start from 0 if you need to start from data preparation
+stop_stage=5
+
+pretrained_model_supplier=Qwen
+pretrained_model_name=Qwen-7B
+
+final_checkpoint_dir=final
+final_model_name=qwen_7b_modern_poetry
+
+patience=0
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+
+$verbose && echo "system_version: ${system_version}"
+
+work_dir="$(pwd)"
+file_dir="${work_dir}/file_dir"
+cache_dir="${file_dir}/cache_dir"
+serialization_dir="${file_dir}/serialization_dir"
+
+pretrained_models_dir="${work_dir}/../../../pretrained_models/huggingface/${pretrained_model_supplier}"
+final_model_dir="${work_dir}/../../../trained_models/${final_model_name}";
+
+mkdir -p "${file_dir}"
+mkdir -p "${cache_dir}"
+mkdir -p "${serialization_dir}"
+mkdir -p "${pretrained_models_dir}"
+mkdir -p "${final_model_dir}"
+
+export PYTHONPATH="${work_dir}/../../.."
+
+
+if [ $system_version == "windows" ]; then
+  alias python3='C:/Users/tianx/PycharmProjects/virtualenv/Transformers/Scripts/python.exe'
+elif [ $system_version == "centos" ]; then
+  # conda activate Transformers
+  alias python3='/usr/local/miniconda3/envs/Transformers/bin/python3'
+elif [ $system_version == "ubuntu" ]; then
+  alias python3='/usr/local/miniconda3/envs/Transformers/bin/python3'
+elif [ $system_version == "macos" ]; then
+  alias python3='/Users/honey/PycharmProjects/virtualenv/TrainLLM/bin/python'
+fi
+
+
+function search_best_ckpt() {
+  patience="$1";
+
+  cd "${serialization_dir}" || exit 1
+  last_epoch=$(ls . | \
+               grep "checkpoint-*" | \
+               awk -F'[-]' '{print$2}' | \
+               sort -n | \
+               awk 'END {print}')
+
+  target_dir=
+  if [ -n "${last_epoch}" ]; then
+    target_epoch=$((last_epoch - patience))
+
+    for epoch_idx in $(ls . | grep "checkpoint-*" | awk -F'[-]' '{print$2}' | sort -nr):
+    do
+      if [ "${epoch_idx}" -le "${target_epoch}" ]; then
+        target_dir="checkpoint-${epoch_idx}";
+        break;
+      fi
+    done
+  fi
+
+  echo "${target_dir}"
+}
+
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+  $verbose && echo "stage 0: download pretrained model"
+  cd "${pretrained_models_dir}" || exit 1;
+
+  if [ ! -d "${pretrained_model_name}" ]; then
+    git clone "https://huggingface.co/${pretrained_model_supplier}/${pretrained_model_name}/"
+
+    rm -rf .git
+    rm -rf .gitattributes
+    rm -rf flax_model.msgpack
+    rm -rf model.safetensors
+  fi
+
+  cd "${pretrained_models_dir}/${pretrained_model_name}" || exit 1;
+
+  # pytorch_model.bin
+  if [ -e "pytorch_model.bin" ]; then
+    data_size=$(ls -l pytorch_model.bin | awk '{print $5}')
+    if [ "${data_size}" == "135" ]; then
+      rm -rf pytorch_model.bin;
+    fi
+  fi
+  if [ ! -e "pytorch_model.bin" ]; then
+    wget -c "https://huggingface.co/${pretrained_model_supplier}/${pretrained_model_name}/resolve/main/pytorch_model.bin"
+  fi
+
+  # tokenizer.json
+  if [ -e "tokenizer.json" ]; then
+    data_size=$(ls -l tokenizer.json | awk '{print $5}')
+    if [ "${data_size}" == "135" ]; then
+      rm -rf tokenizer.json;
+    fi
+  fi
+  if [ ! -e "tokenizer.json" ]; then
+      wget -c "https://huggingface.co/${pretrained_model_supplier}/${pretrained_model_name}/resolve/main/tokenizer.json"
+  fi
+
+fi
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 1: prepare data"
+  cd "${work_dir}" || exit 1;
+
+  python3 1.prepare_data.py
+
+fi
+
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  $verbose && echo "stage 2: train model"
+  cd "${work_dir}" || exit 1;
+
+   python3 2.train_model.py \
+   --pretrained_model_name_or_path "${pretrained_models_dir}/${pretrained_model_name}" \
+   --cache_dir "${cache_dir}" \
+   --output_dir "${serialization_dir}"
+
+fi
+
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  $verbose && echo "stage 3: merge lora"
+  cd "${work_dir}" || exit 1;
+
+  python3 3.merge_lora.py \
+  --pretrained_model_name_or_path "${pretrained_models_dir}/${pretrained_model_name}" \
+  --adapter_name_or_path "${serialization_dir}/${final_checkpoint_dir}" \
+  --save_directory "${final_model_dir}"
+
+fi
+
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  $verbose && echo "stage 4: collect files"
+  cd "${work_dir}" || exit 1;
+
+  cp "${pretrained_models_dir}/${pretrained_model_name}/configuration_qwen.py" "${final_model_dir}/configuration_qwen.py"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/modeling_qwen.py" "${final_model_dir}/modeling_qwen.py"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/qwen_generation_utils.py" "${final_model_dir}/qwen_generation_utils.py"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/tokenization_qwen.py" "${final_model_dir}/tokenization_qwen.py"
+
+fi

examples/exercises/chinese_modern_poetry/stop.sh

@@ -0,0 +1 @@
+#!/usr/bin/env bash

main.py

@@ -0,0 +1,211 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+
+import gradio as gr
+from transformers import AutoModel, AutoTokenizer
+from transformers.models.auto import AutoModelForCausalLM, AutoTokenizer
+# from transformers.utils.quantization_config import BitsAndBytesConfig
+import torch
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--train_subset", default="train.jsonl", type=str)
+    parser.add_argument("--valid_subset", default="valid.jsonl", type=str)
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        # default="YeungNLP/firefly-chatglm2-6b",
+        default=(project_path / "trained_models/firefly_chatglm2_6b_intent").as_posix(),
+        type=str
+    )
+    parser.add_argument("--output_file", default="result.xlsx", type=str)
+
+    parser.add_argument("--max_new_tokens", default=512, type=int)
+    parser.add_argument("--top_p", default=0.9, type=float)
+    parser.add_argument("--temperature", default=0.35, type=float)
+    parser.add_argument("--repetition_penalty", default=1.0, type=float)
+    parser.add_argument('--device', default="cuda" if torch.cuda.is_available() else "cpu", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+description = """
+## ChatGLM-6B
+
+基于 [firefly-chatglm2-6b](https://huggingface.co/YeungNLP/firefly-chatglm2-6b) 模型, 在 [telemarketing_intent](https://huggingface.co/datasets/qgyd2021/telemarketing_intent/tree/main/data/prompt) 的 prompt 数据集上训练, 目的是实现 `电话营销` 场景的 1-shot 意图识别. 
+
+该分类任务有一百多个类别, 但标注数据总是只有 3 万, 并且有一半是 "无关领域", 实现思路是: 
+1. 首先采用传统算法做硬分类, 然后提取概率 top 10 的标签. 
+2. 将 top 10 的标签作为候选标签, 并为每个标签提供一个句子示例. 
+3. 要求 LLM 输出目标句子的类别. 
+
+Gradio 布署代码参考了: https://huggingface.co/spaces/aodianyun/ChatGLM-6B
+
+"""
+
+
+examples = [
+    """我们在做电话营销场景的意图识别任务, 可选的意图如下: 
+否定(不是); 礼貌用语; 否定答复; 肯定(需要); 用户正忙; 否定(不需要); 无关领域; 否定(没有); 否定(不用了); 价格太高
+
+如果你认为给定的句子不属于这些意图中的任务一个, 你可以回答: 不知道. 
+
+Tips: 
+1. 如果候选意图中有 "无关领域", 当你不知道时, 则它有可能属于无关领域. 
+
+
+Examples: 
+
+---------
+
+ExampleSentence: 其实不是
+ExampleIntent: 否定(不是)
+
+ExampleSentence: 嗯!嘿嘿!早点休息,晚安咯
+ExampleIntent: 礼貌用语
+
+ExampleSentence: 没问诶
+ExampleIntent: 否定答复
+
+ExampleSentence: 不好意思都需要谢谢
+ExampleIntent: 肯定(需要)
+
+ExampleSentence: 对呀我在忙
+ExampleIntent: 用户正忙
+
+ExampleSentence: 。嗯也也不需要吧唉呀现在不需要那个啊嗯
+ExampleIntent: 否定(不需要)
+
+ExampleSentence: 我的处理器需要很少的电源。
+ExampleIntent: 无关领域
+
+ExampleSentence: 。呃我好像没有在太平洋买过保险，吧拜拜
+ExampleIntent: 否定(没有)
+
+ExampleSentence: 嗯不用谢谢
+ExampleIntent: 否定(不用了)
+
+ExampleSentence: 费用贵。
+ExampleIntent: 价格太高
+
+---------
+
+Sentence: 。嗯各位不需要，啊谢谢
+Intent:""",
+    """我们在做电话营销场景的意图识别任务, 可选的意图如下: 
+语音信箱; 无关领域; 查物品信息; 污言秽语; 疑问(时间); 疑问(数值); 答时间; 查收费方式; 价格太高; 答数值
+
+如果你认为给定的句子不属于这些意图中的任务一个, 你可以回答: 不知道. 
+
+Tips: 
+1. 如果候选意图中有 "无关领域", 当你不知道时, 则它有可能属于无关领域. 
+
+
+Examples: 
+---------
+
+ExampleSentence: 我们留言。
+ExampleIntent: 语音信箱
+
+ExampleSentence: 很刚刚打
+ExampleIntent: 无关领域
+
+ExampleSentence: 什么东西我听
+ExampleIntent: 查物品信息
+
+ExampleSentence: 知道!AV女优!日本人的骄傲!
+ExampleIntent: 污言秽语
+
+ExampleSentence: 最后期限
+ExampleIntent: 疑问(时间)
+
+ExampleSentence: 一共借了多少钱
+ExampleIntent: 疑问(数值)
+
+ExampleSentence: 22号
+ExampleIntent: 答时间
+
+ExampleSentence: 运费
+ExampleIntent: 查收费方式
+
+ExampleSentence: 利息高
+ExampleIntent: 价格太高
+
+ExampleSentence: 20。
+ExampleIntent: 答数值
+
+---------
+
+Sentence: 。对啊什么东西啊我6月份出来的
+Intent:"""
+]
+
+
+def main():
+    args = get_args()
+
+    use_cpu = os.environ.get("USE_CPU", "all")
+
+    tokenizer = AutoTokenizer.from_pretrained(args.pretrained_model_name_or_path, trust_remote_code=True)
+    # QWenTokenizer比较特殊, pad_token_id, bos_token_id, eos_token_id 均 为None. eod_id对应的token为<|endoftext|>
+    if tokenizer.__class__.__name__ == "QWenTokenizer":
+        tokenizer.pad_token_id = tokenizer.eod_id
+        tokenizer.bos_token_id = tokenizer.eod_id
+        tokenizer.eos_token_id = tokenizer.eod_id
+
+    if not use_cpu:
+        model = AutoModel.from_pretrained(
+            args.pretrained_model_name_or_path,
+            trust_remote_code=True
+        ).half().cuda()
+    else:
+        model = AutoModelForCausalLM.from_pretrained(
+            args.pretrained_model_name_or_path,
+            trust_remote_code=True,
+            low_cpu_mem_usage=True,
+            torch_dtype=torch.bfloat16,
+            device_map="auto",
+            offload_folder="./offload",
+            offload_state_dict=True,
+            # load_in_4bit=True,
+        )
+    model = model.eval()
+
+    def fn(inputs, history=None):
+        if history is None:
+            history = list()
+
+        with torch.no_grad():
+            response, history = model.chat(tokenizer, inputs, history)
+
+        return history, history
+
+    with gr.Blocks() as blocks:
+        gr.Markdown(value=description)
+
+        state = gr.State([])
+
+        chatbot = gr.Chatbot([], elem_id="chatbot").style(height=400)
+        with gr.Row():
+            with gr.Column(scale=4):
+                text = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False)
+            with gr.Column(scale=1):
+                button = gr.Button("Generate")
+
+        gr.Examples(examples, text)
+
+        text.submit(fn, [text, state], [chatbot, state])
+        button.click(fn, [text, state], [chatbot, state])
+
+    blocks.queue().launch()
+
+    return
+
+
+if __name__ == '__main__':
+    main()

project_settings.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+from pathlib import Path
+
+from toolbox.os.environment import EnvironmentManager
+
+
+project_path = os.path.abspath(os.path.dirname(__file__))
+project_path = Path(project_path)
+
+
+environment = EnvironmentManager(
+    path=os.path.join(project_path, 'dotenv'),
+    env=os.environ.get('environment', 'dev'),
+)
+
+
+if __name__ == '__main__':
+    pass

requirements.txt

@@ -0,0 +1,16 @@
+gradio==3.20.1
+pydantic==1.10.12
+thinc==7.4.6
+spacy==2.3.9
+accelerate==0.21.0
+transformers==4.30.2
+peft==0.4.0
+bitsandbytes==0.39.0
+numpy==1.21.4
+pandas==1.2.5
+tqdm==4.62.3
+torch==1.13.0
+datasets
+python-dotenv==1.0.0
+sentencepiece==0.1.99
+scipy==1.10.1

script/install_bitsandbytes.sh

@@ -0,0 +1,232 @@
+#!/usr/bin/env bash
+
+#bitsandbytes
+#https://github.com/TimDettmers/bitsandbytes
+#
+### 安装bitsandbytes
+#
+#bitsandbytes 是 CUDA 自定义函数的轻量级包装器, 特别是 8 位优化器, 矩阵乘法 (LLM.int8()) 和量化函数.
+#
+#### 安装
+#
+#通过 `pip3 install bitsandbytes` 来安装.
+#
+#安装之后通过 `python -m bitsandbytes` 来验证安装是否成功.
+#
+#在某些情况下可能需要从源代码进行编译.
+#
+#```text
+#git clone https://github.com/timdettmers/bitsandbytes.git
+#cd bitsandbytes
+#
+## CUDA_VERSIONS in {110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 120}
+## make argument in {cuda110, cuda11x, cuda12x}
+## if you do not know what CUDA you have, try looking at the output of: python -m bitsandbytes
+#CUDA_VERSION=117 make cuda11x
+#python setup.py install
+#```
+#
+#### 备注
+#
+##### 必须安装与 GPU 版本相匹配的 CUDA
+#
+#我的情况如下:
+#
+#**GPU 和 CUDA 版本. **
+#
+#```text
+## nvidia-smi
+#Mon Aug 28 14:38:32 2023
+#+-----------------------------------------------------------------------------+
+#| NVIDIA-SMI 515.105.01   Driver Version: 515.105.01   CUDA Version: 11.7     |
+#|-------------------------------+----------------------+----------------------+
+#| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+#| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+#|                               |                      |               MIG M. |
+#|===============================+======================+======================|
+#|   0  Tesla V100S-PCI...  Off  | 00000000:0B:00.0 Off |                    0 |
+#| N/A   48C    P0    40W / 250W |  14154MiB / 32768MiB |      0%      Default |
+#|                               |                      |                  N/A |
+#+-------------------------------+----------------------+----------------------+
+#
+#+-----------------------------------------------------------------------------+
+#| Processes:                                                                  |
+#|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
+#|        ID   ID                                                   Usage      |
+#|=============================================================================|
+#|    0   N/A  N/A     11127      C   python3                         12973MiB |
+#|    0   N/A  N/A     25921      C   python3                          1177MiB |
+#+-----------------------------------------------------------------------------+
+#```
+#
+#**CUDA 版本. **
+#
+#我的经历, 安装 nvidia driver 驱动后 PyTorch 就可以使用 GPU 了, 同时 `nvidia-smi` 命令中也会显示 `CUDA Version: 11.7`.
+#
+#但是 `/usr/local/cuda:/usr/local/cuda-11.7` 是不存在的. 这个需要单独安装 (即: 安装 CUDA).
+#
+#```text
+## ll /usr/local/ | grep cuda
+#lrwxrwxrwx   1 root root  20 Aug 15 19:12 cuda -> /usr/local/cuda-11.7
+#drwxr-xr-x  14 root root 268 Aug 15 18:31 cuda-11.7
+#```
+#
+##### 从编译安装
+#
+#从编译安装使用的命令如下:
+#
+#```
+#CUDA SETUP: Something unexpected happened. Please compile from source:
+#git clone git@github.com:TimDettmers/bitsandbytes.git
+#cd bitsandbytes
+#CUDA_VERSION=117 make cuda11x_nomatmul
+#python setup.py install
+#```
+#
+#我的情况是没有使用容器, 在宿主机上安装的.
+#
+#1. 之前机器上安装的是 `/usr/local/cuda-10.4` 编译不通过. 因为 `/usr/local/cuda-10.4/bin` 下的 `nvcc` 编译器与 GPU 所需的 `CUDA Version: 11.7` 是不匹配的.
+#2. 后来安装了 `/usr/local/cuda-11.7` 并删除 `/usr/local/cuda-10.4`, 但还是安装不成功.
+#
+#偶然的一次尝试:
+#
+#**需要使用conda虚拟环境**, 在 python 的 virtualenv 中安装失败了.
+#
+#**执行 `CUDA_VERSION=117 make cuda11x_nomatmul` 命令时, 确保以下几项正确**. 即:
+#
+#* `NVCC path`: 指向了 cuda 中的 nvcc 编译器. (`nvcc` 是 cuda 提供的一款编译器).
+#
+#* `CUDA_HOME`: cuda 安装的目录, 一般安装时会自动确定在 `/usr/local/cuda`,
+#
+#* `CONDA_PREFIX`: 是 `conda` 下创建的虚拟环境.
+#
+#* `PATH`: 应包含 cuda 的 bin 目录.
+#
+#* `LD_LIBARY_PATH`: 应包含 cuda 的 lib 目录.
+#
+#```
+#(Transformers) [root@nlp bitsandbytes-0.39.1]# CUDA_VERSION=117 make cuda11x_nomatmul
+#ENVIRONMENT
+#============================
+#CUDA_VERSION: 117
+#============================
+#NVCC path: /usr/local/cuda/bin/nvcc
+#GPP path: /usr/bin/g++ VERSION: g++ (GCC) 11.1.0
+#CUDA_HOME: /usr/local/cuda
+#CONDA_PREFIX: /usr/local/miniconda3/envs/Transformers
+#PATH: /usr/local/miniconda3/envs/Transformers/bin:/usr/local/miniconda3/condabin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/cuda/bin:/root/bin
+#LD_LIBRARY_PATH: /usr/local/cuda/lib64
+#============================
+#```
+#
+#我的情况是, 本应该是 `LD_LIBRARY_PATH: /usr/local/cuda/lib64` 的项变成了 `LD_LIBRARY_PATH:`.
+#
+#检查 `cat ~/.bashrc` 中包含:
+#
+#```text
+#CUDA_HOME="/usr/local/cuda"
+#PATH=/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/miniconda3/bin:/usr/local/cuda/bin
+#LD_LIBRARY_PATH=/usr/local/cuda/lib64
+#```
+#
+#同时再执行
+#
+#```text
+#export LD_LIBRARY_PATH="/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+#```
+#
+#之后就编译成功了.
+#
+#**检查是否安装成功**
+#
+#重新连接 Terminal 之后, 在执行 `python -m bitsandbytes` 之前先执行以下命令.
+#
+#```text
+#export LD_LIBRARY_PATH="/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+#```
+#
+#这一步非常奇怪, 因为 `echo $LD_LIBRARY_PATH`, 可以看到 `/usr/local/cuda/lib64` 路径在其中.  `echo $PATH` 都可以看到 `/usr/local/cuda` 在其中.
+#
+#但是执行 `CUDA_VERSION=117 make cuda11x_nomatmul` 时会发现原本应该是 `LD_LIBRARY_PATH: /usr/local/cuda/lib64` 的项变成了 `LD_LIBRARY_PATH:`.
+#
+#只要再执行一次以上命名, 再执行以下命令, 就可以成功.
+#
+#安装后执行以下命令, 检查是否安装成功.
+#
+#```
+#python -m bitsandbytes
+#```
+#
+#如过出现以下内容, 说明安装成功了.
+#
+#```text
+#...
+#...
+#...
+#Running a quick check that:
+#    + library is importable
+#    + CUDA function is callable
+#
+#
+#WARNING: Please be sure to sanitize sensible info from any such env vars!
+#
+#SUCCESS!
+#Installation was successful!
+#```
+
+
+# sh install_bitsandbytes.sh --stage 0 --stop_stage 0
+
+
+verbose=true;
+stage=0 # start from 0 if you need to start from data preparation
+stop_stage=5
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+
+work_dir="$(pwd)"
+thirdparty_dir="${work_dir}/thirdparty"
+
+mkdir -p "${thirdparty_dir}"
+
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+  $verbose && echo "stage 0: download bitsandbytes"
+  cd "${thirdparty_dir}" || exit 1;
+
+  wget https://github.com/TimDettmers/bitsandbytes/archive/refs/tags/0.39.1.zip
+  unzip 0.39.1.zip
+  rm -rf 0.39.1.zip
+
+fi
+
+

script/install_conda.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# https://www.5axxw.com/questions/simple/umiecs
+
+# params:
+system_version="centos";
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+
+echo "system_version: ${system_version}";
+
+
+if [ ${system_version} = "centos" ]; then
+  wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+
+  bash Miniconda3-latest-Linux-x86_64.sh
+
+  /usr/local/miniconda3/bin/conda --version
+
+  cat ~/.bashrc
+  echo "PATH=$PATH:/usr/local/miniconda3/bin" >> /root/.bashrc
+  source ~/.bashrc
+
+  conda --version
+
+fi
+
+
+

script/install_cuda.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+# 查看系统架构 Architecture
+# >>> uname -a
+# Linux nlp 3.10.0-1160.66.1.el7.x86_64 #1 SMP Wed May 18 16:02:34 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
+# >>> uname -m
+# x86_64
+
+
+#cuda驱动就像普通的软件一样, 可以安装多个.
+
+
+#在以下路径找到对应版本, 获得安装命令.
+#https://developer.nvidia.com/cuda-toolkit-archive
+#
+#参考链接:
+#https://www.cnblogs.com/yuezc/p/12937239.html
+#https://blog.csdn.net/pursuit_zhangyu/article/details/117073126
+#
+#[root@nlp dep]# sh cuda_10.2.89_440.33.01_linux.run --override
+#(执行以上命令后, 安提示操作, 以下是安装完成后的信息).
+#===========
+#= Summary =
+#===========
+#
+#Driver:   Installed
+#Toolkit:  Installed in /usr/local/cuda-10.2/
+#Samples:  Installed in /home/admin/, but missing recommended libraries
+#
+#Please make sure that
+# -   PATH includes /usr/local/cuda-10.2/bin
+# -   LD_LIBRARY_PATH includes /usr/local/cuda-10.2/lib64, or, add /usr/local/cuda-10.2/lib64 to /etc/ld.so.conf and run ldconfig as root
+#
+#To uninstall the CUDA Toolkit, run cuda-uninstaller in /usr/local/cuda-10.2/bin
+#To uninstall the NVIDIA Driver, run nvidia-uninstall
+#
+#Please see CUDA_Installation_Guide_Linux.pdf in /usr/local/cuda-10.2/doc/pdf for detailed information on setting up CUDA.
+#Logfile is /var/log/cuda-installer.log
+
+
+# params:
+system_version="centos";
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+
+echo "system_version: ${system_version}";
+
+
+if [ ${system_version} = "centos" ]; then
+  #runfile(local)
+  wget https://developer.download.nvidia.com/compute/cuda/11.7.0/local_installers/cuda_11.7.0_515.43.04_linux.run
+  sudo sh cuda_11.7.0_515.43.04_linux.run --override
+
+  #只选择安装 CUDA Toolkit 11.7 其它取消选择.
+
+  rm -rf /usr/local/cuda
+  ln -snf /usr/local/cuda-11.7 /usr/local/cuda
+
+  #export CUDA_HOME=/usr/local/cuda
+  #export PATH="${CUDA_HOME}/bin${PATH:+:$PATH}"
+  #export LD_LIBRARY_PATH="/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+  #export PATH=$PATH:/usr/local/cuda/bin
+  #export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64
+
+  cat ~/.bashrc
+  echo "PATH=$PATH:/usr/local/cuda/bin" >> /root/.bashrc
+  echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64" >> /root/.bashrc
+  source ~/.bashrc
+
+  #查看cuda版本
+  nvcc -V
+
+fi

script/install_nvidia_driver.sh

@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+#GPU驱动安装需要先将原有的显示关闭, 重启机器, 再进行安装.
+#参考链接:
+#https://blog.csdn.net/kingschan/article/details/19033595
+#https://blog.csdn.net/HaixWang/article/details/90408538
+#
+#>>> yum install -y pciutils
+#查看 linux 机器上是否有 GPU
+#lspci |grep -i nvidia
+#
+#>>> lspci |grep -i nvidia
+#00:08.0 3D controller: NVIDIA Corporation TU104GL [Tesla T4] (rev a1)
+#
+#
+#NVIDIA 驱动程序下载
+#先在 pytorch 上查看应该用什么 cuda 版本, 再安装对应的 cuda-toolkit cuda.
+#再根据 gpu 版本下载安装对应的 nvidia 驱动
+#
+## pytorch 版本
+#https://pytorch.org/get-started/locally/
+#
+## CUDA 下载 (好像不需要这个)
+#https://developer.nvidia.com/cuda-toolkit-archive
+#
+## nvidia 驱动
+#https://www.nvidia.cn/Download/index.aspx?lang=cn
+#http://www.nvidia.com/Download/index.aspx
+#
+#在下方的下拉列表中进行选择，针对您的 NVIDIA 产品确定合适的驱动。
+#产品类型:
+#Data Center / Tesla
+#产品系列:
+#T-Series
+#产品家族:
+#Tesla T4
+#操作系统:
+#Linux 64-bit
+#CUDA Toolkit:
+#10.2
+#语言:
+#Chinese (Simpleified)
+#
+#
+#>>> mkdir -p /data/tianxing
+#>>> cd /data/tianxing
+#>>> wget https://cn.download.nvidia.com/tesla/440.118.02/NVIDIA-Linux-x86_64-440.118.02.run
+#>>> sh NVIDIA-Linux-x86_64-440.118.02.run
+#
+## 异常:
+#ERROR: The Nouveau kernel driver is currently in use by your system.  This driver is incompatible with the NVIDIA driver, and must be disabled before proceeding.  Please consult the NVIDIA driver README and your
+#Linux distribution's documentation for details on how to correctly disable the Nouveau kernel driver.
+#[OK]
+#
+#For some distributions, Nouveau can be disabled by adding a file in the modprobe configuration directory.  Would you like nvidia-installer to attempt to create this modprobe file for you?
+#[NO]
+#
+#ERROR: Installation has failed.  Please see the file '/var/log/nvidia-installer.log' for details.  You may find suggestions on fixing installation problems in the README available on the Linux driver download
+#page at www.nvidia.com.
+#[OK]
+#
+## 参考链接:
+#https://blog.csdn.net/kingschan/article/details/19033595
+#
+## 禁用原有的显卡驱动 nouveau
+#>>> echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
+#>>> sudo dracut --force
+## 重启
+#>>> reboot
+#
+#>>> init 3
+#>>> sh NVIDIA-Linux-x86_64-440.118.02.run
+#
+## 异常
+#ERROR: Unable to find the kernel source tree for the currently running kernel. Please make sure you have installed the kernel source files for your kernel and that they are properly configured; on Red Hat Linux systems, for example, be sure you have the 'kernel-source' or 'kernel-devel' RPM installed. If you know the correct kernel source files are installed, you may specify the kernel source path with the '--kernel-source-path' command line option.
+#[OK]
+#ERROR: Installation has failed.  Please see the file '/var/log/nvidia-installer.log' for details.  You may find suggestions on fixing installation problems in the README available on the Linux driver download
+#page at www.nvidia.com.
+#[OK]
+#
+## 参考链接
+## https://blog.csdn.net/HaixWang/article/details/90408538
+#
+#>>> uname -r
+#3.10.0-1160.49.1.el7.x86_64
+#>>> yum install kernel-devel kernel-headers -y
+#>>> yum info kernel-devel kernel-headers
+#>>> yum install -y "kernel-devel-uname-r == $(uname -r)"
+#>>> yum -y distro-sync
+#
+#>>> sh NVIDIA-Linux-x86_64-440.118.02.run
+#
+## 安装成功
+#WARNING: nvidia-installer was forced to guess the X library path '/usr/lib64' and X module path '/usr/lib64/xorg/modules'; these paths were not queryable from the system.  If X fails to find the NVIDIA X driver
+#module, please install the `pkg-config` utility and the X.Org SDK/development package for your distribution and reinstall the driver.
+#[OK]
+#Install NVIDIA's 32-bit compatibility libraries?
+#[YES]
+#Installation of the kernel module for the NVIDIA Accelerated Graphics Driver for Linux-x86_64 (version 440.118.02) is now complete.
+#[OK]
+#
+#
+## 查看 GPU 使用情况; watch -n 1 -d nvidia-smi 每1秒刷新一次.
+#>>> nvidia-smi
+#Thu Mar  9 12:00:37 2023
+#+-----------------------------------------------------------------------------+
+#| NVIDIA-SMI 440.118.02   Driver Version: 440.118.02   CUDA Version: 10.2     |
+#|-------------------------------+----------------------+----------------------+
+#| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+#| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+#|===============================+======================+======================|
+#|   0  Tesla T4            Off  | 00000000:00:08.0 Off |                  Off |
+#| N/A   54C    P0    22W /  70W |      0MiB / 16127MiB |      0%      Default |
+#+-------------------------------+----------------------+----------------------+
+#
+#+-----------------------------------------------------------------------------+
+#| Processes:                                                       GPU Memory |
+#|  GPU       PID   Type   Process name                             Usage      |
+#|=============================================================================|
+#|  No running processes found                                                 |
+#+-----------------------------------------------------------------------------+
+#
+#
+
+# params
+stage=1
+nvidia_driver_filename=https://cn.download.nvidia.com/tesla/440.118.02/NVIDIA-Linux-x86_64-440.118.02.run
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+echo "stage: ${stage}";
+
+yum -y install wget
+yum -y install sudo
+
+if [ ${stage} -eq 0 ]; then
+  mkdir -p /data/dep
+  cd /data/dep || echo 1;
+  wget -P /data/dep ${nvidia_driver_filename}
+
+  echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
+  sudo dracut --force
+  # 重启
+  reboot
+elif [ ${stage} -eq 1 ]; then
+  init 3
+
+  yum install -y kernel-devel kernel-headers
+  yum info kernel-devel kernel-headers
+  yum install -y "kernel-devel-uname-r == $(uname -r)"
+  yum -y distro-sync
+
+  cd /data/dep || echo 1;
+
+  # 安装时, 需要回车三下.
+  sh NVIDIA-Linux-x86_64-440.118.02.run
+  nvidia-smi
+fi

script/install_openssl.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# params:
+system_version="centos";
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+echo "system_version: ${system_version}";
+
+
+if [ ${system_version} = "centos" ]; then
+  mkdir -p /data/dep
+  cd /data/dep || exit 1;
+
+  if [ ! -e openssl-1.1.1n.tar.gz ]; then
+    wget https://www.openssl.org/source/openssl-1.1.1n.tar.gz --no-check-certificate
+  fi
+
+  cd /data/dep || exit 1;
+  if [ ! -d openssl-1.1.1n ]; then
+    tar -zxvf openssl-1.1.1n.tar.gz
+
+    cd /data/dep/openssl-1.1.1n || exit 1;
+  fi
+
+  ./Configure --prefix=/usr/local/openssl
+
+  make -j && make install
+
+fi

script/install_python.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+# sh ./script/install_python.sh --system_version "centos" --python_version "3.10.11"
+
+# 参数:
+python_version="3.8.10";
+system_version="centos";
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+echo "python_version: ${python_version}";
+echo "system_version: ${system_version}";
+
+
+if [ ${system_version} = "centos" ]; then
+  # 安装 python 开发编译环境
+  yum -y groupinstall "Development tools"
+  yum -y install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel
+  yum install libffi-devel -y
+  yum install -y wget
+  yum install -y make
+
+  mkdir /data/dep
+  # wget -P /data/dep https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tgz
+  wget -P /data/dep https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz
+
+  cd /data/dep || exit 1;
+  tar -zxvf Python-${python_version}.tgz
+  cd /data/dep/Python-${python_version} || exit 1;
+
+  mkdir /usr/local/python-${python_version}
+  ./configure --prefix=/usr/local/python-${python_version}
+  make && make install
+
+  # /usr/local/python-3.10.11/bin/python3 -V
+  # /usr/local/python-3.10.11/bin/pip3 -V
+  /usr/local/python-${python_version}/bin/python3 -V
+  /usr/local/python-${python_version}/bin/pip3 -V
+
+  rm -rf /usr/local/bin/python3
+  rm -rf /usr/local/bin/pip3
+  # ln -s /usr/local/python-3.10.11/bin/python3 /usr/local/bin/python3
+  # ln -s /usr/local/python-3.10.11/bin/pip3 /usr/local/bin/pip3
+  ln -s /usr/local/python-${python_version}/bin/python3 /usr/local/bin/python3
+  ln -s /usr/local/python-${python_version}/bin/pip3 /usr/local/bin/pip3
+
+  python3 -V
+  pip3 -V
+
+elif [ ${system_version} = "ubuntu" ]; then
+  # 安装 python 开发编译环境
+  # https://zhuanlan.zhihu.com/p/506491209
+
+  # 刷新软件包目录
+  sudo apt update
+  # 列出当前可用的更新
+  sudo apt list --upgradable
+  # 如上一步提示有可以更新的项目，则执行更新
+  sudo apt -y upgrade
+  # 安装 GCC 编译器
+  sudo apt install gcc
+  # 检查安装是否成功
+  gcc -v
+
+  # 安装依赖
+  sudo apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libbz2-dev liblzma-dev sqlite3 libsqlite3-dev tk-dev uuid-dev libgdbm-compat-dev
+
+  mkdir /data/dep
+
+  # sudo wget -P /data/dep https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz
+  sudo wget -P /data/dep https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz
+
+  cd /data/dep || exit 1;
+  # tar -zxvf Python-3.6.5.tgz
+  tar -zxvf Python-${python_version}.tgz
+  # cd /data/dep/Python-3.6.5
+  cd /data/dep/Python-${python_version} || exit 1;
+  # mkdir /usr/local/python-3.6.5
+  mkdir /usr/local/python-${python_version}
+
+  # 检查依赖与配置编译
+  # sudo ./configure --prefix=/usr/local/python-3.6.5 --enable-optimizations --with-lto --enable-shared
+  sudo ./configure --prefix=/usr/local/python-${python_version} --enable-optimizations --with-lto --enable-shared
+  cpu_count=$(cat /proc/cpuinfo | grep processor | wc -l)
+  # sudo make -j 4
+  sudo make -j "${cpu_count}"
+
+  /usr/local/python-${python_version}/bin/python3 -V
+  /usr/local/python-${python_version}/bin/pip3 -V
+
+  rm -rf /usr/local/bin/python3
+  rm -rf /usr/local/bin/pip3
+  ln -s /usr/local/python-${python_version}/bin/python3 /usr/local/bin/python3
+  ln -s /usr/local/python-${python_version}/bin/pip3 /usr/local/bin/pip3
+
+  python3 -V
+  pip3 -V
+fi

toolbox/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/json/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/json/misc.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Callable
+
+
+def traverse(js, callback: Callable, *args, **kwargs):
+    if isinstance(js, list):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return result
+    elif isinstance(js, tuple):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return tuple(result)
+    elif isinstance(js, dict):
+        result = dict()
+        for k, v in js.items():
+            k = traverse(k, callback, *args, **kwargs)
+            v = traverse(v, callback, *args, **kwargs)
+            result[k] = v
+        return result
+    elif isinstance(js, int):
+        return callback(js, *args, **kwargs)
+    elif isinstance(js, str):
+        return callback(js, *args, **kwargs)
+    else:
+        return js
+
+
+def demo1():
+    d = {
+        "env": "ppe",
+        "mysql_connect": {
+            "host": "$mysql_connect_host",
+            "port": 3306,
+            "user": "callbot",
+            "password": "NxcloudAI2021!",
+            "database": "callbot_ppe",
+            "charset": "utf8"
+        },
+        "es_connect": {
+            "hosts": ["10.20.251.8"],
+            "http_auth": ["elastic", "ElasticAI2021!"],
+            "port": 9200
+        }
+    }
+
+    def callback(s):
+        if isinstance(s, str) and s.startswith('$'):
+            return s[1:]
+        return s
+
+    result = traverse(d, callback=callback)
+    print(result)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/os/environment.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+import os
+
+from dotenv import load_dotenv
+from dotenv.main import DotEnv
+
+from toolbox.json.misc import traverse
+
+
+class EnvironmentManager(object):
+    def __init__(self, path, env, override=False):
+        filename = os.path.join(path, '{}.env'.format(env))
+        self.filename = filename
+
+        load_dotenv(
+            dotenv_path=filename,
+            override=override
+        )
+
+        self._environ = dict()
+
+    def open_dotenv(self, filename: str = None):
+        filename = filename or self.filename
+        dotenv = DotEnv(
+            dotenv_path=filename,
+            stream=None,
+            verbose=False,
+            interpolate=False,
+            override=False,
+            encoding="utf-8",
+        )
+        result = dotenv.dict()
+        return result
+
+    def get(self, key, default=None, dtype=str):
+        result = os.environ.get(key)
+        if result is None:
+            if default is None:
+                result = None
+            else:
+                result = default
+        else:
+            result = dtype(result)
+        self._environ[key] = result
+        return result
+
+
+_DEFAULT_DTYPE_MAP = {
+    'int': int,
+    'float': float,
+    'str': str,
+    'json.loads': json.loads
+}
+
+
+class JsonConfig(object):
+    """
+    将 json 中, 形如 `$float:threshold` 的值, 处理为:
+    从环境变量中查到 threshold, 再将其转换为 float 类型.
+    """
+    def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
+        self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
+        self.environment = environment or os.environ
+
+    def sanitize_by_filename(self, filename: str):
+        with open(filename, 'r', encoding='utf-8') as f:
+            js = json.load(f)
+
+        return self.sanitize_by_json(js)
+
+    def sanitize_by_json(self, js):
+        js = traverse(
+            js,
+            callback=self.sanitize,
+            environment=self.environment
+        )
+        return js
+
+    def sanitize(self, string, environment):
+        """支持 $ 符开始的, 环境变量配置"""
+        if isinstance(string, str) and string.startswith('$'):
+            dtype, key = string[1:].split(':')
+            dtype = self.dtype_map[dtype]
+
+            value = environment.get(key)
+            if value is None:
+                raise AssertionError('environment not exist. key: {}'.format(key))
+
+            value = dtype(value)
+            result = value
+        else:
+            result = string
+        return result
+
+
+def demo1():
+    import json
+
+    from project_settings import project_path
+
+    environment = EnvironmentManager(
+        path=os.path.join(project_path, 'server/callbot_server/dotenv'),
+        env='dev',
+    )
+    init_scenes = environment.get(key='init_scenes', dtype=json.loads)
+    print(init_scenes)
+    print(environment._environ)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/other.py

@@ -0,0 +1,9 @@
+import os
+import inspect
+
+
+def pwd():
+    """你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
+    frame = inspect.stack()[1]
+    module = inspect.getmodule(frame[0])
+    return os.path.dirname(os.path.abspath(module.__file__))

toolbox/transformers/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/transformers/data/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/transformers/data/data_collator.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Any, Dict, List
+import torch
+
+
+class SFTDataCollator(object):
+    def __init__(self, tokenizer, max_seq_length):
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+        self.pad_token_id = tokenizer.pad_token_id
+
+    def __call__(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        # 找出batch中的最大长度.
+        lengths = [len(x['input_ids']) for x in batch]
+        # 取出batch中的最大长度, 如果超过max_seq_length, 则取max_seq_length.
+        batch_max_len = min(max(lengths), self.max_seq_length)
+        # batch_max_len = self.max_seq_length
+
+        input_ids_batch, attention_mask_batch, target_mask_batch = [], [], []
+        # truncate and padding
+        for x in batch:
+            input_ids = x["input_ids"]
+            attention_mask = x["attention_mask"]
+            target_mask = x["target_mask"]
+
+            padding_len = batch_max_len - len(input_ids)
+            # padding
+            input_ids = input_ids + [self.pad_token_id] * padding_len
+            attention_mask = attention_mask + [0] * padding_len
+            target_mask = target_mask + [0] * padding_len
+            # truncate
+            input_ids = input_ids[:self.max_seq_length]
+            attention_mask = attention_mask[:self.max_seq_length]
+            target_mask = target_mask[:self.max_seq_length]
+
+            input_ids_batch.append(input_ids)
+            attention_mask_batch.append(attention_mask)
+            target_mask_batch.append(target_mask)
+
+        # 将list转换为tensor, 得到最终的的模型输入.
+        input_ids_batch = torch.tensor(input_ids_batch, dtype=torch.long)
+        attention_mask_batch = torch.tensor(attention_mask_batch, dtype=torch.long)
+        target_mask_batch = torch.tensor(target_mask_batch, dtype=torch.long)
+        inputs = {
+            "input_ids": input_ids_batch,
+            "attention_mask": attention_mask_batch,
+            "target_mask": target_mask_batch
+        }
+        return inputs
+
+
+if __name__ == '__main__':
+    pass

toolbox/transformers/data/dataset/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/transformers/data/dataset/dataset.py

@@ -0,0 +1,79 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+from typing import List
+
+from torch.utils.data import Dataset
+
+
+class SFTDataset(Dataset):
+    def __init__(self):
+        self.samples: List[dict] = list()
+
+    def read(self, filename: str):
+        samples = list()
+        with open(filename, "r", encoding="utf-8") as f:
+            for row in f:
+                row = str(row).strip()
+                row = json.loads(row)
+                samples.append(row)
+        self.samples = samples
+        return self
+
+    def __getitem__(self, index):
+        sample = self.samples[index]
+        return sample
+
+    def __len__(self):
+        return len(self.samples)
+
+
+class ChatGLM2SFTDataset(SFTDataset):
+    def __init__(self, tokenizer, max_seq_length: int):
+        super(ChatGLM2SFTDataset, self).__init__()
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+
+        self.input_format = '[Round {}]\n\n问：{}\n\n答：'
+        self.target_format = "{}"
+
+    def __getitem__(self, index):
+        sample = self.samples[index]
+
+        conversation = sample["conversation"]
+
+        utterances = list()
+        for i, x in enumerate(conversation):
+            human = self.input_format.format(i+1, x["human"])
+            assistant = self.target_format.format(x["assistant"])
+            utterances += ([human, assistant])
+        utterances_ids = self.tokenizer(utterances, add_special_tokens=False).input_ids
+
+        input_ids = list()
+        target_mask = list()
+        for i, utterances_id in enumerate(utterances_ids):
+            input_ids += utterances_id
+            if i % 2 == 0:
+                target_mask += [0] * (len(utterances_id))
+            else:
+                input_ids += [self.tokenizer.eos_token_id]
+                target_mask += [1] * (len(utterances_id) + 1)
+
+        assert len(input_ids) == len(target_mask)
+
+        input_ids = input_ids[:self.max_seq_length]
+        target_mask = target_mask[:self.max_seq_length]
+        attention_mask = [1] * len(input_ids)
+
+        assert len(input_ids) == len(target_mask) == len(attention_mask)
+
+        inputs = {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "target_mask": target_mask
+        }
+        return inputs
+
+
+if __name__ == '__main__':
+    pass

toolbox/transformers/modules/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/transformers/modules/loss.py

@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import torch
+import torch.nn as nn
+
+
+class Loss(object):
+    """
+    所有loss的类父类
+    """
+    def __call__(self, model, inputs, training_args, return_outputs=False):
+        """
+        用于计算loss.
+        看源码发现, return_outputs=True为train时调用, return_outputs=False为eval和predict调用
+        :param model: 模型
+        :param inputs: 模型输入, dict
+        :param training_args: 训练配置参数
+        :param return_outputs: 是否返回模型的输出
+        :return:
+        """
+        raise NotImplemented
+
+
+class TargetLMLoss(Loss):
+
+    def __init__(self, ignore_index):
+        super().__init__()
+        self.ignore_index = ignore_index
+        self.loss_fn = nn.CrossEntropyLoss(ignore_index=ignore_index)
+
+    def __call__(self, model, inputs, training_args, return_outputs=False):
+        input_ids = inputs["input_ids"]
+        attention_mask = inputs["attention_mask"]
+        target_mask = inputs["target_mask"]
+
+        # 模型前馈预测
+        outputs = model(input_ids=input_ids, attention_mask=attention_mask, return_dict=True)
+        logits = outputs["logits"] if isinstance(outputs, dict) else outputs[0]
+
+        # 将labels中不属于target的部分, 设为ignore_index, 只计算target部分的loss.
+        labels = torch.where(target_mask == 1, input_ids, self.ignore_index)
+        shift_logits = logits[..., :-1, :].contiguous()
+        shift_labels = labels[..., 1:].contiguous()
+
+        # Flatten the tokens
+        loss = self.loss_fn(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+        return (loss, outputs) if return_outputs else loss
+
+
+if __name__ == '__main__':
+    pass

toolbox/transformers/trainer.py

@@ -0,0 +1,99 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import transformers
+from transformers import (
+    PreTrainedModel,
+    TrainingArguments,
+    DataCollator,
+    PreTrainedTokenizerBase,
+    EvalPrediction,
+    TrainerCallback,
+)
+from typing import Callable, Dict, List, Optional, Tuple, Union, Any
+from torch import nn
+from torch.utils.data import Dataset, DataLoader
+from transformers.utils import (
+    logging,
+)
+from typing import Optional
+import os
+import torch
+
+
+logger = logging.get_logger(__name__)
+
+# Name of the files used for checkpointing
+TRAINING_ARGS_NAME = "training_args.bin"
+TRAINER_STATE_NAME = "trainer_state.json"
+OPTIMIZER_NAME = "optimizer.pt"
+SCHEDULER_NAME = "scheduler.pt"
+SCALER_NAME = "scaler.pt"
+
+
+class Trainer(transformers.Trainer):
+    """
+    主要修改逻辑: 通过传入compute_loss, 支持自定义loss计算方式.
+    """
+    def __init__(
+        self,
+        model: Union[PreTrainedModel, nn.Module] = None,
+        args: TrainingArguments = None,
+        data_collator: Optional[DataCollator] = None,
+        train_dataset: Optional[Dataset] = None,
+        eval_dataset: Optional[Dataset] = None,
+        tokenizer: Optional[PreTrainedTokenizerBase] = None,
+        model_init: Callable[[], PreTrainedModel] = None,
+        compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None,
+        callbacks: Optional[List[TrainerCallback]] = None,
+        optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None),
+        preprocess_logits_for_metrics: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = None,
+        compute_loss=None,
+    ):
+        super(Trainer, self).__init__(
+            model=model,
+            args=args,
+            data_collator=data_collator,
+            train_dataset=train_dataset,
+            eval_dataset=eval_dataset,
+            tokenizer=tokenizer,
+            model_init=model_init,
+            compute_metrics=compute_metrics,
+            callbacks=callbacks,
+            optimizers=optimizers,
+            preprocess_logits_for_metrics=preprocess_logits_for_metrics,
+        )
+        self.loss_func = compute_loss
+
+    def compute_loss(self, model, inputs, return_outputs=False):
+        """
+        重写loss的计算方式
+        How the loss is computed by Trainer. By default, all models return the loss in the first element.
+
+        Subclass and override for custom behavior.
+        """
+        return self.loss_func(model, inputs, self.args, return_outputs)
+
+
+class LoRATrainer(Trainer):
+    """
+    修改checkpoint的保存逻辑, 只保存lora.
+    """
+    def _save(self, output_dir: Optional[str] = None, state_dict=None):
+        # If we are executing this function, we are the process zero, so we don't check for that.
+        output_dir = output_dir if output_dir is not None else self.args.output_dir
+        os.makedirs(output_dir, exist_ok=True)
+        logger.info(f"Saving model checkpoint to {output_dir}")
+        # 保存lora权重和配置
+        self.model.save_pretrained(
+            output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
+        )
+
+        if self.tokenizer is not None:
+            self.tokenizer.save_pretrained(output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(self.args, os.path.join(output_dir, TRAINING_ARGS_NAME))
+
+
+if __name__ == '__main__':
+    pass