Update README.md

README.md

@@ -20,3 +20,185 @@ language:
 This gemma2 model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library.
 
 [<img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20made%20with%20love.png" width="200"/>](https://github.com/unslothai/unsloth)
+
+訓練用データ
+- https://huggingface.co/datasets/llm-jp/magpie-sft-v1.0
+- https://huggingface.co/datasets/Aratako/Magpie-Tanuki-8B-annotated-96k
+共にapache-2.0
+編集中
+
+実行コード
+Google Colab用
+リンク先：
+＊リンク先のGoogle Colabノートを実行してください
+
+以下に同じノートを同じコードを掲載します
+```python
+!pip install -q transformers==4.46.3 accelerate bitsandbytes
+!pip install -q tqdm
+!pip install flash-attn --no-build-isolation
+
+import os
+import torch
+import json
+from tqdm import tqdm
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+print("【重要】以下の手順でHugging Faceトークンを設定してください：")
+print("1. 左メニューの'シークレット'タブを開く")
+print("2. '新しいシークレット'をクリック")
+print("3. 名前に'HF_TOKEN'を入力")
+print("4. 値にHugging Faceトークンを入力して保存")
+
+# シークレットからHF_TOKENを取得
+from google.colab import userdata
+HF_TOKEN = userdata.get('HF_TOKEN')
+
+if HF_TOKEN is None:
+    raise ValueError("HF_TOKENが設定されていません。上記の手順でトークンを設定してください。")
+
+
+
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+def load_model_and_tokenizer():
+    model = AutoModelForCausalLM.from_pretrained(
+        "Chrom256/gemma-2-9b-it-lora_20241216_033631",
+        quantization_config=quantization_config,
+        device_map="auto",
+        trust_remote_code=True,
+        torch_dtype=torch.bfloat16,
+        attn_implementation="eager",
+        low_cpu_mem_usage=True
+    )
+    
+    tokenizer = AutoTokenizer.from_pretrained(
+        "Chrom256/gemma-2-9b-it-lora_20241216_033631",
+        trust_remote_code=True
+    )
+    
+    model.config.use_cache = True
+    model.eval()
+    torch.cuda.empty_cache()
+    
+    return model, tokenizer
+
+GENERATION_CONFIG = {
+！編集中！
+}
+
+def load_input_data(file_path):
+    tokenized_inputs = []
+    with open(file_path, "r") as f:
+        for line in f:
+            if line.strip():
+                dt = json.loads(line)
+                tokenized_inputs.append({
+                    "task_id": dt["task_id"],
+                    "input": dt["input"]
+                })
+    return tokenized_inputs
+
+def prepare_inputs(batch_data, tokenizer, max_length=1024):
+    batch_inputs = []
+    for data in batch_data:
+        prompt = f"""<start_of_turn>system
+簡潔に回答してください。装飾や特殊記号は使用しないでください。
+<end_of_turn>
+<start_of_turn>user
+{data["input"]}
+<end_of_turn>
+<start_of_turn>model
+"""
+        inputs = tokenizer(
+            prompt,
+            return_tensors="pt",
+            padding="max_length",
+            truncation=True,
+            max_length=max_length
+        )
+        batch_inputs.append(inputs)
+
+    return {
+        "input_ids": torch.cat([inp["input_ids"] for inp in batch_inputs], dim=0),
+        "attention_mask": torch.cat([inp["attention_mask"] for inp in batch_inputs], dim=0)
+    }
+
+def post_process_output(response):
+    response = response.strip()
+    symbols_to_replace = ['**', '`', '|', '```', '---', '===']
+    for symbol in symbols_to_replace:
+        response = response.replace(symbol, ' ')
+    return ' '.join(response.split())
+
+#メモリが足りない場合は、batch_sizeを下げてください(2など)
+def run_inference(model, tokenizer, tokenized_inputs, generation_config, batch_size=4):
+    results = []
+    
+    for i in tqdm(range(0, len(tokenized_inputs), batch_size)):
+        batch = tokenized_inputs[i:i+batch_size]
+        
+        with torch.no_grad():
+            inputs = prepare_inputs(batch, tokenizer)
+            inputs = {k: v.to(model.device, non_blocking=True) for k, v in inputs.items()}
+            
+            outputs = model.generate(
+                **inputs,
+                pad_token_id=tokenizer.pad_token_id,
+                eos_token_id=tokenizer.eos_token_id,
+                **generation_config
+            )
+            
+            for idx, b in enumerate(batch):
+                full_output = tokenizer.decode(outputs[idx], skip_special_tokens=True)
+                
+                if 'model\n' in full_output:
+                    response = full_output.split('model\n')[-1].strip()
+                elif 'model' in full_output:
+                    response = full_output.split('model')[-1].strip()
+                else:
+                    response = full_output.strip()
+                
+                processed_response = post_process_output(response)
+                
+                results.append({
+                    "task_id": b["task_id"],
+                    "input": b["input"],
+                    "output": processed_response,
+                })
+            
+            del outputs
+            torch.cuda.empty_cache()
+    
+    return results
+
+def save_results(results, output_dir):
+    os.makedirs(output_dir, exist_ok=True)
+    jsonl_path = os.path.join(output_dir, "Output.jsonl")
+    
+    with open(jsonl_path, 'w', encoding='utf-8') as f:
+        for item in results:
+            json.dump(item, f, ensure_ascii=False)
+            f.write('\n')
+    
+    print(f"Saved results to: {jsonl_path}")
+
+def main():
+    model, tokenizer = load_model_and_tokenizer()
+    
+    # 入力データの読み込み
+    #Google colabのファイルにアップロードした際のpathにしてあります
+    #必要に応じてpathの修正をお願いします
+    tokenized_inputs = load_input_data("/content/elyza-tasks-100-TV_0.jsonl")
+    
+    results = run_inference(model, tokenizer, tokenized_inputs, GENERATION_CONFIG)
+    save_results(results, "output")
+
+if __name__ == "__main__":
+    main()
+```