Himalayan-wildcat/gemma-2-9b-finetune

Model Description

This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated.

• Developed by: Hiroaki Hara(@Himalayan-wildcat)
• Language(s) (NLP): ja
• License: MIT
• Finetuned from model: Himalayan-wildcat/gemma-2-9b-finetune
• Datasets: DeL-TaiseiOzaki/Tengentoppa-sft-v1.0

Uses

pip install peft~=0.14 tqdm~=4.67 transformers~=4.47

import json
import re

import torch
from peft import PeftModel
from tqdm import tqdm
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
)


model_id = "Himalayan-wildcat/gemma-2-9b-finetune"
hf_token = "/YOUR_HUGGING_FACE_TOKEN/"
test_jsonl_data = "elyza-tasks-100-TV_0.jsonl"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16)

model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    device_map="auto",
    token = hf_token)

tokenizer = AutoTokenizer.from_pretrained(
    model_id,
    trust_remote_code=True,
    token=hf_token)

datasets = []
with open(test_jsonl_data) as f:
    item = ""
    for line in f:
      line = line.strip()
      item += line
      if item.endswith("}"):
        datasets.append(json.loads(item))
        item = ""

results = []
for data in tqdm(datasets):
    input_: str = data["input"]

    prompt = f"""
[要件]
- 与えられた質問と同じ言語で回答をしてください。
- 回答が分からない場合は、虚偽をせず、「分かりません。」と回答をしてください。

[質問]
{input_}

[回答]"""

    tokenized_input = tokenizer(prompt, return_tensors="pt").to("cuda")
    
    with torch.no_grad():
        generated_ids = model.generate(
            tokenized_input.input_ids,            
            attention_mask=tokenized_input.attention_mask,
            max_new_tokens=500,
            do_sample=False,
            repetition_penalty=1.2,
            pad_token_id=tokenizer.eos_token_id)

    generated_ids = [
        output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_input.input_ids, generated_ids)
    ]
    output = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
    results.append({"task_id": data["task_id"], "input": input_, "output": output})

jsonl_id = re.sub(".*/", "", model_id)
with open(f"./{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
    for result in results:
        json.dump(result, f, ensure_ascii=False)
        f.write('\n')