|
--- |
|
library_name: peft |
|
--- |
|
## Training procedure |
|
|
|
|
|
The following `bitsandbytes` quantization config was used during training: |
|
- quant_method: bitsandbytes |
|
- load_in_8bit: False |
|
- load_in_4bit: True |
|
- llm_int8_threshold: 6.0 |
|
- llm_int8_skip_modules: None |
|
- llm_int8_enable_fp32_cpu_offload: False |
|
- llm_int8_has_fp16_weight: False |
|
- bnb_4bit_quant_type: nf4 |
|
- bnb_4bit_use_double_quant: False |
|
- bnb_4bit_compute_dtype: float16 |
|
### Framework versions |
|
|
|
|
|
- PEFT 0.5.0 |
|
|
|
## Get it started |
|
```python |
|
import torch |
|
from datasets import Dataset |
|
from huggingface_hub import login |
|
from peft import PeftModel, PeftConfig |
|
from transformers import AutoModelForCausalLM, AutoTokenizer, AddedToken |
|
|
|
# load model and tokenizer |
|
login("[YOUR HF TOKEN HERE FOR USING LLAMA]") |
|
config = PeftConfig.from_pretrained("ChangeIsKey/llama-7b-lexical-substitution") |
|
base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf", device_map='auto') |
|
|
|
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf", use_fast=False, trust_remote_code=True) |
|
tokenizer.add_special_tokens({ "additional_special_tokens":[AddedToken("<|s|>"), AddedToken("<|answer|>"), AddedToken("<|end|>")]}) |
|
if tokenizer.pad_token is None: |
|
tokenizer.add_special_tokens({'pad_token': '[PAD]'}) |
|
tokenizer.padding_side = 'left' |
|
base_model.resize_token_embeddings(len(tokenizer)) |
|
|
|
model = PeftModel.from_pretrained(base_model, "ChangeIsKey/llama-7b-lexical-substitution") |
|
model.eval() |
|
|
|
# let's use this model |
|
def formatting_func(records): |
|
text_batch = [] |
|
|
|
for i in range(len(records['example'])): |
|
example = records[i]['example'] |
|
start, end = records[i]['start'], records[i]['end'] |
|
|
|
target = f'**{example[start:end]}**' |
|
input_text = f'{example[:start]} {target} {example[end:]}' |
|
text_batch.append(f"{input_text}<|answer|>") |
|
|
|
return text_batch |
|
|
|
def tokenization(dataset): |
|
return tokenizer(formatting_func(dataset), |
|
truncation=True, |
|
max_length=512, |
|
padding=True, |
|
return_tensors="pt").to("cuda") |
|
|
|
|
|
# a toy example |
|
examples = [{'example': 'The traffic jam on the highway made everyone late for work.', 'start': 12, 'end': 15}, |
|
{'example': 'I spread a generous layer of strawberry jam on my toast this morning', 'start': 40, 'end': 43}] |
|
dataset = Dataset.from_list(examples) |
|
|
|
|
|
batch_size = 32 |
|
output = list() |
|
|
|
with torch.no_grad(): |
|
for i in range(0, len(dataset), batch_size): |
|
model_input = tokenization(dataset.select(range(i, min(dataset.num_rows, i + batch_size)))) |
|
|
|
output_ids = model.generate(**model_input, |
|
do_sample=True, |
|
num_return_sequences=1, |
|
max_new_tokens=30, |
|
temperature=0.00001, |
|
repetition_penalty=1/0.85, |
|
top_k=40, |
|
top_p=0.1) |
|
|
|
answers = tokenizer.batch_decode(output_ids, skip_special_tokens=False) |
|
|
|
for answer in answers: |
|
answer = " ".join(answer.split('<|answer|>')[1:]) |
|
substitutes = [s.strip() for s in answer.split('<|end|>')[:-1] if s.strip() != ""] |
|
output.append(", ".join(substitutes)) |
|
|
|
# output |
|
dataset = dataset.add_column('substitutes', output) |
|
for row in dataset: |
|
target = row['example'][row['start']:row['end']] |
|
print(f"Target: {target}\nExample: {row['example']}\nSubstitutes: {row['substitutes']}\n") |
|
``` |
