README.md

metadata

license: apache-2.0

from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel import torch

device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Using device: {device}")

model = AutoModelForCausalLM.from_pretrained("orionweller/test-flex-gpt", trust_remote_code=True) model = model.to(device) tokenizer = AutoTokenizer.from_pretrained("orionweller/test-flex-gpt", trust_remote_code=True)

test it out and encode some text

prompt = "The capital of France is" inputs = tokenizer(prompt, return_tensors="pt").input_ids

put the input ids on the right device

inputs = inputs.to(device) outputs = model.generate(inputs, max_new_tokens=5, do_sample=True, top_p=0.95) print(tokenizer.batch_decode(outputs, skip_special_tokens=True))