microsoft/Phi-3-mini-4k-instruct

You can use quantization, but you must adjust the hyper parameters.
class Phi3Mini4B4Q:
# Load 2.159 GB
def __init__(self, repo_id = 'microsoft/Phi-3-mini-4k-instruct'):
    # Definindo a configuração para evitar fragmentação
    # os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
    # Model name
    model_name = repo_id
    self.device = "cuda" if torch.cuda.is_available() else "cpu"
    self.tokenizer, self.model = self.initialize_model(model_name)

def initialize_model(self, model_name):
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
    )

    # Tokenizer initialization
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(model_name, device_map=self.device, torch_dtype=torch.bfloat16, quantization_config=bnb_config,trust_remote_code=True)
    print('GPU alocada ', torch.cuda.memory_allocated()/ (1024**2))
    return tokenizer, model

def prompt(self, context):

    text = self.tokenizer.apply_chat_template(
        context,
        tokenize=False,
        add_generation_prompt=True
    )
    model_inputs = self.tokenizer([text], return_tensors="pt").to(self.device)
    generated_ids = self.model.generate(
        model_inputs.input_ids,
        max_new_tokens=512,
        do_sample=False,
        eos_token_id=[
            self.tokenizer.eos_token_id,
        ]
    )
    generated_ids = [
        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
    ]
    response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
 
    return response
microsoft
/

Phi-3-mini-4k-instruct

Quantization
