app.py

from ctransformers import AutoModelForCausalLM

 
# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.
#llm = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-OpenOrca-GGUF", model_file="mistral-7b-openorca.Q4_K_M.gguf", model_type="mistral", gpu_layers=50)


#llm = AutoModelForCausalLM.from_pretrained("Open-Orca/Mistral-7B-OpenOrca",gpu_layers=0, max_new_tokens = 1000, context_length = 10000)

#llm = AutoModelForCausalLM.from_pretrained("NousResearch/Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",gpu_layers=0, max_new_tokens = 500, cotext_length = 2048)

llm = AutoModelForCausalLM.from_pretrained("NousResearch/Nous-Hermes-2-Mistral-7B-DPO",gpu_layers=0, max_new_tokens = 500, context_length = 2048)

# llm = AutoModelForCausalLM.from_pretrained("/home/ubuntu/.cache/gpt4all/Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",gpu_layers=0, max_new_tokens = 500, context_length = 6000)
#llm = AutoModelForCausalLM.from_pretrained("Meta-Llama-3-8B-Instruct.Q4_0.gguf",gpu_layers=0, max_new_tokens = 500, context_length = 6000)
#Meta-Llama-3-8B-Instruct.Q4_0.gguf





#llm = AutoModelForCausalLM.from_pretrained("TheBloke/zephyr-7B-beta-GGUF",model_file="zephyr-7b-beta.Q5_K_M.gguf", model_type="mistral",gpu_layers=0, max_new_tokens = 1000, context_length = 6000)

contenuto="che ore sono?"
print(contenuto)

if __name__ == '__main__':
    print(llm(contenuto,threads=16,temperature=0.7,top_k=40,top_p=0.4,repetition_penalty=1.18))