Spaces:
Running
Running
File size: 1,497 Bytes
d1bcdef 3a655b5 d1bcdef e13c97e 0787c0d d1bcdef |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
from ctransformers import AutoModelForCausalLM
# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.
#llm = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-OpenOrca-GGUF", model_file="mistral-7b-openorca.Q4_K_M.gguf", model_type="mistral", gpu_layers=50)
#llm = AutoModelForCausalLM.from_pretrained("Open-Orca/Mistral-7B-OpenOrca",gpu_layers=0, max_new_tokens = 1000, context_length = 10000)
#llm = AutoModelForCausalLM.from_pretrained("NousResearch/Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",gpu_layers=0, max_new_tokens = 500, cotext_length = 2048)
llm = AutoModelForCausalLM.from_pretrained("NousResearch/Nous-Hermes-2-Mistral-7B-DPO",gpu_layers=0, max_new_tokens = 500, cotext_length = 2048)
# llm = AutoModelForCausalLM.from_pretrained("/home/ubuntu/.cache/gpt4all/Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",gpu_layers=0, max_new_tokens = 500, context_length = 6000)
#llm = AutoModelForCausalLM.from_pretrained("Meta-Llama-3-8B-Instruct.Q4_0.gguf",gpu_layers=0, max_new_tokens = 500, context_length = 6000)
#Meta-Llama-3-8B-Instruct.Q4_0.gguf
#llm = AutoModelForCausalLM.from_pretrained("TheBloke/zephyr-7B-beta-GGUF",model_file="zephyr-7b-beta.Q5_K_M.gguf", model_type="mistral",gpu_layers=0, max_new_tokens = 1000, context_length = 6000)
contenuto="che ore sono?"
print(contenuto)
if __name__ == '__main__':
print(llm(contenuto,threads=16,temperature=0.7,top_k=40,top_p=0.4,repetition_penalty=1.18))
|