Spaces:
Runtime error
Runtime error
File size: 2,017 Bytes
762c182 354bd03 762c182 354bd03 ba444f5 354bd03 ba444f5 354bd03 ba444f5 354bd03 ed610fa 354bd03 ba444f5 354bd03 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
import gradio as gr
import copy
import random
import os
import requests
import time
import sys
from huggingface_hub import snapshot_download
from llama_cpp import Llama
repo_name = "kirp/TinyLlama-1.1B-Chat-v0.2-gguf"
model_name = "ggml-model-q4_k_m.gguf"
snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
model = Llama(
model_path=model_name,
n_ctx=2048,
n_parts=1,
)
template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
def generate(
input=None,
temperature=0.1,
top_p=0.75,
top_k=40,
max_new_tokens=512,
):
prompt = template.format(input)
output = model(prompt,
temperature = temperature,
top_k = top_k,
top_p = top_p,
max_tokens = max_new_tokens + len(input),
stop=["<|im_end|>"],
echo=True)
output = output["choices"][0]['text']
return output.split("assistant\n")[1]
g = gr.Interface(
fn=generate,
inputs=[
gr.components.Textbox(
lines=2, label="Prompt", placeholder="Tell me about huggingface."
),
gr.components.Slider(minimum=0, maximum=1, value=0.7, label="Temperature"),
gr.components.Slider(minimum=0, maximum=1, value=0.8, label="Top p"),
gr.components.Slider(minimum=0, maximum=100, step=1, value=50, label="Top k"),
gr.components.Slider(
minimum=1, maximum=2048, step=1, value=512, label="Max tokens"
),
],
outputs=[
gr.Textbox(
lines=10,
label="Output",
)
],
title = "TinyLlama 1.1B Chat GGUF",
description = """
original model: [PY007/TinyLlama-1.1B-Chat-v0.2](https://huggingface.co/PY007/TinyLlama-1.1B-Chat-v0.2)
quantized_model: [kirp/TinyLlama-1.1B-Chat-v0.2-gguf](https://huggingface.co/kirp/TinyLlama-1.1B-Chat-v0.2-gguf)
"""
)
g.queue(concurrency_count=2)
g.launch()
|