File size: 658 Bytes
424d873 5ef5b00 424d873 5ef5b00 424d873 5ef5b00 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
import os
import requests
from flask import Flask, render_template, request
from llama_cpp import Llama
from dotenv import load_dotenv
load_dotenv()
llm = Llama(model_path="./model.gguf", n_ctx=2048)
app = Flask(__name__, template_folder='./')
@app.route("/")
def home():
return render_template("index.html")
@app.route("/respond")
def respond():
input = str(request.args.get('input'))
maxTok = int(str(request.args.get('maxTok')))
output = llm(input, max_tokens=maxTok, stop=["<|im_end|>"], echo=True)
return output['choices'][0]['text']
if __name__ == '__main__':
app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 7860))) |