Create app.py

app.py

@@ -0,0 +1,56 @@
+from langchain.llms import LlamaCpp
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+import gradio as gr
+import re
+import os
+
+
+
+n_gpu_layers = 40 # Change this value based on your model and your GPU VRAM pool.
+n_batch = 512 # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.
+n_ctx=2048
+
+callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
+path = "Dorna-Llama3-8B-Instruct-GGUF"
+
+llm = LlamaCpp(
+    model_path= path,
+    n_gpu_layers=n_gpu_layers, n_batch=n_batch,
+    callback_manager=callback_manager,
+    verbose=True,
+    n_ctx=n_ctx,
+    temperature=0.2,
+    max_tokens=200,
+    top_p=1,
+)
+
+prompt = """Below is an instruction that describes a task.
+Write a response that appropriately completes the request.\n\n
+### Instruction:\n\n{}\n\n\n### Response:\n\n\n"""
+def generate_output(text):
+    result = ""
+    for s in llm.stream(prompt.format(text)):
+        result += s
+        yield result
+
+
+def clear():
+    return "", ""
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    with gr.Row():
+        inputs=gr.Textbox(label="ورودی",placeholder="سوال خود را وارد کنید",rtl=True)
+        
+    with gr.Row():
+        submit_btn= gr.Button("ارسال", variant="primary")
+        clear_btn = gr.ClearButton(value="پاک کردن", variant="secondary")
+    with gr.Row():
+        outputs=gr.Textbox(label="خروجی",rtl=True)
+    submit_btn.click(fn=generate_output,
+                    inputs= [inputs],
+                    outputs= [outputs])
+    clear_btn.click(fn=clear, inputs=[], outputs=[inputs, outputs])
+
+    
+demo.launch(server_name='0.0.0.0',share=True)