Update app.py

app.py

@@ -15,6 +15,8 @@ print("Embedding model loaded...")
 
 # Load the LLM
 callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
+
+'''
 llm = AutoModelForCausalLM.from_pretrained(
     "TheBloke/Llama-2-7B-Chat-GGUF",
     model_file="llama-2-7b-chat.Q3_K_S.gguf",
@@ -23,6 +25,16 @@ llm = AutoModelForCausalLM.from_pretrained(
     repetition_penalty=1.5,
     max_new_tokens=300,
 )
+'''
+llm = LlamaCpp(
+    model_path="./llama-2-7b-chat.Q3_K_S.gguf",
+    temperature = 0.2,
+    n_ctx=2048,
+    f16_kv=True,  # MUST set to True, otherwise you will run into problem after a couple of calls
+    max_tokens = 500,
+    callback_manager=callback_manager,
+    verbose=True,
+)
 print("LLM loaded...")
 
 client = QdrantClient(path="./db")