feat: add stream

main.py

@@ -44,7 +44,7 @@ DB = Chroma(
 
 RETRIEVER = DB.as_retriever()
 
-LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME)
+LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME, stream=False)
 prompt, memory = get_prompt_template(promptTemplate_type="llama", history=False)
 
 template = """you are a helpful, respectful and honest assistant.

run_localGPT.py

@@ -36,9 +36,9 @@ from constants import (
     MODELS_PATH,
 )
 
-streamer = TextStreamer(tokenizer, skip_prompt=True)
 
-def load_model(device_type, model_id, model_basename=None, LOGGING=logging):
+
+def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stream=False):
     """
     Select a model for text generation using the HuggingFace library.
     If you are running this for the first time, it will download a model for you.
@@ -56,6 +56,8 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging):
     Raises:
         ValueError: If an unsupported model or device type is provided.
     """
+    streamer = TextStreamer(tokenizer, skip_prompt=stream)
+
     logging.info(f"Loading Model: {model_id}, on: {device_type}")
     logging.info("This action can take a few minutes!")