feat: add streem

run_localGPT.py

@@ -14,11 +14,11 @@ callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
 
 from prompt_template_utils import get_prompt_template
 
-# from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain.vectorstores import Chroma
 from transformers import (
     GenerationConfig,
     pipeline,
+    TextStreamer
 )
 
 from load_models import (
@@ -36,6 +36,7 @@ from constants import (
     MODELS_PATH,
 )
 
+streamer = TextStreamer(tokenizer, skip_prompt=True)
 
 def load_model(device_type, model_id, model_basename=None, LOGGING=logging):
     """
@@ -76,6 +77,7 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging):
     # main_classes/text_generation#transformers.GenerationConfig.from_pretrained.returns
 
     # Create a pipeline for text generation
+
     pipe = pipeline(
         "text-generation",
         model=model,
@@ -86,7 +88,7 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging):
         top_k=40,
         repetition_penalty=1.0,
         generation_config=generation_config,
-        streamer=True
+        streamer=streamer
     )
 
     local_llm = HuggingFacePipeline(pipeline=pipe)