Update core/chatbot/retrieval_chatbot.py

core/chatbot/retrieval_chatbot.py

@@ -58,7 +58,7 @@ class RetrievalChatbot(BaseChatbot):
     
     def response(self, message: str, image_path=None, return_logs=False) -> str:
         print("Query: {message}".format(message=message))
-        question = self.decomposer.refine(message,image_path)
+        question = self.decomposer.refine(message, None, image_path)
         print(question)
         question = question.replace('"', "'").replace("', '", "','").lstrip("['").rstrip("']")
         sub_questions = question.split("','")
@@ -76,7 +76,11 @@ class RetrievalChatbot(BaseChatbot):
             # context = self.memory.messages + [{"role": "user", "content": "References: {references}\nQuestion: {question}".format(references=reference, question=sub_question)}]
             # sub_answer = self.model.respond(context)
             sub_answerer_context = "Sub Question References: {sub_retrieve_reference}\nQuestion: {question}\n".format(sub_retrieve_reference=sub_retrieve_reference, question=sub_question)
-            sub_answer = self.answerer.refine(sub_answerer_context,image_path)
+
+            print(sub_answerer_context)
+            print(self.memory)
+            print(image_path)
+            sub_answer = self.answerer.refine(sub_answerer_context, self.memory, image_path)
             
             print(f"Subanswer: {sub_answer}")
             
@@ -87,13 +91,18 @@ class RetrievalChatbot(BaseChatbot):
             references += "Related research for the user query: {ref}\n".format(ref=ref)
 
         summarizer_context = "Question References: {references}\nQuestion: {message}\n".format(references=references, message=message)
-        answer = self.summarizer.refine(summarizer_context,image_path)
+        answer = self.summarizer.refine(summarizer_context, None, image_path)
         
         #todo 记忆管理
-        self.memory.append([{"role": "user", "content": [
-                        {"type": "text", "text": f"{message}"},
-                        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encode_image(image_path)}"}},
-                    ]}, {"role": "assistant", "content": answer}])
+        if image_path is None:
+            self.memory.append([{"role": "user", "content": [
+                            {"type": "text", "text": f"{message}"},
+                        ]}, {"role": "assistant", "content": answer}])
+        else:
+            self.memory.append([{"role": "user", "content": [
+                            {"type": "text", "text": f"{message}"},
+                            {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encode_image(image_path)}"}},
+                        ]}, {"role": "assistant", "content": answer}])
         print("="*20)
         print(f"Final answer: {answer}".format(answer=answer))