added files

just adding files

Astronomy_BH_hybrid_RAG.py

@@ -0,0 +1,321 @@
+#############################Imports##############################################
+## General imports
+import os
+import sys
+import zipfile
+import logging
+import IPython
+from IPython.display import display
+from pyvis.network import Network
+import dotenv
+from google.cloud import storage
+
+dotenv.load_dotenv()
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "tidy-resolver-411707-0f032726c297.json"
+
+
+def download_blob(bucket_name, source_blob_name, destination_file_name):
+    """Downloads a blob from the bucket."""
+    storage_client = storage.Client()
+    bucket = storage_client.bucket(bucket_name)
+    blob = bucket.blob(source_blob_name)
+    blob.download_to_filename(destination_file_name)
+    print(f"Downloaded storage object {source_blob_name} from bucket {bucket_name} to local file {destination_file_name}.")
+
+if not (os.path.exists("storage_file/bm25") and os.path.exists("storage_file/kg")):
+    # List of file names to download
+    file_names = [
+        "default__vector_store.json",
+        "docstore.json",
+        "graph_store.json",
+        "image__vector_store.json",
+        "index_store.json"
+    ]
+
+    # Bucket name
+    bucket_name = "title_tailors_bucket"
+
+    # Create the destination directory if it doesn't exist
+    os.makedirs("storage_file/bm25", exist_ok=True)
+
+    # Loop through the file names and download each one
+    for file_name in file_names:
+        source_blob_name = f"storage/bm25/{file_name}"
+        destination_file_name = f"storage_file/bm25/{file_name}"
+        download_blob(bucket_name, source_blob_name, destination_file_name)
+        
+    # List of file names to download
+    file_names = [
+        "default__vector_store.json",
+        "docstore.json",
+        "graph_store.json",
+        "image__vector_store.json",
+        "index_store.json"
+    ]
+
+    # Bucket name
+    bucket_name = "title_tailors_bucket"
+
+    # Create the destination directory if it doesn't exist
+    os.makedirs("storage_file/kg", exist_ok=True)
+
+    # Loop through the file names and download each one
+    for file_name in file_names:
+        source_blob_name = f"storage/kg/{file_name}"
+        destination_file_name = f"storage_file/kg/{file_name}"
+        download_blob(bucket_name, source_blob_name, destination_file_name)
+else:
+    print("Files already exist in the storage_file directory.")
+
+
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+
+MISTRAL_API = os.environ.get("MISTRAL_API", None)
+print(f"MISTRAL_API: {MISTRAL_API}")
+
+## logger
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logging.getLogger().handlers = []
+logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
+
+# Knowledge graph imports
+from llama_index.core import (
+    SimpleDirectoryReader,
+    StorageContext,
+    KnowledgeGraphIndex,
+    load_index_from_storage,
+    Settings,
+)
+from llama_index.core.graph_stores import SimpleGraphStore
+from llama_index.embeddings.mistralai import MistralAIEmbedding
+from llama_index.llms.mistralai import MistralAI
+
+## BM25 imports
+from llama_index.core import (
+    VectorStoreIndex,
+    QueryBundle,
+)
+from llama_index.retrievers.bm25 import BM25Retriever
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core.retrievers import (
+    BaseRetriever,
+    VectorIndexRetriever,
+    QueryFusionRetriever,
+    )
+from llama_index.core.schema import NodeWithScore
+from llama_index.core.query_engine import RetrieverQueryEngine
+from llama_index.core.postprocessor import SentenceTransformerRerank
+from llama_index.core.response.notebook_utils import (
+    display_response,
+    display_source_node,
+)
+
+# Chat engine
+from llama_index.core import PromptTemplate
+from llama_index.core import chat_engine
+from llama_index.core import memory
+
+import nest_asyncio
+nest_asyncio.apply()
+
+################### Loading the LLM via Mistral API
+llm = MistralAI(api_key=MISTRAL_API, model="open-mixtral-8x7b")
+### Loading the Embedding via Mistral API
+embed_model = MistralAIEmbedding(api_key=MISTRAL_API, model = "mistral-embed")
+
+################### Knowledge Graph Index#################
+########### While loading from persist only ###############
+Settings.llm = llm
+Settings.embed_model = embed_model
+kg_storage_context = StorageContext.from_defaults(persist_dir="storage/kg")
+kg_index = load_index_from_storage(kg_storage_context)
+
+kg_retriever = kg_index.as_retriever(include_text=True,
+                                     response_mode ="tree_summarize",
+                                     embedding_mode="hybrid",
+                                     similarity_top_k=10)
+
+################### BM25 Index #################
+####################### While loading Indices from persist #######################
+# Settings.llm = llm
+# Settings.embed_model = embed_model
+storage_context_v = StorageContext.from_defaults(persist_dir="storage/bm25")
+index_v = load_index_from_storage(storage_context_v)
+
+vector_retriever = index_v.as_retriever(similarity_top_k=10)
+bm25_retriever = BM25Retriever.from_defaults(index=index_v, similarity_top_k=10, verbose=False) ## loading after persist
+
+
+###################### Reranker from HuggingFace 🤗
+reranker = SentenceTransformerRerank(top_n=5, model="mixedbread-ai/mxbai-rerank-base-v1", keep_retrieval_score=False)
+
+################ Query Fusion Retriever ##################
+QUERY_GEN_PROMPT = (
+    "You are a helpful assistant that generates multiple search queries on astronomy based on a "
+    "single input query. Generate {num_queries} search queries for astronomy, one on each line, "
+    "related to the following input query:\n"
+    "Query: {query}\n"
+    "Queries:\n"
+)
+
+#### Hybrid (Dense + BM25 + KG) Retriever ####
+hybrid_retriever = QueryFusionRetriever(
+    retrievers = [vector_retriever, bm25_retriever, kg_retriever],
+    retriever_weights = [0.25, 0.25, 0.50],
+    similarity_top_k=5,
+    llm=llm,
+    num_queries=3,  # set this to 1 to disable query generation
+    mode="reciprocal_rerank",
+    use_async=True,
+    verbose=False,
+    query_gen_prompt=QUERY_GEN_PROMPT,
+)
+
+#### KG Retriever ####
+kg_retriever = QueryFusionRetriever(
+    retrievers = [kg_retriever],
+    # retriever_weights = [0.25, 0.25, 0.50],
+    similarity_top_k=5,
+    llm=llm,
+    num_queries=3,  # set this to 1 to disable query generation
+    mode="reciprocal_rerank",
+    use_async=True,
+    verbose=False,
+    query_gen_prompt=QUERY_GEN_PROMPT,
+)
+
+#### Hybrid BM25 (Dense + BM25) Retriever ####
+hybrid_bm25_retriever = QueryFusionRetriever(
+    retrievers = [vector_retriever, bm25_retriever],
+    # retriever_weights = [0.25, 0.25, 0.50],
+    similarity_top_k=5,
+    llm=llm,
+    num_queries=3,  # set this to 1 to disable query generation
+    mode="reciprocal_rerank",
+    use_async=True,
+    verbose=False,
+    query_gen_prompt=QUERY_GEN_PROMPT,
+)
+
+################# Hybrid Chat Engine ###################
+system_prompt_template = """You are a helpful AI assistant for reaearcher or enthusiast in the domain of astronomy.
+Please check if the following pieces of context has any mention of the keywords provided in the Question. If not then don't know the answer, just say that you don't know.Stop there. Please donot try to make up an answer."""
+
+from llama_index.core import chat_engine
+from llama_index.core import memory
+
+######## Hybrid (Dense + BM25 + KG) chat engine #########
+hybrid_memory = memory.ChatMemoryBuffer.from_defaults(token_limit=30000)
+Hybrid_chat_engine = chat_engine.CondensePlusContextChatEngine(retriever=hybrid_retriever,
+                                          llm=llm,
+                                          memory=hybrid_memory,
+                                          # context_prompt=,
+                                          # condense_prompt=,
+                                          system_prompt=system_prompt_template,
+                                          node_postprocessors=[reranker],
+                                          verbose=False,
+                                          )
+
+######## Hybrid (Dense + BM25 + KG) chat engine #########
+kg_memory = memory.ChatMemoryBuffer.from_defaults(token_limit=30000)
+kg_chat_engine = chat_engine.CondensePlusContextChatEngine(retriever=kg_retriever,
+                                          llm=llm,
+                                          memory=kg_memory,
+                                          # context_prompt=,
+                                          # condense_prompt=,
+                                          system_prompt=system_prompt_template,
+                                          node_postprocessors=[reranker],
+                                          verbose=False,
+                                          )
+
+######## Hybrid (Dense + BM25 + KG) chat engine #########
+hybrid_bm25_memory = memory.ChatMemoryBuffer.from_defaults(token_limit=30000)
+hybrid_bm25_chat_engine = chat_engine.CondensePlusContextChatEngine(retriever=hybrid_bm25_retriever,
+                                          llm=llm,
+                                          memory=hybrid_bm25_memory,
+                                          # context_prompt=,
+                                          # condense_prompt=,
+                                          system_prompt=system_prompt_template,
+                                          node_postprocessors=[reranker],
+                                          verbose=False,
+                                          )
+
+################## Post Processors ###################
+def get_documents(response):
+    """Get the reference documents for a chat engine response.
+    
+    Args:
+        response (ChatResponse): The chat engine response object.
+    
+    Returns:
+        str: A formatted string containing the reference document paths and page numbers.
+    """
+    plist = []
+    reference_docs={}
+    for idx, content in enumerate(response.sources[0].content.split("\n\n")):
+        if "page_label" in content:
+            plist.append(content)
+    for i, entry in enumerate(plist, start=1):
+        lines = entry.split('\n')
+        page_label = int(lines[0].split(': ')[1])
+        file_path = lines[1].split(': ')[1]
+
+        reference_docs[f"doc{i}"] = {
+            "document_path": file_path,
+            "page": page_label
+        }
+    reference = f"""
+    \n Reference Docs (document paths, page number):
+        ...{reference_docs['doc1']['document_path']}, page {reference_docs['doc1']['page']}
+        ...{reference_docs['doc2']['document_path']}, page {reference_docs['doc2']['page']}
+        ...{reference_docs['doc3']['document_path']}, page {reference_docs['doc3']['page']}
+        ...{reference_docs['doc4']['document_path']}, page {reference_docs['doc4']['page']}
+        ...{reference_docs['doc5']['document_path']}, page {reference_docs['doc5']['page']}
+        """
+    return reference
+
+##################### Query Function #####################
+def get_query(query=""):
+    """Get a response from the Hybrid chat engine and format the result with reference documents.
+    
+    Args:
+        query (str): The query to send to the chat engine.
+    
+    Returns:
+        str: The chat engine response with the reference documents appended.
+    """
+    response_1 = Hybrid_chat_engine.chat(query)
+    response_2 = kg_chat_engine.chat(query)
+    response_3 = hybrid_bm25_chat_engine.chat(query)
+    reference_1 = get_documents(response_1)
+    reference_2 = get_documents(response_2)
+    reference_3 = get_documents(response_3)
+    reply_1 = response_1.response
+    reply_2 = response_2.response
+    reply_3 = response_3.response
+    result_hybrid = reply_1 + reference_1
+    result_kg = reply_2 + reference_2
+    result_hybrid_bm25 = reply_3 + reference_3
+    return result_hybrid, result_kg, result_hybrid_bm25
+
+
+
+# print(get_query("what is difference between class I and class II ?"))
+# Based on the provided documents, Class I and Class II are two distinct evolutionary stages of young stellar objects (YSOs) in the process of forming stars. The documents describe the evolutionary stages of YSOs as follows:
+
+# * Class 0: The formation of a YSO in the central region of a protostellar core with an envelope mass that is much in excess of the YSO mass.
+# * Class I: The collapse of the envelope onto the central object, with the transition between Class 0 and Class I being the point in time at which the envelope mass and the mass of the protostar are nearly equal.
+# * Class II: The emergence of a disk around the central star.
+# * Class III: The dissipation of the disk by various processes such as the formation of planets, photo-evaporation, and tidal stripping.
+
+# The documents also mention that an intermediate class between Class 0 and Class I has been proposed, but it is considered to be close to Class I in terms of the evolutionary status of the YSOs.
+
+# Regarding the difference between Class I and Class II, the documents do not provide a detailed explanation, but it is suggested that the main difference lies in the presence of a disk around the central star in Class II, which is not present in Class I. The emergence of a disk around the central star is a sign of a more evolved stage in the star formation process. The documents also mention that the exact duration of each evolutionary stage for YSOs is relatively uncertain and depends on the number of objects found in each class, which may be affected by misclassifications due to YSOs being seen edge-on.
+#   Reference Docs (document paths, page number):
+#     .../content/documents/2405.00095v1.pdf, page 8
+#     .../content/documents/2405.00095v1.pdf, page 3
+#     .../content/documents/2405.00095v1.pdf, page 9
+#     .../content/documents/2405.00095v1.pdf, page 2
+#     .../content/documents/2405.00095v1.pdf, page 2
+    

Dockerfile

@@ -0,0 +1,14 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["streamlit", "run", "streamlit_app:app", "--server.port", "7860"]

Licence.md

@@ -0,0 +1,9 @@
+# GNU AFFERO GENERAL PUBLIC LICENSE
+Version 3, 19 November 2007
+
+Copyright (C) [2024] [Swarnava Bhattacharjee]
+
+Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
+
+---
+For the full license text, please refer to [GNU AGPL v3.0](https://www.gnu.org/licenses/agpl-3.0.html).

README.md

@@ -1,11 +1,11 @@
----
-title: Dep Teat
-emoji: 📚
-colorFrom: blue
-colorTo: yellow
-sdk: docker
-pinned: false
-license: openrail
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Dep Teat
+emoji: 📚
+colorFrom: blue
+colorTo: yellow
+sdk: docker
+pinned: false
+license: openrail
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

requirements.txt

@@ -0,0 +1,19 @@
+llama_index 
+pyvis 
+Ipython 
+PyPDF2
+llama-index-llms-huggingface
+llama-index-llms-mistralai
+llama-index-embeddings-mistralai
+sentence-transformers
+llama-index-retrievers-bm25
+matplotlib
+python-dotenv
+google-api-core==2.15.0
+google-auth==2.26.2
+google-cloud-core==2.4.1
+google-cloud-storage==2.14.0
+google-crc32c==1.5.0
+google-resumable-media==2.7.0
+googleapis-common-protos==1.62.0
+streamlit

streamlit_app.py

@@ -0,0 +1,104 @@
+import streamlit as st
+import random
+import os
+import time
+# from google.cloud import storage
+from Astronomy_BH_hybrid_RAG import get_query
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "tidy-resolver-411707-0f032726c297.json"
+
+
+# def download_blob(bucket_name, source_blob_name, destination_file_name):
+#     """Downloads a blob from the bucket."""
+#     storage_client = storage.Client()
+#     bucket = storage_client.bucket(bucket_name)
+#     blob = bucket.blob(source_blob_name)
+#     blob.download_to_filename(destination_file_name)
+#     print(f"Downloaded storage object {source_blob_name} from bucket {bucket_name} to local file {destination_file_name}.")
+
+# if not (os.path.exists("storage_file/bm25") and os.path.exists("storage_file/kg")):
+#     # List of file names to download
+#     file_names = [
+#         "default__vector_store.json",
+#         "docstore.json",
+#         "graph_store.json",
+#         "image__vector_store.json",
+#         "index_store.json"
+#     ]
+
+#     # Bucket name
+#     bucket_name = "title_tailors_bucket"
+
+#     # Create the destination directory if it doesn't exist
+#     os.makedirs("storage_file/bm25", exist_ok=True)
+
+#     # Loop through the file names and download each one
+#     for file_name in file_names:
+#         source_blob_name = f"storage/bm25/{file_name}"
+#         destination_file_name = f"storage_file/bm25/{file_name}"
+#         download_blob(bucket_name, source_blob_name, destination_file_name)
+        
+#     # List of file names to download
+#     file_names = [
+#         "default__vector_store.json",
+#         "docstore.json",
+#         "graph_store.json",
+#         "image__vector_store.json",
+#         "index_store.json"
+#     ]
+
+#     # Bucket name
+#     bucket_name = "title_tailors_bucket"
+
+#     # Create the destination directory if it doesn't exist
+#     os.makedirs("storage_file/kg", exist_ok=True)
+
+#     # Loop through the file names and download each one
+#     for file_name in file_names:
+#         source_blob_name = f"storage/kg/{file_name}"
+#         destination_file_name = f"storage_file/kg/{file_name}"
+#         download_blob(bucket_name, source_blob_name, destination_file_name)
+# else:
+#     print("Files already exist in the storage_file directory.")
+
+# Streamed response emulator
+def response_generator(text):
+    output = get_query(text)
+    responses = {
+        "Knowledge Graph Response" : output[1],
+        "Dense + BM25 without KG Response" : output[2],
+        "Dense + BM25 with KG Response" : output[2]
+    }
+    return responses
+
+st.title("Context-aware Astronomy ChatBot")
+
+# Initialize chat history
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display chat messages from history on app rerun
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+
+# Accept user input
+if prompt := st.chat_input("What is up?"):
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    # Display user message in chat message container
+    with st.chat_message("user"):
+        st.markdown(prompt)
+
+    # Generate response
+    response_dict = response_generator(prompt)
+
+    # Display each part of the response in separate markdown blocks with headings
+    with st.chat_message("assistant"):
+        for key, value in response_dict.items():
+            st.markdown(f"### {key}")
+            st.markdown(value)
+
+    # Add assistant response to chat history
+    for key, value in response_dict.items():
+        st.session_state.messages.append({"role": "assistant", "content": f"### {key}\n{value}"})

tidy-resolver-411707-0f032726c297.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "tidy-resolver-411707",
+  "private_key_id": "0f032726c297c9deb5845d42d1dadeb19facdfcf",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDWFU4ARCH2MzH9\nV5a93dGCXmj9HFdZfqp16MOk73s6PLgf3evh+yutQY+v774vvy9aUZ83BsTIWI2R\nKdfHYl5dpvBASV0Sui4P/uibtKYg5fJAbHjH5VwgDAzj9tQcK6j/S/9khR/GD/hn\nj6Ulq9ljFAFr3wp/79E3P79oElqjPWZjkZm96smgGX9BkA63zFVZRJcEsk5YvLdX\n3qtkc8UD7wfCGVjdt3v95DVMYQ5/xfe4O/9CImlnjRlgZ1OdLdTP0Te5jMSX/+un\n4G5G7W57VrVZN+jO94O1k5IFEBA5lu2soX/IHGyKI0MrCR5AES5eRxxdNaCkwAVM\ns3sVngUfAgMBAAECggEAFi9kA2wMFu7sPcQ/Z1Ke3QM6nww6UxhowspJyARXQUw8\ninTddnAocQ5rvQA7tcf6uMHfnXyylM50cTgkV9GuVMcxe3/+yKuJjXfNlCEu3Bbo\nOVvkUlbG6r1E8rTH+1lc7IPsJfcTuVs7U0QUCaja4MUYdpzJAkQQasjM5ZSNVwAb\n2hXulzruDLtoRZFEaM13yQshMiSpnxBkaPYl8BsPgXqSDcFIGjb3OBQmRGzp607g\nLa7S4j9MNwWrzoQVLEbz5x8Fya0FBOWsQBD8LLfvBDyBq6DJZbtcL6phm5GR8gp/\nh8lVGivN79yy2mPNZrCx4tGYcQiBq9U/dJJueAMxMQKBgQDxsFfovLIv9FgChGUV\ndCI/ZcNoTFrNvSv8D3CYhaxLLaQE0Ee+1c8tGupUuFPEpCnrt1HU5DUi/+hVm1Es\nXa3bhgnjj4ABukhg5trqlLMxkAX1uknmSA+6KcV8u3BoS2JtueD5IVxAHMlD0HSZ\nmpRFkQyFcv9Prv/o4TrAwcP7sQKBgQDiwn/ote0bbVEokZxb0Ce8aIKzOULfqfH2\n9VS6aUlY1ZfBcE7KF1JhzrOaLn85mnGt3DNTzxjGiL5PGW5x3nOVnzvwawERkxzq\nOW09XrsWLplBvG1SLI/GRAhiwEJmafA2z5HYj4jab6B/DvDgc20rUAJvKVoUj6PU\nWvc/LsrRzwKBgQC1csDs+A2GtxkD+sWxD5lOo2XK/dgGMgm9mRHdUC4D2uYSvxO+\nD+MUZ+qZZDFvphfa4axL+nByMFILQynz9vi5oK337BocMfB435hnGPBXO1teGle0\nzVERYJ7lgAtqIX5qBm0CXKtXbsUjZnLDhyvd9oHCBo3rEuUJv15OrKooYQKBgQDU\neEhfZS9w9oIIED/Aq46/8/EbO3kUl2lX69HjBMosCZ6zKqc4ppeTe5k/y3d8IuGH\ndX9GdRMS24fuF7crzLLirBv+jlSnAgAi24Im8b887pa9SG/qgkSJErAEz36n9XWV\n9fIDR6KEgfmRdA9xT9YnskVFoSp9f4WTcVSgCBkUcwKBgQDTcOyFNl/T/2bMqPo4\n1zriaG0HReus1SatcqpVtStfmyhCzM3KVL9cKy4mgJJLi6+J/G/0Mv+WALcMmZ2O\nWzZ6hLVYkZJj3IUtVPGxF//+DcI9g9hE+JnyNfYCW4zWdg/ZIoUGC5/gJ5lj59yp\nrd02LqhZzHIq4tw7jWqL31mF5A==\n-----END PRIVATE KEY-----\n",
+  "client_email": "bucketacc@tidy-resolver-411707.iam.gserviceaccount.com",
+  "client_id": "115681490178017892115",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/bucketacc%40tidy-resolver-411707.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}