Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.github/workflows/main.yml

@@ -0,0 +1,26 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: LFS Install
+        run: git lfs install
+      - name: LFS Track
+        run: git lfs track *.pdf
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push --force https://fadliaulawi:$HF_TOKEN@huggingface.co/spaces/KalbeDigitalLab/nutrigenme-paper-extractor main

.gitignore

@@ -0,0 +1,6 @@
+__pycache__
+.env
+.vscode
+resources/images/
+resources/papers/
+result/

Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.9
+
+RUN apt update && \ 
+    apt install -y bash \
+                   poppler-utils \
+                   tesseract-ocr \
+                   libtesseract-dev \
+                   build-essential \
+                   git \
+                   curl \
+                   ca-certificates \
+                   python3 \
+                   python3-pip && \
+    rm -rf /var/lib/apt/lists
+
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+RUN [ "python", "-c", "import nltk; nltk.download('punkt')" ]
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+COPY . .
+
+CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0", "--server.enableXsrfProtection=false"]

README.md

@@ -0,0 +1,62 @@
+---
+title: NutriGenMe PaperExtractor
+emoji: 📄
+colorFrom: green
+colorTo: blue
+sdk: docker
+pinned: false
+license: apache-2.0
+app_port: 8501
+---
+
+# NutriGenMe Paper Extractor
+
+## Overview
+The NutriGenMe Paper Extractor is a tool designed to extract relevant information from genomic papers related to the NutriGenMe project. It utilizes natural language processing techniques to parse through documents and extract key data points, enabling researchers and practitioners to efficiently gather insights from a large corpus of literature.
+
+## Features
+- **Automated Extraction**: Extracts various entities, such as title, authors, and conclusion of the study, from academic papers automatically.
+- **Fast Extraction**: Capable of extracting information from complex papers in under 10 minutes.
+- **Table Extraction**: Extracts values from tables, particularly focusing on gene names, SNPs, and associated diseases.
+- **Export to Excel**: Export extraction results to Excel format for easy integration and further analysis.
+
+## Usage
+1. Clone this repository:
+```bash
+git clone https://github.com/KalbeDigitalLab/nutrigenme-paper-extractor
+```
+
+2. Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+3. Prepare environment keys:
+```
+# Credentials for GPT-4 Model
+OPENAI_API_KEY=<openai_api_key>
+
+# (Optional) Tracking your extraction process with LangSmith
+LANGCHAIN_TRACING_V2='true'
+LANGCHAIN_API_KEY=<langchain_api_key>
+LANGCHAIN_ENDPOINT='https://api.smith.langchain.com'
+LANGCHAIN_PROJECT=<project_name>
+```
+4. Run the application with `streamlit`:
+```bash
+streamlit run app.py
+```
+
+This program is also already deployed in 🤗HuggingFace [Space](https://huggingface.co/spaces/KalbeDigitalLab/nutrigenme-paper-extractor/).
+
+## Documentation
+**app.py**: Designs the user interface and guides the application flow, calling on other scripts for specific tasks.
+
+**process.py**: Orchestrates the information extraction by delegating tasks to other scripts and handling the overall workflow.
+
+**prompt.py**: Stores prompts crafted for Large Language Models (LLMs) to target specific information during extraction.
+
+**table_detector.py**: Focuses on extracting info from Optical Character Recognition (OCR) tables, using functions to detect and process them.
+
+## Contributing
+Contributions are welcome! If you'd like to contribute to this project, feel free to create pull requests.

app.py

@@ -0,0 +1,131 @@
+import io
+import os
+import pandas as pd
+import streamlit as st
+
+from datetime import datetime
+from langchain_community.document_loaders.pdf import PyPDFLoader
+from langchain_core.documents.base import Document
+from langchain_text_splitters import TokenTextSplitter
+from process import get_entity, get_entity_one, get_table, validate
+from tempfile import NamedTemporaryFile
+from stqdm import stqdm
+from threading import Thread
+
+class CustomThread(Thread):
+    def __init__(self, func, chunk):
+        super().__init__()
+        self.func = func
+        self.chunk = chunk
+        self.result = ''
+
+    def run(self):
+        self.result = self.func(self.chunk)
+
+buffer = io.BytesIO()
+
+st.cache_data()
+st.set_page_config(page_title="NutriGenMe Paper Extractor")
+st.title("NutriGenMe - Paper Extraction")
+st.markdown("<div style='text-align: left; color: white; font-size: 16px'>In its latest version, the app is equipped to extract essential information from papers, including tables in both horizontal and vertical orientations, images, and text exclusively.</div><br>", unsafe_allow_html=True)
+
+uploaded_files = st.file_uploader("Upload Paper(s) here :", type="pdf", accept_multiple_files=True)
+
+chunk_option = st.selectbox(
+    'Tokens amounts per process :',
+    (32000, 16000, 8000, 0), key='table_hv'
+)
+chunk_overlap = 0
+
+if uploaded_files:
+    journals = []
+    parseButtonHV = st.button("Get Result", key='table_HV')
+
+    if parseButtonHV:
+        with st.status("Extraction in progress ...", expanded=True) as status:
+            start_time = datetime.now()
+
+            csv = pd.DataFrame()
+            for uploaded_file in stqdm(uploaded_files):
+                with NamedTemporaryFile(dir='.', suffix=".pdf", delete=eval(os.getenv('DELETE_TEMP_PDF', 'True'))) as pdf:
+                    pdf.write(uploaded_file.getbuffer())
+                    loader = PyPDFLoader(pdf.name)
+                    pages = loader.load()
+
+                    chunk_size = 120000
+                    chunk_overlap = 0
+                    docs = pages
+
+                    if chunk_option:
+                        docs = [Document('\n'.join([page.page_content for page in pages]))]
+                        docs[0].metadata = {'source': pages[0].metadata['source']}
+
+                        chunk_size = chunk_option
+                        chunk_overlap = int(0.25 * chunk_size)
+
+                    text_splitter = TokenTextSplitter.from_tiktoken_encoder(
+                        chunk_size=chunk_size, chunk_overlap=chunk_overlap
+                    )
+                    chunks = text_splitter.split_documents(docs)
+
+                    threads = []
+                    threads.append(CustomThread(get_entity, (chunks, 'gsd')))
+                    threads.append(CustomThread(get_entity, (chunks, 'summ')))
+                    threads.append(CustomThread(get_entity, (chunks, 'all')))
+                    threads.append(CustomThread(get_entity_one, [c.page_content for c in chunks[:1]]))
+                    threads.append(CustomThread(get_table, pdf.name))
+
+                    [t.start() for t in threads]
+                    [t.join() for t in threads]
+                    
+                    result_gsd = threads[0].result
+                    result_summ = threads[1].result
+                    result = threads[2].result
+                    result_one = threads[3].result
+                    res_gene, res_snp, res_dis = threads[4].result
+
+                    # Combine
+                    result['Genes'] = res_gene + result_gsd['Genes']
+                    result['SNPs'] = res_snp + result_gsd['SNPs']
+                    result['Diseases'] = res_dis + result_gsd['Diseases']
+                    result['Conclusion'] = result_summ
+                    for k in result_one.keys():
+                        result[k] = result_one[k]
+
+                    if len(result['Genes']) == 0:
+                        result['Genes'] = ['']
+                    
+                    num_rows = max(max(len(result['Genes']), len(result['SNPs'])), len(result['Diseases']))
+
+                    # Adjust Genes, SNPs, Diseases
+                    for k in ['Genes', 'SNPs', 'Diseases']:
+                        while len(result[k]) < num_rows:
+                            result[k].append('')
+
+                        # Temporary handling
+                        result[k] = result[k][:num_rows]
+
+                    # Key Column
+                    result = {key: value if isinstance(value, list) else [value] * num_rows for key, value in result.items()}
+
+                    dataframe = pd.DataFrame(result)
+                    dataframe = dataframe[['Genes', 'SNPs', 'Diseases', 'Title', 'Authors', 'Publisher Name', 'Publication Year', 'Population', 'Sample Size', 'Study Methodology', 'Study Level', 'Conclusion']]
+                    dataframe.drop_duplicates(['Genes', 'SNPs'], inplace=True)
+                    dataframe.reset_index(drop=True, inplace=True)
+                    cleaned_dataframe = validate(dataframe)
+
+                    end_time = datetime.now()
+                    st.write("Success in ", round((end_time.timestamp() - start_time.timestamp()) / 60, 2), "minutes")
+
+                    st.dataframe(cleaned_dataframe)
+                    with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer:
+                        cleaned_dataframe.to_excel(writer, sheet_name='Result')
+                        dataframe.to_excel(writer, sheet_name='Original')
+                        writer.close()
+
+                    st.download_button(
+                        label="Save Result",
+                        data=buffer,
+                        file_name=f"{uploaded_file.name.replace('.pdf', '')}_{chunk_option}.xlsx",
+                        mime='application/vnd.ms-excel'
+                    )

process.py

@@ -0,0 +1,222 @@
+from datetime import datetime
+from dotenv import load_dotenv
+from img2table.document import Image
+from langchain.chains.combine_documents.map_reduce import MapReduceDocumentsChain
+from langchain.chains.combine_documents.reduce import ReduceDocumentsChain
+from langchain.chains.combine_documents.stuff import StuffDocumentsChain
+from langchain.chains.llm import LLMChain
+from langchain.prompts import PromptTemplate
+from langchain_openai import ChatOpenAI
+from pdf2image import convert_from_path
+from prompt import prompt_entity_gsd_chunk, prompt_entity_gsd_combine, prompt_entity_summ_chunk, prompt_entity_summ_combine, prompt_entities_chunk, prompt_entities_combine, prompt_entity_one_chunk, prompt_table
+from table_detector import detection_transform, device, model, ocr, outputs_to_objects
+
+import io
+import json
+import os
+import pandas as pd
+import re
+import torch
+
+load_dotenv()
+
+llm = ChatOpenAI(temperature=0, model_name="gpt-4-0125-preview")
+llm_p = ChatOpenAI(temperature=0, api_key=os.environ['PERPLEXITY_API_KEY'], base_url="https://api.perplexity.ai")
+
+prompts = {
+    'gsd': [prompt_entity_gsd_chunk, prompt_entity_gsd_combine],
+    'summ': [prompt_entity_summ_chunk, prompt_entity_summ_combine],
+    'all': [prompt_entities_chunk, prompt_entities_combine]
+}
+
+def get_entity(data):
+
+    chunks, types = data
+
+    map_template = prompts[types][0]
+    map_prompt = PromptTemplate.from_template(map_template)
+    map_chain = LLMChain(llm=llm, prompt=map_prompt)
+
+    reduce_template = prompts[types][1]
+    reduce_prompt = PromptTemplate.from_template(reduce_template)
+    reduce_chain = LLMChain(llm=llm, prompt=reduce_prompt)
+
+    combine_chain = StuffDocumentsChain(
+        llm_chain=reduce_chain, document_variable_name="doc_summaries"
+    )
+
+    reduce_documents_chain = ReduceDocumentsChain(
+        combine_documents_chain=combine_chain,
+        collapse_documents_chain=combine_chain,
+        token_max=100000,
+    )
+
+    map_reduce_chain = MapReduceDocumentsChain(
+        llm_chain=map_chain,
+        reduce_documents_chain=reduce_documents_chain,
+        document_variable_name="docs",
+        return_intermediate_steps=False,
+    )
+
+    result = map_reduce_chain.invoke(chunks)['output_text']
+    print(types)
+    print(result)
+    if types != 'summ':
+        result = re.findall('(\{[^}]+\})', result)[0]
+        return eval(result)
+    
+    return result
+
+def get_entity_one(chunks):
+
+    result = llm.invoke(prompt_entity_one_chunk.format(chunks)).content
+
+    print('One')
+    print(result)
+    result = re.findall('(\{[^}]+\})', result)[0]
+
+    return eval(result)
+
+def get_table(path):
+
+    start_time = datetime.now()
+    images = convert_from_path(path)
+    print('PDF to Image', round((datetime.now().timestamp() - start_time.timestamp()) / 60, 2), "minutes")
+    tables = []
+
+    # Loop pages
+    for image in images:
+
+        pixel_values = detection_transform(image).unsqueeze(0).to(device)
+        with torch.no_grad():
+            outputs = model(pixel_values)
+
+        id2label = model.config.id2label
+        id2label[len(model.config.id2label)] = "no object"
+        detected_tables = outputs_to_objects(outputs, image.size, id2label)
+
+        # Loop table in page (if any)
+        for idx in range(len(detected_tables)):
+            cropped_table = image.crop(detected_tables[idx]["bbox"])
+            if detected_tables[idx]["label"] == 'table rotated':
+                cropped_table = cropped_table.rotate(270, expand=True)
+
+            # TODO: what is the perfect threshold?
+            if detected_tables[idx]['score'] > 0.9:
+                print(detected_tables[idx])
+                tables.append(cropped_table)
+
+    print('Detect table from image', round((datetime.now().timestamp() - start_time.timestamp()) / 60, 2), "minutes")
+    genes = []
+    snps = []
+    diseases = []
+
+    # Loop tables
+    for table in tables:
+
+        buffer = io.BytesIO()
+        table.save(buffer, format='PNG')
+        image = Image(buffer)
+        
+        # Extract to dataframe
+        extracted_tables = image.extract_tables(ocr=ocr, implicit_rows=True, borderless_tables=True, min_confidence=0)
+
+        if len(extracted_tables) == 0:
+            continue
+
+        # Combine multiple dataframe
+        df_table = extracted_tables[0].df
+        for extracted_table in extracted_tables[1:]:
+            df_table = pd.concat([df_table, extracted_table.df]).reset_index(drop=True)
+
+        df_table.loc[0] = df_table.loc[0].fillna('')
+
+        # Identify multiple rows (in dataframe) as one row (in image)
+        rows = []
+        indexes = []
+        for i in df_table.index:
+            if not df_table.loc[i].isna().any():
+                if len(indexes) > 0:
+                    rows.append(indexes)
+                indexes = []
+            indexes.append(i)
+        rows.append(indexes)
+
+        df_table_cleaned = pd.DataFrame(columns=df_table.columns)
+        for row in rows:
+            row_str = df_table.loc[row[0]]
+            for idx in row[1:]:
+                row_str += ' ' + df_table.loc[idx].fillna('')
+            row_str = row_str.str.strip()
+            df_table_cleaned.loc[len(df_table_cleaned)] = row_str
+
+        # Ask LLM with JSON data
+        json_table = df_table_cleaned.to_json(orient='records')
+        str_json_table = json.dumps(json.loads(json_table), indent=2)
+
+        result = llm.invoke(prompt_table.format(str_json_table)).content
+        print('table')
+        print(result)
+        result = result[result.find('['):result.rfind(']')+1]
+        try:
+            result = eval(result)
+        except SyntaxError:
+            result = []
+
+        for res in result:
+            res_gene = res['Genes']
+            res_snp = res['SNPs']
+            res_disease = res['Diseases']
+
+            for snp in res_snp:
+                genes.append(res_gene)
+                snps.append(snp)
+                diseases.append(res_disease)
+
+    print('OCR table to extract', round((datetime.now().timestamp() - start_time.timestamp()) / 60, 2), "minutes")
+    print(genes, snps, diseases)
+    
+    return genes, snps, diseases
+
+def validate(df):
+
+    df = df.fillna('')
+    df['Genes'] = df['Genes'].str.upper()
+    df['SNPs'] = df['SNPs'].str.lower()
+
+    # Check if there is two gene names
+    sym = ['-', '/', '|']
+    for i in df.index:
+        gene = df.loc[i, 'Genes']
+        for s in sym:
+            if s in gene:
+                genes = gene.split(s)
+                df.loc[len(df)] = df.loc[i]
+                df.loc[i, 'Genes'] = genes[0]
+                df.loc[len(df) - 1, 'Genes'] = genes[1]
+
+    # Check if there is SNPs without 'rs'
+    for i in df.index:
+        safe = True
+        snp = df.loc[i, 'SNPs']
+        if not re.fullmatch('rs(\d)+|', snp):
+            if not re.fullmatch('s(\d)+', snp):
+                if not re.fullmatch('(\d)+', snp):
+                    safe = False
+                    df = df.drop(i)
+                else:
+                    snp = 'rs' + snp
+            else:
+                snp = 'r' + snp
+
+        if safe:
+            df.loc[i, 'SNPs'] = snp
+
+    df.reset_index(drop=True, inplace=True)
+
+    # TODO: How to validate genes and SNPs?
+
+    # TODO: Validate genes and diseases with LLM
+    result = llm_p.invoke(model='mistral-7b-instruct', input='How many stars?')
+
+    return df

prompt.py

@@ -0,0 +1,289 @@
+prompt_entity_gsd_chunk = """
+# CONTEXT #
+In my capacity as a genomics specialist, I have recently completed the review of a scholarly publication. I am interested in extracting specific genomic information, or entities, from the body of the paper. 
+To facilitate this process, I have constructed a predefined schema that outlines the desired entities and their corresponding description.
+
+This is the schema provided:
+
+{{
+    "Genes" : {{
+        "type" : "list of strings",
+        "description" : "All relevant genes mentioned in the text. Gene names can only contain uppercase letters and digits."
+    }},
+    "SNPs" : {{
+        "type" : "list of strings",
+        "description" : "Unique identifier associated with each value in Genes schema. These identifiers typically begin with 'rs' and appear near the gene name in the text."
+    }},
+    "Diseases" : {{
+        "type" : "list of strings",
+        "description" : "Type of diseases that related to each value in Genes, typically appear near the gene name in the text."
+    }}
+}}
+
+Note that the values within each list of Genes, SNPs, and Diseases columns correspond directly to each other. Consequently, the lengths of these lists must be identical. 
+
+This is a passage from the paper: {docs} 
+
+# OBJECTIVE #
+Given a predefined schema outlining relevant entities, meticulously extract these entities from the provided text passage. Exercise caution when handling the reference section of the document, abstain from extracting information from this section. 
+IMPORTANT: It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility. If an entity is entirely absent from the passage, just leave the corresponding field blank with an empty string ('').
+
+# RESPONSE #
+The extracted information will be utilizing the JSON format. Information within strings will be demarcated by double quotes (" "), while quotes contained within strings will be denoted by single quotes (' '). List data will be enclosed within square brackets ([]).
+This is the example of the respose:
+
+{{
+    "Genes": ["A", "B", "C"],
+    "SNPs": ["rs1", "rs2", "rs3"],
+    "Diseases": ["X", "Y", "Z"]
+}}
+
+If there is no specific extracted entities, just leave the corresponding field blank with an empty lists ([]).
+"""
+
+prompt_entity_gsd_combine = """
+# CONTEXT #
+In my role as a genomics specialist, I have extracted specific entities from a scholarly publication. These entities were identified and retrieved from various sections throughout the document. My current objective is to consolidate this extracted information into a concise summary, facilitating a comprehensive understanding of the publication's key findings. 
+To achieve this, I have constructed a predefined schema that outlines the desired entities and the methods for their summarization.
+
+{{
+    "Genes" : {{
+        "type" : "list of strings",
+        "description" : "Identify the most relevant gene from the compiled gene list across all sections."
+    }},
+    "SNPs" : {{
+        "type" : "list of strings",
+        "description" : "Upon completion of the gene combination process, associate each resulting value with its unique identifier."
+    }},
+    "Diseases" : {{
+        "type" : "list of strings",
+        "description" : "Upon completion of the gene combination process, associate each resulting value with its diseases."
+    }}
+}}
+
+This is a set of summaries: {doc_summaries} 
+
+If there is no extracted entities, just leave the corresponding result with an empty list ([]) later.
+
+# OBJECTIVE #
+In the context of a predefined schema that specifies entities and their corresponding operations, construct a comprehensive synopsis that incorporates all critical details gleaned from each section. 
+IMPORTANT: It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility.
+
+# RESPONSE #
+The extracted information will be utilizing the JSON format. Information within strings will be demarcated by double quotes (" "), while quotes contained within strings will be denoted by single quotes (' '). List data will be enclosed within square brackets ([]).
+This is the example of the respose:
+
+{{
+    "Genes": ["A", "B", "C"],
+    "SNPs": ["rs1", "rs2", "rs3"],
+    "Diseases": ["X", "Y", "Z"]
+}}
+"""
+
+prompt_entity_summ_chunk = """
+# CONTEXT #
+In my capacity as a genomics specialist, I have recently completed the review of a scholarly publication. I am interested in extracting the summary from the body of the paper.
+
+This is a passage from the paper: {docs} 
+
+# OBJECTIVE #
+Extract the summary or the conclusion from the provided text passage. Exercise caution when handling the reference section of the document, abstain from extracting information from this section. 
+IMPORTANT: It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility.
+
+# RESPONSE #
+Provide the information in a concise way, using four concise paragraphs. The text should be presented in a continuous format, omitting introductory elements like numbers or titles within each paragraph.
+
+1. Overview. Explanation of the provided documents and their exploration of the genetic underpinnings of the disease, and understanding of genetic factors in disease pathology.
+2. Main Themes. Identification of genetic variants and mutations contributing to disease susceptibility. Role of specific genes and genetic pathways in disease development and progression.
+3. Key Genetic Factors and Their Implications. Highlighting specific genes or genetic variants associated with the disease. Discussion of how these genetic factors may influence disease susceptibility, severity, or treatment response.
+4. Conclusion. Recap of the key findings regarding genetic factors and disease mechanisms. Suggestions for future research directions or clinical applications based on the insights gained from genetic analysis.
+"""
+
+prompt_entity_summ_combine = """
+# CONTEXT #
+In my role as a genomics specialist, I have extracted some summaries from a scholarly publication. These summaries were identified and retrieved from various sections throughout the document. 
+My current objective is to consolidate this extracted information into a concise summary, facilitating a comprehensive understanding of the publication's key findings. 
+
+This is a set of summaries: {doc_summaries} 
+
+# OBJECTIVE #
+Construct a comprehensive synopsis that incorporates all critical details gleaned from the summaries of each section. 
+IMPORTANT: It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility.
+
+# RESPONSE #
+Provide the information in a concise way, using four concise paragraphs. The text should be presented in a continuous format, omitting introductory elements like numbers or titles within each paragraph.. 
+
+1. Overview. Explanation of the provided documents and their exploration of the genetic underpinnings of the disease, and understanding of genetic factors in disease pathology.
+2. Main Themes. Identification of genetic variants and mutations contributing to disease susceptibility. Role of specific genes and genetic pathways in disease development and progression.
+3. Key Genetic Factors and Their Implications. Highlighting specific genes or genetic variants associated with the disease. Discussion of how these genetic factors may influence disease susceptibility, severity, or treatment response.
+4. Conclusion. Recap of the key findings regarding genetic factors and disease mechanisms. Suggestions for future research directions or clinical applications based on the insights gained from genetic analysis.
+"""
+
+prompt_entities_chunk = """
+# CONTEXT #
+In my capacity as a genomics specialist, I have recently completed the review of a scholarly publication. I am interested in extracting specific genomic information, or entities, from the body of the paper. 
+To facilitate this process, I have constructed a predefined schema that outlines the desired entities and their corresponding description.
+
+This is the schema provided:
+
+{{
+    "Population" : {{
+        "type" : "string",
+        "description" : "Population / race used by the author in the given text."
+    }},
+    "Sample Size" : {{
+        "type" : "string",
+        "description" : "Sample size of the population used in the research that mentioned in the paper."
+    }},
+    "Study Methodology" : {{
+        "type" : "string",
+        "description" : "Study methodology mentioned in the text."
+    }},
+    "Study Level" : {{
+        "type" : "string",
+        "description" : "Study level mentioned in the text."
+    }}
+}}
+
+This is a passage from the paper: {docs} 
+
+# OBJECTIVE #
+Given a predefined schema outlining relevant entities, meticulously extract these entities from the provided text passage. Exercise caution when handling the reference section of the document, abstain from extracting information from this section. 
+IMPORTANT: It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility. If an entity is entirely absent from the passage, just leave the corresponding field blank with an empty string ('').
+
+# RESPONSE #
+The extracted information will be utilizing the JSON format. Information within strings will be demarcated by double quotes (" "), while quotes contained within strings will be denoted by single quotes (' ').
+This is the example of the respose:
+
+{{
+    "Population": "South Asian",
+    "Sample Size": "403 Relatively Small",
+    "Study Methodology": "Double-Blind Randomized Controlled Trial",
+    "Study Level": "Postdoctoral"
+}}
+
+If there is no specific extracted entities, just leave the corresponding field blank with an empty string ("").
+"""
+
+prompt_entities_combine = """
+# CONTEXT #
+In my role as a genomics specialist, I have extracted some summaries from a scholarly publication. These summaries were identified and retrieved from various sections throughout the document. My current objective is to consolidate this extracted information into a concise summary, facilitating a comprehensive understanding of the publication's key findings. 
+
+This is a set of summaries: {doc_summaries} 
+
+If there is no extracted entities, just leave the corresponding result with an empty string ("") later.
+
+# OBJECTIVE #
+Construct a comprehensive synopsis that incorporates all critical details gleaned from the summaries of each section. 
+IMPORTANT: It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility.
+
+# RESPONSE #
+The extracted information will be utilizing the JSON format. Information within strings will be demarcated by double quotes (" "), while quotes contained within strings will be denoted by single quotes (' ').
+This is the example of the respose:
+
+{{
+    "Population": "South Asian",
+    "Sample Size": "403 Relatively Small",
+    "Study Methodology": "Double-Blind Randomized Controlled Trial",
+    "Study Level": "Postdoctoral"
+}}
+"""
+
+prompt_entity_one_chunk = """
+# CONTEXT #
+In my capacity as a genomics specialist, I have recently completed the review of a scholarly publication. I am interested in extracting specific genomic information, or entities, from the body of the paper. To facilitate this process, I have constructed a predefined schema that outlines the desired entities and their corresponding description.
+
+This is the schema provided:
+
+{{
+    "Title" : {{
+        "type" : "string",
+        "description" : "Title of the given text."
+    }},
+    "Authors" : {{
+        "type" : "string",
+        "description" : "Authors / writers of the given text. To maintain readability, consider only the first 10 author names, ensuring the other key informations and response format are clear."
+    }},
+    "Publisher Name" : {{
+        "type" : "string",
+        "description" : "Publisher name of the given text."
+    }},
+    "Publication Year" : {{
+        "type" : "string",
+        "description" : "The year when the given text published."
+    }},
+}}
+
+This is a passage from the paper: {} 
+
+# OBJECTIVE #
+Given a predefined schema outlining relevant entities, meticulously extract these entities from the provided text passage. 
+IMPORTANT: It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility. If an entity is entirely absent from the passage, just leave the corresponding field blank with an empty string ('').
+
+# RESPONSE #
+The extracted information will be utilizing the JSON format. Information within strings will be demarcated by double quotes (" "), while quotes contained within strings will be denoted by single quotes (' ').
+This is the example of the respose:
+
+{{
+    "Title": "Lorem Ipsum",
+    "Authors": "John Doe, Jane Doe, Alias",
+    "Publisher Name": "Journal of Internal Medicine",
+    "Publication Year": "2024"
+}}
+
+If there is no specific extracted entities, just leave the corresponding field blank with an empty string ("").
+"""
+
+prompt_table = """
+# CONTEXT #
+In my capacity as a genomics specialist, I have table data obtained from a published research paper in the field of genomics. The data is provided in a list of JSONs format, with each JSON object representing a single row in a tabular structure. The first JSON element in the list represents the header row of the table, containing the names of each column.
+This is the data:
+{}
+
+# OBJECTIVE #
+Given the provided table data, the following tasks need to be completed:
+
+1. Identify all unique gene names present within the table. Each row can contains more than one gene name.
+2. If present, extract any entries starting with "rs" (presumably representing Single Nucleotide Polymorphisms or rsIDs) that correspond to the same row as their associated gene names. Each gene name can correspond with more than one SNPs.
+3. If available, extract any disease information associated with both the gene name and its corresponding SNP/rsID.
+
+It is crucial to maintain the utmost accuracy in this process, as any false or fabricated information (hallucination) can have severe consequences for academic integrity and research credibility.
+If an SNPs or Diseases is absent from the table, leave the corresponding field blank with an empty string ('').
+
+# RESPONSE #
+The output should only be a string containing list of JSON objects, each representing an entry with the following structure:
+[
+    {{
+        "Genes": "A",
+        "SNPs": ["rs123", "rs456"],
+        "Diseases": "A disease"
+    }}
+]
+
+If there is no specific extracted entities provided from the table, just leave the response with an empty lists ([]).
+"""
+
+prompt_validation = """
+# CONTEXT #
+In my capacity as a genomics specialist, I have table data containing gene names with its corresponding SNPs and diseases. The data is provided in a list of JSONs format, with each JSON object representing a single row in a tabular structure. 
+The problem is because the data is extracetd using OCR, some gene names and SNPs maybe have a typo.
+
+This is the data:
+{}
+
+# OBJECTIVE #
+Given the provided table data, the following tasks need to be completed:
+
+1. Check whether the gene name is a correct gene name. If the gene name is suspected to be a typo, fix it into a correct form. If not, eliminate this row data because the gene name is not valid. 
+2. If diseases not empty, check whether the gene name is correspond with the gene names. Fix it with the correct diseases if the original disease is wrong.
+
+# RESPONSE #
+The output should only be a string containing list of JSON objects, each representing an validated entry with the following structure:
+[
+    {{
+        "Genes": "A",
+        "SNPs": "rs123",
+        "Diseases": "A disease"
+    }}
+]
+"""

requirements.txt

@@ -0,0 +1,21 @@
+pikepdf
+stqdm
+pdf2image
+nltk
+pandas
+streamlit
+xlsxwriter
+openai
+biopython
+langchain
+pypdf
+tiktoken
+pillow-heif
+torchvision
+transformers
+python-dotenv
+rapidocr-onnxruntime
+langchain-openai
+img2table
+timm
+python-doctr

resources/experiment.py

@@ -0,0 +1,64 @@
+import pandas as pd
+import base64
+
+from img2table.document import Image
+from img2table.ocr import DocTR
+from langchain.schema.messages import HumanMessage, AIMessage
+from langchain_experimental.agents import create_pandas_dataframe_agent
+from langchain_openai import ChatOpenAI
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+path = '../NutriGenMe-Testing/ukmss-1.png'
+
+vision = ChatOpenAI(model="gpt-4-vision-preview", max_tokens=4096)
+
+img = encode_image(path)
+msg = vision.invoke(
+    [
+        AIMessage(content="You are an experienced doctor specializing in genomics and want to identify names of genes, SNPs, and their related diseases based on the tables given."),
+        HumanMessage(
+            content=[
+                {   "type": "text", 
+                    "text": 'You will be provided with the image of a table. Extract all genes / locus names with its respective rsID / SNP and potential diseases in curly brackets like this: {"Genes" : "", "SNPs" : "", "Diseases" : ""}.'
+                },
+                {   
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{img}",
+                        "detail": "low"
+                    },
+                },
+            ]
+        )
+    ]
+)
+
+print(msg.content)
+
+# exit()
+
+image = Image(path)
+ocr = DocTR()
+
+extracted_tables = image.extract_tables(ocr=ocr,
+                                      implicit_rows=True,
+                                      borderless_tables=True,
+                                      min_confidence=0)
+
+df = extracted_tables[0].df
+for et in extracted_tables[1:]:
+    df = pd.concat([df, et.df]).reset_index(drop=True)
+
+print(df)
+
+llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
+
+agent = create_pandas_dataframe_agent(llm, df, verbose=True)
+agent_output = agent.invoke("Is this table contain Gene names?")
+print(agent_output)

table_detector.py

@@ -0,0 +1,68 @@
+from img2table.ocr import DocTR
+from torchvision import transforms
+from transformers import AutoModelForObjectDetection
+
+import torch
+
+def box_cxcywh_to_xyxy(x):
+    x_c, y_c, w, h = x.unbind(-1)
+    b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)]
+    return torch.stack(b, dim=1)
+
+def rescale_bboxes(out_bbox, size):
+    width, height = size
+    boxes = box_cxcywh_to_xyxy(out_bbox)
+    boxes = boxes * torch.tensor(
+        [width, height, width, height], dtype=torch.float32
+    )
+    return boxes
+
+def outputs_to_objects(outputs, img_size, id2label):
+    m = outputs.logits.softmax(-1).max(-1)
+    pred_labels = list(m.indices.detach().cpu().numpy())[0]
+    pred_scores = list(m.values.detach().cpu().numpy())[0]
+    pred_bboxes = outputs["pred_boxes"].detach().cpu()[0]
+    pred_bboxes = [
+        elem.tolist() for elem in rescale_bboxes(pred_bboxes, img_size)
+    ]
+
+    objects = []
+    for label, score, bbox in zip(pred_labels, pred_scores, pred_bboxes):
+        class_label = id2label[int(label)]
+        if not class_label == "no object":
+            objects.append(
+                {
+                    "label": class_label,
+                    "score": float(score),
+                    "bbox": [float(elem) for elem in bbox],
+                }
+            )
+
+    return objects
+
+class MaxResize(object):
+    def __init__(self, max_size=800):
+        self.max_size = max_size
+
+    def __call__(self, image):
+        width, height = image.size
+        current_max_size = max(width, height)
+        scale = self.max_size / current_max_size
+        resized_image = image.resize(
+            (int(round(scale * width)), int(round(scale * height)))
+        )
+
+        return resized_image
+
+detection_transform = transforms.Compose(
+    [
+        MaxResize(800),
+        transforms.ToTensor(),
+        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+    ]
+)
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = AutoModelForObjectDetection.from_pretrained("microsoft/table-transformer-detection", revision="no_timm").to(device)
+
+ocr = DocTR()