Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tables_folder/MATSA_fetaqa.json filter=lfs diff=lfs merge=lfs -text
+wkhtmltox_0.12.6-1.bionic_amd64.deb filter=lfs diff=lfs merge=lfs -text

.github/workflows/update_space.yml

@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy

.ipynb_checkpoints/config_gpt35-checkpoint.yaml

@@ -0,0 +1,3 @@
+API_BASE: https://dil-research.openai.azure.com/
+API_KEY: de6e495251174ceab84f290cd3925b07
+API_VERSION: '2023-05-15'

.ipynb_checkpoints/config_gpt4-checkpoint.yaml

@@ -0,0 +1,3 @@
+API_BASE: https://dil-research-sweden-central.openai.azure.com/
+API_KEY: 0df7237e0fc243a8830a23b2dbba2dcb
+API_VERSION: "2023-12-01-preview"

.ipynb_checkpoints/demo-checkpoint.py

@@ -0,0 +1,221 @@
+import gradio as gr
+import json
+from bs4 import BeautifulSoup
+from matsa import MATSA, InputInstance
+import imgkit
+import tempfile
+import time
+import threading
+
+TABLE_FOLDER = "./tables_folder/MATSA_fetaqa.json"
+# Load data from JSON file
+def load_data():
+    with open(TABLE_FOLDER, 'r') as json_file:
+        return json.load(json_file)
+
+# Global variable to store the loaded data
+TABLE_DATA = load_data()
+
+def get_table_names():
+    return [f"tab_{i+1}" for i in range(len(TABLE_DATA))]
+
+def html_to_image(html_content):
+    with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as temp_img:
+        imgkit.from_string(html_content, temp_img.name)
+        return temp_img.name
+
+def highlight_table(html_table, row_ids, col_ids):
+    soup = BeautifulSoup(html_table, 'html.parser')
+    row_sel = []
+
+    # Highlight rows
+    for row_id in row_ids:
+        row = soup.find('tr', id=row_id)
+        if row:
+            row_sel.append(row)
+
+    for col_id in col_ids:
+        col_index = int(col_id.split('-')[1]) - 1  # Convert col-1 to index 0, col-2 to index 1, etc.
+        for row in soup.find_all('tr'):
+            cells = row.find_all(['td', 'th'])
+            if row in row_sel:
+                if col_index < len(cells):
+                    cells[col_index]['style'] = 'background-color: rgba(173, 216, 230, 0.7);'
+            else:
+                if col_index < len(cells):
+                    cells[col_index]['style'] = 'background-color: rgba(211, 211, 211, 0.6);'  
+
+    return str(soup)
+
+def load_table_data(table_name):
+    if not table_name:
+        return None, "", ""
+    index = int(table_name.split('_')[1]) - 1
+    data = TABLE_DATA[index]
+    
+    html_content = data['html_table']
+    question = data.get("question", "") #data['question'] 
+    if question is None:
+        question = ""
+    answer = data['answer_statement']
+    
+    image_path = html_to_image(html_content)
+    
+    return image_path, question, answer
+
+def process_input(table_name, question, answer):
+    if not table_name:
+        return "Please select a table from the dropdown."
+    
+    # Get the data for the selected table
+    index = int(table_name.split('_')[1]) - 1
+    data = TABLE_DATA[index]
+    
+    html_content = data['html_table']
+
+    print("html_content: ", html_content)
+    print("question: ", question)
+    print("answer: ", answer)
+    
+    # Initialize MATSA
+    matsa_agent = MATSA()
+    
+    # Create input instance
+    instance = InputInstance(html_table=html_content, question=question, answer=answer)
+    
+    # Apply MATSA pipeline
+    # formatted_table = matsa_agent.table_formatting_agent(instance.html_table)
+    augmented_table = matsa_agent.description_augmentation_agent(instance.html_table)
+    print("augmented_table: ", augmented_table)
+    fact_list = matsa_agent.answer_decomposition_agent(instance.answer)
+    print("fact_list: ", fact_list)
+    attributed_table, _, _ = matsa_agent.semantic_retreival_agent(augmented_table, fact_list)
+    print("attributed_table: ", attributed_table)
+    attribution_fxn = matsa_agent.sufficiency_attribution_agent(fact_list, attributed_table)
+    print("attribution_fxn: ", attribution_fxn)
+    
+    # Get row and column attributions
+    row_attribution_set = attribution_fxn["Row Citations"]
+    col_attribution_set = attribution_fxn["Column Citations"]
+    explnation = attribution_fxn.get("Explanation", "")
+    print("row_attribution_set: ", row_attribution_set)
+    print("col_attribution_set: ", col_attribution_set)
+    print("Explanation: ", attribution_fxn.get("Explanation", ""))
+    
+    # Convert string representations to lists
+    if isinstance(row_attribution_set, str):
+        row_ids = eval(row_attribution_set)
+    else:
+        row_ids = row_attribution_set
+    
+    if isinstance(col_attribution_set, str):
+        col_ids = eval(col_attribution_set)
+    else:
+        col_ids = col_attribution_set
+    
+    # Highlight the table
+    highlighted_table = highlight_table(instance.html_table, row_ids, col_ids)
+    
+    result = {
+        "highlighted_table": highlighted_table,
+        "facts": attribution_fxn.get("List of Facts", []),
+        "row_citations": row_attribution_set,
+        "column_citations": col_attribution_set,
+        "Explanation": explnation
+    }
+    
+    return json.dumps(result)
+
+# Define Gradio interface
+with gr.Blocks() as iface:
+    gr.Markdown("# MATSA: Table Question Answering with Attribution")
+    gr.Markdown("Select a table from dropdown load table image, question, and answer.")
+    gr.Markdown("Attributions are provided as per answer. You may change the question/answer as per your need.")
+    
+    table_dropdown = gr.Dropdown(choices=get_table_names(), label="Select Table")
+    original_table = gr.Image(type="filepath", label="Original Table")
+    question_box = gr.Textbox(label="Question")
+    answer_box = gr.Textbox(label="Answer")
+    
+    gr.Markdown("Click 'Process' to see the highlighted relevant parts. Click 'Reset' to start over.")
+    
+    process_button = gr.Button("Process")
+    reset_button = gr.Button("Reset")
+    processing_time = gr.Textbox(label="Processing Time", value="0 seconds")
+    highlighted_table = gr.HTML(label="Highlighted Table")
+    explanation_box = gr.Textbox(label="Explanation")
+
+    def update_table_data(table_name):
+        image_path, question, answer = load_table_data(table_name)
+        return image_path, question, answer, gr.update(interactive=True)
+    
+    def reset_app():
+        return (
+            gr.update(value="", interactive=True),  # table_dropdown
+            None,  # original_table
+            "",  # question_box
+            "",  # answer_box
+            "",  # highlighted_table
+            "",  # explanation_box
+            gr.update(interactive=True),  # process_button
+            "0 seconds",  # processing_time
+        )
+    
+    def process_and_disable(table_name, question, answer):
+        processing = True
+        counter = 0
+        
+        def update_counter():
+            nonlocal counter
+            while processing:
+                counter += 1
+                yield counter
+                time.sleep(1)
+        
+        counter_thread = threading.Thread(target=update_counter)
+        counter_thread.start()
+        
+        # Disable the dropdown and process button during processing
+        yield (
+            gr.update(interactive=False),  # table_dropdown
+            gr.update(interactive=False),  # process_button
+            gr.update(value="Processing..."),  # processing_time
+            gr.update(),  # highlighted_table
+            gr.update(),  # explanation_box
+        )
+        
+        # Process the input
+        result = process_input(table_name, question, answer)
+        result_dict = json.loads(result)
+        
+        # Stop the counter
+        processing = False
+        counter_thread.join()
+        
+        # Re-enable the dropdown and process button, update processing time, and return the result
+        yield (
+            gr.update(interactive=True),  # table_dropdown
+            gr.update(interactive=True),  # process_button
+            f"Processed in {counter} seconds",  # processing_time
+            gr.update(value=result_dict['highlighted_table']),  # highlighted_table
+            gr.update(value=result_dict.get('Explanation', '')),  # explanation_box
+            {
+                "Row Citations": result_dict['row_citations'],
+                "Column Citations": result_dict['column_citations']
+            }
+        )
+    
+    table_dropdown.change(update_table_data, 
+                          inputs=[table_dropdown], 
+                          outputs=[original_table, question_box, answer_box, process_button])
+    
+    process_button.click(process_and_disable, 
+                         inputs=[table_dropdown, question_box, answer_box], 
+                         outputs=[table_dropdown, process_button, processing_time, highlighted_table, explanation_box])
+    
+    reset_button.click(reset_app, 
+                       inputs=[], 
+                       outputs=[table_dropdown, original_table, question_box, answer_box, highlighted_table, explanation_box, process_button, processing_time])
+
+# Launch the interface
+iface.launch(share=True)

.ipynb_checkpoints/llm_query_api-checkpoint.py

@@ -0,0 +1,216 @@
+import base64
+import copy
+import io
+import os
+import random
+import time
+import re
+import json
+import argparse
+import yaml
+import openai
+from openai import AzureOpenAI
+from prompts import *
+import base64
+from mimetypes import guess_type
+# from img2table.document import Image
+# from img2table.document import PDF
+# from img2table.ocr import TesseractOCR
+# from img2table.ocr import EasyOCR
+from PIL import Image as PILImage
+
+class LLMQueryAPI:
+
+    def __init__(self) -> None:
+        pass
+
+    def gpt4_chat_completion(self, query):
+
+        with open('config_gpt4.yaml', 'r') as f:
+            config = yaml.safe_load(f)
+
+        API_KEY = config.get('API_KEY')
+        API_VERSION = config.get('API_VERSION')
+        API_BASE = config.get('API_BASE')
+
+        client = AzureOpenAI(
+            azure_endpoint= API_BASE,
+            api_version= API_VERSION,
+            api_key = API_KEY
+            )
+            
+        deployment_name='gpt-4-2024-04-09'
+
+        response = client.chat.completions.create(
+            model=deployment_name, 
+            messages=query)
+        
+        return response.choices[0].message.content
+
+    def gpt35_chat_completion(self, query):
+    
+        with open('config_gpt35.yaml', 'r') as f:
+            config = yaml.safe_load(f)
+
+        API_KEY = config.get('API_KEY')
+        API_VERSION = config.get('API_VERSION')
+        API_BASE = config.get('API_BASE')
+
+        response = openai.ChatCompletion.create(
+            engine='gpt-35-turbo-0613',
+            messages=query,
+            request_timeout=60,
+            api_key = API_KEY,
+            api_version = API_VERSION,
+            api_type = "azure",
+            api_base = API_BASE,
+        )
+
+        return response['choices'][0]['message']
+
+    def copilot_chat_completion(self, query):
+
+        with open('config_gpt4.yaml', 'r') as f:
+            config = yaml.safe_load(f)
+
+        API_KEY = config.get('API_KEY')
+        API_VERSION = config.get('API_VERSION')
+        API_BASE = config.get('API_BASE')
+
+        response = openai.ChatCompletion.create(
+            engine='gpt-4-0613',
+            messages=query,
+            request_timeout=60,
+            api_key = API_KEY,
+            api_version = API_VERSION,
+            api_type = "azure",
+            api_base = API_BASE,
+        )
+        return response['choices'][0]['message']
+
+    def LLM_chat_query(self, query, llm):
+
+        if llm == 'gpt-3.5-turbo':
+            return self.gpt35_chat_completion(query)
+        elif llm == "gpt-4":
+            return self.gpt4_chat_completion(query)
+            # return self.copilot_chat_completion(query)
+    
+    def get_llm_response(self, llm, query):
+        chat_completion = []
+        chat_completion.append({"role": "system", "content": query})
+        res = self.LLM_chat_query(chat_completion, llm)
+        return res
+
+class LLMProxyQueryAPI:
+
+    def __init__(self) -> None:
+        pass
+
+    def gpt35_chat_completion(self, query):
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-3.5-turbo-16k",
+            messages=query,
+        )
+        return response.choices[0].message.content
+    
+    def gpt4o_chat_completion(self, query):
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-4o",
+            messages=query,
+            )
+        return response.choices[0].message.content
+    
+    def gpt4_chat_completion(self, query):
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-4-1106-preview",
+            messages=query,
+            )
+        return response.choices[0].message.content
+    
+    def gpt4_vision(self, query, image_path):
+
+        print(query)
+        
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-4-vision-preview",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": query
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": image_path
+                            }
+                        }
+                    ]
+                }
+            ],
+            max_tokens=4096,
+            stream=False
+        )
+        return response.choices[0].message.content
+    
+    def LLM_chat_query(self, llm, query, image_path=None):
+
+        if llm == 'gpt-3.5-turbo':
+            return self.gpt35_chat_completion(query)
+        
+        elif llm == "gpt-4":
+            return self.gpt4_chat_completion(query)
+    
+        elif llm == "gpt-4o":
+            return self.gpt4o_chat_completion(query)
+        
+        elif llm == "gpt-4V":
+            return self.gpt4_vision(query, image_path)
+    
+    def get_llm_response(self, llm, query, image_path=None):
+
+        if llm == "gpt-4V" and image_path:
+            res = self.LLM_chat_query(llm, query, image_path)
+            return res
+        
+        chat_completion = []
+        chat_completion.append({"role": "system", "content": query})
+        res = self.LLM_chat_query(llm, chat_completion)
+        return res
+    
+# if __name__ == '__main__':
+
+#     llm_query_api = LLMProxyQueryAPI()
+
+#     def local_image_to_data_url(image_path):
+#         mime_type, _ = guess_type(image_path)
+#         if mime_type is None:
+#             mime_type = 'application/octet-stream'
+
+#         with open(image_path, "rb") as image_file:
+#             base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')
+
+#         return f"data:{mime_type};base64,{base64_encoded_data}"
+    
+#     tesseract = TesseractOCR()
+#     pdf = PDF(src="temp3.pdf", pages=[0, 0])
+#     extracted_tables = pdf.extract_tables(ocr=tesseract,
+#                         implicit_rows=True,
+#                         borderless_tables=True,)
+#     html_table = extracted_tables[0][0].html_repr()
+#     print(html_table)
+
+#     table_image_path = "./temp3.jpeg"
+#     table_image_data_url = local_image_to_data_url(table_image_path)
+#     print(table_image_data_url)
+#     query = table_image_to_html_prompt.replace("{{html_table}}", html_table)
+#     html_table_refined = llm_query_api.get_llm_response("gpt-4V", query, table_image_data_url)
+#     print(html_table_refined)
+

.ipynb_checkpoints/matsa-checkpoint.py

@@ -0,0 +1,182 @@
+import base64
+import json
+from prompts import *
+import ast
+from bs4 import BeautifulSoup
+from semantic_retrieval import *
+from llm_query_api import *
+import base64
+from mimetypes import guess_type
+
+class InputInstance:
+    def __init__(self, id=None, html_table=None, question=None, answer=None):
+        self.id = id
+        self.html_table = html_table
+        self.question = question
+        self.answer = answer
+
+        return
+
+class MATSA:
+    def __init__(self, llm = "gpt-4"):
+        self.llm = llm
+        self.llm_query_api = LLMQueryAPI() #LLMProxyQueryAPI()
+        pass
+
+    def table_formatting_agent(self, html_table = None, table_image_path = None):
+
+        def local_image_to_data_url(image_path):
+            mime_type, _ = guess_type(image_path)
+            if mime_type is None:
+                mime_type = 'application/octet-stream'
+
+            with open(image_path, "rb") as image_file:
+                base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')
+
+            return f"data:{mime_type};base64,{base64_encoded_data}"
+        
+        if table_image_path != None:
+            tesseract = TesseractOCR()
+            pdf = PDF(src=table_image_path, pages=[0, 0])
+            extracted_tables = pdf.extract_tables(ocr=tesseract,
+                                implicit_rows=True,
+                                borderless_tables=True,)
+            html_table = extracted_tables[0][0].html_repr()
+
+            table_image_data_url = local_image_to_data_url(table_image_path)
+            query = table_image_to_html_prompt.replace("{{html_table}}", html_table)
+            html_table = llm_query_api.get_llm_response("gpt-4V", query, table_image_data_url)
+
+        soup = BeautifulSoup(html_table, 'html.parser')
+        tr_tags = soup.find_all('tr')
+        for i, tr_tag in enumerate(tr_tags):
+            tr_tag['id'] = f"row-{i + 1}"  # Assign unique ID using 'row-i' format
+
+            if i == 0:
+                th_tags = tr_tag.find_all('th')
+                for i, th_tag in enumerate(th_tags):
+                    th_tag['id'] = f"col-{i + 1}"  # Assign unique ID using 'col-i' format
+        
+        return str(soup)
+
+    def description_augmentation_agent(self, html_table):
+
+        query = col_description_prompt.replace("{{html_table}}", str(html_table))
+        col_augmented_html_table = self.llm_query_api.get_llm_response(self.llm, query)
+
+        query = row_description_prompt.replace("{{html_table}}", str(col_augmented_html_table))
+        row_augmented_html_table = self.llm_query_api.get_llm_response(self.llm, query)
+        
+        query = trend_description_prompt.replace("{{html_table}}", str(row_augmented_html_table))
+        trend_augmented_html_table = self.llm_query_api.get_llm_response(self.llm, query)
+
+        return trend_augmented_html_table
+
+    def answer_decomposition_agent(self, answer):
+
+        prompt = answer_decomposition_prompt
+        query = prompt.replace("{{answer}}", answer)
+        res = self.llm_query_api.get_llm_response(self.llm, query)
+        res = ast.literal_eval(res)
+        if isinstance(res, list):
+            return res
+        else:
+            return None
+
+    def semantic_retreival_agent(self, html_table, fact_list, topK=5):
+
+        attributed_html_table, row_attribution_ids, col_attribution_ids = get_embedding_attribution(html_table, fact_list, topK)
+        return attributed_html_table, row_attribution_ids, col_attribution_ids
+    
+    def sufficiency_attribution_agent(self, fact_list, attributed_html_table):
+
+        fact_verification_function = {}
+
+        fact_verification_list = []
+
+        for i in range(len(fact_list)):
+            fact=fact_list[i]
+            fxn = {}
+            fxn["Fact " + str(i+1)+":"] = str(fact)
+            # fxn["Verified"] = "..."
+            fact_verification_list.append(fxn)
+        
+        fact_verification_function["List of Fact"] = fact_verification_list
+
+        fact_verification_function["Row Citations"] = "[..., ..., ...]"
+        fact_verification_function["Column Citations"] = "[..., ..., ...]"
+        fact_verification_function["Explanation"] = "..."
+        
+        fact_verification_function_string = json.dumps(fact_verification_function)
+
+        query = functional_attribution_prompt.replace("{{attributed_html_table}}", str(attributed_html_table)).replace("{{fact_verification_function}}", fact_verification_function_string)
+        attribution_fxn = self.llm_query_api.get_llm_response(self.llm, query)
+        
+        attribution_fxn = attribution_fxn.replace("```json", "")
+        attribution_fxn = attribution_fxn.replace("```", "")
+        print(attribution_fxn)
+        attribution_fxn = json.loads(attribution_fxn)
+
+        if isinstance(attribution_fxn, dict):
+            return attribution_fxn
+        else:
+            return None
+
+if __name__ == '__main__':
+
+    html_table = """<table>
+      <tr>
+        <th rowspan="1">Sr. Number</th>
+        <th colspan="3">Types</th>
+        <th rowspan="1">Remark</th>
+      </tr>
+      <tr>
+        <th> </th>
+        <th>A</th>
+        <th>B</th>
+        <th>C</th>
+        <th> </th>
+      </tr>
+      <tr>
+        <td>1</td>
+        <td>Mitten</td>
+        <td>Kity</td>
+        <td>Teddy</td>
+        <td>Names of cats</td>
+      </tr>
+      <tr>
+        <td>1</td>
+        <td>Tommy</td>
+        <td>Rudolph</td>
+        <td>Jerry</td>
+        <td>Names of dogs</td>
+      </tr>
+    </table>"""
+
+    answer = "Tommy is a dog but Mitten is a cat."
+     
+    
+    x = InputInstance(html_table=html_table, answer=answer)
+
+    matsa_agent = MATSA()
+
+    x_reformulated = matsa_agent.table_formatting_agent(x.html_table)
+    print(x_reformulated)
+
+    x_descriptions = matsa_agent.description_augmentation_agent(x_reformulated)
+    print(x_descriptions)
+
+    fact_list = matsa_agent.answer_decomposition_agent(x.answer)
+    print(fact_list)
+
+    attributed_html_table, row_attribution_ids, col_attribution_ids = matsa_agent.semantic_retreival_agent(x_descriptions, fact_list)
+    print(attributed_html_table)
+
+    attribution_fxn = matsa_agent.sufficiency_attribution_agent(fact_list, attributed_html_table)
+    print(attribution_fxn)
+    
+    row_attribution_set = attribution_fxn["Row Citations"]
+    col_attribution_set = attribution_fxn["Column Citations"]
+
+    print(row_attribution_set)
+    print(col_attribution_set)

.ipynb_checkpoints/semantic_retrieval-checkpoint.py

@@ -0,0 +1,146 @@
+import numpy as np
+from bs4 import BeautifulSoup
+from sklearn.preprocessing import minmax_scale
+from sentence_transformers import SentenceTransformer, util
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+sbert = SentenceTransformer("all-MiniLM-L6-v2")
+from llm_query_api import *
+
+def get_row_embedding(html_table):
+    
+    def get_row_elements(html_table):
+        tr_elements = []
+        soup = BeautifulSoup(html_table, 'html.parser')
+        tr_tags = soup.find_all('tr')
+        for t in tr_tags:
+            temp = " " + str(t.get('description'))
+            try:
+                tr_elements.append({'id':str(t.get('id')), 'text': temp})
+            except:
+                pass
+        return tr_elements
+    
+    rows = get_row_elements(html_table)
+    
+    all_elements = rows
+    sentences = []
+    element_ids = []
+    for i in range(len(all_elements)):
+        sentences.append(all_elements[i]['text'])
+        element_ids.append(all_elements[i]['id'])
+    
+    embeddings = sbert.encode(sentences, convert_to_tensor=True).cpu().numpy()
+    return embeddings, element_ids
+    
+def get_col_embedding(html_table):
+    
+    def get_column_elements(html_table):
+        th_elements = []
+        soup = BeautifulSoup(html_table, 'html.parser')
+        th_tags = soup.find_all('th')
+        for t in th_tags:
+            temp = " " + str(t.get('description'))
+            try:
+                th_elements.append({'id':str(t.get('id')), 'text': temp})
+            except:
+                pass
+            
+        return th_elements
+    
+    cols = get_column_elements(html_table)
+    
+    all_elements = cols
+    sentences = []
+    element_ids = []
+    for i in range(len(all_elements)):
+        sentences.append(all_elements[i]['text'])
+        element_ids.append(all_elements[i]['id'])
+    
+    embeddings = sbert.encode(sentences, convert_to_tensor=True).cpu().numpy()
+    return embeddings, element_ids
+
+def normalize_list_numpy(list_numpy):
+    normalized_list = minmax_scale(list_numpy)
+    return normalized_list
+
+def get_answer_embedding(answer):
+    return sbert.encode([answer], convert_to_tensor=True).cpu().numpy()
+
+def row_attribution(answer, html_table, topk=5, threshold = 0.7):
+    
+    answer_embedding = get_answer_embedding(answer)
+    row_embedding = get_row_embedding(html_table)
+
+    similarities = cosine_similarity(row_embedding[0], answer_embedding.reshape(1, -1))
+    sims = similarities.flatten()
+    sims = normalize_list_numpy(sims)
+    #if no of rows >= 5, take max of (5, 1/3 x rows)
+    #else if no of rows < 5, take least of (5,  rows)
+    k = max(topk, int(0.3*len(sims)))
+    k = min(k, len(sims))
+    top_k_indices = np.argpartition(sims, -k)[-k:]    
+    sorted_indices = top_k_indices[np.argsort(sims[top_k_indices])][::-1]
+    top_k_results = [row_embedding[1][idx] for idx in sorted_indices]
+    
+    return top_k_results
+
+def col_attribution(answer, html_table, topk=5, threshold = 0.7):
+    
+    answer_embedding = get_answer_embedding(answer)
+    col_embedding = get_col_embedding(html_table)
+    
+    similarities = cosine_similarity(col_embedding[0], answer_embedding.reshape(1, -1))
+    sims = similarities.flatten()
+    sims = normalize_list_numpy(sims)
+    #if no of cols >= 5, take max of (5, 1/3 x cols)
+    #else if no of cols < 5, take least of (5,  cols)
+    k = max(topk, int(0.3*len(sims)))
+    k = min(k, len(sims))
+    top_k_indices = np.argpartition(sims, -k)[-k:]    
+    sorted_indices = top_k_indices[np.argsort(sims[top_k_indices])][::-1]
+    top_k_results = [col_embedding[1][idx] for idx in sorted_indices]
+    
+    return top_k_results
+
+def retain_rows_and_columns(augmented_html_table, row_ids, column_ids):
+    soup = BeautifulSoup(augmented_html_table, 'html.parser')
+
+    row_ids = list(set(row_ids))
+    column_ids = list(set(column_ids))
+
+    # Retain specified rows and remove others
+    all_rows = soup.find_all('tr')
+    for row in all_rows:
+        if row.get('id') not in row_ids:
+            row.decompose()
+
+    # Retain specified columns and remove others
+    if all_rows:
+        all_columns = all_rows[0].find_all(['th'])
+        for i, col in enumerate(all_columns):
+            if col.get('id') not in column_ids:
+                for row in soup.find_all('tr'):
+                    cells = row.find_all(['td', 'th'])
+                    if len(cells) > i:
+                        cells[i].decompose()
+
+    return str(soup)
+
+def get_embedding_attribution(augmented_html_table, decomposed_fact_list, topK=5, threshold = 0.7):
+    
+    row_attribution_ids = []
+    col_attribution_ids = []
+
+    for i in range(len(decomposed_fact_list)):
+        answer = decomposed_fact_list[i]
+        
+        rorAttr = row_attribution(answer, augmented_html_table, topK)
+        colAttr = col_attribution(answer, augmented_html_table, topK)
+
+        row_attribution_ids.extend(rorAttr)
+        col_attribution_ids.extend(colAttr)
+
+        attributed_html_table = retain_rows_and_columns(augmented_html_table, row_attribution_ids, col_attribution_ids)
+           
+    return attributed_html_table, row_attribution_ids, col_attribution_ids

README.md

@@ -1,12 +1,6 @@
 ---
-title: Matsa Demo
-emoji: 👀
-colorFrom: pink
-colorTo: red
+title: Matsa-demo
+app_file: demo.py
 sdk: gradio
-sdk_version: 4.42.0
-app_file: app.py
-pinned: false
+sdk_version: 4.41.0
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

config_gpt35.yaml

@@ -0,0 +1,3 @@
+API_BASE: https://dil-research.openai.azure.com/
+API_KEY: de6e495251174ceab84f290cd3925b07
+API_VERSION: '2023-05-15'

config_gpt4.yaml

@@ -0,0 +1,3 @@
+API_BASE: https://dil-research-sweden-central.openai.azure.com/
+API_KEY: 0df7237e0fc243a8830a23b2dbba2dcb
+API_VERSION: "2023-12-01-preview"

demo.py

@@ -0,0 +1,221 @@
+import gradio as gr
+import json
+from bs4 import BeautifulSoup
+from matsa import MATSA, InputInstance
+import imgkit
+import tempfile
+import time
+import threading
+
+TABLE_FOLDER = "./tables_folder/MATSA_fetaqa.json"
+# Load data from JSON file
+def load_data():
+    with open(TABLE_FOLDER, 'r') as json_file:
+        return json.load(json_file)
+
+# Global variable to store the loaded data
+TABLE_DATA = load_data()
+
+def get_table_names():
+    return [f"tab_{i+1}" for i in range(len(TABLE_DATA))]
+
+def html_to_image(html_content):
+    with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as temp_img:
+        imgkit.from_string(html_content, temp_img.name)
+        return temp_img.name
+
+def highlight_table(html_table, row_ids, col_ids):
+    soup = BeautifulSoup(html_table, 'html.parser')
+    row_sel = []
+
+    # Highlight rows
+    for row_id in row_ids:
+        row = soup.find('tr', id=row_id)
+        if row:
+            row_sel.append(row)
+
+    for col_id in col_ids:
+        col_index = int(col_id.split('-')[1]) - 1  # Convert col-1 to index 0, col-2 to index 1, etc.
+        for row in soup.find_all('tr'):
+            cells = row.find_all(['td', 'th'])
+            if row in row_sel:
+                if col_index < len(cells):
+                    cells[col_index]['style'] = 'background-color: rgba(173, 216, 230, 0.7);'
+            else:
+                if col_index < len(cells):
+                    cells[col_index]['style'] = 'background-color: rgba(211, 211, 211, 0.6);'  
+
+    return str(soup)
+
+def load_table_data(table_name):
+    if not table_name:
+        return None, "", ""
+    index = int(table_name.split('_')[1]) - 1
+    data = TABLE_DATA[index]
+    
+    html_content = data['html_table']
+    question = data.get("question", "") #data['question'] 
+    if question is None:
+        question = ""
+    answer = data['answer_statement']
+    
+    image_path = html_to_image(html_content)
+    
+    return image_path, question, answer
+
+def process_input(table_name, question, answer):
+    if not table_name:
+        return "Please select a table from the dropdown."
+    
+    # Get the data for the selected table
+    index = int(table_name.split('_')[1]) - 1
+    data = TABLE_DATA[index]
+    
+    html_content = data['html_table']
+
+    print("html_content: ", html_content)
+    print("question: ", question)
+    print("answer: ", answer)
+    
+    # Initialize MATSA
+    matsa_agent = MATSA()
+    
+    # Create input instance
+    instance = InputInstance(html_table=html_content, question=question, answer=answer)
+    
+    # Apply MATSA pipeline
+    # formatted_table = matsa_agent.table_formatting_agent(instance.html_table)
+    augmented_table = matsa_agent.description_augmentation_agent(instance.html_table)
+    print("augmented_table: ", augmented_table)
+    fact_list = matsa_agent.answer_decomposition_agent(instance.answer)
+    print("fact_list: ", fact_list)
+    attributed_table, _, _ = matsa_agent.semantic_retreival_agent(augmented_table, fact_list)
+    print("attributed_table: ", attributed_table)
+    attribution_fxn = matsa_agent.sufficiency_attribution_agent(fact_list, attributed_table)
+    print("attribution_fxn: ", attribution_fxn)
+    
+    # Get row and column attributions
+    row_attribution_set = attribution_fxn["Row Citations"]
+    col_attribution_set = attribution_fxn["Column Citations"]
+    explnation = attribution_fxn.get("Explanation", "")
+    print("row_attribution_set: ", row_attribution_set)
+    print("col_attribution_set: ", col_attribution_set)
+    print("Explanation: ", attribution_fxn.get("Explanation", ""))
+    
+    # Convert string representations to lists
+    if isinstance(row_attribution_set, str):
+        row_ids = eval(row_attribution_set)
+    else:
+        row_ids = row_attribution_set
+    
+    if isinstance(col_attribution_set, str):
+        col_ids = eval(col_attribution_set)
+    else:
+        col_ids = col_attribution_set
+    
+    # Highlight the table
+    highlighted_table = highlight_table(instance.html_table, row_ids, col_ids)
+    
+    result = {
+        "highlighted_table": highlighted_table,
+        "facts": attribution_fxn.get("List of Facts", []),
+        "row_citations": row_attribution_set,
+        "column_citations": col_attribution_set,
+        "Explanation": explnation
+    }
+    
+    return json.dumps(result)
+
+# Define Gradio interface
+with gr.Blocks() as iface:
+    gr.Markdown("# MATSA: Table Question Answering with Attribution")
+    gr.Markdown("Select a table from dropdown load table image, question, and answer.")
+    gr.Markdown("Attributions are provided as per answer. You may change the question/answer as per your need.")
+    
+    table_dropdown = gr.Dropdown(choices=get_table_names(), label="Select Table")
+    original_table = gr.Image(type="filepath", label="Original Table")
+    question_box = gr.Textbox(label="Question")
+    answer_box = gr.Textbox(label="Answer")
+    
+    gr.Markdown("Click 'Process' to see the highlighted relevant parts. Click 'Reset' to start over.")
+    
+    process_button = gr.Button("Process")
+    reset_button = gr.Button("Reset")
+    processing_time = gr.Textbox(label="Processing Time", value="0 seconds")
+    highlighted_table = gr.HTML(label="Highlighted Table")
+    explanation_box = gr.Textbox(label="Explanation")
+
+    def update_table_data(table_name):
+        image_path, question, answer = load_table_data(table_name)
+        return image_path, question, answer, gr.update(interactive=True)
+    
+    def reset_app():
+        return (
+            gr.update(value="", interactive=True),  # table_dropdown
+            None,  # original_table
+            "",  # question_box
+            "",  # answer_box
+            "",  # highlighted_table
+            "",  # explanation_box
+            gr.update(interactive=True),  # process_button
+            "0 seconds",  # processing_time
+        )
+    
+    def process_and_disable(table_name, question, answer):
+        processing = True
+        counter = 0
+        
+        def update_counter():
+            nonlocal counter
+            while processing:
+                counter += 1
+                yield counter
+                time.sleep(1)
+        
+        counter_thread = threading.Thread(target=update_counter)
+        counter_thread.start()
+        
+        # Disable the dropdown and process button during processing
+        yield (
+            gr.update(interactive=False),  # table_dropdown
+            gr.update(interactive=False),  # process_button
+            gr.update(value="Processing..."),  # processing_time
+            gr.update(),  # highlighted_table
+            gr.update(),  # explanation_box
+        )
+        
+        # Process the input
+        result = process_input(table_name, question, answer)
+        result_dict = json.loads(result)
+        
+        # Stop the counter
+        processing = False
+        counter_thread.join()
+        
+        # Re-enable the dropdown and process button, update processing time, and return the result
+        yield (
+            gr.update(interactive=True),  # table_dropdown
+            gr.update(interactive=True),  # process_button
+            f"Processed in {counter} seconds",  # processing_time
+            gr.update(value=result_dict['highlighted_table']),  # highlighted_table
+            gr.update(value=result_dict.get('Explanation', '')),  # explanation_box
+            {
+                "Row Citations": result_dict['row_citations'],
+                "Column Citations": result_dict['column_citations']
+            }
+        )
+    
+    table_dropdown.change(update_table_data, 
+                          inputs=[table_dropdown], 
+                          outputs=[original_table, question_box, answer_box, process_button])
+    
+    process_button.click(process_and_disable, 
+                         inputs=[table_dropdown, question_box, answer_box], 
+                         outputs=[table_dropdown, process_button, processing_time, highlighted_table, explanation_box])
+    
+    reset_button.click(reset_app, 
+                       inputs=[], 
+                       outputs=[table_dropdown, original_table, question_box, answer_box, highlighted_table, explanation_box, process_button, processing_time])
+
+# Launch the interface
+iface.launch(share=True)

llm_query_api.py

@@ -0,0 +1,216 @@
+import base64
+import copy
+import io
+import os
+import random
+import time
+import re
+import json
+import argparse
+import yaml
+import openai
+from openai import AzureOpenAI
+from prompts import *
+import base64
+from mimetypes import guess_type
+# from img2table.document import Image
+# from img2table.document import PDF
+# from img2table.ocr import TesseractOCR
+# from img2table.ocr import EasyOCR
+from PIL import Image as PILImage
+
+class LLMQueryAPI:
+
+    def __init__(self) -> None:
+        pass
+
+    def gpt4_chat_completion(self, query):
+
+        with open('config_gpt4.yaml', 'r') as f:
+            config = yaml.safe_load(f)
+
+        API_KEY = config.get('API_KEY')
+        API_VERSION = config.get('API_VERSION')
+        API_BASE = config.get('API_BASE')
+
+        client = AzureOpenAI(
+            azure_endpoint= API_BASE,
+            api_version= API_VERSION,
+            api_key = API_KEY
+            )
+            
+        deployment_name='gpt-4-2024-04-09'
+
+        response = client.chat.completions.create(
+            model=deployment_name, 
+            messages=query)
+        
+        return response.choices[0].message.content
+
+    def gpt35_chat_completion(self, query):
+    
+        with open('config_gpt35.yaml', 'r') as f:
+            config = yaml.safe_load(f)
+
+        API_KEY = config.get('API_KEY')
+        API_VERSION = config.get('API_VERSION')
+        API_BASE = config.get('API_BASE')
+
+        response = openai.ChatCompletion.create(
+            engine='gpt-35-turbo-0613',
+            messages=query,
+            request_timeout=60,
+            api_key = API_KEY,
+            api_version = API_VERSION,
+            api_type = "azure",
+            api_base = API_BASE,
+        )
+
+        return response['choices'][0]['message']
+
+    def copilot_chat_completion(self, query):
+
+        with open('config_gpt4.yaml', 'r') as f:
+            config = yaml.safe_load(f)
+
+        API_KEY = config.get('API_KEY')
+        API_VERSION = config.get('API_VERSION')
+        API_BASE = config.get('API_BASE')
+
+        response = openai.ChatCompletion.create(
+            engine='gpt-4-0613',
+            messages=query,
+            request_timeout=60,
+            api_key = API_KEY,
+            api_version = API_VERSION,
+            api_type = "azure",
+            api_base = API_BASE,
+        )
+        return response['choices'][0]['message']
+
+    def LLM_chat_query(self, query, llm):
+
+        if llm == 'gpt-3.5-turbo':
+            return self.gpt35_chat_completion(query)
+        elif llm == "gpt-4":
+            return self.gpt4_chat_completion(query)
+            # return self.copilot_chat_completion(query)
+    
+    def get_llm_response(self, llm, query):
+        chat_completion = []
+        chat_completion.append({"role": "system", "content": query})
+        res = self.LLM_chat_query(chat_completion, llm)
+        return res
+
+class LLMProxyQueryAPI:
+
+    def __init__(self) -> None:
+        pass
+
+    def gpt35_chat_completion(self, query):
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-3.5-turbo-16k",
+            messages=query,
+        )
+        return response.choices[0].message.content
+    
+    def gpt4o_chat_completion(self, query):
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-4o",
+            messages=query,
+            )
+        return response.choices[0].message.content
+    
+    def gpt4_chat_completion(self, query):
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-4-1106-preview",
+            messages=query,
+            )
+        return response.choices[0].message.content
+    
+    def gpt4_vision(self, query, image_path):
+
+        print(query)
+        
+        client = openai.Client()
+        response = client.chat.completions.create(
+            model="gpt-4-vision-preview",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": query
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": image_path
+                            }
+                        }
+                    ]
+                }
+            ],
+            max_tokens=4096,
+            stream=False
+        )
+        return response.choices[0].message.content
+    
+    def LLM_chat_query(self, llm, query, image_path=None):
+
+        if llm == 'gpt-3.5-turbo':
+            return self.gpt35_chat_completion(query)
+        
+        elif llm == "gpt-4":
+            return self.gpt4_chat_completion(query)
+    
+        elif llm == "gpt-4o":
+            return self.gpt4o_chat_completion(query)
+        
+        elif llm == "gpt-4V":
+            return self.gpt4_vision(query, image_path)
+    
+    def get_llm_response(self, llm, query, image_path=None):
+
+        if llm == "gpt-4V" and image_path:
+            res = self.LLM_chat_query(llm, query, image_path)
+            return res
+        
+        chat_completion = []
+        chat_completion.append({"role": "system", "content": query})
+        res = self.LLM_chat_query(llm, chat_completion)
+        return res
+    
+# if __name__ == '__main__':
+
+#     llm_query_api = LLMProxyQueryAPI()
+
+#     def local_image_to_data_url(image_path):
+#         mime_type, _ = guess_type(image_path)
+#         if mime_type is None:
+#             mime_type = 'application/octet-stream'
+
+#         with open(image_path, "rb") as image_file:
+#             base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')
+
+#         return f"data:{mime_type};base64,{base64_encoded_data}"
+    
+#     tesseract = TesseractOCR()
+#     pdf = PDF(src="temp3.pdf", pages=[0, 0])
+#     extracted_tables = pdf.extract_tables(ocr=tesseract,
+#                         implicit_rows=True,
+#                         borderless_tables=True,)
+#     html_table = extracted_tables[0][0].html_repr()
+#     print(html_table)
+
+#     table_image_path = "./temp3.jpeg"
+#     table_image_data_url = local_image_to_data_url(table_image_path)
+#     print(table_image_data_url)
+#     query = table_image_to_html_prompt.replace("{{html_table}}", html_table)
+#     html_table_refined = llm_query_api.get_llm_response("gpt-4V", query, table_image_data_url)
+#     print(html_table_refined)
+

matsa.py

@@ -0,0 +1,182 @@
+import base64
+import json
+from prompts import *
+import ast
+from bs4 import BeautifulSoup
+from semantic_retrieval import *
+from llm_query_api import *
+import base64
+from mimetypes import guess_type
+
+class InputInstance:
+    def __init__(self, id=None, html_table=None, question=None, answer=None):
+        self.id = id
+        self.html_table = html_table
+        self.question = question
+        self.answer = answer
+
+        return
+
+class MATSA:
+    def __init__(self, llm = "gpt-4"):
+        self.llm = llm
+        self.llm_query_api = LLMQueryAPI() #LLMProxyQueryAPI()
+        pass
+
+    def table_formatting_agent(self, html_table = None, table_image_path = None):
+
+        def local_image_to_data_url(image_path):
+            mime_type, _ = guess_type(image_path)
+            if mime_type is None:
+                mime_type = 'application/octet-stream'
+
+            with open(image_path, "rb") as image_file:
+                base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')
+
+            return f"data:{mime_type};base64,{base64_encoded_data}"
+        
+        if table_image_path != None:
+            tesseract = TesseractOCR()
+            pdf = PDF(src=table_image_path, pages=[0, 0])
+            extracted_tables = pdf.extract_tables(ocr=tesseract,
+                                implicit_rows=True,
+                                borderless_tables=True,)
+            html_table = extracted_tables[0][0].html_repr()
+
+            table_image_data_url = local_image_to_data_url(table_image_path)
+            query = table_image_to_html_prompt.replace("{{html_table}}", html_table)
+            html_table = llm_query_api.get_llm_response("gpt-4V", query, table_image_data_url)
+
+        soup = BeautifulSoup(html_table, 'html.parser')
+        tr_tags = soup.find_all('tr')
+        for i, tr_tag in enumerate(tr_tags):
+            tr_tag['id'] = f"row-{i + 1}"  # Assign unique ID using 'row-i' format
+
+            if i == 0:
+                th_tags = tr_tag.find_all('th')
+                for i, th_tag in enumerate(th_tags):
+                    th_tag['id'] = f"col-{i + 1}"  # Assign unique ID using 'col-i' format
+        
+        return str(soup)
+
+    def description_augmentation_agent(self, html_table):
+
+        query = col_description_prompt.replace("{{html_table}}", str(html_table))
+        col_augmented_html_table = self.llm_query_api.get_llm_response(self.llm, query)
+
+        query = row_description_prompt.replace("{{html_table}}", str(col_augmented_html_table))
+        row_augmented_html_table = self.llm_query_api.get_llm_response(self.llm, query)
+        
+        query = trend_description_prompt.replace("{{html_table}}", str(row_augmented_html_table))
+        trend_augmented_html_table = self.llm_query_api.get_llm_response(self.llm, query)
+
+        return trend_augmented_html_table
+
+    def answer_decomposition_agent(self, answer):
+
+        prompt = answer_decomposition_prompt
+        query = prompt.replace("{{answer}}", answer)
+        res = self.llm_query_api.get_llm_response(self.llm, query)
+        res = ast.literal_eval(res)
+        if isinstance(res, list):
+            return res
+        else:
+            return None
+
+    def semantic_retreival_agent(self, html_table, fact_list, topK=5):
+
+        attributed_html_table, row_attribution_ids, col_attribution_ids = get_embedding_attribution(html_table, fact_list, topK)
+        return attributed_html_table, row_attribution_ids, col_attribution_ids
+    
+    def sufficiency_attribution_agent(self, fact_list, attributed_html_table):
+
+        fact_verification_function = {}
+
+        fact_verification_list = []
+
+        for i in range(len(fact_list)):
+            fact=fact_list[i]
+            fxn = {}
+            fxn["Fact " + str(i+1)+":"] = str(fact)
+            # fxn["Verified"] = "..."
+            fact_verification_list.append(fxn)
+        
+        fact_verification_function["List of Fact"] = fact_verification_list
+
+        fact_verification_function["Row Citations"] = "[..., ..., ...]"
+        fact_verification_function["Column Citations"] = "[..., ..., ...]"
+        fact_verification_function["Explanation"] = "..."
+        
+        fact_verification_function_string = json.dumps(fact_verification_function)
+
+        query = functional_attribution_prompt.replace("{{attributed_html_table}}", str(attributed_html_table)).replace("{{fact_verification_function}}", fact_verification_function_string)
+        attribution_fxn = self.llm_query_api.get_llm_response(self.llm, query)
+        
+        attribution_fxn = attribution_fxn.replace("```json", "")
+        attribution_fxn = attribution_fxn.replace("```", "")
+        print(attribution_fxn)
+        attribution_fxn = json.loads(attribution_fxn)
+
+        if isinstance(attribution_fxn, dict):
+            return attribution_fxn
+        else:
+            return None
+
+if __name__ == '__main__':
+
+    html_table = """<table>
+      <tr>
+        <th rowspan="1">Sr. Number</th>
+        <th colspan="3">Types</th>
+        <th rowspan="1">Remark</th>
+      </tr>
+      <tr>
+        <th> </th>
+        <th>A</th>
+        <th>B</th>
+        <th>C</th>
+        <th> </th>
+      </tr>
+      <tr>
+        <td>1</td>
+        <td>Mitten</td>
+        <td>Kity</td>
+        <td>Teddy</td>
+        <td>Names of cats</td>
+      </tr>
+      <tr>
+        <td>1</td>
+        <td>Tommy</td>
+        <td>Rudolph</td>
+        <td>Jerry</td>
+        <td>Names of dogs</td>
+      </tr>
+    </table>"""
+
+    answer = "Tommy is a dog but Mitten is a cat."
+     
+    
+    x = InputInstance(html_table=html_table, answer=answer)
+
+    matsa_agent = MATSA()
+
+    x_reformulated = matsa_agent.table_formatting_agent(x.html_table)
+    print(x_reformulated)
+
+    x_descriptions = matsa_agent.description_augmentation_agent(x_reformulated)
+    print(x_descriptions)
+
+    fact_list = matsa_agent.answer_decomposition_agent(x.answer)
+    print(fact_list)
+
+    attributed_html_table, row_attribution_ids, col_attribution_ids = matsa_agent.semantic_retreival_agent(x_descriptions, fact_list)
+    print(attributed_html_table)
+
+    attribution_fxn = matsa_agent.sufficiency_attribution_agent(fact_list, attributed_html_table)
+    print(attribution_fxn)
+    
+    row_attribution_set = attribution_fxn["Row Citations"]
+    col_attribution_set = attribution_fxn["Column Citations"]
+
+    print(row_attribution_set)
+    print(col_attribution_set)

prompts.py

@@ -0,0 +1,69 @@
+row_description_prompt = """You are a brilliant table assistant with the capabilities information retrieval, table parsing, and semantic understanding of the structural information of the table.
+
+Here is a table in html format: \"{{html_table}}\"
+
+We need to add a detailed description for each row of the table denoted by the "<tr>" tag element. The description should discuss the overall information inferred from the row. It should also mention all the elements, numbers, and figures present in the row. Also, include any hierarchical row/column header information.
+    
+Add this description as a "description" attribute in the <tr> tag. Repeat this process for ALL <tr> tags in the provided HTML. Do NOT delete any information that already exists in the other tags. Just print the html and do NOT output any other message. The HTML structure should NOT be changed.
+"""
+
+col_description_prompt = """You are a brilliant table assistant with the capabilities information retrieval, table parsing, and semantic understanding of the structural information of the table. 
+
+Here is a table in html format: \"{{html_table}}\"
+
+First, write a caption for the entire table that captures the general information being presented by all rows and columns in this table. Add it using the <caption> tag in the HTML.
+
+Next, need to add a detailed description for each column of the table denoted by the "<th>" tag element. The description should discuss what elements are present in the column and the overall information that can be inferred by the column, including any hierarchical column header information.
+
+Add this description as a "description" in the <th> tag. Repeat this process for ALL <th> tags in the provided HTML. Do NOT delete any information that already exists in the other tags.  Just print the html and do NOT output any other message. The HTML structure should NOT be changed.
+"""
+
+trend_description_prompt = """You are a brilliant table assistant with the capabilities information retrieval, table parsing, semantic understanding, and trend analysis of the structural information of the table. 
+    
+Here is a table in html format: \"{{html_table}}\".
+        
+We need to add a trend analysis on the elements in the given row compared to its own constituent cells and other rows in the table. The description should discuss semantic descriptions of numerical data, summarizing key quantitative characteristics and tendencies across the table row and across different columns.
+    
+Add this analysis in the \"description\" of the <tr> tag. Repeat this process for ALL <tr> tags in the provided HTML. Do NOT delete any information that already exists in the other tags.  Just print the html and do NOT output any other message. The HTML strcuture should NOT be changed.
+"""
+
+functional_attribution_prompt = """You are a brilliant assistant with the capabilities information retrieval, fact checking, and semantic understanding of tabular data.
+
+Here is the html table - \"{{attributed_html_table}}\"
+
+We have a list of facts pertaining to this table present in this JSON structure - \"{{fact_verification_function}}"\.
+The JSON structure contains three empty fields - "Row Citations", "Column Citations", and "Explanation" that need to filled with relevant information.
+
+We want to identify all the ROWS in the table that are important to support these facts. In other words, which rows are needed to collectively verify the facts. Please copy the "row-id" of all relevant table rows in the "Row Citation" field of the JSON structure. All rows IDs should be added in the form of a LIST "[... , ... , ...]" with no value repeated. Here is a sample: "Row Citations": ["row-2", "row-3"].
+    
+Similar to rows, we want to identify all the COLUMNS in the table that are important to support these facts. In other words, which columns are needed to collectively verify the facts. Please copy the "col-id" of all relevant table rows in the "Column Citation" field of the JSON structure. All column IDs should be added in the form of a LIST "[... , ... , ...]" with no value repeated. Here is a sample: "Column Citations": ["col-1", "col-5", "col-7"]
+
+"Explanation" field should contain a detailed explanation of how the rows and columns identified in the "Row Citations" and "Column Citations" fields respectively, are important to verify the facts present in the JSON structure. The explanation should be coherent and provide a clear rationale for the selection of rows and columns.
+
+The final result should be a complete JSON structure ONLY. Do not print any extra information. Make sure to fill the JSON structure accurately as provided in the prompt. 'Row citations',  'Column Citations', and 'Explanation' should not be empty.
+"""
+
+
+answer_decomposition_prompt = """
+
+Here is a passage: \"{{answer}}\"
+
+Convert the given passage into a list of short facts which specifically answer the given question. 
+Make sure that the facts can be found in the given passage. 
+The facts should be coherent and succinct sentences with clear and simple syntax. 
+Do not use pronouns as the subject or object in the syntax of each fact. 
+The facts should be independent to each other. 
+Do not create facts from the passage which are not answering the given question.
+ONLY return a python LIST of strings seperated by comma (,). Do NOT output any extra explanation. 
+"""
+
+table_image_to_html_prompt = """ Here is an image of a table. Please convert this table image into a HTML representation with accurate table cell data.
+In order to help you in this process, here is a noisy HTML representation of the table extracted from the image: \"{{html_table}}\"
+You may use this as a noisy reference and further refine the HTML structure to accurately represent the table data.
+You should also add any information pertaining to row and column spans. In case of nested rows/columns with multiple spans, take be very careful to leave blank cells to ensure the semantic structure is maintained.
+Be careful to handle hierarchical and nested rows/columns.
+Each HTML should start with "<table>" opening tag.
+Each HTML should end with "</table>" closing tag. 
+Do NOT output any other explanation or text apart from the HTML code of the table.
+"""
+    

requirements.txt

@@ -0,0 +1,104 @@
+aiofiles @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_f56ag8l7kr/croot/aiofiles_1683773599608/work
+altair @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_a8x4081_4h/croot/altair_1687526044471/work
+annotated-types @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_1fa2djihwb/croot/annotated-types_1709542925772/work
+anyio @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_a17a7759g2/croot/anyio_1706220182417/work
+argcomplete==3.4.0
+attrs @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_224434dqzl/croot/attrs_1695717839274/work
+beautifulsoup4==4.12.3
+Bottleneck @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_2bxpizxa3c/croot/bottleneck_1707864819812/work
+Brotli @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_27zk0eqdh0/croot/brotli-split_1714483157007/work
+certifi @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_70oty9s9jh/croot/certifi_1720453497032/work/certifi
+charset-normalizer @ file:///croot/charset-normalizer_1721748349566/work
+click==8.1.7
+colorama @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_100_k35lkb/croot/colorama_1672386539781/work
+contourpy @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_041uwyxdzo/croot/contourpy_1700583585236/work
+cycler @ file:///tmp/build/80754af9/cycler_1637851556182/work
+distro==1.9.0
+exceptiongroup @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_b2258scr33/croot/exceptiongroup_1706031391815/work
+fastapi @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_ab51gv9t42/croot/fastapi_1693295410365/work
+ffmpy @ file:///home/conda/feedstock_root/build_artifacts/ffmpy_1659474992694/work
+filelock @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_d3quwmvouf/croot/filelock_1700591194006/work
+fonttools @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_60c8ux4mkl/croot/fonttools_1713551354374/work
+fsspec @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_19mkn689lo/croot/fsspec_1714461553219/work
+gradio @ file:///home/conda/feedstock_root/build_artifacts/gradio_1721770785660/work
+gradio_client @ file:///home/conda/feedstock_root/build_artifacts/gradio-client_1721697178984/work
+h11 @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_110bmw2coo/croot/h11_1706652289620/work
+httpcore @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_fcxiho9nv7/croot/httpcore_1706728465004/work
+httpx @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_727e6zfsxn/croot/httpx_1706887102687/work
+huggingface-hub==0.24.2
+idna @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_a12xpo84t2/croot/idna_1714398852854/work
+img2table==1.2.11
+imgkit==1.2.3
+importlib_resources @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_784gdd8gq5/croot/importlib_resources-suite_1720641109833/work
+Jinja2 @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_44yzu12j7f/croot/jinja2_1716993410427/work
+joblib==1.4.2
+jsonschema @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_27o3go8sqa/croot/jsonschema_1699041627313/work
+jsonschema-specifications @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_d38pclgu95/croot/jsonschema-specifications_1699032390832/work
+kiwisolver @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_93o8te804v/croot/kiwisolver_1672387163224/work
+llvmlite==0.43.0
+markdown-it-py @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_43l_4ajkho/croot/markdown-it-py_1684279912406/work
+MarkupSafe @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_a84ni4pci8/croot/markupsafe_1704206002077/work
+matplotlib @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_a8crvoz7ca/croot/matplotlib-suite_1713336381679/work
+mdurl @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_0a8xm6w4wv/croots/recipe/mdurl_1659716035810/work
+mpmath==1.3.0
+networkx==3.3
+numba==0.60.0
+numexpr @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_45yefq0kt6/croot/numexpr_1696515289183/work
+numpy @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_a51i_mbs7m/croot/numpy_and_numpy_base_1708638620867/work/dist/numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl#sha256=c4b11b3c4d4fdb810039503fe01f311ade06cd1d675fcd6d208800a393f19b69
+openai==1.37.0
+opencv-contrib-python==4.10.0.84
+orjson @ file:///var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_910h7wyrw8/croot/orjson_1711143065818/work/target/wheels/orjson-3.9.15-cp310-cp310-macosx_11_0_arm64.whl#sha256=fb0784f650f58e15827dd32f58284d983bb401ec3b85b38321eea14ebd2d01e9
+packaging==24.1
+pandas @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_b53hgou29t/croot/pandas_1718308972393/work/dist/pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl#sha256=7e70989ec6c6e08f2fd87d3940d106704e9feb936002297690d7462e35b5da35
+pdfkit==1.0.0
+pillow @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_617xe_y58w/croot/pillow_1721059447446/work
+pipx==1.6.0
+platformdirs==4.2.2
+polars==1.2.1
+pyarrow==17.0.0
+pydantic @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_0ai8cvgm2c/croot/pydantic_1709577986211/work
+pydantic_core @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_06smitnu98/croot/pydantic-core_1709573985903/work
+pydub @ file:///home/conda/feedstock_root/build_artifacts/pydub_1615612442567/work
+Pygments @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_29bs9f_dh9/croot/pygments_1684279974747/work
+PyMuPDF==1.24.9
+PyMuPDFb==1.24.9
+pyparsing @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_3b_3vxnd07/croots/recipe/pyparsing_1661452540919/work
+PySocks @ file:///Users/ktietz/ci_310/pysocks_1643961536721/work
+python-dateutil @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_66ud1l42_h/croot/python-dateutil_1716495741162/work
+python-multipart @ file:///home/conda/feedstock_root/build_artifacts/python-multipart_1707760088566/work
+pytz @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_a4b76c83ik/croot/pytz_1713974318928/work
+PyYAML @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_a8_sdgulmz/croot/pyyaml_1698096054705/work
+referencing @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_5cz64gsx70/croot/referencing_1699012046031/work
+regex==2024.7.24
+requests @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_70sm12ba9w/croot/requests_1721414707360/work
+rich @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_bcb8hqb_r4/croot/rich_1720637498249/work
+rpds-py @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_f8jkozoefm/croot/rpds-py_1698945944860/work
+ruff @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_e70v1wc31c/croot/ruff_1713372744076/work
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.14.0
+semantic-version @ file:///tmp/build/80754af9/semantic_version_1613321057691/work
+sentence-transformers==3.0.1
+shellingham @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_9bf5wowles/croot/shellingham_1669142181600/work
+six @ file:///tmp/build/80754af9/six_1644875935023/work
+sniffio @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_1573pknjrg/croot/sniffio_1705431298885/work
+soupsieve==2.5
+starlette @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_7feeoam7sd/croot/starlette-recipe_1692980426894/work
+sympy==1.13.1
+threadpoolctl==3.5.0
+tokenizers==0.19.1
+tomli==2.0.1
+tomlkit @ file:///home/conda/feedstock_root/build_artifacts/tomlkit_1690458286251/work
+toolz @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_362wyqvvgy/croot/toolz_1667464079070/work
+torch==2.4.0
+tqdm @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_f76_dxtcsh/croot/tqdm_1716395948224/work
+transformers==4.43.2
+typer @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_e62p5mo8z5/croot/typer_1684251930377/work
+typing_extensions @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_93dg13ilv4/croot/typing_extensions_1715268840722/work
+tzdata @ file:///croot/python-tzdata_1690578112552/work
+unicodedata2 @ file:///private/var/folders/k1/30mswbxs7r1g6zwn8y4fyt500000gp/T/abs_a3epjto7gs/croot/unicodedata2_1713212955584/work
+urllib3 @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_cao7_u9937/croot/urllib3_1718912649114/work
+userpath==1.9.2
+uvicorn @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_75zglvwlmy/croot/uvicorn-split_1678090090396/work
+websockets @ file:///private/var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_d8ij4gljy1/croot/websockets_1678966799107/work
+XlsxWriter==3.2.0

semantic_retrieval.py

@@ -0,0 +1,146 @@
+import numpy as np
+from bs4 import BeautifulSoup
+from sklearn.preprocessing import minmax_scale
+from sentence_transformers import SentenceTransformer, util
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+sbert = SentenceTransformer("all-MiniLM-L6-v2")
+from llm_query_api import *
+
+def get_row_embedding(html_table):
+    
+    def get_row_elements(html_table):
+        tr_elements = []
+        soup = BeautifulSoup(html_table, 'html.parser')
+        tr_tags = soup.find_all('tr')
+        for t in tr_tags:
+            temp = " " + str(t.get('description'))
+            try:
+                tr_elements.append({'id':str(t.get('id')), 'text': temp})
+            except:
+                pass
+        return tr_elements
+    
+    rows = get_row_elements(html_table)
+    
+    all_elements = rows
+    sentences = []
+    element_ids = []
+    for i in range(len(all_elements)):
+        sentences.append(all_elements[i]['text'])
+        element_ids.append(all_elements[i]['id'])
+    
+    embeddings = sbert.encode(sentences, convert_to_tensor=True).cpu().numpy()
+    return embeddings, element_ids
+    
+def get_col_embedding(html_table):
+    
+    def get_column_elements(html_table):
+        th_elements = []
+        soup = BeautifulSoup(html_table, 'html.parser')
+        th_tags = soup.find_all('th')
+        for t in th_tags:
+            temp = " " + str(t.get('description'))
+            try:
+                th_elements.append({'id':str(t.get('id')), 'text': temp})
+            except:
+                pass
+            
+        return th_elements
+    
+    cols = get_column_elements(html_table)
+    
+    all_elements = cols
+    sentences = []
+    element_ids = []
+    for i in range(len(all_elements)):
+        sentences.append(all_elements[i]['text'])
+        element_ids.append(all_elements[i]['id'])
+    
+    embeddings = sbert.encode(sentences, convert_to_tensor=True).cpu().numpy()
+    return embeddings, element_ids
+
+def normalize_list_numpy(list_numpy):
+    normalized_list = minmax_scale(list_numpy)
+    return normalized_list
+
+def get_answer_embedding(answer):
+    return sbert.encode([answer], convert_to_tensor=True).cpu().numpy()
+
+def row_attribution(answer, html_table, topk=5, threshold = 0.7):
+    
+    answer_embedding = get_answer_embedding(answer)
+    row_embedding = get_row_embedding(html_table)
+
+    similarities = cosine_similarity(row_embedding[0], answer_embedding.reshape(1, -1))
+    sims = similarities.flatten()
+    sims = normalize_list_numpy(sims)
+    #if no of rows >= 5, take max of (5, 1/3 x rows)
+    #else if no of rows < 5, take least of (5,  rows)
+    k = max(topk, int(0.3*len(sims)))
+    k = min(k, len(sims))
+    top_k_indices = np.argpartition(sims, -k)[-k:]    
+    sorted_indices = top_k_indices[np.argsort(sims[top_k_indices])][::-1]
+    top_k_results = [row_embedding[1][idx] for idx in sorted_indices]
+    
+    return top_k_results
+
+def col_attribution(answer, html_table, topk=5, threshold = 0.7):
+    
+    answer_embedding = get_answer_embedding(answer)
+    col_embedding = get_col_embedding(html_table)
+    
+    similarities = cosine_similarity(col_embedding[0], answer_embedding.reshape(1, -1))
+    sims = similarities.flatten()
+    sims = normalize_list_numpy(sims)
+    #if no of cols >= 5, take max of (5, 1/3 x cols)
+    #else if no of cols < 5, take least of (5,  cols)
+    k = max(topk, int(0.3*len(sims)))
+    k = min(k, len(sims))
+    top_k_indices = np.argpartition(sims, -k)[-k:]    
+    sorted_indices = top_k_indices[np.argsort(sims[top_k_indices])][::-1]
+    top_k_results = [col_embedding[1][idx] for idx in sorted_indices]
+    
+    return top_k_results
+
+def retain_rows_and_columns(augmented_html_table, row_ids, column_ids):
+    soup = BeautifulSoup(augmented_html_table, 'html.parser')
+
+    row_ids = list(set(row_ids))
+    column_ids = list(set(column_ids))
+
+    # Retain specified rows and remove others
+    all_rows = soup.find_all('tr')
+    for row in all_rows:
+        if row.get('id') not in row_ids:
+            row.decompose()
+
+    # Retain specified columns and remove others
+    if all_rows:
+        all_columns = all_rows[0].find_all(['th'])
+        for i, col in enumerate(all_columns):
+            if col.get('id') not in column_ids:
+                for row in soup.find_all('tr'):
+                    cells = row.find_all(['td', 'th'])
+                    if len(cells) > i:
+                        cells[i].decompose()
+
+    return str(soup)
+
+def get_embedding_attribution(augmented_html_table, decomposed_fact_list, topK=5, threshold = 0.7):
+    
+    row_attribution_ids = []
+    col_attribution_ids = []
+
+    for i in range(len(decomposed_fact_list)):
+        answer = decomposed_fact_list[i]
+        
+        rorAttr = row_attribution(answer, augmented_html_table, topK)
+        colAttr = col_attribution(answer, augmented_html_table, topK)
+
+        row_attribution_ids.extend(rorAttr)
+        col_attribution_ids.extend(colAttr)
+
+        attributed_html_table = retain_rows_and_columns(augmented_html_table, row_attribution_ids, col_attribution_ids)
+           
+    return attributed_html_table, row_attribution_ids, col_attribution_ids

tables_folder/MATSA_fetaqa.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:365905f7696b0c00498f7b88cf602769677998c5b32614823cb86127ccfd13f9
+size 18562195

wkhtmltox_0.12.6-1.bionic_amd64.deb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:503a8a97fcf8fd397ed52c1789471e0f2513f5752f3e214d3a5eda30caa0354b
+size 15729530