First commit

gradio_app.py

@@ -0,0 +1,641 @@
+import openai
+import gradio as gr
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
+from langchain.vectorstores.faiss import FAISS
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chains import LLMChain
+from langchain.llms import OpenAI
+from langchain import PromptTemplate
+from langchain.docstore.document import Document
+import pandas as pd
+import os
+import scipdf ## You need a Gorbid service available
+import tabula ## You need to have the Java Tabula installed in the environment
+from gradio import DataFrame
+import asyncio
+from transformers import pipeline
+from dotenv import load_dotenv
+import json
+load_dotenv()
+
+## You api key from vendors or hugginface
+openai.api_key=os.getenv("OPEN_AI_API_KEY")
+LLMClient = OpenAI(model_name='text-davinci-003', openai_api_key=openai.api_key,temperature=0)
+
+# Extract text from PDF file using SCIPDF and Gorbid service (you need gorbid to use it)
+def extract_text_from_pdf(file_path):
+    article_dict = scipdf.parse_pdf_to_dict(file_path, soup=True,return_coordinates=False, grobid_url="https://kermitt2-grobid.hf.space") # return dictionary
+    print("parsed")
+    source = article_dict.find("sourcedesc")
+    authors = source.find_all("persname")
+    finaltext = article_dict['title'] + " \n\n " + article_dict['authors'] + " \n\n Abstract: " + article_dict['abstract'] + " \n\n "
+    sections = []
+    for section in article_dict['sections']:
+        sec = section['heading'] + ": "
+        if(isinstance(section['text'], str)):
+            finaltext = finaltext + sec + section['text'] + " \n\n " 
+        else:
+            for text in section['text']:
+                sec = sec + text+ " \n\n " 
+            finaltext = finaltext + sec
+    return finaltext
+
+# Extract and transform the tables of the papers
+async def get_tables(docsearch,chain_table,input_file):   
+    print("Getting tables")
+    table_texts = []
+    dfs = tabula.read_pdf(input_file.name, pages='all')
+    for idx, table in enumerate(dfs):
+        query = "Table "+str(idx+1)+":"
+        docs = docsearch.similarity_search(query, k=4)
+        #result = chain_table({"context":docs,"table":table})
+        table_texts.append(async_table_generate(docs, table,  chain_table))
+        #print(query + " "+ result['text'])
+        #table_texts.append(query + " "+ result['text'])
+    table_texts = await asyncio.gather(*table_texts)
+    for table in table_texts:
+        docsearch.add_texts(table[1])
+    return docsearch
+
+def extract_text_clean(file_path):
+    file_extension = os.path.splitext(file_path.name)[1]
+    if file_extension == ".pdf":
+        all_text = extract_text_from_pdf(file_path.name)
+        return all_text
+    elif file_extension == ".txt":
+        with open(file_path.name) as f:
+            all_text = f.read()
+            return all_text
+
+async def prepare_data(input_file, chain_table, apikey):
+    #with open(input_file.name) as f:
+    #    documentation = f.read()
+    file_name = input_file.name.split("/")[-1]
+ 
+
+    # Process text and get the embeddings
+    filepath = "./vectors/"+file_name
+    if not apikey:
+        apikey = openai.api_key
+        gr.Error("Please set your api key")
+    embeddings = OpenAIEmbeddings(openai_api_key=openai.api_key)
+    if os.path.isfile(filepath+"/index.faiss"):
+
+        # file exists
+        docsearch = FAISS.load_local(filepath,embeddings=embeddings)
+
+        print("We get the embeddings from local store")
+    else:
+        #progress(0.40, desc="Detected new document. Splitting and generating the embeddings")
+        print("We generate the embeddings using thir-party service")
+        # Get extracted running text
+        text = extract_text_clean(input_file)
+
+        # Configure the text splitter and embeddings
+        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=250, chunk_overlap=10, separators=[".", ",", " \n\n "])
+
+        # Split, and clean
+        texts = text_splitter.split_text(text)
+        for idx, text in enumerate(texts):
+            texts[idx] = text.replace('\n',' ')
+        print("Creating embeddings")
+        # Create an index search    
+        docsearch = FAISS.from_texts(texts, embeddings, metadatas=[{"source": i} for i in range(len(texts))])
+        
+        # Extract and prepare tables
+       # progress(0.60, desc="Embeddings generated, parsing and transforming tables")
+        if (os.path.splitext(input_file.name)[1] == '.pdf'):
+            docsearch = await get_tables(docsearch,chain_table,input_file)
+        
+        # Save the index locally
+        FAISS.save_local(docsearch, "./vectors/"+file_name)
+ 
+    return docsearch
+
+def build_chains(apikey):
+    if not apikey:
+        apikey = openai.api_key
+        gr.Error("Please set your api key")
+    LLMClient = OpenAI(model_name='text-davinci-003',openai_api_key=apikey,temperature=0)
+    ## In-context prompt
+    prompt_template = """Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+    Question: {question}
+    ###
+    Context: 
+    {context}
+    ###
+    Helpful answer:
+    """
+    in_context_prompt = PromptTemplate(
+        input_variables=["context","question"],
+        template=prompt_template,
+    )
+    chain_incontext = load_qa_chain(LLMClient, chain_type="stuff", prompt=in_context_prompt)
+
+    # Table extraction prompts
+    ## Table prompt to transform parsed tables in natural text
+    prompt_template = """Given the following table in HTML, and the given context related the table: Translate the content of the table into natural language.
+    ###
+    Context: 
+    {context}
+    ###
+    Table: {table}
+    ###
+    Table translation:
+    """
+    table_prompt = PromptTemplate(
+        input_variables=["context","table"],
+        template=prompt_template,
+    )
+    chain_table = LLMChain(llm=LLMClient, prompt=table_prompt)
+
+    return chain_incontext, chain_table
+
+async def async_table_generate(docs,table,chain):
+
+    resp = await chain.arun({"context": docs, "table": table})
+    #resp = "Description of the team, the type, and the demographics information, Description of the team, the type, and the demographics information"
+    return resp
+
+async def async_generate(dimension, docs,question,chain):
+    resp = await chain.arun({"input_documents": docs, "question": question})
+    #resp = "Description of the team, the type, and the demographics information, Description of the team, the type, and the demographics information"
+    return [dimension, resp]
+
+async def get_gathering_dimension(docsearch, incontext_prompt, retrieved_docs):
+    dimensions = [
+                 {"Gathering description":"""Provide a summary of how the data of the dataset has been collected? Please avoid mention the annotation process or data preparation processes"""},
+                 {"Gathering type":"""Which of the following types corresponds to the gathering process mentioned in the context?
+
+Types: Web API, Web Scrapping, Sensors, Manual Human Curator, Software collection, Surveys, Observations, Interviews, Focus groups, Document analysis, Secondary data analysis, Physical data collection, Self-reporting, Experiments, Direct measurement, Interviews, Document analysis, Secondary data analysis, Physical data collection, Self-reporting, Experiments, Direct measurement, Customer feedback data, Audio or video recordings, Image data, Biometric data, Medical or health data, Financial data, Geographic or spatial data, Time series data, User-generated content data.
+
+Answer with "Others", if you are unsure. Please answer with only the type"""},
+                 {"Gathering team": """Who was the team who collect the data?"""},
+                 {"Team Type": """The data was collected by an internal team, an external team, or crowdsourcing team?""" },
+                 {"Team Demographics": "Are the any demographic information of team gathering the data?"},
+                 {"Timeframe ":""" Which are the timeframe when the data was collected?
+                    If present, answer only with the collection timeframe of the data. If your are not sure, or there is no mention, just answers 'not provided'"""},
+                 {"Sources": """Which is the source of the data during the collection process? Answer solely with the name of the source""" },
+                 {"Infrastructure": """Which tools or infrastructure has been used during the collection process?"""},
+                 {"Localization": """Which are the places where data has been collected?
+                 If present, answer only with the collection timeframe of the data. If your are not sure, or there is no mention, just answers 'not provided'"""}
+    
+    ]
+
+    results = []
+    for dimension in dimensions:
+        for title, question in dimension.items():
+          docs = docsearch.similarity_search(question, k=retrieved_docs)
+          results.append(async_generate(title, docs,question,incontext_prompt))
+
+    answers = await asyncio.gather(*results)
+    return answers
+
+async def get_annotation_dimension(docsearch, incontext_prompt, retrieved_docs):
+    dimensions = [
+                 {"Annotation description":"""How the data of the has been annotated or labelled? Provide a short summary of the annotation process"""},
+                 {"Annotation type":""" Which  of the following category corresponds to the annotation
+                process mentioned in the context? 
+                
+                Categories: Bounding boxes, Lines and splines, Semantinc Segmentation, 3D cuboids, Polygonal segmentation, Landmark and key-point, Image and video annotations, Entity annotation, Content and textual categorization
+                
+                If you are not sure, answer with 'others'. Please answer only with the categories provided in the context. """},
+                 {"Labels":""" Which are the specific labels of the dataset? Can you enumerate it an provide a description of each one?"""},
+                 {"Team Description": """Who has annotated the data?"""},
+                 {"Team type": """The data was annotated by an internal team, an external team, or crowdsourcing team?""" },
+                 {"Team Demographics": """Is there any demographic information about the team who annotate the data?"""},
+                 {"Infrastructure": """Which tool has been used to annotate or label the dataset?"""},
+                 {"Validation": """How the quality of the labels have been validated?""" }
+    ]
+
+    results = []
+    for dimension in dimensions:
+        for title, question in dimension.items():
+          docs = docsearch.similarity_search(question, k=retrieved_docs)
+          results.append(async_generate(title, docs,question,incontext_prompt))
+
+    answers = await asyncio.gather(*results)
+    return answers
+  
+async def get_social_concerns_dimension(docsearch, incontext_prompt, retrieved_docs):
+    dimensions = [
+                 {"Representetiveness":"""Are there any social group that could be misrepresented in the dataset?"""},
+                 {"Biases":"""Is there any potential bias or imbalance in the data?"""},
+                 {"Sensitivity":""" Are there sensitive data, or data that can be offensive for people in the dataset?"""},
+                 {"Privacy":""" Is there any privacy issues on the data?"""},
+                
+    ]
+
+    results = []
+    for dimension in dimensions:
+        for title, question in dimension.items():
+          docs = docsearch.similarity_search(question, k=retrieved_docs)
+          results.append(async_generate(title, docs,question,incontext_prompt))
+
+    answers = await asyncio.gather(*results)
+    return answers
+
+async def get_uses_dimension(docsearch, incontext_prompt, retrieved_docs):
+    dimensions = [
+                 {"Purposes":"""Which are the purpose or purposes of the dataset?"""},
+                 {"Gaps":"""Which are the gaps the  dataset intend to fill?"""},
+                 {"Task":"""Which machine learning tasks the dataset inteded for?:"""},
+                 {"Recommended":"""For which applications the dataset is recommended?"""},
+                 {"Non-Recommneded":"""Is there any non-recommneded application for the dataset? If you are not sure, or there is any non-recommended use of the dataset metioned in the context, just answer with "no"."""},
+    ]
+    results = []
+    for dimension in dimensions:
+        for title, question in dimension.items():
+          docs = docsearch.similarity_search(question, k=retrieved_docs)
+          if (title == "Task"):
+            question = """Which of the following ML tasks for the dataset best matches the context?  
+            
+                    Tasks: text-classification, question-answering, text-generation, token-classification, translation,
+                    fill-mask, text-retrieval, conditional-text-generation, sequence-modeling, summarization, other,
+                    structure-prediction, information-retrieval, text2text-generation, zero-shot-retrieval,
+                    zero-shot-information-retrieval, automatic-speech-recognition, image-classification, speech-processing,
+                    text-scoring, audio-classification, conversational, question-generation, image-to-text, data-to-text,
+                    classification, object-detection, multiple-choice, text-mining, image-segmentation, dialog-response-generation,
+                    named-entity-recognition, sentiment-analysis, machine-translation, tabular-to-text, table-to-text, simplification,
+                    sentence-similarity, zero-shot-classification, visual-question-answering, text_classification, time-series-forecasting,
+                    computer-vision, feature-extraction, symbolic-regression, topic modeling, one liner summary, email subject, meeting title,
+                    text-to-structured, reasoning, paraphrasing, paraphrase, code-generation, tts, image-retrieval, image-captioning,
+                    language-modelling, video-captionning, neural-machine-translation, transkation, text-generation-other-common-sense-inference,
+                    text-generation-other-discourse-analysis, text-to-tabular, text-generation-other-code-modeling, other-text-search
+
+                    If you are not sure answer with just with "others".
+                    Please, answer only with one or some of the provided tasks """
+            
+          results.append(async_generate(title, docs,question,incontext_prompt))
+
+    answers = await asyncio.gather(*results)
+    return answers
+
+async def get_contributors_dimension(docsearch, incontext_prompt, retrieved_docs):
+    dimensions = [
+                 {"Authors":"""Who are the authors of the dataset """},
+                 {"Funders":"""Is there any organization which supported or funded the creation of the dataset?"""},
+                 {"Maintainers":"""Who are the maintainers of the dataset?"""},
+                 {"Erratums":"""Is there any data retention limit in the dataset? If you are not sure, or there is no retention limit just answer with "no"."""},
+                 {"Data Retention Policies":"""Is there any data retention policies policiy of the dataset? If you are not sure, or there is no retention policy just answer with "no"."""},
+    ]
+
+    results = []
+    for dimension in dimensions:
+        for title, question in dimension.items():
+          docs = docsearch.similarity_search(question, k=retrieved_docs)
+          results.append(async_generate(title, docs,question,incontext_prompt))
+
+    answers = await asyncio.gather(*results)
+    return answers
+
+async def get_composition_dimension(docsearch, incontext_prompt, retrieved_docs):
+    dimensions = [
+                 {"File composition":"""Can you provide a description of each files the dataset is composed of?"""},
+                 {"Attributes":"""Can you enumerate the different attributes present in the dataset? """},
+                 {"Trainig splits":"""The paper mentions any recommended data split of the dataset?"""},
+                 {"Relevant statistics":"""Are there relevant statistics or distributions of the dataset? """},
+    ]
+
+    results = []
+    for dimension in dimensions:
+        for title, question in dimension.items():
+          docs = docsearch.similarity_search(question, k=retrieved_docs)
+          results.append(async_generate(title, docs,question,incontext_prompt))
+
+    answers = await asyncio.gather(*results)
+    return answers
+
+async def get_distribution_dimension(docsearch, incontext_prompt, retrieved_docs):
+    dimensions = [
+                 {"Data repository":"""Is there a link to the a repository containing the data? If you are not sure, or there is no link to the repository just answer with "no"."""},
+                 {"Licence":"""Which is the license of the dataset. If you are not sure, or there is mention to a license of the dataset in the context, just answer with "no". """},
+                 {"Deprecation policies":"""Is there any deprecation plan or policy of the dataset?
+                """},
+                
+    ]
+
+    results = []
+    for dimension in dimensions:
+        for title, question in dimension.items():
+          docs = docsearch.similarity_search(question, k=retrieved_docs)
+          results.append(async_generate(title, docs,question,incontext_prompt))
+
+    answers = await asyncio.gather(*results)
+    return answers
+
+def get_warnings(results):
+    warnings = []
+    classifier = pipeline("zero-shot-classification", model="valhalla/distilbart-mnli-12-1")
+    for result in results:  
+        if(result[0] == "Team Demographics"):
+            classifications = classifier(result[1], ["Have demographics information","Do not have demographics information"])
+            if(classifications['labels'][0] == 'Do not have demographics information'):
+                print("Dimension: "+result[0]+" is missing. Inserting a warning")
+                warnings.append(result[0]+" is missing. This information is relevant to evaluate the quality of the labels")
+        if(result[0] == "Localization"):  
+            classifications = classifier(result[1], ["Is a localization","Is not a localization"])
+            if(classifications['labels'][0] == 'Is not a localization'):
+                print("Dimension: "+result[0]+" is missing. Inserting a warning")
+                warnings.append(result[0]+" is missing. Please indicate where the data has been collected")
+        if(result[0] == "Time Localization"):  
+            classifications = classifier(result[1], ["It is a date","It is not a date"])
+            if(classifications['labels'][0] == 'Is not a localization'):
+                print("Dimension: "+result[0]+" is missing. Inserting a warning")
+                warnings.append(result[0]+" is missing. Please indicate when the data has been collected")
+    if len(warnings) == 0:
+        warnings.append("No warnings")
+    return warnings
+
+# Define function to handle the Gradio interface 
+async def annotate_only(input_file, apikey):
+    #progress(0, desc="Starting")
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey) 
+    # Prepare the data
+    #progress(0.20, desc="Preparing data: Generating embeddings, splitting text, and adding transformed tables")
+    docsearch = await prepare_data(input_file, chain_table, apikey)
+    # Get annotation dimensions
+    #progress(0.40, desc="Extracting dimensions")
+    results = await get_annotation_dimension(docsearch,chain_incontext, retrieved_docs=10)
+    # Get warning
+    #progress(0.80, desc="Generating Warning")
+    warnings = get_warnings(results)
+
+    # Build results in the correct format for the Gradio front-end
+    results = pd.DataFrame(results, columns=['Dimension', 'Results'])
+    return results, gr.update(value=pd.DataFrame(warnings,columns=['Warnings:']), visible=True)
+# Define function to handle the Gradio interface
+async def uses_only(input_file, apikey):
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey) 
+    # Prepare the data
+    docsearch = await prepare_data(input_file, chain_table, apikey)
+    # Get annotation dimensions
+    results = await get_uses_dimension(docsearch,chain_incontext, retrieved_docs=10)
+    # Get warning
+    warnings = get_warnings(results)
+
+    # Build results in the correct format for the Gradio front-end
+    results = pd.DataFrame(results, columns=['Dimension', 'Results'])
+    return results, gr.update(value=pd.DataFrame(warnings,columns=['Warnings:']), visible=True)
+
+# Define function to handle the Gradio interface
+async def distribution_only(input_file, apikey):
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey) 
+    # Prepare the data
+    docsearch = await prepare_data(input_file, chain_table, apikey)
+    # Get annotation dimensions
+    results = await get_distribution_dimension(docsearch,chain_incontext, retrieved_docs=10)
+    # Get warning
+    warnings = get_warnings(results)
+
+    # Build results in the correct format for the Gradio front-end
+    results = pd.DataFrame(results, columns=['Dimension', 'Results'])
+    return results, gr.update(value=pd.DataFrame(warnings,columns=['Warnings:']), visible=True)
+# Define function to handle the Gradio interface
+async def social_cocerns_only(input_file, apikey):
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey) 
+    # Prepare the data
+    docsearch = await prepare_data(input_file, chain_table, apikey)
+    # Get annotation dimensions
+    results = await get_social_concerns_dimension(docsearch,chain_incontext, retrieved_docs=10)
+    # Get warning
+    warnings = get_warnings(results)
+
+    # Build results in the correct format for the Gradio front-end
+    results = pd.DataFrame(results, columns=['Dimension', 'Results'])
+    return results, gr.update(value=pd.DataFrame(warnings,columns=['Warnings:']), visible=True)
+# Define function to handle the Gradio interface
+async def composition_only(input_file, apikey):
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey) 
+    # Prepare the data
+    docsearch = await prepare_data(input_file, chain_table, apikey)
+    # Get annotation dimensions
+    results = await get_composition_dimension(docsearch,chain_incontext, retrieved_docs=10)
+    # Get warning
+    warnings = get_warnings(results)
+
+    # Build results in the correct format for the Gradio front-end
+    results = pd.DataFrame(results, columns=['Dimension', 'Results'])
+    return results, gr.update(value=pd.DataFrame(warnings,columns=['Warnings:']), visible=True)
+# Define function to handle the Gradio interface
+async def contributors_only(input_file, apikey):
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey) 
+    # Prepare the data
+    docsearch = await prepare_data(input_file, chain_table, apikey)
+    # Get annotation dimensions
+    results = await get_contributors_dimension(docsearch,chain_incontext, retrieved_docs=10)
+    # Get warning
+    warnings = get_warnings(results)
+
+    # Build results in the correct format for the Gradio front-end
+    results = pd.DataFrame(results, columns=['Dimension', 'Results'])
+    return results, gr.update(value=pd.DataFrame(warnings,columns=['Warnings:']), visible=True)
+
+async def gather_only(input_file, apikey):
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey) 
+    # Prepare the data
+    docsearch = await prepare_data(input_file, chain_table, apikey)
+    # Get annotation dimensions
+    results = await get_gathering_dimension(docsearch,chain_incontext, retrieved_docs=10)
+    # Get warning
+    warnings = get_warnings(results)
+    results = pd.DataFrame(results, columns=['Dimension', 'Results'])
+    return  results, gr.update(value=pd.DataFrame(warnings, columns=['Warnings:']), visible=True)
+
+async def complete(input_file):
+
+    # Build the chains
+    chain_incontext, chain_table = build_chains(apikey=os.getenv("OPEN_AI_API_KEY")) 
+    # Prepare the data
+    docsearch = await prepare_data(input_file, chain_table, apikey=os.getenv("OPEN_AI_API_KEY"))
+    #Retrieve dimensions    
+    results = await asyncio.gather(get_annotation_dimension(docsearch,chain_incontext, retrieved_docs=10),
+                                    get_gathering_dimension(docsearch,chain_incontext, retrieved_docs=10),
+                                    get_uses_dimension(docsearch,chain_incontext, retrieved_docs=10),
+                                    get_contributors_dimension(docsearch,chain_incontext, retrieved_docs=10),
+                                    get_composition_dimension(docsearch,chain_incontext, retrieved_docs=10),
+                                    get_social_concerns_dimension(docsearch,chain_incontext, retrieved_docs=10),
+                                    get_distribution_dimension(docsearch,chain_incontext, retrieved_docs=10))
+    # Get warning from the results
+    warnings = []
+    for result in results:
+        warnings.append(gr.update(value=[get_warnings(result)], visible=True))
+    #warnings_dt = gr.update(value=pd.DataFrame(warnings,columns=['Warnings:'],labels= None), visible=True)
+    results.extend(warnings)
+    return results
+
+async def annotation_api_wrapper(input_file, apikey):
+
+    results, alarms = await annotate_only(input_file, apikey)
+
+    response_results = results.to_dict()
+    response_alarms = alarms['value'].to_dict()
+    api_answer = {'results':response_results, 'warnings': response_alarms}
+    return api_answer
+
+## Building the layout of the app
+css = """.table-wrap.scroll-hide.svelte-8hrj8a.no-wrap {
+    white-space: normal;
+}
+#component-7 .wrap.svelte-xwlu1w {
+    min-height: var(--size-40);
+}
+div#component-2 h2 {
+    color: var(--block-label-text-color);
+    text-align: center;
+    border-bottom: 2px solid;
+    border-radius: 7px;
+    text-align: center;
+    margin: 0 15% 0 15%;
+}
+div#component-5 {
+    border: 1px solid var(--border-color-primary);
+    border-radius: 0 0px 10px 10px;
+    padding: 20px;
+}
+.gradio-container.gradio-container-3-26-0.svelte-ac4rv4.app {
+    max-width: 850px;
+}
+div#component-6 {
+    min-height: 150px;
+}
+button#component-17 {
+    color: var(--block-label-text-color);
+}
+.gradio-container.gradio-container-3-26-0.svelte-ac4rv4.app {
+    max-width: 1100px;
+}
+#component-9 .wrap.svelte-xwlu1w {
+    min-height: var(--size-40);
+}
+div#component-11 {
+    height: var(--size-40);
+}
+div#component-9 {
+    border: 1px solid grey;
+    border-radius: 10px;
+    padding: 3px;
+    text-align: center;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
+    with gr.Row():
+            gr.Markdown("## Dataset documentation analyzer")
+    with gr.Row():
+        gr.Markdown("""Extract, in a structured manner, the **[general guidelines](https://knowingmachines.org/reading-list#dataset_documentation_practices)** from the ML community about dataset documentation practices from its scientific documentation. Study and analyze scientific data published in peer-review journals such as: **[Nature's Scientific Data](https://duckduckgo.com)** and **[Data-in-Brief](https://duckduckgo.com)**. Here you have a **[complete list](https://zenodo.org/record/7082126#.ZDaf-OxBz0p)** of data journals suitable to be analyzed with this tool.
+                 """)
+        
+    with gr.Row():
+        
+        with gr.Column():
+           fileinput = gr.File(label="Upload TXT file"),
+
+        with gr.Column():
+             gr.Markdown(""" <h4 style=text-align:center>Instructions: </h4> 
+     
+         <b>  &#10549; Try the examples </b> at the bottom 
+
+         <b> &#8680; Set your API KEY </b> of OpenAI  
+        
+         <b> &#8678; Upload </b> your data paper (in PDF or TXT)
+
+         <b> &#8681; Click in get insights  </b> in one tab!
+
+         """)
+        with gr.Column():
+            apikey_elem = gr.Text(label="Your OpenAI APIKey")
+         #   gr.Markdown(""" 
+         #                   <h3> Improving your data and assesing your dataset documentation </h3>
+         #                   The generated warning also allows you quicly check the completeness of the documentation, and spotting gaps in the document
+         #                   <h3> Performing studies studies over scientific data </h3>
+         #                   If you need to analyze a large scale of documents, we provide an <strong>API</strong> that can be used programatically. Documentation on how to use it is at the bottom of the page.  """)
+    with gr.Row():
+        with gr.Tab("Annotation"):
+       
+            gr.Markdown("""In this chapter, we get information regarding the annotation process of the data: We provide a description of the process and we infer its type from the documentation. Then we extract the labels, and information about the annotation team, the infrastructure used to annotate the data and the validation process applied over the labels""")
+            result_anot = gr.DataFrame(headers=["dimension","result"],type="array",label="Results of the extraction:")
+            alerts_anot = gr.DataFrame(headers=["warnings"],type="array", visible=False)
+            button_annotation = gr.Button("Get the annotation process insights!")
+                
+        with gr.Tab("Gathering"):
+            gr.Markdown("""In this chapter, we get information regarding the collection process of the data: We provide a description of the process and we infer its type from the documentation. Then we extract information about the collection team, the infrastructure used to collect the data and the sources. Also we get the timeframe of the data collection and its geolocalization.""")
+            result_gather = gr.DataFrame(headers=["dimension","result"],type="array",label="Results of the extraction:")
+            alerts_gather = gr.DataFrame(headers=["warnings"],type="array", visible=False)
+            button_gathering = gr.Button("Get the gathering process insights!")
+        with gr.Tab("Uses"):
+            gr.Markdown("""In this chapter, we extract the design intentios of the authors, we extract the purposes, gaps, and we infer the ML tasks (extracted form hugginface) the dataset is inteded for. Also we get the uses recomendation and the ML Benchmarks if the dataset have been tested with them""")
+            result_uses = gr.DataFrame(headers=["dimension","result"],type="array",label="Results of the extraction:")
+            alerts_uses = gr.DataFrame(headers=["warnings"],type="array", visible=False)
+            button_uses = gr.Button("Get the uses of the dataset!")
+        with gr.Tab("Contributors"):
+            gr.Markdown("""In this chapter, we extract all the contributors, funding information and maintenance of the dataset""")
+            result_contrib = gr.DataFrame(headers=["dimension","result"],type="array",label="Results of the extraction:")
+            alerts_contrib = gr.DataFrame(headers=["warnings"],type="array", visible=False)
+            button_contrib = gr.Button("Get the contributors of the dataset!")
+          
+        with gr.Tab("Composition"):
+            gr.Markdown("""In this chapter, we extract the file structure, we identify the attributes of the dataset, the recommneded trainig splits and the relevant statistics (if provided in the documentation) """)
+            result_comp = gr.DataFrame(headers=["dimension","result"],type="array",label="Results of the extraction:")
+            alerts_comp = gr.DataFrame(headers=["warnings"],type="array", visible=False)
+            button_comp = gr.Button("Get the composition of the dataset!")
+        with gr.Tab("Social Concerns"):
+            gr.Markdown("""In this chapter, we extract social concerns regarding the representativeness of social groups, potential biases, sensitivity issues, and privacy issues. """)
+            result_social = gr.DataFrame(headers=["dimension","result"],type="array",label="Results of the extraction:")
+            alerts_social = gr.DataFrame(headers=["warnings"],type="array", visible=False)
+            button_social = gr.Button("Get the Social Cocerns!")
+
+        with gr.Tab("Distribution"):
+            gr.Markdown("""In this chapter, we aim to extract the legal conditions under the dataset is released) """)
+            result_distri = gr.DataFrame(headers=["dimension","result"],type="array",label="Results of the extraction:")
+            alerts_distribution = gr.DataFrame(headers=["warning"],type="array", visible=False)
+            button_dist = gr.Button("Get the Distribution!")
+    with gr.Row():
+        examples = gr.Examples(
+                examples=["sources/Nature-Scientific-Data/A whole-body FDG-PET:CT.pdf","sources/Nature-Scientific-Data/Lontar-Manuscripts.pdf"],
+                inputs=[fileinput[0]], 
+                fn=complete, 
+                outputs=[
+                    result_anot,
+                    result_gather,
+                    result_uses,
+                    result_contrib,
+                    result_comp,
+                    result_social,
+                    result_distri,
+                    alerts_anot,
+                    alerts_gather,
+                    alerts_uses,
+                    alerts_contrib,
+                    alerts_comp,
+                    alerts_social,
+                    alerts_distribution], 
+                cache_examples=True)
+        button_complete = gr.Button("Get all the dimensions", visible=False)
+    allres = gr.Text(visible=False)
+    ## Events of the apps
+    button_annotation.click(annotate_only,inputs=[fileinput[0],apikey_elem ],outputs=[result_anot,alerts_anot])
+    button_gathering.click(gather_only,inputs=[fileinput[0],apikey_elem ],outputs=[result_gather,alerts_gather])
+    button_uses.click(uses_only,inputs=[fileinput[0],apikey_elem ],outputs=[result_uses,alerts_uses])
+    button_contrib.click(contributors_only,inputs=[fileinput[0],apikey_elem ],outputs=[result_contrib,alerts_contrib])
+    button_comp.click(composition_only,inputs=[fileinput[0],apikey_elem ],outputs=[result_comp,alerts_comp])
+    button_social.click(social_cocerns_only,inputs=[fileinput[0],apikey_elem ],outputs=[result_social,alerts_social])
+    button_dist.click(distribution_only,inputs=[fileinput[0],apikey_elem ],outputs=[result_distri,alerts_distribution])
+   
+
+    ## API endpoints
+    button_complete.click(annotation_api_wrapper,inputs=[fileinput[0],apikey_elem],outputs=allres, api_name="our")
+    
+   
+    # Run the app
+    #demo.queue(concurrency_count=5,max_size=20).launch()
+    demo.launch(share=True,auth=("CKIM2023", "demodemo"))
+        

packages.txt

@@ -0,0 +1 @@
+default-jre

requirements.txt

@@ -0,0 +1,317 @@
+absl-py==1.3.0
+accelerate==0.19.0
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+alembic==1.8.1
+altair==5.0.0
+anyio==3.6.2
+appdirs==1.4.4
+appnope==0.1.3
+asttokens==2.1.0
+astunparse==1.6.3
+async-timeout==4.0.2
+attrs==23.1.0
+azure-ai-formrecognizer==3.2.0
+azure-common==1.1.28
+azure-core==1.26.1
+backcall==0.2.0
+backoff==2.2.1
+beautifulsoup4==4.11.1
+bitsandbytes==0.39.0
+blis==0.7.9
+blobfile==2.0.0
+cachetools==5.2.0
+camelot-py==0.9.0
+catalogue==2.0.8
+certifi==2023.5.7
+cffi==1.15.1
+chardet==5.1.0
+charset-normalizer==3.1.0
+click==8.1.3
+cloudpickle==2.2.0
+cohere==4.1.3
+confection==0.0.4
+construct==2.5.3
+contourpy==1.0.7
+cryptography==40.0.2
+cycler==0.11.0
+cymem==2.0.7
+databricks-cli==0.17.3
+dataclasses-json==0.5.7
+datasets==2.7.1
+debugpy==1.6.3
+decorator==5.1.1
+dill==0.3.6
+Distance==0.1.3
+distlib==0.3.6
+distro==1.8.0
+docker==6.0.1
+docopt==0.6.2
+elasticsearch==7.17.7
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl
+entrypoints==0.4
+et-xmlfile==1.1.0
+executing==1.2.0
+extruct==0.14.0
+faiss-cpu==1.7.2
+farm-haystack==1.12.2
+fastapi==0.95.2
+fastjsonschema==2.16.2
+ffmpy==0.3.0
+filelock==3.12.0
+Flask==2.2.2
+flatbuffers==22.10.26
+fonttools==4.39.4
+frozenlist==1.3.3
+fsspec==2023.5.0
+future==0.18.2
+gast==0.4.0
+gitdb==4.0.9
+GitPython==3.1.29
+google-auth==2.14.1
+google-auth-oauthlib==0.4.6
+google-pasta==0.2.0
+gpt-index==0.5.12
+gradio==3.26.0
+gradio_client==0.1.2
+grobid-client-python==0.0.5
+grpcio==1.50.0
+gunicorn==20.1.0
+h11==0.14.0
+h5py==3.7.0
+haystack==0.42
+html-text==0.5.2
+html5lib==1.1
+htmlmin==0.1.12
+httpcore==0.17.1
+httpx==0.24.1
+huggingface-hub==0.14.1
+idna==3.4
+ImageHash==4.3.1
+importlib-metadata==5.0.0
+inflect==6.0.2
+iniconfig==1.1.1
+ipykernel==6.17.1
+ipython==8.6.0
+ipywidgets==8.0.2
+isodate==0.6.1
+itsdangerous==2.1.2
+jaconv==0.3
+jamo==0.4.1
+jarowinkler==1.2.3
+jedi==0.18.1
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+jstyleson==0.0.2
+jupyter_client==7.4.7
+jupyter_core==5.0.0
+jupyterlab-widgets==3.0.3
+keras==2.10.0
+Keras-Preprocessing==1.1.2
+kiwisolver==1.4.4
+langchain==0.0.173
+langcodes==3.3.0
+langdetect==1.0.9
+libclang==14.0.6
+linkify-it-py==2.0.2
+llvmlite==0.39.1
+loralib==0.1.1
+lxml==4.9.1
+Mako==1.2.4
+Markdown==3.4.1
+markdown-it-py==2.2.0
+markdown2==2.4.6
+MarkupSafe==2.1.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mf2py==1.1.2
+mistune==2.0.4
+mlflow==2.0.1
+mmh3==3.0.0
+monotonic==1.6
+more-itertools==9.0.0
+mpmath==1.2.1
+msgpack==1.0.4
+msrest==0.7.1
+multidict==6.0.4
+multimethod==1.9
+multiprocess==0.70.14
+murmurhash==1.0.9
+mypy-extensions==0.4.3
+nest-asyncio==1.5.6
+networkx==2.8.8
+nltk==3.7
+num2words==0.5.12
+numba==0.56.4
+numexpr==2.8.4
+numpy==1.24.3
+oauthlib==3.2.2
+openai==0.27.2
+openapi-schema-pydantic==1.2.4
+opencv-python==4.7.0.72
+openpyxl==3.0.10
+opt-einsum==3.3.0
+orjson==3.8.12
+outcome==1.2.0
+packaging==23.1
+pandas==2.0.1
+pandas-profiling==3.6.0
+pandocfilters==1.5.0
+parso==0.8.3
+pathspec==0.10.2
+pathy==0.10.1
+patsy==0.5.3
+pdf2image==1.16.0
+pdfminer.six==20221105
+pefile==2022.5.30
+pexpect==4.8.0
+phik==0.12.3
+pickleshare==0.7.5
+Pillow==9.5.0
+platformdirs==2.5.4
+pluggy==1.0.0
+posthog==2.2.0
+preshed==3.0.8
+prompt-toolkit==3.0.32
+protobuf==3.19.6
+psutil==5.9.4
+psycopg2-binary==2.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py==1.11.0
+py-cpuinfo==9.0.0
+pyarrow==10.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.21
+pycryptodomex==3.16.0
+pydantic==1.10.7
+pydub==0.25.1
+Pygments==2.13.0
+PyJWT==2.6.0
+pyparsing==3.0.9
+PyPDF2==2.12.1
+pyphen==0.14.0
+pypinyin==0.44.0
+pyRdfa3==3.5.3
+pyrsistent==0.19.3
+PySocks==1.7.1
+pytesseract==0.3.10
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==0.21.0
+python-magic==0.4.27
+python-multipart==0.0.6
+python-ptrace==0.9.8
+pytorch-wpe==0.0.1
+pytrec-eval==0.5
+pytz==2023.3
+PyWavelets==1.4.1
+PyYAML==6.0
+pyzmq==24.0.1
+quantulum3==0.7.11
+querystring-parser==1.2.4
+rank-bm25==0.2.2
+rapidfuzz==2.7.0
+rdflib==6.2.0
+regex==2022.10.31
+requests==2.30.0
+requests-oauthlib==1.3.1
+responses==0.18.0
+rsa==4.9
+safetensors==0.3.1
+scikit-learn==1.1.3
+scipdf @ git+https://github.com/titipata/scipdf_parser@501cf2547320adca78a94e2b2ba83526e46bd82e
+scipy==1.9.3
+seaborn==0.12.1
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.97
+seqeval==1.2.2
+shap==0.41.0
+six==1.16.0
+slicer==0.0.7
+smart-open==6.3.0
+smmap==5.0.0
+sniffio==1.3.0
+sortedcontainers==2.4.0
+soupsieve==2.3.2.post1
+spacy==3.5.3
+spacy-legacy==3.0.12
+spacy-loggers==1.0.4
+SQLAlchemy==1.4.44
+SQLAlchemy-Utils==0.38.3
+sqlparse==0.4.3
+srsly==2.4.6
+stack-data==0.6.1
+starlette==0.27.0
+statsmodels==0.13.5
+sympy==1.11.1
+tabula-py==2.7.0
+tabulate==0.9.0
+tangled-up-in-unicode==0.2.0
+tenacity==8.2.2
+tensorboard==2.10.1
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorflow-estimator==2.10.0
+tensorflow-macos==2.10.0
+tensorflow-metal==0.6.0
+termcolor==2.1.0
+textstat==0.7.3
+thinc==8.1.10
+threadpoolctl==3.1.0
+tika==1.24
+tiktoken==0.4.0
+tinycss2==1.2.1
+tk==0.1.0
+tokenize-rt==5.0.0
+tokenizers==0.12.1
+toml==0.10.2
+tomli==2.0.1
+tomli_w==1.0.0
+tomlkit==0.11.6
+toolz==0.12.0
+torch==1.12.1
+torch-complex==0.4.3
+torchvision==0.14.0
+tornado==6.2
+tqdm==4.65.0
+traitlets==5.5.0
+transformers @ git+https://github.com/huggingface/transformers@4b6a5a7caa1ea31a3321eb17e6dcc9baff4f55d9
+typeguard==2.13.3
+typer==0.7.0
+typing-inspect==0.8.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+ujson==5.1.0
+Unidecode==1.3.6
+url-normalize==1.4.3
+urllib3==2.0.2
+uvicorn==0.22.0
+validators==0.18.2
+virtualenv==20.16.7
+visions==0.7.5
+w3lib==2.1.1
+wasabi==1.1.1
+watchdog==2.1.9
+wcwidth==0.2.5
+webencodings==0.5.1
+websocket-client==1.4.2
+websockets==11.0.3
+Werkzeug==2.2.2
+widgetsnbextension==4.0.3
+wrapt==1.14.1
+xlwt==1.3.0
+xmltodict==0.13.0
+xxhash==3.1.0
+yapf==0.32.0
+yarl==1.9.2
+zipp==3.10.0