Upload 4 files

app.py

@@ -0,0 +1,69 @@
+import os
+import sys
+from generate_markdown import load_llm_model, generate_markdown
+from generate_mindmap import generate_mindmap_svg
+import gradio as gr
+import subprocess
+
+llm = load_llm_model()
+
+def generate(file):
+    summary = "This is a summary of the research paper"
+    mindmap_markdown = generate_markdown(llm, file)
+    mindmap_svg = generate_mindmap_svg(mindmap_markdown)
+    return summary, mindmap_markdown, mindmap_svg
+
+theme = gr.themes.Soft(
+    primary_hue="purple",
+    secondary_hue="cyan",
+    neutral_hue="slate",
+    font=[gr.themes.GoogleFont('Syne'), gr.themes.GoogleFont('poppins'), gr.themes.GoogleFont('poppins'), gr.themes.GoogleFont('poppins')],
+)
+
+with gr.Blocks(theme=theme, title="Binary Biology") as app:
+    file = gr.File(file_count='single', label='Upload Research Paper PDF file')
+    summary = gr.TextArea(label='Summary', lines=5, interactive=False, show_copy_button=True)
+    markdown_mindmap = gr.Textbox(label='Mindmap', lines=5, interactive=False, show_copy_button=True)
+    graphical_mindmap = gr.Image(label='Graphical Mindmap', interactive=False, show_download_button=True)
+    submit = gr.Button(value='Submit')
+
+    submit.click(generate,
+                inputs=[file],
+                outputs=[summary, markdown_mindmap, graphical_mindmap],
+                scroll_to_output=True,
+                show_progress=True,
+                queue=True,
+    )
+
+if __name__ == "__main__":
+    # Download models
+    subprocess.run(['wget', '-o', 'model.gguf', 'https://huggingface.co/featherless-ai-quants/amazon-MegaBeam-Mistral-7B-300k-GGUF/resolve/main/amazon-MegaBeam-Mistral-7B-300k-Q6_K.gguf?download=true'])
+    # run entire command: "CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python"
+    try:
+        env = os.environ.copy()
+        env["CMAKE_ARGS"] = "-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS"
+        cmd = ["pip", "install", "llama-cpp-python"]
+        subprocess.run(cmd, env=env)
+    except:
+        cmd = ["pip", "install", "llama-cpp-python"]
+        subprocess.run(cmd)
+    try:
+        try:
+            subprocess.run(['apt', 'install', '-y', 'graphviz'])
+            print("Graphviz installed successfully")
+        except:
+            subprocess.run(['sudo', 'apt', 'install', '-y', 'graphviz'])
+            print("Graphviz installed successfully")
+    except:
+        print("Graphviz installation failed")
+        sys.exit(1)
+    print("Model loaded successfully")
+    llm = load_llm_model()
+    print("Model loaded successfully")
+    app.queue(default_concurrency_limit=5).launch(show_error=True)
+    # summary, markdown_mindmap, graphical_mindmap = generate("cr1c00107.pdf")
+    # print(summary)
+    # print("\n\n")
+    # print(markdown_mindmap)
+    # print("\n\n")
+    # print(graphical_mindmap)

generate_markdown.py

@@ -0,0 +1,95 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import PyPDFLoader
+from llama_cpp import Llama
+import llama_cpp
+
+def load_llm_model():
+    llm = Llama(
+        model_path="Llama-3.2-3B-Instruct-Q8_0.gguf",
+        # n_gpu_layers = 20, # Uncomment for GPU
+        n_ctx=200000,
+        n_threads=16,
+        n_batch=512,
+        split_mode=llama_cpp.LLAMA_SPLIT_MODE_LAYER,
+        pooling_type=llama_cpp.LLAMA_POOLING_TYPE_RANK,
+        rope_scaling_type=llama_cpp.LLAMA_ROPE_SCALING_TYPE_LINEAR,
+        # main_gpu=0 # Uncomment for GPU
+    )
+    return llm
+
+def get_text_from_pdf(file):
+    loader = PyPDFLoader(file)
+    pages = loader.load_and_split()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=250, chunk_overlap=50)
+    texts = text_splitter.split_documents(pages)
+    final_text = ""
+    for text in texts:
+        final_text = final_text + text.page_content
+    print(f"Length of final text: {len(final_text)}")
+    with open("final_text.txt", "w") as f:
+        f.write(final_text)
+    research_paper = ""
+    for line in final_text.split("\n"):
+        if line.startswith("REFERENCES"):
+            break
+        else:
+            research_paper = research_paper + line + " " 
+    with open("research_paper.txt", "w") as f:
+        f.write(research_paper)
+    print(f"Length of research paper: {len(research_paper)}")
+    return research_paper
+
+def generate_prompt(final_text):
+    prompt = f'''
+    You have been provided with a research paper in text format. Your task is to generate a mindmap structure in markdown format that summarizes the research paper.
+    Your output should use the language \"en\" 0.3 times the length of the original research paper. Do not include anything in the response, that is not the part of mindmap and use the following template (any node in the mindmap should not exceed 10-12 words, also generate additional headings that aren't present in document if required for elaborative explaination):
+        # {{Title}} (should be the title of the research paper)
+        ## {{Subtitle01}} (as required and as many as required in markdown format)
+        - {{Emoji01}} Bulletpoint01 (as required and as many as required in markdown format)
+            - {{Emoji01.1}} Bulletpoint01.1 (as required and as many as sub levels required in markdown format)
+                - {{Emoji01.1.1}} Bulletpoint01.1.1 (as required and as many as sub levels required in markdown format)
+                - {{Emoji01.1.2}} Bulletpoint01.1.2 (as required and as many as sub levels required in markdown format)
+            - {{Emoji01.2}} Bulletpoint01.2 (as required and as many as sub levels required in markdown format)
+        - {{Emoji02}} Bulletpoint02 (as required and as many as required in markdown format)
+            - {{Emoji02.1}} Bulletpoint02.1 (as required and as many as sub levels required in markdown format)
+            - {{Emoji02.2}} Bulletpoint02.2 (as required and as many as sub levels required in markdown format)
+                - {{Emoji02.2.1}} Bulletpoint02.2.1 (as required and as many as sub levels required in markdown format)
+                - {{Emoji02.2.2}} Bulletpoint02.2.2 (as required and as many as sub levels required in markdown format)
+                - {{Emoji02.2.3}} Bulletpoint02.2.3 (as required and as many as sub levels required in markdown format)
+                - {{Emoji02.2.4}} Bulletpoint02.2.4 (as required and as many as sub levels required in markdown format)
+        ## {{Subtitle02}} (as required and as many as required in markdown format)
+        - {{Emoji03}} Bulletpoint03 (as required and as many as required in markdown format)
+            - {{Emoji03.1}} Bulletpoint03.1 (as required and as many as sub levels required in markdown format)
+            - {{Emoji03.2}} Bulletpoint03.2 (as required and as many as sub levels required in markdown format)
+                - {{Emoji03.2.1}} Bulletpoint03.2.1 (as required and as many as sub levels required in markdown format)
+                - {{Emoji03.2.2}} Bulletpoint03.2.2 (as required and as many as sub levels required in markdown format)
+        - {{Emoji04}} Bulletpoint04 (as required and as many as required in markdown format)
+            - {{Emoji04.1}} Bulletpoint04.1 (as required and as many as sub levels required in markdown format)
+                - {{Emoji04.1.1}} Bulletpoint04.1.1 (as required and as many as sub levels required in markdown format)
+                - {{Emoji04.1.2}} Bulletpoint04.1.2 (as required and as many as sub levels required in markdown format)
+            - {{Emoji04.2}} Bulletpoint04.2 (as required and as many as sub levels required in markdown format)
+                - {{Emoji04.2.1}} Bulletpoint04.2.1 (as required and as many as sub levels required in markdown format)
+                - {{Emoji04.2.2}} Bulletpoint04.2.2 (as required and as many as sub levels required in markdown format)
+        Summarize the text \"{final_text}\" to generate a elaborated hierarchical mindmap structure (any node in the mindmap should not exceed 10-12 words, also generate additional headings that aren't present in document if required for elaborative explaination) markdown using the \"en\" language 0.3 times the length of the original research paper. Do not include anything in the response, that is not the part of mindmap
+    '''
+    return prompt
+
+def generate_mindmap_structure(llm, prompt):
+    response = llm.create_chat_completion(
+        messages = [
+            {'role':'user',
+            'content': prompt}
+        ],
+        temperature=0.7,
+        top_k=200,
+        top_p=3.0,
+    )
+    mindmap_data = response['choices'][0]['message']['content']
+    print(mindmap_data)
+    return mindmap_data
+
+def generate_markdown(llm, file):
+    final_text = get_text_from_pdf(file)
+    prompt = generate_prompt(final_text)
+    mindmap_markdown = generate_mindmap_structure(llm, prompt)
+    return mindmap_markdown

generate_mindmap.py

@@ -0,0 +1,108 @@
+from graphviz import Digraph
+import re
+import random
+
+def parse_markdown_to_dict(md_text):
+    lines = md_text.strip().splitlines()
+    mindmap = {}
+    stack = []
+    for line in lines:
+        heading_match = re.match(r'^(#{1,6})\s+(.*)', line)
+        bullet_match = re.match(r'^\s*-\s+(.*)', line)
+        if heading_match:
+            level = len(heading_match.group(1))
+            title = heading_match.group(2).strip()
+            node = {'title': title, 'children': []}
+            while len(stack) >= level:
+                stack.pop()
+            if stack:
+                stack[-1]['children'].append(node)
+            else:
+                mindmap = node  
+            stack.append(node)
+        elif bullet_match and stack:
+            stack[-1]['children'].append({'title': bullet_match.group(1), 'children': []})
+    return mindmap
+
+generated_colors = set()
+
+def generate_random_color():
+    """Generate a random color that hasn't been generated before."""
+    while True:
+        # Generate a random color in hex format
+        color = "#{:02x}{:02x}{:02x}".format(random.randint(128, 255), random.randint(128, 255), random.randint(128, 255))
+        # If the color is not in the set, it's unique
+        if color not in generated_colors:
+            generated_colors.add(color)  # Add the color to the set of generated colors
+            return color  # Return the unique color
+        else:
+            continue  # Try again
+
+def brighten_color(color, factor=0.15):
+    """Brighten the color by a certain factor (default 10%)"""
+    # Remove the '#' symbol
+    color = color.lstrip('#')
+    
+    # Convert hex to RGB
+    r, g, b = [int(color[i:i+2], 16) for i in (0, 2, 4)]
+    
+    # Increase each component by the factor, but clamp to 255
+    r = min(255, int(r * (1 + factor)))
+    g = min(255, int(g * (1 + factor)))
+    b = min(255, int(b * (1 + factor)))
+    
+    # Convert back to hex
+    return "#{:02x}{:02x}{:02x}".format(r, g, b)
+
+def add_nodes_to_graph(graph, node, parent_id=None, font_size=9, parent_color=None):
+    node_id = str(id(node))
+    title = node['title']
+    if parent_color is None:
+        node_color = "#ADD8E6"  # Light Blue for the main heading
+        border_color = "#000000"  # Dark Blue border for the main heading
+        parent_color = "#ADD8E6"
+    elif parent_color == "#ADD8E6":
+        node_color = generate_random_color()
+        border_color = "#808080"
+        parent_color = node_color
+    else:
+        # Child node and its descendants with the same random color
+        node_color = brighten_color(parent_color, factor=0.15)
+        border_color = "#808080"
+    # Check for markdown links
+    url_match = re.search(r'\[(.*?)\]\((.*?)\)', title)
+    if url_match:
+        prefix_text = title[:url_match.start()].strip()
+        display_text = url_match.group(1)
+        url = url_match.group(2)
+        
+        label = f'{prefix_text} {display_text}'
+        graph.node(node_id, label=label, shape="box", style="rounded,filled", color=border_color, fontcolor="black", fillcolor=node_color, href=url, tooltip=title, fontsize=str(font_size))
+    else:
+        graph.node(node_id, title, shape="box", style="rounded,filled", color=border_color, fontcolor="black", fillcolor=node_color, tooltip=title, fontsize=str(font_size))
+    
+    if parent_id:
+        graph.edge(parent_id, node_id)
+
+    # Recurse to children, passing down color for the child and its descendants
+    for child in node.get('children', []):
+        # Assign a random color to each child node (no inheritance from parent)
+        add_nodes_to_graph(graph, child, node_id, font_size=max(8, font_size - 1), parent_color=parent_color)
+
+def generate_mindmap_svg(md_text):
+    mindmap_dict = parse_markdown_to_dict(md_text)
+    root_title = mindmap_dict.get('title', 'Mindmap')
+    sanitized_title = re.sub(r'[^a-zA-Z0-9_\-]', '', root_title.replace(" ", ""))
+    if output_filename is None:
+        output_filename = sanitized_title
+    graph = Digraph(format='svg')
+    graph.attr(rankdir='LR', size='10,10!', pad="0.5", margin="0.2", ratio="auto")
+    graph.attr('node', fontname="Arial", fontsize="9")
+    add_nodes_to_graph(graph, mindmap_dict)
+    svg_content = graph.pipe(format='svg').decode('utf-8')
+    # Replace %3 with the sanitized filename in the SVG content
+    svg_content = svg_content.replace("%3", root_title)
+    # Save the modified SVG content to a file
+    with open(f'{output_filename}.svg', 'w') as f:
+        f.write(svg_content)
+    return f"{output_filename}".svg

requirements.txt

@@ -0,0 +1,5 @@
+gradio==5.5.0
+langchain==0.3.7
+langchain-community==0.3.7
+graphviz==0.20.3
+llama-cpp-python==0.3.1