app.py

import pandas as pd
import gradio as gr
import numpy as np

df_table_dsc = pd.read_csv('./results/df_table_dsc.csv')
df_table_prd = pd.read_csv('./results/df_table_prd.csv')
n_lines = 12

defect_dict = {'NoDefect':'No defect', 'RO': 'Defect: Roots', 'OB': 'Defect: Surface damage', 
               'RB': 'Defect: Cracks, breaks, and collapses', 'PF': 'Defect: Production error', 
               'DE':'Defect: Deformation'}


css="""
#image-out {
  height: 450px;
  width: 450px;
}
"""

def conv_int(txt):
    try:
        return (int(txt))
    except:
        return 'NaN'

def return_prev(index):
    # Update current index based on navigation input
    index -=1 
    # Ensure index is within bounds
    index = max(0, min(index, len(df_table_dsc) - 1))

    image_path, gpt4, gpt4b, cogvlm, llava, defect_class, predictions, filename = return_image(index)
    return index, index, image_path, gpt4, gpt4b, cogvlm, llava, defect_class, predictions, filename

def return_next(index):
    # Update current index based on navigation input
    index +=1 
    # Ensure index is within bounds
    index = max(0, min(index, len(df_table_dsc) - 1))
    image_path, gpt4, gpt4b, cogvlm, llava, defect_class, predictions, filename = return_image(index)
    return index, index, image_path, gpt4, gpt4b, cogvlm, llava, defect_class, predictions, filename

def return_rand(index):
    # Update current index based on navigation input
    index = np.random.randint(0, len(df_table_dsc))
    image_path, gpt4, gpt4b, cogvlm, llava, defect_class, predictions, filename = return_image(index)
    return index, index, image_path, gpt4, gpt4b, cogvlm, llava, defect_class, predictions, filename
    
def return_image(index):
    # Fetch row from DataFrame
    row = df_table_dsc.iloc[index]
    image_path = f"./images/{row['img_id']}"

    defect_class =  defect_dict[row['defect_type']]    
    
    row_prd = df_table_prd.iloc[index]
    
    predictions = f"XIE:{conv_int(row_prd['Xie']>0.5)}, GPT4-V:{conv_int(row_prd['GPT4'])}, GPT4-Vs:{conv_int(row_prd['GPT4_basic'])}, CogVLM:{conv_int(row_prd['CogVLM'])}, LLaVA:{conv_int(row_prd['LLaVA'])}"

    # Return iamge path and descriptions
    return image_path, row['GPT4'], row['GPT4_basic'], row['CogVLM'], row['LLaVA'], defect_class, predictions, row['img_id']


with gr.Blocks(css=css) as demo: 
    # gr.Markdown("""
                ## Demo for 'The Potential of Generative AI for the Urban Water Sector'
                ### Riccardo Taormina, Delft University of Technology (TU Delft), Department of Water Management                
                ### email: r.taormina@tudelft.nl""")                
    gr.Markdown("""
                ## Testing Large Multimodal Models for Sewer Defect Inspection 
                Press on \<Next\>, \<Previous\> or \<Random\> to start!
                """)                
    index = gr.Number(value=-1, visible=False)      
    with gr.Row():
        with gr.Column(scale=1):
            img_out = gr.Image(type="filepath", label="Image", elem_id="image-out")
            with gr.Row():
                txt_item = gr.Textbox(label="Sample no.", min_width= 20)
                txt_filename = gr.Textbox(label="SewerML file", min_width= 20)
                txt_defect_class = gr.Textbox(label="Defect class", interactive=False, min_width= 150)
            txt_xie_pred = gr.Textbox(label="Predictions (XIE = benchmark*, 0 = No Defect, 1 = Defect)", min_width= 300, interactive=False)
            with gr.Row():
                prev_btn = gr.Button("Previous")            
                next_btn = gr.Button("Next")    
                rand_btn = gr.Button("Random") 
            with gr.Row():
                gr.Markdown('*Xie, Qian, et al. "Automatic detection and classification of sewer defects via hierarchical deep learning." IEEE Transactions on Automation Science and Engineering 16.4 (2019): 1836-1847.')
                
        with gr.Column(scale=1):
            gr.Markdown('Multimodal descriptions')
            with gr.Row():
                txt_out_GPT4 = gr.Textbox(label="GPT4-V", lines= n_lines, max_lines=n_lines)
                txt_out_GPT4s = gr.Textbox(label="GPT4-V simple", lines= n_lines, max_lines=n_lines)    
            with gr.Row():
                txt_out_CogVLM = gr.Textbox(label="CogVLM", lines= n_lines, max_lines=n_lines)
                txt_out_LLaVa = gr.Textbox(label="LLaVa", lines= n_lines, max_lines=n_lines)
            with gr.Row():
                with gr.Accordion("Basic prompt used for GPT4-V", open=False):
                    gr.Markdown("""
                    You are a virtual sewer technician with the capability to analyze images from CCTV cameras taken inside sewer pipes.
                    Your task is to examine each image and provide a concise, yet accurate, summary for retrieval.
                    After summarizing, you must classify the image as DEFECTIVE or NON DEFECTIVE.
                    You will always try to describe the image that you see.
                    You must provide your output in JSON format with DESCRIPTION: <your description of the image>, PREDICTION: <your prediction, either DEFECTIVE or NON DEFECTIVE>"""
                               )
            with gr.Row():
                with gr.Accordion("Prompt used for GPT4-V, CogVLM and LLaVA", open=False):
                    gr.Markdown("""
                    You are a virtual sewer technician with the capability to analyze images from CCTV cameras taken inside sewer pipes.
                    Your task is to examine each image and provide a concise, yet accurate, summary for retrieval. 
                    After summarizing, you must classify the image as DEFECTIVE or NON DEFECTIVE.
                    While providing the summary, remember the following guidelines:"
                    1) Provide a general overview of the image that you see, describing important elements such image clarity, lighting conditions, type of pipe (concrete, PVC, ...), presence of water.
                    2) Check for defects in the sewer pipes in the image.
                    3) Pipes in good condition usually show a smooth, unbroken surface, no visible signs of damage like cracks or collapses, and an absence of blockages such as roots.
                    4) On the other hand, you can have the following defects:
                    4a)Cracks, Breaks, and Collapses: Identify visible cracks along the pipe, instances where the pipe has fractured or completely broken apart, and areas where the pipe has collapsed.
                    This includes longitudinal cracks, circumferential breaks, and complete structural failures that compromise the integrity of the sewer system.
                    4b)Surface Damage: Detect areas of the pipe's interior that exhibit signs of wear, erosion, or damage on the surface.
                    This includes minor scratches, pitting, scaling, or any form of deterioration that affects the pipe's surface but does not necessarily penetrate deeply into the structure.
                    4c) Production Error: Identify defects that originated during the pipe's manufacturing process, such as inconsistent pipe thickness, improper joint alignment, or material imperfections.
                    These are flaws that were introduced before installation and could potentially affect the pipe's performance or longevity.
                    4d) Deformations: Recognize any alterations in the shape of the pipe, such as bending, sagging, or bulging, that indicate a deformation.
                    This includes both minor deformations that may affect flow efficiency and major deformations that threaten the pipe's structural integrity.
                    4e) Roots: Detect the presence of roots infiltrating the sewer pipe, whether through joints, cracks, or other vulnerabilities.
                    This involves identifying both the initial stages of root intrusion and the more advanced stages where roots have significantly obstructed the pipe.
                    5) Additional considerations while analyzing the images: do not consider blurred text or user-defined circled areas in the images.
                    6) You will always try to describe the image that you see.
                    You must provide your output in JSON format with DESCRIPTION: <your description of the image>, PREDICTION: <your prediction, either DEFECTIVE or NON DEFECTIVE>"""
                               )
                               
    
    prev_btn.click(fn=return_prev, inputs=index, outputs=[index, txt_item, img_out, 
                                                            txt_out_GPT4, txt_out_GPT4s, 
                                                            txt_out_CogVLM, txt_out_LLaVa, txt_defect_class, txt_xie_pred, txt_filename])
    next_btn.click(fn=return_next, inputs=index, outputs=[index, txt_item, img_out, 
                                                            txt_out_GPT4, txt_out_GPT4s, 
                                                            txt_out_CogVLM, txt_out_LLaVa, txt_defect_class, txt_xie_pred, txt_filename])
    rand_btn.click(fn=return_rand, inputs=index, outputs=[index, txt_item, img_out, 
                                                            txt_out_GPT4, txt_out_GPT4s, 
                                                            txt_out_CogVLM, txt_out_LLaVa, txt_defect_class, txt_xie_pred, txt_filename])

if __name__ == "__main__":
    demo.launch()