app.py

import gradio as gr
from dotenv import load_dotenv

from utils import get_evaluated_component, get_evaluated_component_adjective, get_result_description, to_title_case, get_prompt_from_test_case, to_snake_case
load_dotenv()
import json
from model import generate_text
from logger import logger
import os 
from gradio_modal import Modal

catalog = {}

with open('catalog.json') as f:
    logger.debug('Loading catalog from json.')
    catalog = json.load(f)

def update_selected_test_case(button_name, state: gr.State, event: gr.EventData):
    target_sub_catalog_name, target_test_case_name = event.target.elem_id.split('---')
    state['selected_sub_catalog'] = target_sub_catalog_name
    state['selected_criteria_name'] = target_test_case_name
    state['selected_test_case'] = [t for sub_catalog_name, sub_catalog in catalog.items() for t in sub_catalog if t['name'] == to_snake_case(button_name) and to_snake_case(sub_catalog_name) == target_sub_catalog_name][0]
    return state

def on_test_case_click(state: gr.State):
    selected_sub_catalog = state['selected_sub_catalog']
    selected_criteria_name = state['selected_criteria_name']
    selected_test_case = state['selected_test_case']
    logger.debug(f'Changing to test case "{selected_criteria_name}" from catalog "{selected_sub_catalog}".')
    return {
        test_case_name: f'<h2>{to_title_case(selected_test_case["name"])}</h2>',
        criteria: selected_test_case['criteria'],
        context: gr.update(value=selected_test_case['context'], visible=True) if selected_test_case['context'] is not None else gr.update(visible=False, value=''),
        user_message: gr.update(value=selected_test_case['user_message'], interactive=True, elem_classes=['input-box']) if selected_sub_catalog != 'harmful_content_in_assistant_message' else gr.update(value=selected_test_case['user_message'], interactive=False, elem_classes=['read-only', 'input-box']),
        assistant_message: gr.update(value=selected_test_case['assistant_message'], visible=True) if selected_test_case['assistant_message'] is not None else gr.update(visible=False, value=''),
        result_text: gr.update(value='', visible=False),
        result_container: gr.update(visible=False),
        evaluation_results_label: gr.update(visible=False)
    }

def change_button_color(event: gr.EventData):
    return [gr.update(elem_classes=['catalog-button', 'selected']) if v.elem_id == event.target.elem_id else gr.update(elem_classes=['catalog-button']) for c in catalog_buttons.values() for v in c.values()]

def on_submit(criteria, context, user_message, assistant_message, state):
    prompt = get_prompt_from_test_case({
        'name': state['selected_criteria_name'],
        'criteria': criteria,
        'context': context,
        'user_message': user_message,
        'assistant_message': assistant_message,
    }, state['selected_sub_catalog'])
    logger.debug(f"Starting evaluation for subcatelog {state['selected_sub_catalog']} and criteria name {state['selected_criteria_name']}")
    result_label = generate_text(prompt)['assessment'] # Yes or No
    # result_str = f"Is the {evaluated_component} message harmful based on the risk definition: {result_label}"
    html_str = f"<p>{get_result_description(state['selected_sub_catalog'], state['selected_criteria_name'])} <strong>{result_label}</strong></p>"
    # html_str = f"Is the {evaluated_component} {evaluated_component_adjective}: {result_label}"
    return [gr.update(value=html_str), gr.update(visible=True), gr.update(visible=True)]

def on_show_prompt_click(criteria, context, user_message, assistant_message, state):
    prompt = get_prompt_from_test_case({
        'name': state['selected_criteria_name'],
        'criteria': criteria,
        'context': context,
        'user_message': user_message,
        'assistant_message': assistant_message,
    }, state['selected_sub_catalog'])
    prompt['content'] = prompt['content'].replace('<', '&lt;').replace('>', '&gt;').replace('\n', '<br>')
    prompt = json.dumps(prompt, indent=4)
    return gr.Markdown(prompt)

ibm_blue = blue = gr.themes.Color(
    name="ibm-blue",
    c50="#eff6ff",
    c100="#dbeafe",
    c200="#bfdbfe",
    c300="#93c5fd",
    c400="#60a5fa",
    c500="#0F62FE",
    c600="#2563eb",
    c700="#1d4ed8",
    c800="#1e40af",
    c900="#1e3a8a",
    c950="#1d3660"
)

with gr.Blocks(
    title='Granite Guardian',
    theme=gr.themes.Soft(
        primary_hue=ibm_blue,
        font=[gr.themes.GoogleFont("IBM Plex Sans"), gr.themes.GoogleFont('Source Sans 3')]),
    css='styles.css') as demo:
    
    with gr.Blocks(elem_classes="roott"):

        state = gr.State(value={
            'selected_sub_catalog': 'harmful_content_in_user_message',
            'selected_criteria_name': 'general_harm'
        })

        starting_test_case = [t for sub_catalog_name, sub_catalog in catalog.items() for t in sub_catalog if t['name'] == state.value['selected_criteria_name'] and sub_catalog_name == state.value['selected_sub_catalog']][0]
        
        with gr.Row(elem_classes='title-row'):
            with gr.Column(scale=4):
                gr.HTML('<h2>IBM Granite Guardian 3.0</h2>', elem_classes='title')
                gr.HTML(elem_classes='system-description', value='<p>Granite Guardian models are specialized language models in the Granite family that allow you to detect harms and risks in generative AI systems. The Granite Guardian models can be used with any other large language models to make interactions with generative AI systems safe. Select an example in the left panel to see how the model evaluates harms and risks in user messages, assistant messages, and for hallucinations in retrieval-augmented generation. In this demo, we use granite-guardian-3.0-8B.</p>')
        with gr.Row(elem_classes='column-gap'):
            with gr.Column(scale=0, elem_classes='no-gap'):
                title_display_left = gr.HTML("<h2>Harms & Risks</h2>", elem_classes=['subtitle', 'subtitle-harms'])
                accordions = []
                catalog_buttons: dict[str,dict[str,gr.Button]] = {}
                for i, (sub_catalog_name, sub_catalog) in enumerate(catalog.items()):
                    with gr.Accordion(to_title_case(sub_catalog_name), open=(i==0), elem_classes='accordion') as accordion:
                        for test_case in sub_catalog:
                            elem_classes=['catalog-button']
                            elem_id=f"{sub_catalog_name}---{test_case['name']}"
                            if starting_test_case == test_case:
                                elem_classes.append('selected')

                            if not sub_catalog_name in catalog_buttons:
                                catalog_buttons[sub_catalog_name] = {}

                            catalog_buttons[sub_catalog_name][test_case['name']] = \
                                gr.Button(to_title_case(test_case['name']), elem_classes=elem_classes, variant='secondary', size='sm', elem_id=elem_id)
                            
                        accordions.append(accordion)

            with gr.Column(visible=True, scale=1) as test_case_content:
                with gr.Row():
                    test_case_name = gr.HTML(f'<h2>{to_title_case(starting_test_case["name"])}</h2>', elem_classes='subtitle')
                    show_propt_button = gr.Button('Show prompt', size='sm', scale=0, min_width=110)

                criteria = gr.Textbox(label="Evaluation Criteria", lines=3, interactive=False, value=starting_test_case['criteria'], elem_classes=['read-only', 'input-box', 'margin-bottom'])
                gr.HTML(elem_classes=['block', 'content-gap'])
                context = gr.Textbox(label="Context", lines=3, interactive=True, value=starting_test_case['context'], visible=False, elem_classes=['input-box'])
                user_message = gr.Textbox(label="User Message", lines=3, interactive=True, value=starting_test_case['user_message'], elem_classes=['input-box'])
                assistant_message = gr.Textbox(label="Assistant Message", lines=3, interactive=True, visible=False, value=starting_test_case['assistant_message'], elem_classes=['input-box'])

                submit_button = gr.Button("Evaluate", variant='primary',icon=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'send-white.png'), elem_classes='submit-button')
                
                with gr.Column(elem_classes="result-container", visible=False) as result_container:
                    evaluation_results_label = gr.HTML("<span>Results</span>", elem_classes='result-title', visible=False)
                    result_text = gr.HTML(label="Result", elem_classes=['result-text', 'input-box', 'read-only', 'block'], visible=False, value='')

                with Modal(visible=False, elem_classes='modal') as modal:
                    prompt = gr.Markdown('')

    show_propt_button.click(
        on_show_prompt_click,
        inputs=[criteria, context, user_message, assistant_message, state],
        outputs=prompt
    ).then(lambda: gr.update(visible=True), None, modal)

    submit_button \
        .click(
            lambda: [gr.update(visible=True), gr.update(visible=True)],
            inputs=None,
            outputs=[result_container, result_text]) \
        .then(
            on_submit,
            inputs=[criteria, context, user_message, assistant_message, state],
            outputs=[result_text, evaluation_results_label, result_container])
      
    for button in [t for sub_catalog_name, sub_catalog_buttons in catalog_buttons.items() for t in sub_catalog_buttons.values()]:
        button.click(update_selected_test_case, inputs=[button, state], outputs=[state])\
            .then(on_test_case_click, inputs=state, outputs={test_case_name, criteria, context, user_message, assistant_message, result_text, result_container, evaluation_results_label}) \
            .then(change_button_color, None, [v for c in catalog_buttons.values() for v in c.values()])
    
demo.launch(server_name='0.0.0.0')