import gradio as gr from utils import process_paper import os import logging import html from logging_config import setup_logging from aws_utils import update_leaderboard, get_leaderboard, write_request from flask import request import hashlib import uuid setup_logging() paper_dir = 'path_to_temp_storage' prompt_dir = 'iclr2024' api_keys = { 'openai_api_key': os.environ.get('openai_api_key'), 'claude_api_key': os.environ.get('anthropic_api_key'), 'gemini_api_key': os.environ.get('google_api_key'), 'commandr_api_key': os.environ.get('cohere_api_key') } use_real_api = False # Function to generate a paper_id using SHA-512 hash def generate_paper_id(paper_content): return hashlib.sha512(paper_content).hexdigest() # Function to get user IP address def get_user_ip(): return request.remote_addr def review_papers(pdf_file): logging.info(f"Received file type: {type(pdf_file)}") paper_content = pdf_file.read() # Read the content of the uploaded PDF file if use_real_api: reviews, selected_models = process_paper( pdf_file, paper_dir, prompt_dir, api_keys) processed_reviews = [] for review in reviews: processed_review = {} for section in review: if ':' in section: key, value = section.split(':', 1) processed_value = value.strip().replace('\n', '
') processed_review[key.strip()] = html.escape( processed_value) processed_reviews.append(processed_review) reviews = processed_reviews else: reviews = [ # Placeholder reviews ] selected_models = ['model1-placeholder', 'model2-placeholder'] review_texts = [] for review in reviews: formatted_review = "
" for section, content in review.items(): formatted_review += f"
{section}: {html.unescape(content)}
" formatted_review += "
" review_texts.append(formatted_review) model_a = selected_models[0] model_b = selected_models[1] logging.debug(f"Final formatted reviews: {review_texts}") return review_texts[0], review_texts[1], gr.update(visible=True), gr.update(visible=True), model_a, model_b, paper_content def handle_vote(vote, model_a, model_b, paper_content): user_id = get_user_ip() # Get the user IP address as user_id paper_id = generate_paper_id(paper_content) # Generate paper_id from paper content # Write the request write_request(user_id, paper_id, model_a, model_b, vote) # Update the leaderboard update_leaderboard(model_a, model_b, vote) # Fetch the updated leaderboard (optional, if you want to display it immediately) leaderboard = get_leaderboard() message = f"

Thank you for your vote!

Model A: {model_a}

Model B: {model_b}

" return gr.update(value=message, visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True) def setup_interface(): logging.debug("Setting up Gradio interface.") css = """ .review-container { padding: 10px; margin-bottom: 20px; border: 1px solid #ccc; background-color: #f9f9f9; } .review-section { margin-bottom: 12px; padding: 8px; background-color: #ffffff; border-left: 4px solid #007BFF; padding-left: 10px; } .review-section strong { color: #333; font-weight: bold; display: block; margin-bottom: 5px; } .review-section span, .gr-markdown { color: #000; font-size: 14px; line-height: 1.5; display: block; white-space: normal; opacity: 1; } .model-label { font-size: 18px; font-weight: bold; color: #007BFF; margin-bottom: 10px; } .gr-file, .gr-button, .gr-radio { width: 300px; margin: auto; } .gr-button-small { width: 150px; height: 40px; font-size: 16px; } """ with gr.Blocks(css=css) as demo: with gr.Tabs(): with gr.TabItem("Reviewer Arena"): gr.Markdown("## Reviewer Arena") gr.Markdown("Upload an academic paper to get reviews from two randomly selected LLMs.") with gr.Row(): file_input = gr.File(label="Upload Academic Paper") submit_button = gr.Button("Submit!", elem_id="submit-button") with gr.Row(): with gr.Column(): gr.HTML("
Model A
") review1 = gr.Markdown() with gr.Column(): gr.HTML("
Model B
") review2 = gr.Markdown() vote_options = ["👍 A is better", "👍 B is better", "👔 Tie", "👎 Both are bad"] vote = gr.Radio(label="Vote on the best model", choices=vote_options, value="Tie", visible=False) vote_button = gr.Button("Submit Vote", visible=False) vote_message = gr.HTML("", visible=False) another_paper_button = gr.Button("Review another paper", visible=False) model_identity_message = gr.HTML("", visible=False) def handle_vote_interface(vote, model_identity_message_a, model_identity_message_b, paper_content): return handle_vote(vote, model_identity_message_a, model_identity_message_b, paper_content) submit_button.click(fn=review_papers, inputs=[file_input], outputs=[review1, review2, vote, vote_button, model_identity_message, model_identity_message]) vote_button.click(fn=handle_vote_interface, inputs=[vote, model_identity_message, model_identity_message], outputs=[vote_message, vote, vote_button, another_paper_button]) another_paper_button.click(fn=lambda: None, inputs=None, outputs=None, js="() => { location.reload(); }") with gr.TabItem("Leaderboard"): gr.Markdown("## Leaderboard") # Fetch the leaderboard data from the database leaderboard_data = get_leaderboard() leaderboard_html = """
Rank Model Arena Elo 95% CI Votes Organization License Knowledge Cutoff
1 GPT-4-Turbo-2024-04-09 1258 +3/-3 44592 OpenAI Proprietary 2023/12
2 GPT-4-1106-preview 1252 +2/-3 76173 OpenAI Proprietary 2023/4
2 Gemini 1.5 Pro API-0409-Preview 1249 +3/-3 61011 Google Proprietary 2023/11
2 Claude 3 Opus 1248 +2/-2 101063 Anthropic Proprietary 2023/8
3 GPT-4-0125-preview 1246 +3/-2 70239 OpenAI Proprietary 2023/12
""" leaderboard_html += """ """ gr.HTML(leaderboard_html) logging.debug("Gradio interface setup complete.") return demo if __name__ == "__main__": logging.basicConfig(level=logging.INFO) demo = setup_interface() demo.launch()