import os import gradio as gr from utils import * with gr.Blocks(title='LEO Demo') as demo: gr.HTML(value="

An Embodied Generalist Agent in 3D World

") gr.HTML(value="
") # gr.HTML(value="Teaser") gr.HTML(value="

arXiv | Project Page | Code

") gr.HTML(value="

LEO: an embodied generalist agent capable of perceiving, grounding, reasoning, planning, and acting in 3D world.

") with gr.Row(): with gr.Column(scale=5): dropdown_scene = gr.Dropdown( choices=MESH_NAMES, value='3RScan-Office', interactive=True, label='Select a 3D scene', ) model_3d = gr.Model3D( value=os.path.join(MESH_DIR, f'3RScan-Office.glb'), clear_color=[0.0, 0.0, 0.0, 0.0], label='3D Scene', camera_position=(90, 30, 10), height=659, ) gr.HTML( """
👆 SCROLL and DRAG on the 3D Scene to zoom in/out and rotate. Press CTRL and DRAG to pan.
""" ) with gr.Column(scale=5): dropdown_conversation_mode = gr.Dropdown( choices=['Single-round mode', 'Multi-round mode'], value='Single-round mode', interactive=True, label='Select conversation mode', ) chatbot = gr.Chatbot(label='Chat with LEO') with gr.Row(): with gr.Column(scale=8): user_chat_input = gr.Textbox( placeholder="Enter text here to chat with LEO", show_label=False, autofocus=True, ) with gr.Column(scale=2, min_width=0): send_button = gr.Button('Send', variant='primary', scale=2) with gr.Row(): upvote_button = gr.Button(value='👍 Upvote', interactive=False) downvote_button = gr.Button(value='👎 Downvote', interactive=False) flag_button = gr.Button(value='⚠️ Flag', interactive=False) clear_button = gr.Button(value='🗑️ Clear', interactive=False) with gr.Row(): with gr.Accordion(label="Examples for user instruction:", open=True): gr.Examples( examples=[ ["What color is the floor?"], ["Is there a blackboard in the room?"], ["How many chairs are there in this room?"], ["Describe this scene."], ["What is this room used for?"], ["Plan for the task: tidy up and arrange this room."], ], inputs=user_chat_input, ) # generation_config with gr.Accordion('Parameters', open=False): repetition_penalty = gr.Slider( minimum=0.0, maximum=10.0, value=3.0, step=1.0, interactive=True, label='Repetition penalty', ) length_penalty = gr.Slider( minimum=0.0, maximum=10.0, value=1.0, step=1.0, interactive=True, label="Length penalty", ) gr.Markdown("### Terms of Service") gr.HTML( """By using this service, users are required to agree to the following terms: the service is a research preview intended for non-commercial use only and may collect user dialogue data for future research.""" ) gr.Markdown("### Acknowledgment") gr.HTML( """Template adapted from LLaVA and LLM-Grounder.""" ) # Event handling button_list = [upvote_button, downvote_button, flag_button, clear_button] dropdown_scene.change( fn=change_scene, inputs=[dropdown_scene], outputs=[model_3d, chatbot], queue=False, ) dropdown_conversation_mode.change( fn=clear_history, inputs=[], outputs=[chatbot, user_chat_input] + button_list, queue=False, ) user_chat_input.submit( fn=receive_instruction, inputs=[chatbot, user_chat_input], outputs=[chatbot, user_chat_input, send_button] + button_list, queue=False, ).then( fn=generate_response, inputs=[ chatbot, dropdown_scene, dropdown_conversation_mode, repetition_penalty, length_penalty, ], outputs=[chatbot, send_button] + button_list, scroll_to_output=True, ) send_button.click( fn=receive_instruction, inputs=[chatbot, user_chat_input], outputs=[chatbot, user_chat_input, send_button] + button_list, queue=False, ).then( fn=generate_response, inputs=[ chatbot, dropdown_scene, dropdown_conversation_mode, repetition_penalty, length_penalty, ], outputs=[chatbot, send_button] + button_list, scroll_to_output=True, ) upvote_button.click( upvote_response, [chatbot, dropdown_scene, dropdown_conversation_mode], [user_chat_input, upvote_button, downvote_button, flag_button], queue=False, ) downvote_button.click( downvote_response, [chatbot, dropdown_scene, dropdown_conversation_mode], [user_chat_input, upvote_button, downvote_button, flag_button], queue=False, ) flag_button.click( flag_response, [chatbot, dropdown_scene, dropdown_conversation_mode], [user_chat_input, upvote_button, downvote_button, flag_button], queue=False, ) clear_button.click( fn=clear_history, inputs=[], outputs=[chatbot, user_chat_input] + button_list, queue=False, ) demo.queue().launch(share=True, allowed_paths=['assets'])