import os
import gradio as gr
from utils import *
with gr.Blocks(title='LEO Demo') as demo:
gr.HTML(value="
An Embodied Generalist Agent in 3D World
")
gr.HTML(value="")
# gr.HTML(value="")
gr.HTML(value="arXiv | Project Page | Code
")
gr.HTML(value="LEO: an embodied generalist agent capable of perceiving, grounding, reasoning, planning, and acting in 3D world.
")
with gr.Row():
with gr.Column(scale=5):
dropdown_scene = gr.Dropdown(
choices=MESH_NAMES,
value='3RScan-Office',
interactive=True,
label='Select a 3D scene',
)
model_3d = gr.Model3D(
value=os.path.join(MESH_DIR, f'3RScan-Office.glb'),
clear_color=[0.0, 0.0, 0.0, 0.0],
label='3D Scene',
camera_position=(90, 30, 10),
height=659,
)
gr.HTML(
"""
👆 SCROLL and DRAG on the 3D Scene
to zoom in/out and rotate. Press CTRL and DRAG to pan.
"""
)
with gr.Column(scale=5):
dropdown_conversation_mode = gr.Dropdown(
choices=['Single-round mode', 'Multi-round mode'],
value='Single-round mode',
interactive=True,
label='Select conversation mode',
)
chatbot = gr.Chatbot(label='Chat with LEO')
with gr.Row():
with gr.Column(scale=8):
user_chat_input = gr.Textbox(
placeholder="Enter text here to chat with LEO",
show_label=False,
autofocus=True,
)
with gr.Column(scale=2, min_width=0):
send_button = gr.Button('Send', variant='primary', scale=2)
with gr.Row():
upvote_button = gr.Button(value='👍 Upvote', interactive=False)
downvote_button = gr.Button(value='👎 Downvote', interactive=False)
flag_button = gr.Button(value='⚠️ Flag', interactive=False)
clear_button = gr.Button(value='🗑️ Clear', interactive=False)
with gr.Row():
with gr.Accordion(label="Examples for user instruction:", open=True):
gr.Examples(
examples=[
["What color is the floor?"],
["Is there a blackboard in the room?"],
["How many chairs are there in this room?"],
["Describe this scene."],
["What is this room used for?"],
["Plan for the task: tidy up and arrange this room."],
],
inputs=user_chat_input,
)
# generation_config
with gr.Accordion('Parameters', open=False):
repetition_penalty = gr.Slider(
minimum=0.0,
maximum=10.0,
value=3.0,
step=1.0,
interactive=True,
label='Repetition penalty',
)
length_penalty = gr.Slider(
minimum=0.0,
maximum=10.0,
value=1.0,
step=1.0,
interactive=True,
label="Length penalty",
)
gr.Markdown("### Terms of Service")
gr.HTML(
"""By using this service, users are required to agree to the following terms:
the service is a research preview intended for non-commercial use only
and may collect user dialogue data for future research."""
)
gr.Markdown("### Acknowledgment")
gr.HTML(
"""Template adapted from LLaVA and
LLM-Grounder."""
)
# Event handling
button_list = [upvote_button, downvote_button, flag_button, clear_button]
dropdown_scene.change(
fn=change_scene,
inputs=[dropdown_scene],
outputs=[model_3d, chatbot],
queue=False,
)
dropdown_conversation_mode.change(
fn=clear_history,
inputs=[],
outputs=[chatbot, user_chat_input] + button_list,
queue=False,
)
user_chat_input.submit(
fn=receive_instruction,
inputs=[chatbot, user_chat_input],
outputs=[chatbot, user_chat_input, send_button] + button_list,
queue=False,
).then(
fn=generate_response,
inputs=[
chatbot,
dropdown_scene,
dropdown_conversation_mode,
repetition_penalty,
length_penalty,
],
outputs=[chatbot, send_button] + button_list,
scroll_to_output=True,
)
send_button.click(
fn=receive_instruction,
inputs=[chatbot, user_chat_input],
outputs=[chatbot, user_chat_input, send_button] + button_list,
queue=False,
).then(
fn=generate_response,
inputs=[
chatbot,
dropdown_scene,
dropdown_conversation_mode,
repetition_penalty,
length_penalty,
],
outputs=[chatbot, send_button] + button_list,
scroll_to_output=True,
)
upvote_button.click(
upvote_response,
[chatbot, dropdown_scene, dropdown_conversation_mode],
[user_chat_input, upvote_button, downvote_button, flag_button],
queue=False,
)
downvote_button.click(
downvote_response,
[chatbot, dropdown_scene, dropdown_conversation_mode],
[user_chat_input, upvote_button, downvote_button, flag_button],
queue=False,
)
flag_button.click(
flag_response,
[chatbot, dropdown_scene, dropdown_conversation_mode],
[user_chat_input, upvote_button, downvote_button, flag_button],
queue=False,
)
clear_button.click(
fn=clear_history,
inputs=[],
outputs=[chatbot, user_chat_input] + button_list,
queue=False,
)
demo.queue().launch(share=True, allowed_paths=['assets'])