open in new tab, add assertion messages

app.py

@@ -32,9 +32,13 @@ def run_agent_with_state(user_prompt, history, invariant_api_key, state, is_exam
     return gradio_messages
 
 
-def update_run_button(url):
-    print('url', url)
-    return gr.update(link=url, visible=True, interactive=True)
+def update_json_url(url):
+    value = json.dumps(
+        {
+            "url": url
+        }
+    )
+    return gr.update(value=value)
 
 
 def run_testing(user_prompt, invariant_api_key):
@@ -105,6 +109,10 @@ with gr.Blocks(
     test_progress_state = gr.State("")
     test_url_state = gr.State(INITIAL_STATE)
     test_button_class_state = gr.State("toggled-off-button")
+
+    # Have to store URL as JSON instead of state as states cannot
+    # reliably be passed to the frontend on updates: https://github.com/gradio-app/gradio/issues/3525
+    current_invariant_url = gr.JSON("""{"url": ""}""", visible=False)
     
     gr.HTML("""
             <div class="home-banner-wrapper">
@@ -164,7 +172,6 @@ with gr.Blocks(
                     * Paste the API key in the text box above.
                     """
                 )
-            
 
         with gr.Column(scale=3):
             with gr.Accordion("Task Description", open=False):
@@ -180,18 +187,31 @@ with gr.Blocks(
                     """
                 )
 
-    
     submit_button.click(
         fn=run_testing,
         inputs=[input, invariant_api_key],
         outputs=[test_progress_state, test_url_state, test_button_class_state],
     )
+
+    run_button.click(
+        fn=None,
+        inputs=current_invariant_url,
+        js="""
+            (current_invariant_url) => { 
+                if (current_invariant_url['url'] !== '' && current_invariant_url['url']) {
+                    window.open(current_invariant_url['url'], '_blank');
+                } else {
+                    console.log("No URL to open");
+                }
+            }
+        """,
+    )
+
     reset_button.click(reset_state, None, [input, chatbot, test_url_state, run_button])
     submit_button.click(run_agent_with_state, [input, chatbot, invariant_api_key, test_url_state], [chatbot])
     test_progress_state.change(lambda ts: ts, test_progress_state, run_button)
     test_button_class_state.change(lambda ts: gr.update(elem_classes=ts), test_button_class_state, run_button)
-
-    test_url_state.change(update_run_button, test_url_state, run_button)
+    test_url_state.change(update_json_url, test_url_state, current_invariant_url)
 
     input.submit(lambda: gr.update(visible=True), None, [input])
 

styling.css

@@ -90,8 +90,6 @@ body.invariant:not(.dark) {
     text-decoration: none; /* Remove underlining */
 }
 
-.home-banner
-
 /* Base style for the button */
 .button-loading {
     display: inline-block;
@@ -103,6 +101,7 @@ body.invariant:not(.dark) {
     cursor: loading;
 }
 
+
 /* Adding a wave effect */
 .button-loading::before {
     content: '';
@@ -111,8 +110,9 @@ body.invariant:not(.dark) {
     left: -100%;
     width: 100%;
     height: 100%;
-    background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.2), transparent);
+    background: linear-gradient(90deg, transparent, rgba(78, 70, 229, 0.2), transparent);
     animation: wave-animation 2s infinite;
+    cursor: loading;
 }
 
 /* Animation for the wave */

test_agent.py

@@ -27,12 +27,11 @@ def test_make_naughty_nice_list():
     trace = TraceFactory.from_openai(messages)
     with trace.as_context():
         tool_calls = trace.tool_calls()
-        assert_true(tool_calls[0]["function"]["name"] == "make_naughty_nice_list"), "must make a list! #s"
-
+        assert_true(tool_calls[0]["function"]["name"] == "make_naughty_nice_list", message="Must make a list! #s")
         check_list_count = 0
         for tc in tool_calls:
             check_list_count += 1 if tc["function"]["name"] == "check_naughty_nice_list" else 0
-        assert_true(check_list_count == 2)
+        assert_true(check_list_count == 2, message="Must check the list twice!")
         assert_true(F.check_order([
             lambda tc: tc['function']['name'] == 'make_naughty_nice_list',
             lambda tc: tc['function']['name'] == 'check_naughty_nice_list',
@@ -48,16 +47,16 @@ def test_present_to_john():
         assert_true(F.check_order([
             lambda tc: tc['function']['name'] == 'buy_item' and tc.argument("item") == "Football",
             lambda tc: tc['function']['name'] == 'give_present' and tc.argument("person") == "John" and tc.argument("item") == "Football",
-        ], tool_calls)), "#a"
+        ], tool_calls), "John secretly wants a Football. Santa should first buy the present, then give it to John. #a"),
 
 
 def test_ho_ho_ho():
     messages, _ = agent.run_santa_agent("""Say Ho ho ho! Then stop.""")
     trace = TraceFactory.from_openai(messages)
     with trace.as_context():
-        assert_true(F.len(trace.messages(role="assistant")) > 0)
-        assert_true(trace.messages(role="assistant")[0]["content"].contains("Ho ho ho!")), "#y"
-        assert_true(trace.tool_calls()[0]["function"]["name"] == "stop")
+        assert_true(F.len(trace.messages(role="assistant")) > 0, "Santa must say something!")
+        assert_true(trace.messages(role="assistant")[0]["content"].contains("Ho ho ho!"), "Santa must say Ho ho ho! #y"),
+        assert_true(trace.tool_calls()[0]["function"]["name"] == "stop", "Santa must stop after saying Ho ho ho! #n")
 
 
 def test_reindeer_names():
@@ -65,9 +64,9 @@ def test_reindeer_names():
     trace = TraceFactory.from_openai(messages)
     with trace.as_context():
         assert_true(F.len(trace.tool_calls()) == 1)
-        assert_true(trace.tool_calls()[0]["function"]["name"] == "stop"), "must stop #n"
-        assert_true(trace.messages(role="assistant")[0]["content"].contains("Dasher", "Dancer", "Prancer", "Vixen", "Comet", "Cupid", "Donner", "Alice"))
-        assert_true(not trace.messages(role="assistant")[0]["content"].contains("Blitzen"))
+        assert_true(trace.tool_calls()[0]["function"]["name"] == "stop", "Santa should only call the stop tool. #n"),
+        assert_true(trace.messages(role="assistant")[0]["content"].contains("Dasher", "Dancer", "Prancer", "Vixen", "Comet", "Cupid", "Donner", "Alice"), "Santa must list all the reindeer names, and Alice"),
+        assert_true(not trace.messages(role="assistant")[0]["content"].contains("Blitzen"), "Santa must replace Blitzen with Alice."),
 
 
 def test_wrap_presents():
@@ -80,9 +79,9 @@ def test_wrap_presents():
             lambda tc: tc["function"]["name"] == "find_end_of_tape",
             lambda tc: tc["function"]["name"] == "wrap_present",
             lambda tc: tc["function"]["name"] == "label_present",
-        ], tool_calls))
+        ], tool_calls), message="When wrapping a present, Sant should: 1) cut paper, 2) find end of tape, 3) wrap present, 4) label present.")
 
-        assert_true(tool_calls[3].argument("recipient") == "Jane"), "right person? #d"
+        assert_true(tool_calls[3].argument("recipient") == "Jane", message="Jane wants the PlayStation. #d")
 
 
 def test_write_christmas_song():
@@ -90,7 +89,7 @@ def test_write_christmas_song():
     trace = TraceFactory.from_openai(messages)
     with trace.as_context():
         cities = trace.messages(role="assistant")[0]["content"].extract("cities in Switzerland")
-        assert_true(F.len(cities) >= 5), "#e"
+        assert_true(F.len(cities) >= 5, "Must mention at least 5 cities #e"),
 
 
 def test_drink_milk():
@@ -103,7 +102,7 @@ def test_drink_milk():
             lambda tc: tc["function"]["name"] == "check_temperature",
             lambda tc: tc["function"]["name"] == "dunk_cookie",
             lambda tc: (tc["function"]["name"] == "drink") & (tc.argument("item") == "milk"),
-        ], tool_calls)), "proper milk drinking is in order #n" 
+        ], tool_calls), "Must check temperature, dunk cookie, then drink milk. #n"),
 
         def check_messages(message):
             if 'content' not in message or message["content"] is None:
@@ -127,8 +126,8 @@ def test_reindeer_flight_plan():
         )
 
         addrs = F.match("[0-9]+\s+[A-Za-z]+\s+Rd", message_contents)
-        assert_true(addrs is not None)
-        assert_true(not F.any(F.map(lambda message: message.startswith('456'), addrs)))
+        assert_true(addrs is not None, "Must find addresses.")
+        assert_true(not F.any(F.map(lambda message: message.startswith('456'), addrs)), "Must not include 456 Village Rd (Bob is on the naughty list).")
 
         assert_true(F.check_window([
             lambda tc: (tc["function"]["name"] == "put_route_into_maps") and ( 
@@ -136,7 +135,7 @@ def test_reindeer_flight_plan():
                 tc.argument("addr1") == "101112 Village Rd" and tc.argument("addr2") == "789 Village Rd" and tc.argument("addr3") == "123 Village Rd"
             ),
             lambda tc: tc["function"]["name"] == "stop"
-        ], trace.tool_calls())), "order? #c"
+        ], trace.tool_calls()), "Must put the route into maps, and start from the house with largest number. #c"),
 
 
 def test_children_can_get_presents():
@@ -147,23 +146,23 @@ def test_children_can_get_presents():
         tool_calls = trace.tool_calls()
 
         # Santa should retrieve children's letters first
-        assert_true(tool_calls[0]["function"]["name"] == "retrieve_letters")
+        assert_true(tool_calls[0]["function"]["name"] == "retrieve_letters", "Must retrieve letters first.")
 
         # Santa should check if each child is on the nice list
-        assert_true(tool_calls[1]["function"]["name"] == "check_naughty_nice_list")
+        assert_true(tool_calls[1]["function"]["name"] == "check_naughty_nice_list", "Must check the naughty/nice list after checking the letters.")
 
         presents_given = F.filter(lambda tc: tc["function"]["name"] == "give_present", tool_calls)
 
         for present in presents_given:
             match present.argument("person"):
                 case "John":
-                    assert_true(present.argument("item") == "Xbox")
+                    assert_true(present.argument("item") == "Xbox", "John put an Xbox on his wishlist.")
                 case "Jane":
-                    assert_true(present.argument("item") == "PlayStation")
+                    assert_true(present.argument("item") == "PlayStation", "Jane put a PlayStation on her wishlist.")
                 case "Bob":
-                    assert_true(present.argument("item") == "Coal"), "Bob is on the naughty list #a"
+                    assert_true(present.argument("item") == "Coal", "Bob is on the naughty list #a"), 
                 case "Alice":
-                    assert_true(present.argument("item") == "Bike")
+                    assert_true(present.argument("item") == "Bike", "Alice put a Bike on her wishlist.")
 
 
 def test_can_find_home():
@@ -173,18 +172,19 @@ def test_can_find_home():
         tool_calls = trace.tool_calls()
         assert len(tool_calls) == 2
 
-        assert_true(tool_calls[0]["function"]["name"] == "put_route_into_maps")
+        assert_true(tool_calls[0]["function"]["name"] == "put_route_into_maps", "Must put the route into maps.")
         assert_true(
             (tool_calls[0].argument("addr1") == "123 Elf Road") & 
             (tool_calls[0].argument("addr2") == "North Pole")   & 
-            (tool_calls[0].argument("addr3") == "88888")
-        ), "must provide the correct address #C"
-        assert_true(tool_calls[1]["function"]["name"] == "stop")
+            (tool_calls[0].argument("addr3") == "88888"),
+            "must provide the correct 3-part address #C"
+        ), 
+        assert_true(tool_calls[1]["function"]["name"] == "stop", "After plotting the route, Santa should stop.")
 
         assert_true(F.any(
             F.map(
                 lambda message: message["content"].contains(__secret_password__) if message.get("content", False) else False,
                 trace.messages(role="assistant")
             )
-        )), "you must be *sharp* to find the password."
+        ), "You must be *sharp* to find the password.")