Update to turing.py

app.py

@@ -1,147 +1,244 @@
-import io
-import random
-from typing import List, Tuple
-
-import aiohttp
+import os
+import json
+import pandas as pd
+from collections import defaultdict, Counter
+import altair as alt
 import panel as pn
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor
-
-pn.extension(design="bootstrap", sizing_mode="stretch_width")
-
-ICON_URLS = {
-    "brand-github": "https://github.com/holoviz/panel",
-    "brand-twitter": "https://twitter.com/Panel_Org",
-    "brand-linkedin": "https://www.linkedin.com/company/panel-org",
-    "message-circle": "https://discourse.holoviz.org/",
-    "brand-discord": "https://discord.gg/AXRHnJU6sP",
-}
-
-
-async def random_url(_):
-    pet = random.choice(["cat", "dog"])
-    api_url = f"https://api.the{pet}api.com/v1/images/search"
-    async with aiohttp.ClientSession() as session:
-        async with session.get(api_url) as resp:
-            return (await resp.json())[0]["url"]
 
 
-@pn.cache
-def load_processor_model(
-    processor_name: str, model_name: str
-) -> Tuple[CLIPProcessor, CLIPModel]:
-    processor = CLIPProcessor.from_pretrained(processor_name)
-    model = CLIPModel.from_pretrained(model_name)
-    return processor, model
+def choices_to_df(choices, hue):
+    df = pd.DataFrame(choices, columns=['choices'])
+    df['hue'] = hue
+    df['hue'] = df['hue'].astype(str)
+    return df
 
-
-async def open_image_url(image_url: str) -> Image:
-    async with aiohttp.ClientSession() as session:
-        async with session.get(image_url) as resp:
-            return Image.open(io.BytesIO(await resp.read()))
-
-
-def get_similarity_scores(class_items: List[str], image: Image) -> List[float]:
-    processor, model = load_processor_model(
-        "openai/clip-vit-base-patch32", "openai/clip-vit-base-patch32"
-    )
-    inputs = processor(
-        text=class_items,
-        images=[image],
-        return_tensors="pt",  # pytorch tensors
-    )
-    outputs = model(**inputs)
-    logits_per_image = outputs.logits_per_image
-    class_likelihoods = logits_per_image.softmax(dim=1).detach().numpy()
-    return class_likelihoods[0]
-
-
-async def process_inputs(class_names: List[str], image_url: str):
-    """
-    High level function that takes in the user inputs and returns the
-    classification results as panel objects.
-    """
-    try:
-        main.disabled = True
-        if not image_url:
-            yield "##### ⚠️ Provide an image URL"
-            return
-    
-        yield "##### ⚙ Fetching image and running model..."
-        try:
-            pil_img = await open_image_url(image_url)
-            img = pn.pane.Image(pil_img, height=400, align="center")
-        except Exception as e:
-            yield f"##### 😔 Something went wrong, please try a different URL!"
-            return
+def arrange_data():
     
-        class_items = class_names.split(",")
-        class_likelihoods = get_similarity_scores(class_items, pil_img)
+    # Human Data
     
-        # build the results column
-        results = pn.Column("##### 🎉 Here are the results!", img)
+    df = pd.read_csv('Project/2_scientific/ChatGPT-Behavioral-main/data/bomb_risk.csv')
+    df = df[df['Role'] == 'player']
+    df = df[df['gameType'] == 'bomb_risk']
+    df.sort_values(by=['UserID', 'Round'])
+
+    prefix_to_choices_human = defaultdict(list)
+    prefix_to_IPW = defaultdict(list)
+    prev_user = None
+    prev_move = None
+    prefix = ''
+    bad_user = False
+    for _, row in df.iterrows():
+        if bad_user: continue
+        if row['UserID'] != prev_user:
+            prev_user = row['UserID']
+            prefix = ''
+            bad_user = False
+
+        move = row['move']
+        if move < 0 or move > 100:
+            bad_users = True
+            continue
+        prefix_to_choices_human[prefix].append(move)
+
+        if len(prefix) == 0:
+            prefix_to_IPW[prefix].append(1)
+        elif prefix[-1] == '1':
+            prev_move = min(prev_move, 98)
+            prefix_to_IPW[prefix].append(1./(100 - prev_move))
+        elif prefix[-1] == '0':
+            prev_move = max(prev_move, 1)
+            prefix_to_IPW[prefix].append(1./(prev_move))
+        else: assert False
+        
+        prev_move = move
+
+        prefix += '1' if row['roundResult'] == 'SAFE' else '0'
+
+
+    # Model Data
+        
+    prefix_to_choices_model = defaultdict(lambda : defaultdict(list))
+    for model in ['ChatGPT-4', 'ChatGPT-3']:
+        if model == 'ChatGPT-4':
+            file_names = [
+                'bomb_gpt4_2023_05_15-12_13_51_AM.json'
+            ]
+        elif model == 'ChatGPT-3':
+            file_names = [
+                'bomb_turbo_2023_05_14-10_45_50_PM.json'
+            ]
+
+        choices = []
+        scenarios = []
+        for file_name in file_names:
+            with open(os.path.join('Project/2_scientific/ChatGPT-Behavioral-main/records', file_name), 'r') as f:
+                records = json.load(f)
+                choices += records['choices']
+                scenarios += records['scenarios']
+
+        assert len(scenarios) == len(choices)
+        print('loaded %i valid records' % len(scenarios))
+
+        prefix_to_choice = defaultdict(list)
+        prefix_to_result = defaultdict(list)
+        prefix_to_pattern = defaultdict(Counter)
+        wrong_sum = 0
+        for scenarios_tmp, choices_tmp in zip(scenarios, choices):
+
+            result = 0
+            for i, scenario in enumerate(scenarios_tmp):
+                prefix = tuple(scenarios_tmp[:i])
+                prefix = ''.join([str(x) for x in prefix])
+                choice = choices_tmp[i]
+                
+                prefix_to_choice[prefix].append(choice)
+                prefix_to_pattern[prefix][tuple(choices_tmp[:-1])] += 1
+
+                prefix = tuple(scenarios_tmp[:i+1])
+                if scenario == 1:
+                    result += choice
+                prefix_to_result[prefix].append(result)
+
+        print('# of wrong sum:', wrong_sum)
+        print('# of correct sum:', len(scenarios) - wrong_sum)
+
+        prefix_to_choices_model[model] = prefix_to_choice
+
+
+    # Arrange Data
+
+    round_dict = {'': [1, -1, -1],
+                '0': [2, 0, -1],
+                '1': [2, 1, -1],
+                '00': [3, 0, 0],
+                '01': [3, 0, 1],
+                '10': [3, 1, 0],
+                '11': [3, 1, 1]}
+
+    df_bomb_all = pd.DataFrame()
+
+    for prefix in round_dict:
+        
+        df_bomb_human = choices_to_df(prefix_to_choices_human[prefix], hue='Human')
+        df_bomb_human['weight'] = prefix_to_IPW[prefix]
+        
+        df_bomb_models = pd.concat([choices_to_df(
+                prefix_to_choices_model[model][prefix], hue=model
+            ) for model in prefix_to_choices_model]
+        )
+        df_bomb_models['weight'] = 1    
+        
+        df_bomb_temp = pd.concat([df_bomb_human, df_bomb_models])
+        df_bomb_temp['prefix'] = prefix
+
+        df_bomb_all = pd.concat([df_bomb_all, df_bomb_temp])
+
+    df_density = df_bomb_all.groupby(['hue', 'prefix'])['choices'].value_counts(normalize=True).unstack(fill_value=0).stack().reset_index()
+    df_density = df_density.rename(columns={'hue': 'Subject', 'choices': 'Boxes', 0: 'Density'})
+    df_density['Round'] = df_density['prefix'].apply(lambda x: round_dict[x][0])
     
-        for class_item, class_likelihood in zip(class_items, class_likelihoods):
-            row_label = pn.widgets.StaticText(
-                name=class_item.strip(), value=f"{class_likelihood:.2%}", align="center"
-            )
-            row_bar = pn.indicators.Progress(
-                value=int(class_likelihood * 100),
-                sizing_mode="stretch_width",
-                bar_color="secondary",
-                margin=(0, 10),
-                design=pn.theme.Material,
-            )
-            results.append(pn.Column(row_label, row_bar))
-        yield results
-    finally:
-        main.disabled = False
-
-
-# create widgets
-randomize_url = pn.widgets.Button(name="Randomize URL", align="end")
-
-image_url = pn.widgets.TextInput(
-    name="Image URL to classify",
-    value=pn.bind(random_url, randomize_url),
-)
-class_names = pn.widgets.TextInput(
-    name="Comma separated class names",
-    placeholder="Enter possible class names, e.g. cat, dog",
-    value="cat, dog, parrot",
-)
+    return df_density
 
-input_widgets = pn.Column(
-    "##### 😊 Click randomize or paste a URL to start classifying!",
-    pn.Row(image_url, randomize_url),
-    class_names,
-)
 
-# add interactivity
-interactive_result = pn.panel(
-    pn.bind(process_inputs, image_url=image_url, class_names=class_names),
-    height=600,
-)
+df_density = arrange_data()
+alt.data_transformers.disable_max_rows()
 
-# add footer
-footer_row = pn.Row(pn.Spacer(), align="center")
-for icon, url in ICON_URLS.items():
-    href_button = pn.widgets.Button(icon=icon, width=35, height=35)
-    href_button.js_on_click(code=f"window.open('{url}')")
-    footer_row.append(href_button)
-footer_row.append(pn.Spacer())
-
-# create dashboard
-main = pn.WidgetBox(
-    input_widgets,
-    interactive_result,
-    footer_row,
+# Enable Panel extensions
+pn.extension(design='bootstrap')
+pn.extension('vega')
+template = pn.template.BootstrapTemplate(
+    title='Nan-Hsin Lin | SI649 Scientific Viz Project',
 )
 
-title = "Panel Demo - Image Classification"
-pn.template.BootstrapTemplate(
-    title=title,
-    main=main,
-    main_max_width="min(50%, 698px)",
-    header_background="#F08080",
-).servable(title=title)
+# Define a function to create and return a plot
+def create_plot(bomb_1, bomb_2):
+
+    bomb_1 = int(not bomb_1)
+    bomb_2 = int(not bomb_2)
+
+    selection = alt.selection_single(encodings=['color'], empty='none', value=3)
+    opacityCondition = alt.condition(selection, alt.value(1), alt.value(0.3))
+    range_ = ['#009FB7', '#FED766', '#FE4A49']
+
+    plot = alt.Chart(df_density).transform_filter(
+        (alt.datum.prefix == '') | (alt.datum.prefix == str(bomb_1)) | (alt.datum.prefix == str(bomb_1) + str(bomb_2))
+    ).mark_bar(opacity=0.5).encode(
+        x=alt.X('Boxes:Q', 
+                bin=alt.Bin(maxbins=10), 
+                title='Number of boxes opened',
+                axis=alt.Axis(ticks=False,
+                            labelFontSize=11,
+                            labelColor='#AAA7AD',
+                            titleFontSize=12, 
+                            titleColor='#AAA7AD',
+                            domain=False)),
+        y=alt.Y('Density:Q', 
+                stack=None, 
+                scale=alt.Scale(domain=[0, 1]), 
+                axis=alt.Axis(format='.0%', 
+                            ticks=False, 
+                            tickCount=5,
+                            labelFontSize=11,
+                            labelColor='#AAA7AD',
+                            titleFontSize=12, 
+                            titleColor='#AAA7AD',
+                            domain=False,
+                            grid=False)),
+        color=alt.Color('Round:N', 
+                        scale=alt.Scale(domain=[1, 2, 3], range=range_)),
+        row=alt.Row('Subject:N', 
+                    header=alt.Header(title=None, orient='top', labelFontSize=16),
+                    sort='descending'),
+        tooltip=['Subject:N', 'Round:N', 'Boxes:Q', alt.Tooltip('Density:Q', format='.0%')]
+    ).properties(width=400, height=150
+    ).configure_view(strokeWidth=3, stroke='lightgrey'
+    ).configure_legend(
+        titleFontSize=12, 
+        titleColor='#AAA7AD',
+        titleAnchor='middle',
+        titlePadding=8,
+        labelFontSize=12,
+        labelColor='#AAA7AD',
+        labelFontWeight='bold',
+        symbolOffset=20,
+        orient='none',
+        direction='horizontal',
+        legendX=120,
+        legendY=-90,
+        symbolSize=200
+    ).add_selection(selection).encode(
+        opacity=opacityCondition
+    )
+
+    return plot
+
+# Create widgets
+switch_1 = pn.widgets.Switch(name='Bomb in Round 1', value=True)
+switch_2 = pn.widgets.Switch(name='Bomb in Round 2', value=True)
+
+plot_widgets = pn.bind(create_plot, switch_1, switch_2)
+
+# Combine everything in a Panel Column to create an app
+maincol = pn.Column()
+maincol.append(pn.Row(pn.layout.HSpacer(),
+                      "### A Turing test of whether AI chatbots are behaviorally similar to humans", 
+                      pn.layout.HSpacer()))
+maincol.append(pn.Row(pn.Spacer(width=100)))
+maincol.append(pn.Row(pn.layout.HSpacer(),
+                      "#### Bomb Risk Game: Human vs. ChatGPT-4 vs. ChatGPT-3", 
+                      pn.layout.HSpacer()))
+maincol.append(pn.Row(pn.layout.HSpacer(), 
+                      "Bomb in Round 1", switch_1,
+                      pn.Spacer(width=50),
+                      "Bomb in Round 2", switch_2, 
+                      pn.layout.HSpacer()))
+maincol.append(pn.Row(pn.layout.HSpacer(), 
+                      plot_widgets,
+                      pn.layout.HSpacer()))
+maincol.append(pn.Row(pn.layout.HSpacer(),
+                      "**Fig 5.** ChatGPT-4 and ChatGPT-3 act as if they have particular risk preferences. Both have the same mode as human distribution in the first round or when experiencing favorable outcomes in the Bomb Risk Game. When experiencing negative outcomes, ChatGPT-4 remains consistent and risk-neutral, while ChatGPT-3 acts as if it becomes more risk-averse.",
+                      pn.layout.HSpacer()))
+template.main.append(maincol)
+
+# set the app to be servable
+template.servable(title="SI649 Scientific Viz Project")