import os import json import pandas as pd from collections import defaultdict, Counter import altair as alt import panel as pn def choices_to_df(choices, hue): df = pd.DataFrame(choices, columns=['choices']) df['hue'] = hue df['hue'] = df['hue'].astype(str) return df def arrange_data(): # Human Data df = pd.read_csv('Project/2_scientific/ChatGPT-Behavioral-main/data/bomb_risk.csv') df = df[df['Role'] == 'player'] df = df[df['gameType'] == 'bomb_risk'] df.sort_values(by=['UserID', 'Round']) prefix_to_choices_human = defaultdict(list) prefix_to_IPW = defaultdict(list) prev_user = None prev_move = None prefix = '' bad_user = False for _, row in df.iterrows(): if bad_user: continue if row['UserID'] != prev_user: prev_user = row['UserID'] prefix = '' bad_user = False move = row['move'] if move < 0 or move > 100: bad_users = True continue prefix_to_choices_human[prefix].append(move) if len(prefix) == 0: prefix_to_IPW[prefix].append(1) elif prefix[-1] == '1': prev_move = min(prev_move, 98) prefix_to_IPW[prefix].append(1./(100 - prev_move)) elif prefix[-1] == '0': prev_move = max(prev_move, 1) prefix_to_IPW[prefix].append(1./(prev_move)) else: assert False prev_move = move prefix += '1' if row['roundResult'] == 'SAFE' else '0' # Model Data prefix_to_choices_model = defaultdict(lambda : defaultdict(list)) for model in ['ChatGPT-4', 'ChatGPT-3']: if model == 'ChatGPT-4': file_names = [ 'bomb_gpt4_2023_05_15-12_13_51_AM.json' ] elif model == 'ChatGPT-3': file_names = [ 'bomb_turbo_2023_05_14-10_45_50_PM.json' ] choices = [] scenarios = [] for file_name in file_names: with open(os.path.join('Project/2_scientific/ChatGPT-Behavioral-main/records', file_name), 'r') as f: records = json.load(f) choices += records['choices'] scenarios += records['scenarios'] assert len(scenarios) == len(choices) print('loaded %i valid records' % len(scenarios)) prefix_to_choice = defaultdict(list) prefix_to_result = defaultdict(list) prefix_to_pattern = defaultdict(Counter) wrong_sum = 0 for scenarios_tmp, choices_tmp in zip(scenarios, choices): result = 0 for i, scenario in enumerate(scenarios_tmp): prefix = tuple(scenarios_tmp[:i]) prefix = ''.join([str(x) for x in prefix]) choice = choices_tmp[i] prefix_to_choice[prefix].append(choice) prefix_to_pattern[prefix][tuple(choices_tmp[:-1])] += 1 prefix = tuple(scenarios_tmp[:i+1]) if scenario == 1: result += choice prefix_to_result[prefix].append(result) print('# of wrong sum:', wrong_sum) print('# of correct sum:', len(scenarios) - wrong_sum) prefix_to_choices_model[model] = prefix_to_choice # Arrange Data round_dict = {'': [1, -1, -1], '0': [2, 0, -1], '1': [2, 1, -1], '00': [3, 0, 0], '01': [3, 0, 1], '10': [3, 1, 0], '11': [3, 1, 1]} df_bomb_all = pd.DataFrame() for prefix in round_dict: df_bomb_human = choices_to_df(prefix_to_choices_human[prefix], hue='Human') df_bomb_human['weight'] = prefix_to_IPW[prefix] df_bomb_models = pd.concat([choices_to_df( prefix_to_choices_model[model][prefix], hue=model ) for model in prefix_to_choices_model] ) df_bomb_models['weight'] = 1 df_bomb_temp = pd.concat([df_bomb_human, df_bomb_models]) df_bomb_temp['prefix'] = prefix df_bomb_all = pd.concat([df_bomb_all, df_bomb_temp]) df_density = df_bomb_all.groupby(['hue', 'prefix'])['choices'].value_counts(normalize=True).unstack(fill_value=0).stack().reset_index() df_density = df_density.rename(columns={'hue': 'Subject', 'choices': 'Boxes', 0: 'Density'}) df_density['Round'] = df_density['prefix'].apply(lambda x: round_dict[x][0]) return df_density df_density = arrange_data() alt.data_transformers.disable_max_rows() # Enable Panel extensions pn.extension(design='bootstrap') pn.extension('vega') template = pn.template.BootstrapTemplate( title='Nan-Hsin Lin | SI649 Scientific Viz Project', ) # Define a function to create and return a plot def create_plot(bomb_1, bomb_2): bomb_1 = int(not bomb_1) bomb_2 = int(not bomb_2) selection = alt.selection_single(encodings=['color'], empty='none', value=3) opacityCondition = alt.condition(selection, alt.value(1), alt.value(0.3)) range_ = ['#009FB7', '#FED766', '#FE4A49'] plot = alt.Chart(df_density).transform_filter( (alt.datum.prefix == '') | (alt.datum.prefix == str(bomb_1)) | (alt.datum.prefix == str(bomb_1) + str(bomb_2)) ).mark_bar(opacity=0.5).encode( x=alt.X('Boxes:Q', bin=alt.Bin(maxbins=10), title='Number of boxes opened', axis=alt.Axis(ticks=False, labelFontSize=11, labelColor='#AAA7AD', titleFontSize=12, titleColor='#AAA7AD', domain=False)), y=alt.Y('Density:Q', stack=None, scale=alt.Scale(domain=[0, 1]), axis=alt.Axis(format='.0%', ticks=False, tickCount=5, labelFontSize=11, labelColor='#AAA7AD', titleFontSize=12, titleColor='#AAA7AD', domain=False, grid=False)), color=alt.Color('Round:N', scale=alt.Scale(domain=[1, 2, 3], range=range_)), row=alt.Row('Subject:N', header=alt.Header(title=None, orient='top', labelFontSize=16), sort='descending'), tooltip=['Subject:N', 'Round:N', 'Boxes:Q', alt.Tooltip('Density:Q', format='.0%')] ).properties(width=400, height=150 ).configure_view(strokeWidth=3, stroke='lightgrey' ).configure_legend( titleFontSize=12, titleColor='#AAA7AD', titleAnchor='middle', titlePadding=8, labelFontSize=12, labelColor='#AAA7AD', labelFontWeight='bold', symbolOffset=20, orient='none', direction='horizontal', legendX=120, legendY=-90, symbolSize=200 ).add_selection(selection).encode( opacity=opacityCondition ) return plot # Create widgets switch_1 = pn.widgets.Switch(name='Bomb in Round 1', value=True) switch_2 = pn.widgets.Switch(name='Bomb in Round 2', value=True) plot_widgets = pn.bind(create_plot, switch_1, switch_2) # Combine everything in a Panel Column to create an app maincol = pn.Column() maincol.append(pn.Row(pn.layout.HSpacer(), "### A Turing test of whether AI chatbots are behaviorally similar to humans", pn.layout.HSpacer())) maincol.append(pn.Row(pn.Spacer(width=100))) maincol.append(pn.Row(pn.layout.HSpacer(), "#### Bomb Risk Game: Human vs. ChatGPT-4 vs. ChatGPT-3", pn.layout.HSpacer())) maincol.append(pn.Row(pn.layout.HSpacer(), "Bomb in Round 1", switch_1, pn.Spacer(width=50), "Bomb in Round 2", switch_2, pn.layout.HSpacer())) maincol.append(pn.Row(pn.layout.HSpacer(), plot_widgets, pn.layout.HSpacer())) maincol.append(pn.Row(pn.layout.HSpacer(), "**Fig 5.** ChatGPT-4 and ChatGPT-3 act as if they have particular risk preferences. Both have the same mode as human distribution in the first round or when experiencing favorable outcomes in the Bomb Risk Game. When experiencing negative outcomes, ChatGPT-4 remains consistent and risk-neutral, while ChatGPT-3 acts as if it becomes more risk-averse.", pn.layout.HSpacer())) template.main.append(maincol) # set the app to be servable template.servable(title="SI649 Scientific Viz Project")