import transformers import gradio as gr import pandas as pd from text_gen import generate_text def get_answer(question, answer, options): '''File given answer''' global questions global SEPARATOR answer = options.split(SEPARATOR)[answer] questions['Choice'][questions.Question == question] = answer def find_score(column_name, questions): '''Count chosen answers in a given column''' actual = questions[column_name].to_list() chosen = questions.Choice.to_list() return sum(a == c for a, c in zip(actual, chosen)) def set_score(): '''Return the score (for the human and for the models)''' global questions global NUM_QUESTIONS score = find_score(answers[0], questions) ai_score = {name:find_score(name, questions) for name in answers[1:]} start = '##

' end = '

' if score == NUM_QUESTIONS: text = f'Perfect score!' else: if score == 0: text = 'Not a single right answer. Are you doing this on purpose?' elif score <= NUM_QUESTIONS / 2 + 1: text = f'Only {score} right answers out of {NUM_QUESTIONS}. You ought to pay more attention.' else: text = f'{score} right answers out of {NUM_QUESTIONS}. It\'s probably alright.' for name in ai_score.keys(): text += f'\n{name} got {ai_score[name]}.' # Add a section to display correct/incorrect answers text += "\n\nList of questions with correct answers:\n" i = 0 for idx, row in questions.iterrows(): i += 1 question = row['Question'] answer = row['Answer'] chosen_answer = row['Choice'] status = "✅" if answer == chosen_answer else "❌" text += f"\n\n{status} {i}. {question} (Correct: {answer})\n" return start + text + end if __name__ == "__main__": NUM_QUESTIONS = 10 SEPARATOR = '<[._.]>' # Adding/replacing models may require adjustments to text_gen.py model_names = ['microsoft/GODEL-v1_1-large-seq2seq', 'facebook/blenderbot-1B-distill', 'facebook/blenderbot_small-90M'] tokenizers = [transformers.AutoTokenizer.from_pretrained(model_names[0]), transformers.BlenderbotTokenizer.from_pretrained(model_names[1]), transformers.BlenderbotSmallTokenizer.from_pretrained(model_names[2])] model = [transformers.AutoModelForSeq2SeqLM.from_pretrained(model_names[0]), transformers.BlenderbotForConditionalGeneration.from_pretrained(model_names[1]), transformers.BlenderbotSmallForConditionalGeneration.from_pretrained(model_names[2])] # Semi-randomly choose questions questions = pd.read_csv('lines_2.txt', sep='*') questions = pd.concat([questions[questions.Part == 'Start'].sample(1), questions[questions.Part == 'Middle_1'].sample(1), questions[questions.Part == 'Middle_2'].sample(1), questions[questions.Part == 'Middle_3'].sample(1), questions[questions.Part == 'Middle_4'].sample(NUM_QUESTIONS-5), questions[questions.Part == 'End'].sample(1)] ).fillna('') # Generate answers for i in range(len(model_names)): questions[model_names[i]] = questions.Question.apply( lambda x: generate_text( x, questions.Context[questions.Question == x].iat[0], model_names[i], model[i], tokenizers[i], minimum=len(questions.Answer[questions.Question == x].iat[0].split())+8)) questions['Choice'] = '' answers = ['Answer'] + model_names # App with gr.Blocks() as test_gen: with gr.Row(): with gr.Column(scale=1): pass with gr.Column(scale=2): gr.Markdown(f'##

IMITATION GAME

\n' + f'###

Choose answers NOT given by an AI model.') for ind, question in enumerate(questions.Question.to_list(), start=1): letters = list('ABCD') options = list(set([questions[col][questions.Question == question].iat[0] for col in answers])) gr.Markdown(f'###

{ind}. {question}

') for letter, option in zip(letters, options): gr.Markdown(f'{letter}. {option}') options = gr.State(SEPARATOR.join(options)) radio = gr.Radio(letters, type='index', show_label=False) question = gr.State(question) radio.change(fn=get_answer, inputs=[question, radio, options]) button = gr.Button(value='Get score') score = gr.Markdown() button.click(fn=set_score, outputs=score) with gr.Column(scale=1): pass test_gen.launch()