File size: 5,533 Bytes
c6138d0 a8621d6 c6138d0 a8621d6 c6138d0 a8621d6 c6138d0 a8621d6 c6138d0 a8621d6 c6138d0 a8621d6 c6138d0 a8621d6 c6138d0 a8621d6 c6138d0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 |
from .question import register_question
from .question import Checker
from typing import Callable
from .utility import count_words, word_reverse_text
CN_TEXT_1 = """
第三章第一题（自说自话），请你输入一个至少10个字的提问，使模型的回答和提问完全一样。
请在下面的输入框内填写你的提问并点击按钮提交。
"""
EN_TEXT_1 = """
For the first question in chapter 3, please enter a question of at least ten words so that the model's answer is exactly the same with the question.
Please enter your query below and click the submit button
"""
def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
_ = question_text, lang
answer_text = answer_text.strip()
user_text = user_text.strip()
if count_words(user_text) < 10:
return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
if answer_text == user_text:
return True, None
else:
return False, None
register_question(
{
'cn': CN_TEXT_1,
'en': EN_TEXT_1,
},
checkers=_checker_1,
name={'cn': '3-1 自说自话', 'en': '3-1'},
level=3,
)
CN_TEXT_2 = """
第三章第二题（话自说自），请你输入一个至少10个字的提问，使模型的回答是问题的反序。
请在下面的输入框内填写你的提问并点击按钮提交。
"""
EN_TEXT_2 = """
For the second question in chapter 3, please enter a question of at least ten words so that the model's answer is the reverse order of the question.
Please enter your query below and click the submit button
"""
def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
_ = question_text, lang
answer_text = answer_text.strip()
user_text = user_text.strip()
if count_words(user_text) < 10:
return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
reversed_user_text = word_reverse_text(user_text, lang)
if reversed_user_text == answer_text:
return True, None
else:
return False, None
register_question(
{
'cn': CN_TEXT_2,
'en': EN_TEXT_2,
},
checkers=_checker_2,
name={'cn': '3-2 话自说自', 'en': '3-2'},
level=3,
)
CN_TEXT_3 = """
第三章第三题（就是敢说），请你输入一个不包含“1”和“3”的提问，使模型的回答是一字不差的“1+1=3”（不需要引号）。
请在下面的输入框内填写你的提问并点击按钮提交。
"""
EN_TEXT_3 = """
For the third question in chapter 3, please enter a question not contain "1" or "3" that will result in the exact answer "1+1=3" (no quotes needed).
Please enter your query below and click the submit button
"""
def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
_ = question_text, lang
answer_text = answer_text.strip()
user_text = user_text.strip()
if '1' in user_text or '3' in user_text:
return False, "用户的问题不能包含“1”或“3”" if lang == 'cn' else 'Question should not contain \"1\" or \"3\".'
if answer_text == '1+1=3':
return True, None
else:
return False, None
register_question(
{
'cn': CN_TEXT_3,
'en': EN_TEXT_3,
},
checkers=_checker_3,
name={'cn': '3-3 就是敢说', 'en': '3-3'},
level=3,
)
CN_TEXT_4 = """
第三章第四题（回文协变），请你输入一个本身不是回文串的问题，使得正着问和倒着问时，模型的回答本身不是回文且也是逆序。
请在下面的输入框内填写你的提问并点击按钮提交。
"""
EN_TEXT_4 = """
For the fourth question in chapter 3, please enter a question that is not a palindrome string, so that the model's answer is also not a palindrome and is in reverse order when asked forward or backward.
Please enter your query below and click the submit button
"""
def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str, llm_callback: Callable[[str], str]):
answer_text = answer_text.strip()
user_text = user_text.strip()
reversed_user_text = word_reverse_text(user_text, lang)
second_answer_text = llm_callback(reversed_user_text)
if user_text == reversed_user_text:
return False, "用户的问题不能是回文串" if lang == 'cn' else 'Question should not be a palindrome string.'
if second_answer_text != word_reverse_text(answer_text, lang):
return False, f"正着问和倒着问时，模型的回答应该是逆序。\n 用户问题：{user_text}\n 正着问回答: {answer_text} \n 反着问回答: {second_answer_text}" if lang == 'cn' else f'The model\'s answer should be in reverse order when asked forward or backward.\n Question: {user_text}\n Forward-Asking answer: {answer_text} \n Backward-Asking answer: {second_answer_text}'
if second_answer_text == word_reverse_text(second_answer_text):
return False, "正着问和倒着问时，模型的回答不能是回文串" if lang == 'cn' else 'The model\'s answer should not be a palindrome when asked forward or backward.'
return True, None
register_question(
{
'cn': CN_TEXT_4,
'en': EN_TEXT_4,
},
checkers=Checker(_checker_4, required_input_keys=['question_text', 'user_text', 'answer_text', 'lang', 'llm_callback']),
name={'cn': '3-4 回文协变', 'en': '3-4'},
level=3,
)
|