Spaces:
Sleeping
Sleeping
Diego Carpintero
commited on
Commit
·
e42055e
1
Parent(s):
4533ee3
refactor
Browse files- minerva.ipynb +146 -192
minerva.ipynb
CHANGED
@@ -28,7 +28,7 @@
|
|
28 |
},
|
29 |
{
|
30 |
"cell_type": "code",
|
31 |
-
"execution_count":
|
32 |
"metadata": {},
|
33 |
"outputs": [
|
34 |
{
|
@@ -40,23 +40,26 @@
|
|
40 |
}
|
41 |
],
|
42 |
"source": [
|
43 |
-
"%pip install -q
|
44 |
]
|
45 |
},
|
46 |
{
|
47 |
"cell_type": "code",
|
48 |
-
"execution_count":
|
49 |
"metadata": {},
|
50 |
"outputs": [],
|
51 |
"source": [
|
|
|
|
|
52 |
"import autogen\n",
|
|
|
53 |
"from IPython.display import Image as IPImage\n",
|
54 |
"from IPython.display import display"
|
55 |
]
|
56 |
},
|
57 |
{
|
58 |
"cell_type": "code",
|
59 |
-
"execution_count":
|
60 |
"metadata": {},
|
61 |
"outputs": [],
|
62 |
"source": [
|
@@ -65,37 +68,17 @@
|
|
65 |
"\n",
|
66 |
"load_dotenv(find_dotenv())\n",
|
67 |
"\n",
|
68 |
-
"
|
69 |
" {\n",
|
70 |
" \"model\": \"gpt-4o-mini\",\n",
|
71 |
" \"api_key\": os.getenv(\"OPENAI_API_KEY\")\n",
|
72 |
" }\n",
|
73 |
-
"]"
|
74 |
-
]
|
75 |
-
},
|
76 |
-
{
|
77 |
-
"cell_type": "markdown",
|
78 |
-
"metadata": {},
|
79 |
-
"source": [
|
80 |
-
"## Tools Definition"
|
81 |
-
]
|
82 |
-
},
|
83 |
-
{
|
84 |
-
"cell_type": "code",
|
85 |
-
"execution_count": 46,
|
86 |
-
"metadata": {},
|
87 |
-
"outputs": [],
|
88 |
-
"source": [
|
89 |
-
"def ocr(image_path: str) -> str:\n",
|
90 |
-
" from PIL import Image\n",
|
91 |
-
" import pytesseract\n",
|
92 |
"\n",
|
93 |
-
"
|
94 |
-
"
|
95 |
-
"
|
96 |
-
"
|
97 |
-
" except Exception as e:\n",
|
98 |
-
" return f\"Error in text extraction: {str(e)}\""
|
99 |
]
|
100 |
},
|
101 |
{
|
@@ -107,23 +90,21 @@
|
|
107 |
},
|
108 |
{
|
109 |
"cell_type": "code",
|
110 |
-
"execution_count":
|
111 |
"metadata": {},
|
112 |
"outputs": [],
|
113 |
"source": [
|
114 |
"def create_agents():\n",
|
115 |
" \"\"\"Create and initialize the specialized agents.\"\"\"\n",
|
116 |
-
"
|
117 |
-
" # OCR Agent - Extracts and processes text from images\n",
|
118 |
" ocr_agent = autogen.AssistantAgent(\n",
|
119 |
" name=\"OCR_Specialist\",\n",
|
120 |
" system_message=\"\"\"You are an OCR specialist. Your role is to:\n",
|
121 |
" 1. Extract text from an image path using pytesseract\n",
|
122 |
" 2. Clean and format the extracted text for further analysis\"\"\",\n",
|
123 |
-
" llm_config=
|
124 |
" )\n",
|
125 |
-
"\n",
|
126 |
-
" # Content Analysis Agent - Evaluates text content for scam signals\n",
|
127 |
" content_agent = autogen.AssistantAgent(\n",
|
128 |
" name=\"Content_Analyst\",\n",
|
129 |
" system_message=\"\"\"You are a content analysis specialist. Your role is to:\n",
|
@@ -132,10 +113,9 @@
|
|
132 |
" 3. Identify urgency indicators, threats, or pressure tactics\n",
|
133 |
" 5. Check for inconsistencies in messaging\n",
|
134 |
" 6. Evaluate legitimacy of any claims or offers\"\"\",\n",
|
135 |
-
" llm_config=
|
136 |
" )\n",
|
137 |
"\n",
|
138 |
-
" # Decision Making Agent - Makes final determination based on all analyses\n",
|
139 |
" decision_agent = autogen.AssistantAgent(\n",
|
140 |
" name=\"Decision_Maker\",\n",
|
141 |
" system_message=\"\"\"You are the final decision maker. Your role is to:\n",
|
@@ -143,7 +123,7 @@
|
|
143 |
" 2. Make a final determination on scam probability\n",
|
144 |
" 3. Provide detailed explanation of the decision\n",
|
145 |
" 4. End your explanation with the label as 'TASK_COMPLETE' when done\"\"\",\n",
|
146 |
-
" llm_config=
|
147 |
" )\n",
|
148 |
"\n",
|
149 |
" summary_agent = autogen.AssistantAgent(\n",
|
@@ -154,19 +134,28 @@
|
|
154 |
" 3. Provide actionable recommendations\n",
|
155 |
" 4. Use clear, non-technical language while maintaining accuracy\n",
|
156 |
" 5. Format information in a way that's easy to read and understand\"\"\",\n",
|
157 |
-
" llm_config=
|
158 |
" )\n",
|
159 |
"\n",
|
160 |
-
" # User proxy for automated interaction and code execution\n",
|
161 |
" user_proxy = autogen.UserProxyAgent(\n",
|
162 |
-
" name=\"
|
163 |
-
" human_input_mode=\"NEVER\",\n",
|
164 |
-
" max_consecutive_auto_reply=5,\n",
|
165 |
" is_termination_msg=lambda x: \"TASK_COMPLETE\" in x.get(\"content\", \"\"),\n",
|
|
|
|
|
166 |
" )\n",
|
167 |
"\n",
|
168 |
-
"
|
169 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
170 |
" \n",
|
171 |
" return ocr_agent, content_agent, decision_agent, user_proxy "
|
172 |
]
|
@@ -180,7 +169,7 @@
|
|
180 |
},
|
181 |
{
|
182 |
"cell_type": "code",
|
183 |
-
"execution_count":
|
184 |
"metadata": {},
|
185 |
"outputs": [],
|
186 |
"source": [
|
@@ -196,14 +185,14 @@
|
|
196 |
" groupchat = autogen.GroupChat(\n",
|
197 |
" agents=[self.ocr_agent, self.content_agent, self.decision_agent, self.user_proxy],\n",
|
198 |
" messages=[],\n",
|
199 |
-
" max_round=
|
200 |
" )\n",
|
201 |
" manager = autogen.GroupChatManager(groupchat=groupchat)\n",
|
202 |
"\n",
|
203 |
" # Start the collaborative analysis\n",
|
204 |
" messages = self.user_proxy.initiate_chat(\n",
|
205 |
" manager,\n",
|
206 |
-
" message=f\"\"\"
|
207 |
" 1. OCR Agent: Extract text from this image: {image_path}\n",
|
208 |
" 2. Content Agent: Evaluate the messaging and claims\n",
|
209 |
" 3. Decision Maker: Synthesize all analyses and make final determination\"\"\",\n",
|
@@ -214,7 +203,7 @@
|
|
214 |
},
|
215 |
{
|
216 |
"cell_type": "code",
|
217 |
-
"execution_count":
|
218 |
"metadata": {},
|
219 |
"outputs": [
|
220 |
{
|
@@ -231,9 +220,9 @@
|
|
231 |
"name": "stdout",
|
232 |
"output_type": "stream",
|
233 |
"text": [
|
234 |
-
"\u001b[
|
|
|
235 |
"\n",
|
236 |
-
"Please analyze the content of an image for scam indicators:\n",
|
237 |
" 1. OCR Agent: Extract text from this image: ./samples/02.giftcard.message.scam.png\n",
|
238 |
" 2. Content Agent: Evaluate the messaging and claims\n",
|
239 |
" 3. Decision Maker: Synthesize all analyses and make final determination\n",
|
@@ -244,20 +233,20 @@
|
|
244 |
"\u001b[0m\n",
|
245 |
"\u001b[33mOCR_Specialist\u001b[0m (to chat_manager):\n",
|
246 |
"\n",
|
247 |
-
"\u001b[32m***** Suggested tool call (
|
248 |
"Arguments: \n",
|
249 |
"{\"image_path\":\"./samples/02.giftcard.message.scam.png\"}\n",
|
250 |
"\u001b[32m********************************************************************\u001b[0m\n",
|
251 |
"\n",
|
252 |
"--------------------------------------------------------------------------------\n",
|
253 |
"\u001b[32m\n",
|
254 |
-
"Next speaker:
|
255 |
"\u001b[0m\n",
|
256 |
"\u001b[35m\n",
|
257 |
">>>>>>>> EXECUTING FUNCTION ocr...\u001b[0m\n",
|
258 |
-
"\u001b[
|
259 |
"\n",
|
260 |
-
"\u001b[32m***** Response from calling tool (
|
261 |
"Congratulations!\n",
|
262 |
"You've won a $1,000\n",
|
263 |
"Walmart gift card. Go\n",
|
@@ -273,84 +262,39 @@
|
|
273 |
"\u001b[0m\n",
|
274 |
"\u001b[33mOCR_Specialist\u001b[0m (to chat_manager):\n",
|
275 |
"\n",
|
276 |
-
"
|
277 |
-
"Arguments: \n",
|
278 |
-
"{\"image_path\": \"./samples/02.giftcard.message.scam.png\"}\n",
|
279 |
-
"\u001b[32m********************************************************************\u001b[0m\n",
|
280 |
-
"\u001b[32m***** Suggested tool call (call_4P6ZQAv9jirUcmieUUSavrIX): ocr *****\u001b[0m\n",
|
281 |
-
"Arguments: \n",
|
282 |
-
"{\"image_path\": \"./samples/02.giftcard.message.scam.png\"}\n",
|
283 |
-
"\u001b[32m********************************************************************\u001b[0m\n",
|
284 |
"\n",
|
285 |
-
"
|
286 |
-
"\u001b[32m\n",
|
287 |
-
"Next speaker: User_Proxy\n",
|
288 |
-
"\u001b[0m\n",
|
289 |
-
"\u001b[35m\n",
|
290 |
-
">>>>>>>> EXECUTING FUNCTION ocr...\u001b[0m\n",
|
291 |
-
"\u001b[35m\n",
|
292 |
-
">>>>>>>> EXECUTING FUNCTION ocr...\u001b[0m\n",
|
293 |
-
"\u001b[33mUser_Proxy\u001b[0m (to chat_manager):\n",
|
294 |
"\n",
|
295 |
-
"\
|
296 |
-
"
|
297 |
-
"
|
298 |
-
"
|
299 |
-
"\n",
|
300 |
-
"http://bit.ly/123456\n",
|
301 |
"tp claim now.\n",
|
302 |
-
"\f\n",
|
303 |
-
"\u001b[32m**********************************************************************\u001b[0m\n",
|
304 |
"\n",
|
305 |
-
"
|
306 |
-
"\u001b[32m***** Response from calling tool (call_4P6ZQAv9jirUcmieUUSavrIX) *****\u001b[0m\n",
|
307 |
-
"Congratulations!\n",
|
308 |
-
"You've won a $1,000\n",
|
309 |
-
"Walmart gift card. Go\n",
|
310 |
"\n",
|
311 |
-
"
|
312 |
-
"tp claim now.\n",
|
313 |
-
"\f\n",
|
314 |
-
"\u001b[32m**********************************************************************\u001b[0m\n",
|
315 |
"\n",
|
316 |
-
"
|
317 |
-
"\
|
318 |
-
"
|
319 |
-
"\
|
320 |
-
"
|
321 |
"\n",
|
322 |
-
"
|
323 |
-
"The extracted text from the image is:\n",
|
324 |
"\n",
|
325 |
"```\n",
|
326 |
-
"Congratulations
|
327 |
-
"You've won a $1,000
|
328 |
-
"
|
329 |
-
"http://bit.ly/123456\n",
|
330 |
-
"tp claim now.\n",
|
331 |
"```\n",
|
332 |
"\n",
|
333 |
-
"###
|
334 |
-
"1. **
|
335 |
-
"\n",
|
336 |
-
"2. **Urgency and Action Directive**: The phrase \"Go tp claim now\" creates urgency, pushing the reader to act quickly. Scammers often employ urgency to provoke emotional responses leading to hasty decisions.\n",
|
337 |
-
"\n",
|
338 |
-
"3. **Use of Link**: The URL uses a shortened link (bit.ly). Scammers frequently use such links to obscure the final destination, making it hard for users to see where they are being directed. This is a major red flag.\n",
|
339 |
-
"\n",
|
340 |
-
"4. **Lack of Personalization**: The message does not address the recipient by name or provide any details about how they \"won,\" which is typical of generic scam messages.\n",
|
341 |
-
"\n",
|
342 |
-
"5. **Formatting and Miscues**: The text contains formatting issues, such as \"tp\" instead of \"to,\" which is unprofessional and suggests a lack of authenticity.\n",
|
343 |
"\n",
|
344 |
-
"
|
345 |
-
"Based on the characteristics of the extracted message:\n",
|
346 |
-
"\n",
|
347 |
-
"- The claim of winning a prize without any entry process.\n",
|
348 |
-
"- The use of urgency and action-oriented language.\n",
|
349 |
-
"- The inclusion of a suspicious shortened URL.\n",
|
350 |
-
"- The generic nature of the communication.\n",
|
351 |
-
"- The presence of grammatical errors and poor formatting.\n",
|
352 |
-
"\n",
|
353 |
-
"**Conclusion**: This message exhibits strong indicators of a scam. It is advisable to disregard the message and not click on any links provided.\n",
|
354 |
"\n",
|
355 |
"--------------------------------------------------------------------------------\n",
|
356 |
"\u001b[32m\n",
|
@@ -358,38 +302,30 @@
|
|
358 |
"\u001b[0m\n",
|
359 |
"\u001b[33mContent_Analyst\u001b[0m (to chat_manager):\n",
|
360 |
"\n",
|
361 |
-
"###
|
362 |
-
"The extracted text from the image is:\n",
|
363 |
"\n",
|
364 |
-
"
|
365 |
-
"
|
366 |
-
"
|
367 |
-
"Walmart gift card. Go\n",
|
368 |
-
"http://bit.ly/123456\n",
|
369 |
-
"tp claim now.\n",
|
370 |
-
"```\n",
|
371 |
-
"\n",
|
372 |
-
"### Analysis of Content\n",
|
373 |
-
"1. **Claims of Winning**: The message claims that the recipient has won a significant prize ($1,000 Walmart gift card). Such claims are commonly associated with scams, especially when there is no previous participation in a contest.\n",
|
374 |
"\n",
|
375 |
-
"2. **
|
|
|
376 |
"\n",
|
377 |
-
"3. **
|
|
|
378 |
"\n",
|
379 |
-
"4. **
|
|
|
380 |
"\n",
|
381 |
-
"5. **
|
|
|
382 |
"\n",
|
383 |
-
"### Final Determination
|
384 |
-
"Based on the characteristics of the extracted message:\n",
|
385 |
"\n",
|
386 |
-
"
|
387 |
-
"- The use of urgency in the language.\n",
|
388 |
-
"- The presence of a suspicious shortened URL.\n",
|
389 |
-
"- The generic nature and lack of personalization in the message.\n",
|
390 |
-
"- Grammatical errors suggesting a lack of authenticity.\n",
|
391 |
"\n",
|
392 |
-
"
|
|
|
393 |
"\n",
|
394 |
"--------------------------------------------------------------------------------\n",
|
395 |
"\u001b[32m\n",
|
@@ -397,30 +333,39 @@
|
|
397 |
"\u001b[0m\n",
|
398 |
"\u001b[33mDecision_Maker\u001b[0m (to chat_manager):\n",
|
399 |
"\n",
|
400 |
-
"### Final
|
401 |
"\n",
|
402 |
-
"
|
403 |
"\n",
|
404 |
-
"
|
405 |
"\n",
|
406 |
-
"1. **
|
|
|
407 |
"\n",
|
408 |
-
"2. **
|
|
|
409 |
"\n",
|
410 |
-
"3. **
|
|
|
411 |
"\n",
|
412 |
-
"4. **
|
|
|
413 |
"\n",
|
414 |
-
"5. **
|
|
|
415 |
"\n",
|
416 |
-
"###
|
417 |
-
"
|
|
|
|
|
|
|
|
|
418 |
"\n",
|
419 |
"**TASK_COMPLETE**\n",
|
420 |
"\n",
|
421 |
"--------------------------------------------------------------------------------\n",
|
422 |
"\u001b[32m\n",
|
423 |
-
"Next speaker:
|
424 |
"\u001b[0m\n"
|
425 |
]
|
426 |
}
|
@@ -435,54 +380,63 @@
|
|
435 |
},
|
436 |
{
|
437 |
"cell_type": "code",
|
438 |
-
"execution_count":
|
439 |
"metadata": {},
|
440 |
"outputs": [
|
441 |
{
|
442 |
"name": "stdout",
|
443 |
"output_type": "stream",
|
444 |
"text": [
|
445 |
-
"{'content': '### Final
|
|
|
|
|
|
|
|
|
|
|
|
|
446 |
" '\\n'\n",
|
447 |
-
" '**
|
|
|
|
|
|
|
|
|
448 |
" '\\n'\n",
|
449 |
-
"
|
450 |
-
" '
|
451 |
-
" '
|
|
|
|
|
452 |
" '\\n'\n",
|
453 |
-
" '
|
454 |
-
" '
|
455 |
-
" '
|
456 |
-
" '
|
457 |
-
" '
|
|
|
458 |
" '\\n'\n",
|
459 |
-
" '
|
460 |
-
" '
|
461 |
-
" '
|
462 |
-
" '
|
463 |
-
" '
|
464 |
" '\\n'\n",
|
465 |
-
" '
|
466 |
-
" '
|
467 |
-
" '
|
468 |
-
" '
|
469 |
-
" '
|
470 |
" '\\n'\n",
|
471 |
-
" '
|
472 |
-
" 'that pertain to the recipient aligns with conventional scam '\n",
|
473 |
-
" 'communication, indicating it is likely mass-produced.\\n'\n",
|
474 |
" '\\n'\n",
|
475 |
-
" '
|
476 |
-
" '
|
477 |
-
" '
|
478 |
-
" '
|
|
|
479 |
" '\\n'\n",
|
480 |
-
" '### Conclusion
|
481 |
-
" '
|
482 |
-
" '
|
483 |
-
" 'is highly indicative of a scam. Individuals should be cautious, '\n",
|
484 |
-
" 'avoid clicking any hyperlinks, and report the message if received '\n",
|
485 |
-
" 'via electronic means.\\n'\n",
|
486 |
" '\\n'\n",
|
487 |
" '**TASK_COMPLETE**',\n",
|
488 |
" 'name': 'Decision_Maker',\n",
|
@@ -498,7 +452,7 @@
|
|
498 |
},
|
499 |
{
|
500 |
"cell_type": "code",
|
501 |
-
"execution_count":
|
502 |
"metadata": {},
|
503 |
"outputs": [],
|
504 |
"source": [
|
|
|
28 |
},
|
29 |
{
|
30 |
"cell_type": "code",
|
31 |
+
"execution_count": 72,
|
32 |
"metadata": {},
|
33 |
"outputs": [
|
34 |
{
|
|
|
40 |
}
|
41 |
],
|
42 |
"source": [
|
43 |
+
"%pip install -q autogen-agentchat~=0.2 pillow pytesseract"
|
44 |
]
|
45 |
},
|
46 |
{
|
47 |
"cell_type": "code",
|
48 |
+
"execution_count": 73,
|
49 |
"metadata": {},
|
50 |
"outputs": [],
|
51 |
"source": [
|
52 |
+
"from typing_extensions import Annotated\n",
|
53 |
+
"\n",
|
54 |
"import autogen\n",
|
55 |
+
"\n",
|
56 |
"from IPython.display import Image as IPImage\n",
|
57 |
"from IPython.display import display"
|
58 |
]
|
59 |
},
|
60 |
{
|
61 |
"cell_type": "code",
|
62 |
+
"execution_count": 74,
|
63 |
"metadata": {},
|
64 |
"outputs": [],
|
65 |
"source": [
|
|
|
68 |
"\n",
|
69 |
"load_dotenv(find_dotenv())\n",
|
70 |
"\n",
|
71 |
+
"config_list = [\n",
|
72 |
" {\n",
|
73 |
" \"model\": \"gpt-4o-mini\",\n",
|
74 |
" \"api_key\": os.getenv(\"OPENAI_API_KEY\")\n",
|
75 |
" }\n",
|
76 |
+
"]\n",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
77 |
"\n",
|
78 |
+
"llm_config = {\n",
|
79 |
+
" \"config_list\": config_list,\n",
|
80 |
+
" \"timeout\": 120,\n",
|
81 |
+
"}"
|
|
|
|
|
82 |
]
|
83 |
},
|
84 |
{
|
|
|
90 |
},
|
91 |
{
|
92 |
"cell_type": "code",
|
93 |
+
"execution_count": 99,
|
94 |
"metadata": {},
|
95 |
"outputs": [],
|
96 |
"source": [
|
97 |
"def create_agents():\n",
|
98 |
" \"\"\"Create and initialize the specialized agents.\"\"\"\n",
|
99 |
+
"\n",
|
|
|
100 |
" ocr_agent = autogen.AssistantAgent(\n",
|
101 |
" name=\"OCR_Specialist\",\n",
|
102 |
" system_message=\"\"\"You are an OCR specialist. Your role is to:\n",
|
103 |
" 1. Extract text from an image path using pytesseract\n",
|
104 |
" 2. Clean and format the extracted text for further analysis\"\"\",\n",
|
105 |
+
" llm_config=llm_config\n",
|
106 |
" )\n",
|
107 |
+
" \n",
|
|
|
108 |
" content_agent = autogen.AssistantAgent(\n",
|
109 |
" name=\"Content_Analyst\",\n",
|
110 |
" system_message=\"\"\"You are a content analysis specialist. Your role is to:\n",
|
|
|
113 |
" 3. Identify urgency indicators, threats, or pressure tactics\n",
|
114 |
" 5. Check for inconsistencies in messaging\n",
|
115 |
" 6. Evaluate legitimacy of any claims or offers\"\"\",\n",
|
116 |
+
" llm_config=llm_config\n",
|
117 |
" )\n",
|
118 |
"\n",
|
|
|
119 |
" decision_agent = autogen.AssistantAgent(\n",
|
120 |
" name=\"Decision_Maker\",\n",
|
121 |
" system_message=\"\"\"You are the final decision maker. Your role is to:\n",
|
|
|
123 |
" 2. Make a final determination on scam probability\n",
|
124 |
" 3. Provide detailed explanation of the decision\n",
|
125 |
" 4. End your explanation with the label as 'TASK_COMPLETE' when done\"\"\",\n",
|
126 |
+
" llm_config=llm_config\n",
|
127 |
" )\n",
|
128 |
"\n",
|
129 |
" summary_agent = autogen.AssistantAgent(\n",
|
|
|
134 |
" 3. Provide actionable recommendations\n",
|
135 |
" 4. Use clear, non-technical language while maintaining accuracy\n",
|
136 |
" 5. Format information in a way that's easy to read and understand\"\"\",\n",
|
137 |
+
" llm_config=llm_config\n",
|
138 |
" )\n",
|
139 |
"\n",
|
|
|
140 |
" user_proxy = autogen.UserProxyAgent(\n",
|
141 |
+
" name=\"user_proxy\",\n",
|
|
|
|
|
142 |
" is_termination_msg=lambda x: \"TASK_COMPLETE\" in x.get(\"content\", \"\"),\n",
|
143 |
+
" human_input_mode=\"NEVER\",\n",
|
144 |
+
" max_consecutive_auto_reply=10,\n",
|
145 |
" )\n",
|
146 |
"\n",
|
147 |
+
" @user_proxy.register_for_execution()\n",
|
148 |
+
" @ocr_agent.register_for_llm(description=\"Extracts text from an image path\")\n",
|
149 |
+
" def ocr(image_path: str) -> str:\n",
|
150 |
+
" from PIL import Image\n",
|
151 |
+
" import pytesseract\n",
|
152 |
+
"\n",
|
153 |
+
" try:\n",
|
154 |
+
" image = Image.open(image_path)\n",
|
155 |
+
" text = pytesseract.image_to_string(image)\n",
|
156 |
+
" return text\n",
|
157 |
+
" except Exception as e:\n",
|
158 |
+
" return f\"Error in text extraction: {str(e)}\"\n",
|
159 |
" \n",
|
160 |
" return ocr_agent, content_agent, decision_agent, user_proxy "
|
161 |
]
|
|
|
169 |
},
|
170 |
{
|
171 |
"cell_type": "code",
|
172 |
+
"execution_count": 100,
|
173 |
"metadata": {},
|
174 |
"outputs": [],
|
175 |
"source": [
|
|
|
185 |
" groupchat = autogen.GroupChat(\n",
|
186 |
" agents=[self.ocr_agent, self.content_agent, self.decision_agent, self.user_proxy],\n",
|
187 |
" messages=[],\n",
|
188 |
+
" max_round=15,\n",
|
189 |
" )\n",
|
190 |
" manager = autogen.GroupChatManager(groupchat=groupchat)\n",
|
191 |
"\n",
|
192 |
" # Start the collaborative analysis\n",
|
193 |
" messages = self.user_proxy.initiate_chat(\n",
|
194 |
" manager,\n",
|
195 |
+
" message=f\"\"\"\n",
|
196 |
" 1. OCR Agent: Extract text from this image: {image_path}\n",
|
197 |
" 2. Content Agent: Evaluate the messaging and claims\n",
|
198 |
" 3. Decision Maker: Synthesize all analyses and make final determination\"\"\",\n",
|
|
|
203 |
},
|
204 |
{
|
205 |
"cell_type": "code",
|
206 |
+
"execution_count": 101,
|
207 |
"metadata": {},
|
208 |
"outputs": [
|
209 |
{
|
|
|
220 |
"name": "stdout",
|
221 |
"output_type": "stream",
|
222 |
"text": [
|
223 |
+
"\u001b[33muser_proxy\u001b[0m (to chat_manager):\n",
|
224 |
+
"\n",
|
225 |
"\n",
|
|
|
226 |
" 1. OCR Agent: Extract text from this image: ./samples/02.giftcard.message.scam.png\n",
|
227 |
" 2. Content Agent: Evaluate the messaging and claims\n",
|
228 |
" 3. Decision Maker: Synthesize all analyses and make final determination\n",
|
|
|
233 |
"\u001b[0m\n",
|
234 |
"\u001b[33mOCR_Specialist\u001b[0m (to chat_manager):\n",
|
235 |
"\n",
|
236 |
+
"\u001b[32m***** Suggested tool call (call_ta6tNtpskpnHXvaWicWmYHi8): ocr *****\u001b[0m\n",
|
237 |
"Arguments: \n",
|
238 |
"{\"image_path\":\"./samples/02.giftcard.message.scam.png\"}\n",
|
239 |
"\u001b[32m********************************************************************\u001b[0m\n",
|
240 |
"\n",
|
241 |
"--------------------------------------------------------------------------------\n",
|
242 |
"\u001b[32m\n",
|
243 |
+
"Next speaker: user_proxy\n",
|
244 |
"\u001b[0m\n",
|
245 |
"\u001b[35m\n",
|
246 |
">>>>>>>> EXECUTING FUNCTION ocr...\u001b[0m\n",
|
247 |
+
"\u001b[33muser_proxy\u001b[0m (to chat_manager):\n",
|
248 |
"\n",
|
249 |
+
"\u001b[32m***** Response from calling tool (call_ta6tNtpskpnHXvaWicWmYHi8) *****\u001b[0m\n",
|
250 |
"Congratulations!\n",
|
251 |
"You've won a $1,000\n",
|
252 |
"Walmart gift card. Go\n",
|
|
|
262 |
"\u001b[0m\n",
|
263 |
"\u001b[33mOCR_Specialist\u001b[0m (to chat_manager):\n",
|
264 |
"\n",
|
265 |
+
"The extracted text from the image is as follows:\n",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
266 |
"\n",
|
267 |
+
"---\n",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
268 |
"\n",
|
269 |
+
"**Congratulations!** \n",
|
270 |
+
"You've won a **$1,000** \n",
|
271 |
+
"Walmart gift card. Go \n",
|
272 |
+
"[http://bit.ly/123456](http://bit.ly/123456) \n",
|
|
|
|
|
273 |
"tp claim now.\n",
|
|
|
|
|
274 |
"\n",
|
275 |
+
"---\n",
|
|
|
|
|
|
|
|
|
276 |
"\n",
|
277 |
+
"### Cleaning and Formatting the Extracted Text:\n",
|
|
|
|
|
|
|
278 |
"\n",
|
279 |
+
"1. **Remove unnecessary characters:** The text appears to have an odd break (\"tp claim now.\") which might be due to OCR error.\n",
|
280 |
+
" \n",
|
281 |
+
"2. **Correct the message structure:** Creating a more readable format.\n",
|
282 |
+
"\n",
|
283 |
+
"---\n",
|
284 |
"\n",
|
285 |
+
"**Final Cleaned Text:**\n",
|
|
|
286 |
"\n",
|
287 |
"```\n",
|
288 |
+
"Congratulations! \n",
|
289 |
+
"You've won a $1,000 Walmart gift card.\n",
|
290 |
+
"Go to http://bit.ly/123456 to claim now.\n",
|
|
|
|
|
291 |
"```\n",
|
292 |
"\n",
|
293 |
+
"### Next Steps:\n",
|
294 |
+
"1. **Evaluate the messaging and claims** based on typical characteristics of scams. \n",
|
295 |
+
"2. **Synthesize analysis** and provide a final determination on the legitimacy of the message. \n",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
296 |
"\n",
|
297 |
+
"Would you like me to proceed with evaluating the messaging and claims?\n",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
298 |
"\n",
|
299 |
"--------------------------------------------------------------------------------\n",
|
300 |
"\u001b[32m\n",
|
|
|
302 |
"\u001b[0m\n",
|
303 |
"\u001b[33mContent_Analyst\u001b[0m (to chat_manager):\n",
|
304 |
"\n",
|
305 |
+
"### Evaluation of Messaging and Claims:\n",
|
|
|
306 |
"\n",
|
307 |
+
"1. **Common Scam Patterns:**\n",
|
308 |
+
" - **Too Good to Be True:** The claim of winning a $1,000 gift card is a classic scam tactic. Scammers often offer substantial rewards to entice victims.\n",
|
309 |
+
" - **Generic Greeting:** The use of \"Congratulations!\" without personalizing the message is common in mass scam messages.\n",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
310 |
"\n",
|
311 |
+
"2. **Identification of URLs:**\n",
|
312 |
+
" - The URL provided (`http://bit.ly/123456`) is a shortened URL. Scammers often use URL shorteners to obscure the actual destination, which is a red flag. It is advisable to check where this URL would direct a user since it could lead to phishing sites or malicious downloads.\n",
|
313 |
"\n",
|
314 |
+
"3. **Urgency Indicators, Threats, or Pressure Tactics:**\n",
|
315 |
+
" - Phrases like \"Go to claim now\" indicate a sense of urgency, pressuring the recipient to act quickly without thinking critically about the legitimacy of the offer.\n",
|
316 |
"\n",
|
317 |
+
"4. **Inconsistencies in Messaging:**\n",
|
318 |
+
" - The abrupt ending with \"tp claim now.\" suggests a potential OCR error or deliberate attempt to confuse the recipient, further indicating a lack of professionalism associated with scams.\n",
|
319 |
"\n",
|
320 |
+
"5. **Legitimacy of Claims or Offers:**\n",
|
321 |
+
" - Large corporations like Walmart do not typically give out gift cards in such arbitrary fashions. Any real promotions would likely require personalization and would not come through unsolicited messages.\n",
|
322 |
"\n",
|
323 |
+
"### Final Determination:\n",
|
|
|
324 |
"\n",
|
325 |
+
"Based on the analysis of common scam patterns, the use of a shortened URL, urgency indicators, potential errors in the message, and the legitimacy of the claims, it is very likely that this message is a scam. It poses a risk to individuals who might click on the link without verifying the legitimacy of the offer. \n",
|
|
|
|
|
|
|
|
|
326 |
"\n",
|
327 |
+
"### Recommendation:\n",
|
328 |
+
"Do not engage with the message or click on the link. It is advisable to report such scams to appropriate authorities to help prevent others from falling victim to similar tactics.\n",
|
329 |
"\n",
|
330 |
"--------------------------------------------------------------------------------\n",
|
331 |
"\u001b[32m\n",
|
|
|
333 |
"\u001b[0m\n",
|
334 |
"\u001b[33mDecision_Maker\u001b[0m (to chat_manager):\n",
|
335 |
"\n",
|
336 |
+
"### Final Decision on Scam Probability\n",
|
337 |
"\n",
|
338 |
+
"After gathering and synthesizing information from both the OCR extraction and the content evaluation, I conclude that the probability of this message being a scam is very high.\n",
|
339 |
"\n",
|
340 |
+
"#### Detailed Explanation of the Decision:\n",
|
341 |
"\n",
|
342 |
+
"1. **Content of the Message:**\n",
|
343 |
+
" - The message claims that the recipient has \"won\" a significant prize, a $1,000 Walmart gift card. This type of claim is a traditional tactic used by scammers to lure individuals into providing personal information.\n",
|
344 |
"\n",
|
345 |
+
"2. **Use of a Shortened URL:**\n",
|
346 |
+
" - The inclusion of a modified URL (`http://bit.ly/123456`) is a major red flag. Scammers tend to use shortened links to hide the destination, leading individuals to malicious websites that can steal information or install malware.\n",
|
347 |
"\n",
|
348 |
+
"3. **Urgency and Pressure:**\n",
|
349 |
+
" - The message creates a sense of urgency with phrases like \"Go to claim now,\" encouraging immediate action. This tactic is a psychological manipulation commonly seen in scams, meant to prevent the recipient from thinking critically or researching legitimacy.\n",
|
350 |
"\n",
|
351 |
+
"4. **Nature of the Offer:**\n",
|
352 |
+
" - Offers of large monetary gifts or benefits without any prior engagement or purchase are usually inauthentic. Recognized companies such as Walmart typically have established promotional methods and would not distribute gift cards in this manner.\n",
|
353 |
"\n",
|
354 |
+
"5. **Inconsistencies in Messaging:**\n",
|
355 |
+
" - The message has grammatical inconsistencies (e.g., “tp” instead of “to”), which can imply poor crafting and possible scams. Legitimate communications from companies are typically well-written and free of such errors.\n",
|
356 |
"\n",
|
357 |
+
"### Final Determination:\n",
|
358 |
+
"\n",
|
359 |
+
"Given the above factors, I assess the probability of this message being a scam to be exceptionally high. It employs multiple red flags characteristic of phishing and scam communications, suggesting that recipients should avoid engaging further and should report it to the relevant authorities.\n",
|
360 |
+
"\n",
|
361 |
+
"### Conclusion:\n",
|
362 |
+
"Please refrain from interacting with this message and keep an eye out to identify any attempts to impersonate trusted companies.\n",
|
363 |
"\n",
|
364 |
"**TASK_COMPLETE**\n",
|
365 |
"\n",
|
366 |
"--------------------------------------------------------------------------------\n",
|
367 |
"\u001b[32m\n",
|
368 |
+
"Next speaker: user_proxy\n",
|
369 |
"\u001b[0m\n"
|
370 |
]
|
371 |
}
|
|
|
380 |
},
|
381 |
{
|
382 |
"cell_type": "code",
|
383 |
+
"execution_count": 102,
|
384 |
"metadata": {},
|
385 |
"outputs": [
|
386 |
{
|
387 |
"name": "stdout",
|
388 |
"output_type": "stream",
|
389 |
"text": [
|
390 |
+
"{'content': '### Final Decision on Scam Probability\\n'\n",
|
391 |
+
" '\\n'\n",
|
392 |
+
" 'After gathering and synthesizing information from both the OCR '\n",
|
393 |
+
" 'extraction and the content evaluation, I conclude that the '\n",
|
394 |
+
" 'probability of this message being a scam is very high.\\n'\n",
|
395 |
+
" '\\n'\n",
|
396 |
+
" '#### Detailed Explanation of the Decision:\\n'\n",
|
397 |
" '\\n'\n",
|
398 |
+
" '1. **Content of the Message:**\\n'\n",
|
399 |
+
" ' - The message claims that the recipient has \"won\" a '\n",
|
400 |
+
" 'significant prize, a $1,000 Walmart gift card. This type of claim '\n",
|
401 |
+
" 'is a traditional tactic used by scammers to lure individuals into '\n",
|
402 |
+
" 'providing personal information.\\n'\n",
|
403 |
" '\\n'\n",
|
404 |
+
" '2. **Use of a Shortened URL:**\\n'\n",
|
405 |
+
" ' - The inclusion of a modified URL (`http://bit.ly/123456`) is '\n",
|
406 |
+
" 'a major red flag. Scammers tend to use shortened links to hide '\n",
|
407 |
+
" 'the destination, leading individuals to malicious websites that '\n",
|
408 |
+
" 'can steal information or install malware.\\n'\n",
|
409 |
" '\\n'\n",
|
410 |
+
" '3. **Urgency and Pressure:**\\n'\n",
|
411 |
+
" ' - The message creates a sense of urgency with phrases like \"Go '\n",
|
412 |
+
" 'to claim now,\" encouraging immediate action. This tactic is a '\n",
|
413 |
+
" 'psychological manipulation commonly seen in scams, meant to '\n",
|
414 |
+
" 'prevent the recipient from thinking critically or researching '\n",
|
415 |
+
" 'legitimacy.\\n'\n",
|
416 |
" '\\n'\n",
|
417 |
+
" '4. **Nature of the Offer:**\\n'\n",
|
418 |
+
" ' - Offers of large monetary gifts or benefits without any prior '\n",
|
419 |
+
" 'engagement or purchase are usually inauthentic. Recognized '\n",
|
420 |
+
" 'companies such as Walmart typically have established promotional '\n",
|
421 |
+
" 'methods and would not distribute gift cards in this manner.\\n'\n",
|
422 |
" '\\n'\n",
|
423 |
+
" '5. **Inconsistencies in Messaging:**\\n'\n",
|
424 |
+
" ' - The message has grammatical inconsistencies (e.g., “tp” '\n",
|
425 |
+
" 'instead of “to”), which can imply poor crafting and possible '\n",
|
426 |
+
" 'scams. Legitimate communications from companies are typically '\n",
|
427 |
+
" 'well-written and free of such errors.\\n'\n",
|
428 |
" '\\n'\n",
|
429 |
+
" '### Final Determination:\\n'\n",
|
|
|
|
|
430 |
" '\\n'\n",
|
431 |
+
" 'Given the above factors, I assess the probability of this message '\n",
|
432 |
+
" 'being a scam to be exceptionally high. It employs multiple red '\n",
|
433 |
+
" 'flags characteristic of phishing and scam communications, '\n",
|
434 |
+
" 'suggesting that recipients should avoid engaging further and '\n",
|
435 |
+
" 'should report it to the relevant authorities.\\n'\n",
|
436 |
" '\\n'\n",
|
437 |
+
" '### Conclusion:\\n'\n",
|
438 |
+
" 'Please refrain from interacting with this message and keep an eye '\n",
|
439 |
+
" 'out to identify any attempts to impersonate trusted companies.\\n'\n",
|
|
|
|
|
|
|
440 |
" '\\n'\n",
|
441 |
" '**TASK_COMPLETE**',\n",
|
442 |
" 'name': 'Decision_Maker',\n",
|
|
|
452 |
},
|
453 |
{
|
454 |
"cell_type": "code",
|
455 |
+
"execution_count": 98,
|
456 |
"metadata": {},
|
457 |
"outputs": [],
|
458 |
"source": [
|