Fix #31: find & use similar icons when non-existing icon names are generated by the LLM

app.py

@@ -3,9 +3,9 @@ Streamlit app containing the UI and the application logic.
 """
 import datetime
 import logging
-import os
 import pathlib
 import random
+import sys
 import tempfile
 from typing import List, Union
 
@@ -15,6 +15,10 @@ from langchain_community.chat_message_histories import StreamlitChatMessageHisto
 from langchain_core.messages import HumanMessage
 from langchain_core.prompts import ChatPromptTemplate
 
+sys.path.append('..')
+sys.path.append('../..')
+
+import helpers.icons_embeddings as ice
 from global_config import GlobalConfig
 from helpers import llm_helper, pptx_helper, text_helper
 
@@ -68,12 +72,7 @@ def _get_icons_list() -> List[str]:
     :return: A llist of the icons.
     """
 
-    items = pathlib.Path(GlobalConfig.ICONS_DIR).glob('*.png')
-    items = [
-        os.path.basename(str(item)).removesuffix('.png') for item in items
-    ]
-
-    return items
+    return ice.get_icons_list()
 
 
 APP_TEXT = _load_strings()

file_embeddings/embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f11da27237bdf979f27e6b7347e1ea0ac7a375f3492340871ee7f812709f5c9
+size 95360

file_embeddings/icons.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:248012493bfb0d29c38a62a32204544e1a3eab9d0971e9c31d5d99216a8809c4
+size 21704

global_config.py

@@ -20,8 +20,8 @@ class GlobalConfig:
     HF_LLM_MODEL_NAME = 'mistralai/Mistral-Nemo-Instruct-2407'
     LLM_MODEL_TEMPERATURE = 0.2
     LLM_MODEL_MIN_OUTPUT_LENGTH = 100
-    LLM_MODEL_MAX_OUTPUT_LENGTH = 4 * 4096
-    LLM_MODEL_MAX_INPUT_LENGTH = 750
+    LLM_MODEL_MAX_OUTPUT_LENGTH = 4 * 4096  # tokens
+    LLM_MODEL_MAX_INPUT_LENGTH = 750  # characters
 
     HUGGINGFACEHUB_API_TOKEN = os.environ.get('HUGGINGFACEHUB_API_TOKEN', '')
     METAPHOR_API_KEY = os.environ.get('METAPHOR_API_KEY', '')
@@ -36,6 +36,9 @@ class GlobalConfig:
 
     LLM_PROGRESS_MAX = 90
     ICONS_DIR = 'icons/png128/'
+    TINY_BERT_MODEL = 'gaunernst/bert-tiny-uncased'
+    EMBEDDINGS_FILE_NAME = 'file_embeddings/embeddings.npy'
+    ICONS_FILE_NAME = 'file_embeddings/icons.npy'
 
     PPTX_TEMPLATE_FILES = {
         'Basic': {

helpers/icons_embeddings.py

@@ -0,0 +1,122 @@
+"""
+Generate and save the embeddings of a pre-defined list of icons.
+Compare them with keywords embeddings to find most relevant icons.
+"""
+import os
+import pathlib
+import sys
+from typing import List, Tuple
+
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+from transformers import BertTokenizer, BertModel
+
+sys.path.append('..')
+sys.path.append('../..')
+
+from global_config import GlobalConfig
+
+
+tokenizer = BertTokenizer.from_pretrained(GlobalConfig.TINY_BERT_MODEL)
+model = BertModel.from_pretrained(GlobalConfig.TINY_BERT_MODEL)
+
+
+def get_icons_list() -> List[str]:
+    """
+    Get a list of available icons.
+
+    :return: The icons file names.
+    """
+
+    items = pathlib.Path('../' + GlobalConfig.ICONS_DIR).glob('*.png')
+    items = [
+        os.path.basename(str(item)).removesuffix('.png') for item in items
+    ]
+
+    return items
+
+
+def get_embeddings(texts) -> np.ndarray:
+    """
+    Generate embeddings for a list of texts using a pre-trained language model.
+
+    :param texts: A string or a list of strings to be converted into embeddings.
+    :type texts: Union[str, List[str]]
+    :return: A NumPy array containing the embeddings for the input texts.
+    :rtype: numpy.ndarray
+
+    :raises ValueError: If the input is not a string or a list of strings, or if any element
+    in the list is not a string.
+
+    Example usage:
+    >>> keyword = 'neural network'
+    >>> file_names = ['neural_network_icon.png', 'data_analysis_icon.png', 'machine_learning.png']
+    >>> keyword_embeddings = get_embeddings(keyword)
+    >>> file_name_embeddings = get_embeddings(file_names)
+    """
+
+    inputs = tokenizer(texts, return_tensors='pt', padding=True, max_length=128, truncation=True)
+    outputs = model(**inputs)
+
+    return outputs.last_hidden_state.mean(dim=1).detach().numpy()
+
+
+def save_icons_embeddings():
+    """
+    Generate and save the embeddings for the icon file names.
+    """
+
+    file_names = get_icons_list()
+    file_name_embeddings = get_embeddings(file_names)
+
+    # Save embeddings to a file
+    np.save(GlobalConfig.EMBEDDINGS_FILE_NAME, file_name_embeddings)
+    np.save(GlobalConfig.ICONS_FILE_NAME, file_names)  # Save file names for reference
+
+
+def load_saved_embeddings() -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Load precomputed embeddings and icons file names.
+
+    :return: The embeddings and the icon file names.
+    """
+
+    file_name_embeddings = np.load(GlobalConfig.EMBEDDINGS_FILE_NAME)
+    file_names = np.load(GlobalConfig.ICONS_FILE_NAME)
+
+    return file_name_embeddings, file_names
+
+
+def find_icons(keywords: List[str]) -> List[str]:
+    """
+    Find relevant icon file names for a list of keywords.
+
+    :param keywords: The list of one or more keywords.
+    :return: A list of the file names relevant for each keyword.
+    """
+
+    keyword_embeddings = get_embeddings(keywords)
+    file_name_embeddings, file_names = load_saved_embeddings()
+
+    # Compute similarity
+    similarities = cosine_similarity(keyword_embeddings, file_name_embeddings)
+    icon_files = file_names[np.argmax(similarities, axis=-1)]
+
+    return icon_files
+
+
+def main():
+    """
+    Example usage.
+    """
+
+    # Run this again if icons are to be added/removed
+    # save_icons_embeddings()
+
+    keywords = ['deep learning', 'library', 'universe', 'brain', 'cybersecurity', 'gaming', '']
+    icon_files = find_icons(keywords)
+    print(f'The relevant icon files are: {icon_files}')
+
+
+if __name__ == '__main__':
+    main()

helpers/pptx_helper.py

@@ -19,6 +19,7 @@ from pptx.shapes.placeholder import PicturePlaceholder, SlidePlaceholder
 sys.path.append('..')
 sys.path.append('../..')
 
+import helpers.icons_embeddings as ice
 import helpers.image_search as ims
 from global_config import GlobalConfig
 
@@ -493,24 +494,28 @@ def _handle_icons_ideas(
 
         # Calculate the total width of all pictures and the spacing
         total_width = n_items * ICON_SIZE
-        # slide_width = presentation.slide_width
         spacing = (pptx.util.Inches(slide_width_inch) - total_width) / (n_items + 1)
-
-        for idx, item in enumerate(items):
-            # Extract the icon name and text
-            match = ICONS_REGEX.search(item)
-
-            if not match:
-                # print('No icon/text pattern match found...skipping to the next item')
-                continue
-
-            icon_name = match.group(1)
-            accompanying_text = match.group(2)
-            icon_path = f'{GlobalConfig.ICONS_DIR}/{icon_name}.png'
+        top = INCHES_3
+
+        icons_texts = [
+            (match.group(1), match.group(2)) for match in [
+                ICONS_REGEX.search(item) for item in items
+            ]
+        ]
+        fallback_icon_files = ice.find_icons([item[0] for item in icons_texts])
+
+        for idx, item in enumerate(icons_texts):
+            icon, accompanying_text = item
+            icon_path = f'{GlobalConfig.ICONS_DIR}/{icon}.png'
+
+            if not os.path.exists(icon_path):
+                logger.warning(
+                    'Icon not found: %s...using fallback icon: %s',
+                    icon, fallback_icon_files[idx]
+                )
+                icon_path = f'{GlobalConfig.ICONS_DIR}/{fallback_icon_files[idx]}.png'
 
             left = spacing + idx * (ICON_SIZE + spacing)
-            top = INCHES_3
-
             # Calculate the center position for alignment
             center = left + ICON_SIZE / 2
 
@@ -526,30 +531,18 @@ def _handle_icons_ideas(
             shape.shadow.inherit = False
 
             # Set the icon's background shape color
-            color = random.choice(ICON_COLORS)
-            shape.fill.fore_color.rgb = color
-            shape.line.color.rgb = color
+            shape.fill.fore_color.rgb = shape.line.color.rgb = random.choice(ICON_COLORS)
 
             # Add the icon image on top of the colored shape
-            try:
-                slide.shapes.add_picture(icon_path, left, top, height=ICON_SIZE)
-            except FileNotFoundError:
-                logger.error(
-                    'Icon %s not found...using generic step number as icon...',
-                    icon_name
-                )
-                step_icon_path = f'{GlobalConfig.ICONS_DIR}/{idx + 1}-circle.png'
-                if os.path.exists(step_icon_path):
-                    slide.shapes.add_picture(step_icon_path, left, top, height=ICON_SIZE)
+            slide.shapes.add_picture(icon_path, left, top, height=ICON_SIZE)
 
             # Add a text box below the shape
-            text_top = top + ICON_SIZE + INCHES_0_2
-            text_left = center - text_box_size / 2  # Center the text box horizontally
-            # text_box = slide.shapes.add_textbox(text_left, text_top, text_box_size, text_box_size)
             text_box = slide.shapes.add_shape(
                 MSO_AUTO_SHAPE_TYPE.ROUNDED_RECTANGLE,
-                text_left, text_top,
-                text_box_size, text_box_size
+                left=center - text_box_size / 2,  # Center the text box horizontally
+                top=top + ICON_SIZE + INCHES_0_2,
+                width=text_box_size,
+                height=text_box_size
             )
             text_frame = text_box.text_frame
             text_frame.text = accompanying_text
@@ -955,7 +948,7 @@ if __name__ == '__main__':
       "bullet_points": [
         "[[brain]] Human-like intelligence and decision-making",
         "[[robot]] Automation and physical tasks",
-        "[[cloud]] Data processing and cloud computing",
+        "[[]] Data processing and cloud computing",
         "[[lightbulb]] Insights and predictions",
         "[[globe2]] Global connectivity and impact"
       ],

langchain_templates/chat_prompts/initial_template_v4_two_cols_img.txt

@@ -13,22 +13,18 @@ In addition, for each slide, add image keywords based on the content of the resp
 These keywords will be later used to search for images from the Web relevant to the slide content.
 
 In addition, create one slide containing 4 TO 6 icons (pictograms) illustrating some key ideas/aspects/concepts relevant to the topic.
-In this slide, each line of text will begin with the name of an icon enclosed between [[ and ]].
-The name of an icon MUST BE chosen exactly as it is from the <ICONS> section provided below and should be relevant to the aspects described.
-If you need an icon that is unavailable in the <ICONS> section, you may select a conceptually similar icon's name from <ICONS>.
-For example, if an icon for "neural network" is unavailable, you may choose the "deep-learning" icon from <ICONS>.
-However, you MUST NEVER generate any icon name not mentioned in the <ICONS> section.
-
-The content of each slide should be VERBOSE, DESCRIPTIVE, and very DETAILED.
-
-ALWAYS add a concluding slide at the end, containing a list of the key takeaways and an optional call-to-action if relevant to the context.
-Unless explicitly instructed, create 10 TO 12 SLIDES in total.
-
+In this slide, each line of text will begin with the name of a relevant icon enclosed between [[ and ]].
+Select appropriate and exact icon names from the <ICONS> section provided below.
 
 <ICONS>
 {icons_list}
 </ICONS>
 
+The content of each slide should be VERBOSE, DESCRIPTIVE, and very DETAILED. Each bullet point should be detailed and explanatory, not just short phrases.
+
+ALWAYS add a concluding slide at the end, containing a list of the key takeaways and an optional call-to-action if relevant to the context.
+Unless explicitly instructed with the topic, create 10 TO 12 SLIDES in total. You must never create more tha 15 slides.
+
 
 ### Topic:
 {question}
@@ -64,10 +60,10 @@ The output must be only a valid and syntactically correct JSON adhering to the f
         {{
             "heading": "A slide illustrating key ideas/aspects/concepts (Hint: generate an appropriate heading)",
             "bullet_points": [
-                "[[icon_name]] Some text",
-                "[[another_icon_name]] Some words describing this aspect",
-                "[[icon_other_name]] Another aspect highlighted here",
-                "[[an_icon]] Another point here",
+                "[[icon name]] Some text",
+                "[[another icon name]] Some words describing this aspect",
+                "[[icon]] Another aspect highlighted here",
+                "[[an icon]] Another point here",
             ],
             "key_message": "",
             "img_keywords": ""

langchain_templates/chat_prompts/refinement_template_v4_two_cols_img.txt

@@ -14,21 +14,17 @@ In addition, for each slide, add image keywords based on the content of the resp
 These keywords will be later used to search for images from the Web relevant to the slide content.
 
 In addition, create one slide containing 4 TO 6 icons (pictograms) illustrating some key ideas/aspects/concepts relevant to the topic.
-In this slide, each line of text will begin with the name of an icon enclosed between [[ and ]].
-The name of an icon MUST BE chosen exactly as it is from the <ICONS> section provided below and should be relevant to the aspects described.
-If you need an icon that is unavailable in the <ICONS> section, you may select a conceptually similar icon's name from <ICONS>.
-For example, if an icon for "neural network" is unavailable, you may choose the "deep-learning" icon from <ICONS>.
-However, you MUST NEVER generate any icon name not mentioned in the <ICONS> section.
+In this slide, each line of text will begin with the name of a relevant icon enclosed between [[ and ]].
+Select appropriate and exact icon names from the <ICONS> section provided below.
 
-The content of each slide should be VERBOSE, DESCRIPTIVE, and very DETAILED.
-
-ALWAYS add a concluding slide at the end, containing a list of the key takeaways and an optional call-to-action if relevant to the context.
-Unless explicitly instructed, create 10 TO 12 SLIDES in total.
+<ICONS>
+{icons_list}
+</ICONS>
 
+The content of each slide should be VERBOSE, DESCRIPTIVE, and very DETAILED. Each bullet point should be detailed and explanatory, not just short phrases.
 
-<Icons>
-{icons_list}
-</Icons>
+ALWAYS add a concluding slide at the end, containing a list of the key takeaways and an optional call-to-action if relevant to the context.
+Unless explicitly specified in the instructions below, create 10 TO 12 SLIDES in total. You must never create more tha 15 slides.
 
 
 ### List of instructions:
@@ -69,10 +65,10 @@ The output must be only a valid and syntactically correct JSON adhering to the f
         {{
             "heading": "A slide illustrating key ideas/aspects/concepts (Hint: generate an appropriate heading)",
             "bullet_points": [
-                "[[icon_name]] Some text",
-                "[[another_icon_name]] Some words describing this aspect",
-                "[[icon_other_name]] Another aspect highlighted here",
-                "[[an_icon]] Another point here",
+                "[[icon name]] Some text",
+                "[[another icon name]] Some words describing this aspect",
+                "[[icon]] Another aspect highlighted here",
+                "[[an icon]] Another point here",
             ],
             "key_message": "",
             "img_keywords": ""

requirements.txt

@@ -16,9 +16,11 @@ metaphor-python
 json5~=0.9.14
 requests~=2.31.0
 
-transformers~=4.39.2
+transformers~=4.44.0
 langchain-community
 
 urllib3~=2.2.1
 lxml~=4.9.3
-tqdm~=4.64.1
+tqdm~=4.64.1
+numpy~=1.25.2
+scikit-learn~=1.5.1