Upload folder using huggingface_hub

backend/vespa_app.py

@@ -104,54 +104,6 @@ class VespaQueryClient:
         self.logger.debug(result_text)
         return response.json
 
-    async def query_vespa_default(
-        self,
-        query: str,
-        q_emb: torch.Tensor,
-        hits: int = 3,
-        timeout: str = "10s",
-        sim_map: bool = False,
-        **kwargs,
-    ) -> dict:
-        """
-        Query Vespa using the default ranking profile.
-        This corresponds to the "Hybrid ColPali+BM25" radio button in the UI.
-
-        Args:
-            query (str): The query text.
-            q_emb (torch.Tensor): Query embeddings.
-            hits (int, optional): Number of hits to retrieve. Defaults to 3.
-            timeout (str, optional): Query timeout. Defaults to "10s".
-
-        Returns:
-            dict: The formatted query results.
-        """
-        async with self.app.asyncio(connections=1) as session:
-            query_embedding = self.format_q_embs(q_emb)
-
-            start = time.perf_counter()
-            response: VespaQueryResponse = await session.query(
-                body={
-                    "yql": (
-                        f"select {self.get_fields(sim_map=sim_map)} from {self.VESPA_SCHEMA_NAME} where userQuery();"
-                    ),
-                    "ranking": self.get_rank_profile("default", sim_map),
-                    "query": query,
-                    "timeout": timeout,
-                    "hits": hits,
-                    "input.query(qt)": query_embedding,
-                    "presentation.timing": True,
-                    **kwargs,
-                },
-            )
-            assert response.is_successful(), response.json
-            stop = time.perf_counter()
-            self.logger.debug(
-                f"Query time + data transfer took: {stop - start} s, Vespa reported searchtime was "
-                f"{response.json.get('timing', {}).get('searchtime', -1)} s"
-            )
-        return self.format_query_results(query, response)
-
     async def query_vespa_bm25(
         self,
         query: str,
@@ -286,12 +238,14 @@ class VespaQueryClient:
 
         rank_method = ranking.split("_")[0]
         sim_map: bool = len(ranking.split("_")) > 1 and ranking.split("_")[1] == "sim"
-        if rank_method == "nn+colpali":
-            result = await self.query_vespa_nearest_neighbor(
-                query, q_embs, sim_map=sim_map
+        if rank_method == "colpali":  # ColPali
+            result = await self.query_vespa_colpali(
+                query=query, ranking=rank_method, q_emb=q_embs, sim_map=sim_map
+            )
+        elif rank_method == "hybrid":  # Hybrid ColPali+BM25
+            result = await self.query_vespa_colpali(
+                query=query, ranking=rank_method, q_emb=q_embs, sim_map=sim_map
             )
-        elif rank_method == "bm25+colpali":
-            result = await self.query_vespa_default(query, q_embs, sim_map=sim_map)
         elif rank_method == "bm25":
             result = await self.query_vespa_bm25(query, q_embs, sim_map=sim_map)
         else:
@@ -419,9 +373,10 @@ class VespaQueryClient:
         else:
             return ranking
 
-    async def query_vespa_nearest_neighbor(
+    async def query_vespa_colpali(
         self,
         query: str,
+        ranking: str,
         q_emb: torch.Tensor,
         target_hits_per_query_tensor: int = 100,
         hnsw_explore_additional_hits: int = 300,
@@ -467,7 +422,7 @@ class VespaQueryClient:
                         f"select {self.get_fields(sim_map=sim_map)} from {self.VESPA_SCHEMA_NAME} where {nn_string} or userQuery()"
                     ),
                     "ranking.profile": self.get_rank_profile(
-                        "retrieval-and-rerank", sim_map
+                        ranking=ranking, sim_map=sim_map
                     ),
                     "timeout": timeout,
                     "hits": hits,

frontend/app.py

@@ -1,7 +1,30 @@
 from typing import Optional
 from urllib.parse import quote_plus
 
-from fasthtml.components import H1, H2, H3, Br, Div, Form, Img, NotStr, P, Span, Strong
+from fasthtml.components import (
+    H1,
+    H2,
+    H3,
+    Br,
+    Div,
+    Form,
+    Img,
+    NotStr,
+    P,
+    Hr,
+    Span,
+    A,
+    Script,
+    Button,
+    Label,
+    RadioGroup,
+    RadioGroupItem,
+    Separator,
+    Ul,
+    Li,
+    Strong,
+    Iframe,
+)
 from fasthtml.xtend import A, Script
 from lucide_fasthtml import Lucide
 from shad4fast import Badge, Button, Input, Label, RadioGroup, RadioGroupItem, Separator
@@ -175,7 +198,7 @@ def ShareButtons():
     )
 
 
-def SearchBox(with_border=False, query_value="", ranking_value="nn+colpali"):
+def SearchBox(with_border=False, query_value="", ranking_value="hybrid"):
     grid_cls = "grid gap-2 items-center p-3 bg-muted w-full"
 
     if with_border:
@@ -203,7 +226,7 @@ def SearchBox(with_border=False, query_value="", ranking_value="nn+colpali"):
                 Span("Ranking by:", cls="text-muted-foreground text-xs font-semibold"),
                 RadioGroup(
                     Div(
-                        RadioGroupItem(value="nn+colpali", id="nn+colpali"),
+                        RadioGroupItem(value="colpali", id="colpali"),
                         Label("ColPali", htmlFor="ColPali"),
                         cls="flex items-center space-x-2",
                     ),
@@ -213,7 +236,7 @@ def SearchBox(with_border=False, query_value="", ranking_value="nn+colpali"):
                         cls="flex items-center space-x-2",
                     ),
                     Div(
-                        RadioGroupItem(value="bm25+colpali", id="bm25+colpali"),
+                        RadioGroupItem(value="hybrid", id="hybrid"),
                         Label("Hybrid ColPali + BM25", htmlFor="Hybrid ColPali + BM25"),
                         cls="flex items-center space-x-2",
                     ),
@@ -308,48 +331,124 @@ def Home():
     )
 
 
+def LinkResource(text, href):
+    return Li(
+        A(
+            Lucide(icon="external-link", size="18"),
+            text,
+            href=href,
+            target="_blank",
+            cls="flex items-center gap-1.5 hover:underline bold text-md",
+        ),
+    )
+
+
 def AboutThisDemo():
+    resources = [
+        {
+            "text": "Vespa Blog: How we built this demo",
+            "href": "https://blog.vespa.ai/visual-rag-in-practice",
+        },
+        {
+            "text": "Notebook to set up Vespa application and feed dataset",
+            "href": "https://pyvespa.readthedocs.io/en/latest/examples/visual_pdf_rag_with_vespa_colpali_cloud.html",
+        },
+        {
+            "text": "Web App (FastHTML) Code",
+            "href": "https://github.com/vespa-engine/sample-apps/tree/master/visual-retrieval-colpali",
+        },
+        {
+            "text": "Vespa Blog: Scaling ColPali to Billions",
+            "href": "https://blog.vespa.ai/scaling-colpali-to-billions/",
+        },
+        {
+            "text": "Vespa Blog: Retrieval with Vision Language Models",
+            "href": "https://blog.vespa.ai/retrieval-with-vision-language-models-colpali/",
+        },
+    ]
     return Div(
-        Div(
-            Div(
-                H1(
-                    "Vespa.ai + ColPali",
-                    cls="text-5xl font-bold tracking-wide md:tracking-wider",
-                ),
-                P(
-                    "Efficient Document Retrieval with Vision Language Models",
-                    cls="text-lg text-muted-foreground md:tracking-wide",
+        H1(
+            "About This Demo",
+            cls="text-3xl md:text-5xl font-bold tracking-wide md:tracking-wider",
+        ),
+        P(
+            "This demo showcases a Visual Retrieval-Augmented Generation (RAG) application over PDFs using ColPali embeddings in Vespa, built entirely in Python, using FastHTML. The code is fully open source.",
+            cls="text-base",
+        ),
+        Img(
+            src="/static/img/colpali_child.png",
+            alt="Example of token level similarity map",
+            cls="w-full",
+        ),
+        H2("Resources", cls="text-2xl font-semibold"),
+        Ul(
+            *[
+                LinkResource(resource["text"], resource["href"])
+                for resource in resources
+            ],
+            cls="space-y-2 list-disc pl-5",
+        ),
+        H2("Architecture Overview", cls="text-2xl font-semibold"),
+        Img(
+            src="/static/img/visual-retrieval-demoapp-arch.png",
+            alt="Architecture Overview",
+            cls="w-full",
+        ),
+        Ul(
+            Li(
+                Strong("Vespa Application: "),
+                "Vespa Application that handles indexing, search, ranking and queries, leveraging features like phased ranking and multivector MaxSim calculations.",
+            ),
+            Li(
+                Strong("Frontend: "),
+                "Built with FastHTML, offering a professional and responsive user interface without the complexity of separate frontend frameworks.",
+            ),
+            Li(
+                Strong("Backend: "),
+                "Also built with FastHTML. Handles query embedding inference using ColPali, serves static files, and is responsible for orchestrating interactions between Vespa and the frontend.",
+            ),
+            Li(
+                Strong("Gemini API: "),
+                "VLM for the AI response, providing responses based on the top results from Vespa.",
+                cls="list-disc list-inside",
+            ),
+            H2("User Experience Highlights", cls="text-2xl font-semibold"),
+            Ul(
+                Li(
+                    Strong("Fast and Responsive: "),
+                    "Optimized for quick loading times, with phased content delivery to display essential information immediately while loading detailed data in the background.",
                 ),
-                Div(
-                    Img(
-                        src="/static/img/vespa-colpali.png",
-                        alt="Vespa and ColPali",
-                        cls="object-contain h-[377px]",
-                    ),
-                    cls="grid justify-center",
+                Li(
+                    Strong("Similarity Maps: "),
+                    "Provides visual highlights of the most relevant parts of a page in response to a query, enhancing interpretability.",
                 ),
-                Div(
-                    P(
-                        "This is a demo application showcasing the integration of Vespa.ai and ColPali for visual retrieval of documents.",
-                        cls="text-base",
-                    ),
-                    P(
-                        "The application uses a combination of neural networks and traditional search algorithms to retrieve relevant documents based on visual and textual queries.",
-                        cls="text-base",
-                    ),
-                    cls="grid gap-2 text-center",
+                Li(
+                    Strong("Type-Ahead Suggestions: "),
+                    "Offers query suggestions to assist users in formulating effective searches.",
                 ),
-                cls="grid gap-5 text-center",
+                cls="list-disc list-inside",
             ),
-            cls="grid gap-8 content-start mt-[8vh]",
+            cls="grid gap-5",
         ),
-        cls="grid w-full h-full max-w-screen-md gap-4 mx-auto",
+        H2("Dataset", cls="text-2xl font-semibold"),
+        P(
+            "The dataset used in this demo is retrieved from reports published by the Norwegian Government Pension Fund Global. It contains 6,992 pages from 116 PDF reports (2000–2024). The information is often presented in visual formats, making it an ideal dataset for visual retrieval applications.",
+            cls="text-base",
+        ),
+        Iframe(
+            src="https://huggingface.co/datasets/vespa-engine/gpfg-QA/embed/viewer",
+            frameborder="0",
+            width="100%",
+            height="500",
+        ),
+        Hr(),  # To add some margin to bottom. Probably a much better way to do this, but the mb-[16vh] class doesn't seem to be applied
+        cls="w-full h-full max-w-screen-md gap-4 mx-auto mt-[8vh] mb-[16vh] grid gap-8 content-start",
     )
 
 
 def Search(request, search_results=[]):
     query_value = request.query_params.get("query", "").strip()
-    ranking_value = request.query_params.get("ranking", "nn+colpali")
+    ranking_value = request.query_params.get("ranking", "hybrid")
     return Div(
         Div(
             Div(

main.py

@@ -156,7 +156,7 @@ def get():
 
 
 @rt("/search")
-def get(request, query: str = "", ranking: str = "nn+colpali"):
+def get(request, query: str = "", ranking: str = "hybrid"):
     logger.info(f"/search: Fetching results for query: {query}, ranking: {ranking}")
 
     # Always render the SearchBox first