app.py

import spaces
import gradio as gr
import open_clip
import torch
import requests
import numpy as np
from PIL import Image
from io import BytesIO
from items import ecommerce_items
import os
# from dotenv import load_dotenv

# Load environment variables from the .env file
# load_dotenv()

# Sidebar content
sidebar_markdown = """

Note, this demo can classify 200 items. If you didn't find what you're looking for, reach out to us on our [Community](https://join.slack.com/t/marqo-community/shared_invite/zt-2iab0260n-QJrZLUSOJYUifVxf964Gdw) and request an item to be added.

## Documentation
📚 [Blog Post](https://www.marqo.ai/blog/introducing-marqos-ecommerce-embedding-models)

📝 [Classification Use Case Blog Post](https://www.marqo.ai/blog/ecommerce-image-classification-with-huggingface-transformers)

🔎 [Image Search Use Case Blog Post](https://www.marqo.ai/blog/how-to-build-an-ecommerce-image-search-application)

## Code
💻 [GitHub Repo](https://github.com/marqo-ai/marqo-ecommerce-embeddings)

🤝 [Google Colab](https://colab.research.google.com/drive/1ctqDrXs_P-RIOPc9xcUF83WLdYQ0wf-8?usp=sharing)

🤗 [Hugging Face Collection](https://huggingface.co/collections/Marqo/marqo-ecommerce-embeddings-66f611b9bb9d035a8d164fbb)

## Citation
If you use Marqo-Ecommerce-L or Marqo-Ecommerce-B, please cite us:
```
@software{zhu2024marqoecommembed_2024,
    author = {Tianyu Zhu and and Jesse Clark},
    month = oct,
    title = {{Marqo Ecommerce Embeddings - Foundation Model for Product Embeddings}},
    url = {https://github.com/marqo-ai/marqo-ecommerce-embeddings/},
    version = {1.0.0},
    year = {2024}
    }
```
"""

# Function to initialize a model, preprocess, and text features
@spaces.GPU
def initialize_model(model_name, progress=gr.Progress()):
    progress(0, f"Initializing model: {model_name}...")
    model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms(f"hf-hub:Marqo/{model_name}")
    
    progress(0.5, "Loading tokenizer...")
    tokenizer = open_clip.get_tokenizer(f"hf-hub:Marqo/{model_name}")
    
    text = tokenizer(ecommerce_items)
    
    progress(0.75, "Encoding text features...")
    with torch.no_grad(), torch.amp.autocast('cuda'):
        text_features = model.encode_text(text)
        text_features /= text_features.norm(dim=-1, keepdim=True)
    
    progress(1.0, f"Model {model_name} loaded successfully!")
    
    return model, preprocess_val, text_features

# Load L model first, followed by B model
progress_bar = gr.Progress()
model_l, preprocess_val_l, text_features_l = initialize_model("marqo-ecommerce-embeddings-L", progress=progress_bar)
model_b, preprocess_val_b, text_features_b = initialize_model("marqo-ecommerce-embeddings-B", progress=progress_bar)

# Prediction function
@spaces.GPU
def predict(image, url, model_name):
    if model_name == "marqo-ecommerce-embeddings-B":
        model, preprocess_val, text_features = model_b, preprocess_val_b, text_features_b
    else:
        model, preprocess_val, text_features = model_l, preprocess_val_l, text_features_l
    
    if url:
        response = requests.get(url)
        image = Image.open(BytesIO(response.content))
    
    processed_image = preprocess_val(image).unsqueeze(0)

    with torch.no_grad(), torch.amp.autocast('cuda'):
        image_features = model.encode_image(processed_image)
        image_features /= image_features.norm(dim=-1, keepdim=True)

        text_probs = (100 * image_features @ text_features.T).softmax(dim=-1)
        
        sorted_confidences = sorted(
            {ecommerce_items[i]: float(text_probs[0, i]) for i in range(len(ecommerce_items))}.items(), 
            key=lambda x: x[1], 
            reverse=True
        )
        top_10_confidences = dict(sorted_confidences[:10])
        
    return image, top_10_confidences

# Clear function
@spaces.GPU
def clear_fields():
    return None, ""

# Gradio interface
title = "Ecommerce Item Classifier with Marqo-Ecommerce Embedding Models"
description = "Upload an image or provide a URL of an ecommerce item to classify it using Marqo-Ecommerce Models!"

examples = [
    ["images/laptop.png", "Laptop"],
    ["images/grater.png", "Grater"],
    ["images/flip-flops.jpg", "Flip Flops"],
    ["images/bike-helmet.png", "Bike Helmet"],
    ["images/sleeping-bag.png", "Sleeping Bag"],
    ["images/cutting-board.png", "Cutting Board"],
    ["images/iron.png", "Iron"],
    ["images/coffee.png", "Coffee"],
]

with gr.Blocks(css="""
    .remove-btn {
        font-size: 24px !important; /* Increase the font size of the cross button */
        line-height: 24px !important;
        width: 30px !important; /* Increase the width */
        height: 30px !important; /* Increase the height */
    }
""") as demo:
    with gr.Row():
        with gr.Column(scale=1):
            gr.Markdown(f"# {title}")
            gr.Markdown(description)
            gr.Markdown(sidebar_markdown)
            gr.Markdown(" ", elem_id="vertical-line")  # Add an empty Markdown with a custom ID
        with gr.Column(scale=2):
            input_image = gr.Image(type="pil", label="Upload Ecommerce Item Image", height=312)
            input_url = gr.Textbox(label="Or provide an image URL")
            model_selector = gr.Dropdown(
                choices=["marqo-ecommerce-embeddings-L", "marqo-ecommerce-embeddings-B"],
                value="marqo-ecommerce-embeddings-L",
                label="Select Model"
            )
            with gr.Row():
                predict_button = gr.Button("Classify")
                clear_button = gr.Button("Clear")
            gr.Markdown("Or click on one of the images below to classify it:")
            gr.Examples(examples=examples, inputs=input_image)
            output_label = gr.Label(num_top_classes=6)
            predict_button.click(predict, inputs=[input_image, input_url, model_selector], outputs=[input_image, output_label])
            clear_button.click(clear_fields, outputs=[input_image, input_url, model_selector])

# Launch the interface
demo.launch()