Update merged_app2.py

merged_app2.py

@@ -8,7 +8,6 @@ from PIL import Image, ImageDraw
 from huggingface_hub import hf_hub_download
 import spaces
 
-import spaces
 import argparse
 import random
 
@@ -64,8 +63,19 @@ from PIL import Image, ImageDraw
 import numpy as np
 import spaces
 from huggingface_hub import hf_hub_download
+import openai
+from openai import OpenAI
+import gradio as gr
+import os
+from PIL import Image
+import numpy as np
+import io
+import base64
 
 
+MAX_IMAGE_WIDTH = 2048
+IMAGE_FORMAT = "JPEG"
+
 
 
 client = httpx.Client(timeout=httpx.Timeout(10.0))  # Set timeout to 10 seconds
@@ -1040,6 +1050,58 @@ def compress_image(image):
 
 def use_orientation(selected_image:gr.SelectData):
     return selected_image.value['image']['path']
+
+
+
+def generate_description(object_description,image, detail="high", max_tokens=250):
+    client = OpenAI(api_key=os.getenv["OPENAI_API_KEY"])
+
+    if image is not None:
+        try:
+            img = image  # No need to open, directly use the PIL Image object
+
+            buffered = io.BytesIO()
+            img.save(buffered, format=IMAGE_FORMAT)
+            img_base64 = base64.b64encode(buffered.getvalue()).decode()
+
+            prompt = f"As if you were describing the interior design, make a detailed caption of this image in one large paragraph. Highlighting textures, furnitures, locations. This object should be included in the description :{object_description}"
+
+            payload = {
+                "model": "gpt-4o-mini",
+                "messages": [{
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {"type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{img_base64}", "detail": detail}}
+                    ]
+                }],
+                "max_tokens": max_tokens
+            }
+
+            response = client.chat.completions.create(**payload)
+            return response.choices[0].message.content
+        except Exception as e:
+            print(e)
+    else:
+        try:
+            prompt = f"Description: {object_description}. As if you were designing an interior, improve this sentence in one large paragraph. Highlighting textures, furnitures, locations, such that you create a coherent, visually pleasing setting."
+
+            payload = {
+                "model": "gpt-4o-mini",
+                "messages": [{
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                    ]
+                }],
+                "max_tokens": max_tokens
+            }
+
+            response = client.chat.completions.create(**payload)
+            return response.choices[0].message.content
+        except Exception as e:
+            print(e)    
     
 
 @spaces.GPU(duration=60)
@@ -1875,12 +1937,15 @@ with gr.Blocks() as app:
                     image_to_describe = gr.Image(type="pil", label="Image", height=480)
                 with gr.Row():
                     with gr.Group():
-                        describe_button = gr.Button(value="Describe Image")
+                        with gr.Column():
+                            describe_button = gr.Button(value="Describe Image")
+                            text_to_describe = gr.Textbox(value="Describe object or scene")
                         description_text = gr.Textbox(
                                 label="Output", 
                                 placeholder="",
                                 value=""
                             )
+                        
 
         def send_img(img_result):
                 return img_result
@@ -1889,6 +1954,11 @@ with gr.Blocks() as app:
 
         # describe_button.click(describe_image, [image_to_describe], [description_text])
 
+        describe_button.click(
+            fn=generate_description,
+            inputs=[text_to_describe,image_to_describe],
+            outputs=description_text
+        )
                 
         generate_btn.click(
             fn=generate_image,