use binary for all

experimental/clip_api_app.py

@@ -1,9 +1,12 @@
-from typing import List
+# File name: model.py
+import json
+import os
 import numpy as np
 import torch
+from starlette.requests import Request
+from PIL import Image
 import ray
 from ray import serve
-from PIL import Image
 from clip_retrieval.load_clip import load_clip, get_tokenizer 
 # from clip_retrieval.clip_client import ClipClient, Modality
 
@@ -21,14 +24,11 @@ class CLIPTransform:
 
         print ("using device", self.device)        
 
-    @serve.batch(max_batch_size=32)
-    # def text_to_embeddings(self, prompts: List[str]) -> torch.Tensor:
-    def text_to_embeddings(self, prompts: List[str]) -> List[np.ndarray]:
-        text = self.tokenizer(prompts).to(self.device)
+    def text_to_embeddings(self, prompt):
+        text = self.tokenizer([prompt]).to(self.device)
         with torch.no_grad():
             prompt_embededdings = self.model.encode_text(text)
         prompt_embededdings /= prompt_embededdings.norm(dim=-1, keepdim=True)
-        prompt_embededdings = prompt_embededdings.cpu().numpy().tolist()
         return(prompt_embededdings)
     
     def image_to_embeddings(self, input_im):
@@ -45,31 +45,91 @@ class CLIPTransform:
         image_embeddings /= image_embeddings.norm(dim=-1, keepdim=True)
         return(image_embeddings)    
 
-    # async def __call__(self, http_request: Request) -> str:
-    #     request = await http_request.json()
-    #     # print(type(request))
-    #     # print(str(request))
-    #     # switch based if we are using text or image
-    #     embeddings = None
-    #     if "text" in request:
-    #         prompt = request["text"]
-    #         embeddings = self.text_to_embeddings(prompt)
-    #     elif "image" in request:
-    #         image_url = request["image_url"]
-    #         # download image from url
-    #         import requests
-    #         from io import BytesIO
-    #         input_image = Image.open(BytesIO(image_url))
-    #         input_image = input_image.convert('RGB')
-    #         input_image = np.array(input_image)
-    #         embeddings = self.image_to_embeddings(input_image)
-    #     elif "preprocessed_image" in request:
-    #         prepro = request["preprocessed_image"]
-    #         # create torch tensor on the device
-    #         prepro = torch.tensor(prepro).to(self.device)
-    #         embeddings = self.preprocessed_image_to_emdeddings(prepro)
-    #     else:
-    #         raise Exception("Invalid request")
-    #     return embeddings.cpu().numpy().tolist()
+    async def __call__(self, http_request: Request) -> str:
+        form_data = await http_request.form()
+
+        embeddings = None
+        if "text" in form_data:
+            prompt = (await form_data["text"].read()).decode()
+            print (type(prompt))
+            print (str(prompt))
+            embeddings = self.text_to_embeddings(prompt)
+        elif "image_url" in form_data:
+            image_url = (await form_data["image_url"].read()).decode()
+            # download image from url
+            import requests
+            from io import BytesIO
+            image_bytes = requests.get(image_url).content
+            input_image = Image.open(BytesIO(image_bytes))
+            input_image = input_image.convert('RGB')
+            input_image = np.array(input_image)
+            embeddings = self.image_to_embeddings(input_image)
+        elif "preprocessed_image" in form_data:
+            tensor_bytes = await form_data["preprocessed_image"].read()
+            shape_bytes = await form_data["shape"].read()
+            dtype_bytes = await form_data["dtype"].read()
+
+            # Convert bytes back to original form
+            dtype_mapping = {
+                "torch.float32": torch.float32,
+                "torch.float64": torch.float64,
+                "torch.float16": torch.float16,
+                "torch.uint8": torch.uint8,
+                "torch.int8": torch.int8,
+                "torch.int16": torch.int16,
+                "torch.int32": torch.int32,
+                "torch.int64": torch.int64,
+                torch.float32: np.float32,
+                torch.float64: np.float64,
+                torch.float16: np.float16,
+                torch.uint8: np.uint8,
+                torch.int8: np.int8,
+                torch.int16: np.int16,
+                torch.int32: np.int32,
+                torch.int64: np.int64,
+                # add more if needed
+            }
+            dtype_str = dtype_bytes.decode()
+            dtype_torch = dtype_mapping[dtype_str]
+            dtype_numpy = dtype_mapping[dtype_torch]
+            # shape = np.frombuffer(shape_bytes, dtype=np.int64)
+            # TODO: fix shape so it is passed nicely
+            shape = tuple([1, 3, 224, 224])
+
+            tensor_numpy = np.frombuffer(tensor_bytes, dtype=dtype_numpy).reshape(shape)
+            tensor = torch.from_numpy(tensor_numpy)
+            prepro = tensor.to(self.device)
+            embeddings = self.preprocessed_image_to_emdeddings(prepro)
+        else:
+            print ("Invalid request")
+            raise Exception("Invalid request")
+        return embeddings.cpu().numpy().tolist()
+
+        request = await http_request.json()
+        # print(type(request))
+        # print(str(request))
+        # switch based if we are using text or image
+        embeddings = None
+        if "text" in request:
+            prompt = request["text"]
+            embeddings = self.text_to_embeddings(prompt)
+        elif "image_url" in request:
+            image_url = request["image_url"]
+            # download image from url
+            import requests
+            from io import BytesIO
+            image_bytes = requests.get(image_url).content
+            input_image = Image.open(BytesIO(image_bytes))
+            input_image = input_image.convert('RGB')
+            input_image = np.array(input_image)
+            embeddings = self.image_to_embeddings(input_image)
+        elif "preprocessed_image" in request:
+            prepro = request["preprocessed_image"]
+            # create torch tensor on the device
+            prepro = torch.tensor(prepro).to(self.device)
+            embeddings = self.preprocessed_image_to_emdeddings(prepro)
+        else:
+            raise Exception("Invalid request")
+        return embeddings.cpu().numpy().tolist()
 
 deployment_graph = CLIPTransform.bind()

experimental/clip_app_client.py

@@ -38,30 +38,41 @@ def preprocess_image(image_url):
     # convert image to numpy array
     input_image = np.array(input_image)
     input_im = Image.fromarray(input_image)
-    prepro = preprocess(input_im).unsqueeze(0).to(device)
+    prepro = preprocess(input_im).unsqueeze(0).cpu()
     return prepro
 
 preprocessed_image = preprocess_image(test_image_url)
 
 def send_text_request(number):
-    data = {"text": english_text}
+    payload = {
+        "text": ('str', english_text, 'application/octet-stream'),
+    }
     url = os.environ.get("HTTP_ADDRESS", "http://127.0.0.1:8000/")
-    response = requests.post(url, json=data)
+    response = requests.post(url, files=payload)
     embeddings = response.text
     return number, embeddings
 
 def send_image_url_request(number):
-    data = {"image_url": test_image_url}
+    payload = {
+        "image_url": ('str', test_image_url, 'application/octet-stream'),
+    }
     url = os.environ.get("HTTP_ADDRESS", "http://127.0.0.1:8000/")
-    response = requests.post(url, json=data)
+    response = requests.post(url, files=payload)
     embeddings = response.text
     return number, embeddings
 
 def send_preprocessed_image_request(number):
-    nested_list = preprocessed_image.tolist()
-    data = {"preprocessed_image": nested_list}
+    key = "preprocessed_image"
+    data_bytes = preprocessed_image.numpy().tobytes()
+    shape_bytes = np.array(preprocessed_image.shape).tobytes()
+    dtype_bytes = str(preprocessed_image.dtype).encode()
+    payload = {
+        key: ('tensor', data_bytes, 'application/octet-stream'),
+        'shape': ('shape', shape_bytes, 'application/octet-stream'),
+        'dtype': ('dtype', dtype_bytes, 'application/octet-stream'),
+    }
     url = os.environ.get("HTTP_ADDRESS", "http://127.0.0.1:8000/")
-    response = requests.post(url, json=data)
+    response = requests.post(url, files=payload)
     embeddings = response.text
     return number, embeddings
 
@@ -80,7 +91,7 @@ def process(numbers, send_func, max_workers=10):
 #         print (f"{n_result} : {len(result[0])}")
 
 if __name__ == "__main__":
-    n_calls = 10000
+    n_calls = 300
 
     # test text
     # n_calls = 1
@@ -95,8 +106,6 @@ if __name__ == "__main__":
     print(f" Average time taken: {avg_time_ms:.2f} ms")
     print(f" Number of calls per second: {calls_per_sec:.2f}")    
 
-    n_calls = 100
-
     # test image url
     # n_calls = 1
     numbers = list(range(n_calls))
@@ -119,6 +128,6 @@ if __name__ == "__main__":
     total_time = end_time - start_time
     avg_time_ms = total_time / n_calls * 1000
     calls_per_sec = n_calls / total_time
-    print(f"Text...")
+    print(f"Preprocessed image...")
     print(f" Average time taken: {avg_time_ms:.2f} ms")
     print(f" Number of calls per second: {calls_per_sec:.2f}")