Done

Dockerfile

@@ -1,8 +1,5 @@
 FROM python:3.10
 
-# Use the official Python 3.9 image
-# FROM python:3.9
-
 # Set the working directory to /code
 WORKDIR /code
 
@@ -26,4 +23,4 @@ WORKDIR $HOME/app
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
 COPY --chown=user . $HOME/app
 
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["flask", "run", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -8,7 +8,7 @@ from transformers import AutoProcessor, LlavaForConditionalGeneration, BitsAndBy
 from deep_translator import GoogleTranslator
 from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
 import warnings 
-# from flask import Flask
+from flask import Flask
 
 # from flask_ngrok import run_with_ngrok
 app = Flask(__name__)
@@ -16,16 +16,16 @@ app = Flask(__name__)
     
 warnings.filterwarnings('ignore') 
 
-# model_id = "HuggingFaceH4/vsft-llava-1.5-7b-hf-trl"
-# quantization_config = BitsAndBytesConfig(load_in_4bit=True)
-# base_model = LlavaForConditionalGeneration.from_pretrained(model_id, quantization_config=quantization_config, torch_dtype=torch.float16)
+model_id = "HuggingFaceH4/vsft-llava-1.5-7b-hf-trl"
+quantization_config = BitsAndBytesConfig(load_in_4bit=True)
+base_model = LlavaForConditionalGeneration.from_pretrained(model_id, quantization_config=quantization_config, torch_dtype=torch.float16)
 
-# # Load the PEFT Lora adapter
-# peft_lora_adapter_path = "Praveen0309/llava-1.5-7b-hf-ft-mix-vsft-3"
-# peft_lora_adapter = PeftModel.from_pretrained(base_model, peft_lora_adapter_path, adapter_name="lora_adapter")
-# base_model.load_adapter(peft_lora_adapter_path, adapter_name="lora_adapter")
+# Load the PEFT Lora adapter
+peft_lora_adapter_path = "Praveen0309/llava-1.5-7b-hf-ft-mix-vsft-3"
+peft_lora_adapter = PeftModel.from_pretrained(base_model, peft_lora_adapter_path, adapter_name="lora_adapter")
+base_model.load_adapter(peft_lora_adapter_path, adapter_name="lora_adapter")
 
-# processor = AutoProcessor.from_pretrained("HuggingFaceH4/vsft-llava-1.5-7b-hf-trl")
+processor = AutoProcessor.from_pretrained("HuggingFaceH4/vsft-llava-1.5-7b-hf-trl")
 # model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M")
 # tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M")
 
@@ -49,7 +49,7 @@ warnings.filterwarnings('ignore')
 def inference(image_prompt, image):
     prompt = f"USER: <image>\n{image_prompt} ASSISTANT:"
     inputs = processor(text=prompt, images=image, return_tensors="pt")
-    generate_ids = base_model.generate(**inputs, max_new_tokens=1024)
+    generate_ids = base_model.generate(**inputs, max_new_tokens=15)
     decoded_response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
 
 #     prompt = "USER: <image>\nWhat's the content of the image? ASSISTANT:"
@@ -144,8 +144,8 @@ def get_bot_response():
           image = image_cache['image']
           # print(image)
           query = request.args.get('msg')
-          output = query
-          # output = google_response(image, query)
+          # output = query
+          output = google_response(image, query)
           return output
       else:
           return "Please upload an image to continue"
@@ -154,8 +154,8 @@ def get_bot_response():
 
 
 # Run the Flask app
-# if __name__ == "__main__":
-    app.run(debug = True)
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5000, debug=True)
 
 
 # from pymongo import MongoClient