visual_foundation_models.py

from diffusers import StableDiffusionPipeline
from diffusers import StableDiffusionInpaintPipeline
from diffusers import StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
from controlnet_aux import OpenposeDetector, MLSDdetector, HEDdetector

from transformers import AutoModelForCausalLM, AutoTokenizer, CLIPSegProcessor, CLIPSegForImageSegmentation
from transformers import pipeline, BlipProcessor, BlipForConditionalGeneration, BlipForQuestionAnswering
from transformers import AutoImageProcessor, UperNetForSemanticSegmentation

from PIL import Image
import torch
import numpy as np
import uuid
from pytorch_lightning import seed_everything
import cv2
import random
import os

def ade_palette():
    return [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50],
            [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255],
            [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7],
            [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82],
            [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3],
            [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255],
            [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220],
            [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224],
            [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255],
            [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7],
            [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153],
            [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255],
            [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0],
            [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255],
            [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255],
            [11, 200, 200], [255, 82, 0], [0, 255, 245], [0, 61, 255],
            [0, 255, 112], [0, 255, 133], [255, 0, 0], [255, 163, 0],
            [255, 102, 0], [194, 255, 0], [0, 143, 255], [51, 255, 0],
            [0, 82, 255], [0, 255, 41], [0, 255, 173], [10, 0, 255],
            [173, 255, 0], [0, 255, 153], [255, 92, 0], [255, 0, 255],
            [255, 0, 245], [255, 0, 102], [255, 173, 0], [255, 0, 20],
            [255, 184, 184], [0, 31, 255], [0, 255, 61], [0, 71, 255],
            [255, 0, 204], [0, 255, 194], [0, 255, 82], [0, 10, 255],
            [0, 112, 255], [51, 0, 255], [0, 194, 255], [0, 122, 255],
            [0, 255, 163], [255, 153, 0], [0, 255, 10], [255, 112, 0],
            [143, 255, 0], [82, 0, 255], [163, 255, 0], [255, 235, 0],
            [8, 184, 170], [133, 0, 255], [0, 255, 92], [184, 0, 255],
            [255, 0, 31], [0, 184, 255], [0, 214, 255], [255, 0, 112],
            [92, 255, 0], [0, 224, 255], [112, 224, 255], [70, 184, 160],
            [163, 0, 255], [153, 0, 255], [71, 255, 0], [255, 0, 163],
            [255, 204, 0], [255, 0, 143], [0, 255, 235], [133, 255, 0],
            [255, 0, 235], [245, 0, 255], [255, 0, 122], [255, 245, 0],
            [10, 190, 212], [214, 255, 0], [0, 204, 255], [20, 0, 255],
            [255, 255, 0], [0, 153, 255], [0, 41, 255], [0, 255, 204],
            [41, 0, 255], [41, 255, 0], [173, 0, 255], [0, 245, 255],
            [71, 0, 255], [122, 0, 255], [0, 255, 184], [0, 92, 255],
            [184, 255, 0], [0, 133, 255], [255, 214, 0], [25, 194, 194],
            [102, 255, 0], [92, 0, 255]]

def HWC3(x):
    assert x.dtype == np.uint8
    if x.ndim == 2:
        x = x[:, :, None]
    assert x.ndim == 3
    H, W, C = x.shape
    assert C == 1 or C == 3 or C == 4
    if C == 3:
        return x
    if C == 1:
        return np.concatenate([x, x, x], axis=2)
    if C == 4:
        color = x[:, :, 0:3].astype(np.float32)
        alpha = x[:, :, 3:4].astype(np.float32) / 255.0
        y = color * alpha + 255.0 * (1.0 - alpha)
        y = y.clip(0, 255).astype(np.uint8)
        return y

def resize_image(input_image, resolution):
    H, W, C = input_image.shape
    H = float(H)
    W = float(W)
    k = float(resolution) / min(H, W)
    H *= k
    W *= k
    H = int(np.round(H / 64.0)) * 64
    W = int(np.round(W / 64.0)) * 64
    img = cv2.resize(input_image, (W, H), interpolation=cv2.INTER_LANCZOS4 if k > 1 else cv2.INTER_AREA)
    return img

def get_new_image_name(org_img_name, func_name="update"):
    head_tail = os.path.split(org_img_name)
    head = head_tail[0]
    tail = head_tail[1]
    name_split = tail.split('.')[0].split('_')
    this_new_uuid = str(uuid.uuid4())[0:4]
    if len(name_split) == 1:
        most_org_file_name = name_split[0]
        recent_prev_file_name = name_split[0]
        new_file_name = '{}_{}_{}_{}.png'.format(this_new_uuid, func_name, recent_prev_file_name, most_org_file_name)
    else:
        assert len(name_split) == 4
        most_org_file_name = name_split[3]
        recent_prev_file_name = name_split[0]
        new_file_name = '{}_{}_{}_{}.png'.format(this_new_uuid, func_name, recent_prev_file_name, most_org_file_name)
    return os.path.join(head, new_file_name)

class MaskFormer:
    def __init__(self, device):
        self.device = device
        self.processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined", torch_dtype=torch.float16)
        self.model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined", torch_dtype=torch.float16).to(device)

    def inference(self, image_path, text):
        threshold = 0.5
        min_area = 0.02
        padding = 20
        original_image = Image.open(image_path)
        image = original_image.resize((512, 512))
        inputs = self.processor(text=text, images=image, padding="max_length", return_tensors="pt",).to(self.device)
        with torch.no_grad():
            outputs = self.model(**inputs)
        mask = torch.sigmoid(outputs[0]).squeeze().cpu().numpy() > threshold
        area_ratio = len(np.argwhere(mask)) / (mask.shape[0] * mask.shape[1])
        if area_ratio < min_area:
            return None
        true_indices = np.argwhere(mask)
        mask_array = np.zeros_like(mask, dtype=bool)
        for idx in true_indices:
            padded_slice = tuple(slice(max(0, i - padding), i + padding + 1) for i in idx)
            mask_array[padded_slice] = True
        visual_mask = (mask_array * 255).astype(np.uint8)
        image_mask = Image.fromarray(visual_mask)
        return image_mask.resize(image.size)

class ImageEditing:
    def __init__(self, device):
        print("Initializing StableDiffusionInpaint to %s" % device)
        self.device = device
        self.mask_former = MaskFormer(device=self.device)
        self.inpainting = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16).to(device)

    def remove_part_of_image(self, input):
        image_path, to_be_removed_txt = input.split(",")
        print(f'remove_part_of_image: to_be_removed {to_be_removed_txt}')
        return self.replace_part_of_image(f"{image_path},{to_be_removed_txt},background")

    def replace_part_of_image(self, input):
        image_path, to_be_replaced_txt, replace_with_txt = input.split(",")
        print(f'replace_part_of_image: replace_with_txt {replace_with_txt}')
        original_image = Image.open(image_path)
        mask_image = self.mask_former.inference(image_path, to_be_replaced_txt)
        updated_image = self.inpainting(prompt=replace_with_txt, image=original_image, mask_image=mask_image).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="replace-something")
        updated_image.save(updated_image_path)
        return updated_image_path

class Pix2Pix:
    def __init__(self, device):
        print("Initializing Pix2Pix to %s" % device)
        self.device = device
        self.pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained("timbrooks/instruct-pix2pix", torch_dtype=torch.float16, safety_checker=None).to(device)
        self.pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(self.pipe.scheduler.config)

    def inference(self, inputs):
        """Change style of image."""
        print("===>Starting Pix2Pix Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        original_image = Image.open(image_path)
        image = self.pipe(instruct_text,image=original_image,num_inference_steps=40,image_guidance_scale=1.2,).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="pix2pix")
        image.save(updated_image_path)
        return updated_image_path

class T2I:
    def __init__(self, device):
        print("Initializing T2I to %s" % device)
        self.device = device
        self.pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
        self.text_refine_tokenizer = AutoTokenizer.from_pretrained("Gustavosta/MagicPrompt-Stable-Diffusion")
        self.text_refine_model = AutoModelForCausalLM.from_pretrained("Gustavosta/MagicPrompt-Stable-Diffusion", torch_dtype=torch.float16)
        self.text_refine_gpt2_pipe = pipeline("text-generation", model=self.text_refine_model, tokenizer=self.text_refine_tokenizer, device=self.device, torch_dtype=torch.float16)
        self.pipe.to(device)

    def inference(self, text):
        image_filename = os.path.join('image', str(uuid.uuid4())[0:8] + ".png")
        refined_text = self.text_refine_gpt2_pipe(text)[0]["generated_text"]
        print(f'{text} refined to {refined_text}')
        image = self.pipe(refined_text).images[0]
        image.save(image_filename)
        print(f"Processed T2I.run, text: {text}, image_filename: {image_filename}")
        return image_filename

class ImageCaptioning:
    def __init__(self, device):
        print("Initializing ImageCaptioning to %s" % device)
        self.device = device
        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base", torch_dtype=torch.float16)
        self.model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base", torch_dtype=torch.float16).to(self.device)

    def inference(self, image_path):
        inputs = self.processor(Image.open(image_path), return_tensors="pt").to(self.device)
        out = self.model.generate(**inputs)
        captions = self.processor.decode(out[0], skip_special_tokens=True)
        return captions

class image2canny_new:
    def __init__(self):
        print("Direct detect canny.")
        self.low_threshold = 100
        self.high_threshold = 200

    def inference(self, inputs):
        print("===>Starting image2canny Inference")
        image = Image.open(inputs)
        image = np.array(image)
        canny = cv2.Canny(image, self.low_threshold, self.high_threshold)
        canny = canny[:, :, None]
        canny = np.concatenate([canny, canny, canny], axis=2)
        canny = 255 - canny
        canny = Image.fromarray(canny)
        updated_image_path = get_new_image_name(inputs, func_name="edge")
        canny.save(updated_image_path)
        return updated_image_path

class canny2image_new:
    def __init__(self, device):
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-canny", torch_dtype=torch.float16
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None, torch_dtype=torch.float16
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting canny2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        image = 255 - image
        prompt = instruct_text
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)
        prompt = prompt + ', ' + self.a_prompt
        image = self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt, guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="canny2image")
        image.save(updated_image_path)
        return updated_image_path


# class image2canny:
#     def __init__(self):
#         print("Direct detect canny.")
#         self.detector = CannyDetector()
#         self.low_thresh = 100
#         self.high_thresh = 200
#
#     def inference(self, inputs):
#         print("===>Starting image2canny Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         canny = self.detector(image, self.low_thresh, self.high_thresh)
#         canny = 255 - canny
#         image = Image.fromarray(canny)
#         updated_image_path = get_new_image_name(inputs, func_name="edge")
#         image.save(updated_image_path)
#         return updated_image_path
#
# class canny2image:
#     def __init__(self, device):
#         print("Initialize the canny2image model.")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_canny.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting canny2image Inference")
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         image = 255 - image
#         prompt = instruct_text
#         img = resize_image(HWC3(image), self.image_resolution)
#         H, W, C = img.shape
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)  # Magic number. IDK why. Perhaps because 0.825**12<0.01 but 0.826**12>0.01
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="canny2image")
#         real_image = Image.fromarray(x_samples[0])  # get default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path
class image2line_new:
    def __init__(self):
        self.detector = MLSDdetector.from_pretrained('lllyasviel/ControlNet')
        self.value_thresh = 0.1
        self.dis_thresh = 0.1
        self.resolution = 512

    def inference(self, inputs):
        print("===>Starting image2line Inference")
        image = Image.open(inputs)
        image = np.array(image)
        image = HWC3(image)
        mlsd = self.detector(resize_image(image, self.resolution), thr_v=self.value_thresh, thr_d=self.dis_thresh)
        mlsd = np.array(mlsd)
        mlsd = 255 - mlsd
        mlsd = Image.fromarray(mlsd)
        updated_image_path = get_new_image_name(inputs, func_name="line-of")
        mlsd.save(updated_image_path)
        return updated_image_path

class line2image_new:
    def __init__(self, device):
        print("Initialize the line2image model...")
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-mlsd"
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting line2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        image = 255 - image
        prompt = instruct_text
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)

        prompt = prompt + ', ' + self.a_prompt
        image = self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt, guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="line2image")
        image.save(updated_image_path)
        return updated_image_path


# class image2line:
#     def __init__(self):
#         print("Direct detect straight line...")
#         self.detector = MLSDdetector()
#         self.value_thresh = 0.1
#         self.dis_thresh = 0.1
#         self.resolution = 512
#
#     def inference(self, inputs):
#         print("===>Starting image2hough Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         image = HWC3(image)
#         hough = self.detector(resize_image(image, self.resolution), self.value_thresh, self.dis_thresh)
#         updated_image_path = get_new_image_name(inputs, func_name="line-of")
#         hough = 255 - cv2.dilate(hough, np.ones(shape=(3, 3), dtype=np.uint8), iterations=1)
#         image = Image.fromarray(hough)
#         image.save(updated_image_path)
#         return updated_image_path
#
#
# class line2image:
#     def __init__(self, device):
#         print("Initialize the line2image model...")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_mlsd.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting line2image Inference")
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         image = 255 - image
#         prompt = instruct_text
#         img = resize_image(HWC3(image), self.image_resolution)
#         H, W, C = img.shape
#         img = cv2.resize(img, (W, H), interpolation=cv2.INTER_NEAREST)
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)  # Magic number. IDK why. Perhaps because 0.825**12<0.01 but 0.826**12>0.01
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).\
#             cpu().numpy().clip(0,255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="line2image")
#         real_image = Image.fromarray(x_samples[0])  # default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path

class image2hed_new:
    def __init__(self):
        print("Direct detect soft HED boundary...")
        self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')
        self.resolution = 512

    def inference(self, inputs):
        print("===>Starting image2hed Inference")
        image = Image.open(inputs)
        image = np.array(image)
        image = HWC3(image)
        image = Image.fromarray(resize_image(image, self.resolution))
        hed = self.detector(image)

        updated_image_path = get_new_image_name(inputs, func_name="hed-boundary")
        hed.save(updated_image_path)
        return updated_image_path

class hed2image_new:
    def __init__(self, device):
        print("Initialize the hed2image model...")
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-hed"
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting hed2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)

        prompt = instruct_text
        prompt = prompt + ', ' + self.a_prompt
        image = \
        self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt,
                  guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="hed2image")
        image.save(updated_image_path)
        return updated_image_path

# class image2hed:
#     def __init__(self):
#         print("Direct detect soft HED boundary...")
#         self.detector = HEDdetector()
#         self.resolution = 512
#
#     def inference(self, inputs):
#         print("===>Starting image2hed Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         image = HWC3(image)
#         hed = self.detector(resize_image(image, self.resolution))
#         updated_image_path = get_new_image_name(inputs, func_name="hed-boundary")
#         image = Image.fromarray(hed)
#         image.save(updated_image_path)
#         return updated_image_path
#
#
# class hed2image:
#     def __init__(self, device):
#         print("Initialize the hed2image model...")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_hed.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting hed2image Inference")
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         prompt = instruct_text
#         img = resize_image(HWC3(image), self.image_resolution)
#         H, W, C = img.shape
#         img = cv2.resize(img, (W, H), interpolation=cv2.INTER_NEAREST)
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="hed2image")
#         real_image = Image.fromarray(x_samples[0])  # default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path
class image2scribble_new:
    def __init__(self):
        print("Direct detect scribble.")
        self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')
        self.resolution = 512

    def inference(self, inputs):
        print("===>Starting image2scribble Inference")
        image = Image.open(inputs)
        image = np.array(image)
        image = HWC3(image)
        image = resize_image(image, self.resolution)
        image = Image.fromarray(image)
        scribble = self.detector(image, scribble=True)
        scribble = np.array(scribble)
        scribble = 255 - scribble
        scribble = Image.fromarray(scribble)
        updated_image_path = get_new_image_name(inputs, func_name="scribble")
        scribble.save(updated_image_path)
        return updated_image_path

class scribble2image_new:
    def __init__(self, device):
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-scribble"
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting scribble2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        image = 255 - image
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)

        prompt = instruct_text
        prompt = prompt + ', ' + self.a_prompt
        image = \
            self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt,
                      guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="scribble2image")
        image.save(updated_image_path)
        return updated_image_path

# class image2scribble:
#     def __init__(self):
#         print("Direct detect scribble.")
#         self.detector = HEDdetector()
#         self.resolution = 512
#
#     def inference(self, inputs):
#         print("===>Starting image2scribble Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         image = HWC3(image)
#         detected_map = self.detector(resize_image(image, self.resolution))
#         detected_map = HWC3(detected_map)
#         image = resize_image(image, self.resolution)
#         H, W, C = image.shape
#         detected_map = cv2.resize(detected_map, (W, H), interpolation=cv2.INTER_LINEAR)
#         detected_map = nms(detected_map, 127, 3.0)
#         detected_map = cv2.GaussianBlur(detected_map, (0, 0), 3.0)
#         detected_map[detected_map > 4] = 255
#         detected_map[detected_map < 255] = 0
#         detected_map = 255 - detected_map
#         updated_image_path = get_new_image_name(inputs, func_name="scribble")
#         image = Image.fromarray(detected_map)
#         image.save(updated_image_path)
#         return updated_image_path
#
# class scribble2image:
#     def __init__(self, device):
#         print("Initialize the scribble2image model...")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_scribble.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting scribble2image Inference")
#         print(f'sketch device {self.device}')
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         prompt = instruct_text
#         image = 255 - image
#         img = resize_image(HWC3(image), self.image_resolution)
#         H, W, C = img.shape
#         img = cv2.resize(img, (W, H), interpolation=cv2.INTER_NEAREST)
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="scribble2image")
#         real_image = Image.fromarray(x_samples[0])  # default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path

class image2pose_new:
    def __init__(self):
        self.detector = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')
        self.resolution = 512

    def inference(self, inputs):
        print("===>Starting image2pose Inference")
        image = Image.open(inputs)
        image = np.array(image)
        image = HWC3(image)
        image = resize_image(image, self.resolution)
        image = Image.fromarray(image)
        pose = self.detector(image)

        updated_image_path = get_new_image_name(inputs, func_name="human-pose")
        pose.save(updated_image_path)
        return updated_image_path

class pose2image_new:
    def __init__(self, device):
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-openpose"
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting pose2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)

        prompt = instruct_text
        prompt = prompt + ', ' + self.a_prompt
        image = \
            self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt,
                      guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="pose2image")
        image.save(updated_image_path)
        return updated_image_path


# class image2pose:
#     def __init__(self):
#         print("Direct human pose.")
#         self.detector = OpenposeDetector()
#         self.resolution = 512
#
#     def inference(self, inputs):
#         print("===>Starting image2pose Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         image = HWC3(image)
#         detected_map, _ = self.detector(resize_image(image, self.resolution))
#         detected_map = HWC3(detected_map)
#         image = resize_image(image, self.resolution)
#         H, W, C = image.shape
#         detected_map = cv2.resize(detected_map, (W, H), interpolation=cv2.INTER_LINEAR)
#         updated_image_path = get_new_image_name(inputs, func_name="human-pose")
#         image = Image.fromarray(detected_map)
#         image.save(updated_image_path)
#         return updated_image_path
#
# class pose2image:
#     def __init__(self, device):
#         print("Initialize the pose2image model...")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_openpose.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting pose2image Inference")
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         prompt = instruct_text
#         img = resize_image(HWC3(image), self.image_resolution)
#         H, W, C = img.shape
#         img = cv2.resize(img, (W, H), interpolation=cv2.INTER_NEAREST)
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [ self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="pose2image")
#         real_image = Image.fromarray(x_samples[0])  # default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path
class image2seg_new:
    def __init__(self):
        print("Initialize image2segmentation Inference")
        self.image_processor = AutoImageProcessor.from_pretrained("openmmlab/upernet-convnext-small")
        self.image_segmentor = UperNetForSemanticSegmentation.from_pretrained("openmmlab/upernet-convnext-small")
        self.resolution = 512

    def inference(self, inputs):
        image = Image.open(inputs)
        image = np.array(image)
        image = HWC3(image)
        image = resize_image(image, self.resolution)
        image = Image.fromarray(image)
        pixel_values = self.image_processor(image, return_tensors="pt").pixel_values

        with torch.no_grad():
            outputs = self.image_segmentor(pixel_values)

        seg = self.image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0]

        color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)  # height, width, 3

        palette = np.array(ade_palette())

        for label, color in enumerate(palette):
            color_seg[seg == label, :] = color

        color_seg = color_seg.astype(np.uint8)

        segmentation = Image.fromarray(color_seg)
        updated_image_path = get_new_image_name(inputs, func_name="segmentation")
        segmentation.save(updated_image_path)
        return updated_image_path

class seg2image_new:
    def __init__(self, device):
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-seg"
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting seg2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)

        prompt = instruct_text
        prompt = prompt + ', ' + self.a_prompt
        image = \
            self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt,
                      guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="segment2image")
        image.save(updated_image_path)
        return updated_image_path



# class image2seg:
#     def __init__(self):
#         print("===>Starting image2seg Inference")
#         print("Direct segmentations.")
#         self.detector = UniformerDetector()
#         self.resolution = 512
#
#     def inference(self, inputs):
#         print("===>Starting image2seg Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         image = HWC3(image)
#         detected_map = self.detector(resize_image(image, self.resolution))
#         detected_map = HWC3(detected_map)
#         image = resize_image(image, self.resolution)
#         H, W, C = image.shape
#         detected_map = cv2.resize(detected_map, (W, H), interpolation=cv2.INTER_LINEAR)
#         updated_image_path = get_new_image_name(inputs, func_name="segmentation")
#         image = Image.fromarray(detected_map)
#         image.save(updated_image_path)
#         return updated_image_path
#
# class seg2image:
#     def __init__(self, device):
#         print("Initialize the seg2image model...")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_seg.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting seg2image Inference")
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         prompt = instruct_text
#         img = resize_image(HWC3(image), self.image_resolution)
#         H, W, C = img.shape
#         img = cv2.resize(img, (W, H), interpolation=cv2.INTER_NEAREST)
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="segment2image")
#         real_image = Image.fromarray(x_samples[0])  # default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path
class image2depth_new:
    def __init__(self):
        print("initialize depth estimation")
        self.depth_estimator = pipeline('depth-estimation')
        self.resolution = 512

    def inference(self, inputs):
        image = Image.open(inputs)
        image = np.array(image)
        image = HWC3(image)
        image = resize_image(image, self.resolution)
        image = Image.fromarray(image)
        depth = self.depth_estimator(image)['depth']
        depth = np.array(depth)
        depth = depth[:, :, None]
        depth = np.concatenate([depth, depth, depth], axis=2)
        depth = Image.fromarray(depth)
        updated_image_path = get_new_image_name(inputs, func_name="depth")
        depth.save(updated_image_path)
        return updated_image_path

class depth2image_new:
    def __init__(self, device):
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-depth"
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting depth2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)

        prompt = instruct_text
        prompt = prompt + ', ' + self.a_prompt
        image = \
            self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt,
                      guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="depth2image")
        image.save(updated_image_path)
        return updated_image_path

# class image2depth:
#     def __init__(self):
#         print("Direct depth estimation.")
#         self.detector = MidasDetector()
#         self.resolution = 512
#
#     def inference(self, inputs):
#         print("===>Starting image2depth Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         image = HWC3(image)
#         detected_map, _ = self.detector(resize_image(image, self.resolution))
#         detected_map = HWC3(detected_map)
#         image = resize_image(image, self.resolution)
#         H, W, C = image.shape
#         detected_map = cv2.resize(detected_map, (W, H), interpolation=cv2.INTER_LINEAR)
#         updated_image_path = get_new_image_name(inputs, func_name="depth")
#         image = Image.fromarray(detected_map)
#         image.save(updated_image_path)
#         return updated_image_path
#
# class depth2image:
#     def __init__(self, device):
#         print("Initialize depth2image model...")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_depth.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting depth2image Inference")
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         prompt = instruct_text
#         img = resize_image(HWC3(image), self.image_resolution)
#         H, W, C = img.shape
#         img = cv2.resize(img, (W, H), interpolation=cv2.INTER_NEAREST)
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [ self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)  # Magic number. IDK why. Perhaps because 0.825**12<0.01 but 0.826**12>0.01
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="depth2image")
#         real_image = Image.fromarray(x_samples[0])  # default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path

class image2normal_new:
    def __init__(self):
        print("normal estimation")
        self.depth_estimator = pipeline("depth-estimation", model="Intel/dpt-hybrid-midas")
        self.resolution = 512
        self.bg_threhold = 0.4

    def inference(self, inputs):
        image = Image.open(inputs)
        image = np.array(image)
        image = HWC3(image)
        image = resize_image(image, self.resolution)
        image = Image.fromarray(image)
        image = self.depth_estimator(image)['predicted_depth'][0]

        image = image.numpy()

        image_depth = image.copy()
        image_depth -= np.min(image_depth)
        image_depth /= np.max(image_depth)

        bg_threhold = 0.4

        x = cv2.Sobel(image, cv2.CV_32F, 1, 0, ksize=3)
        x[image_depth < bg_threhold] = 0

        y = cv2.Sobel(image, cv2.CV_32F, 0, 1, ksize=3)
        y[image_depth < bg_threhold] = 0

        z = np.ones_like(x) * np.pi * 2.0

        image = np.stack([x, y, z], axis=2)
        image /= np.sum(image ** 2.0, axis=2, keepdims=True) ** 0.5
        image = (image * 127.5 + 127.5).clip(0, 255).astype(np.uint8)
        image = Image.fromarray(image)
        updated_image_path = get_new_image_name(inputs, func_name="normal-map")
        image.save(updated_image_path)
        return updated_image_path

class normal2image_new:
    def __init__(self, device):
        self.controlnet = ControlNetModel.from_pretrained(
            "fusing/stable-diffusion-v1-5-controlnet-normal"
        )

        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None
        )

        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.to(device)
        self.image_resolution = 512
        self.num_inference_steps = 20
        self.seed = -1
        self.unconditional_guidance_scale = 9.0
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    def inference(self, inputs):
        print("===>Starting normal2image Inference")
        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        image = Image.open(image_path)
        image = np.array(image)
        img = resize_image(HWC3(image), self.image_resolution)
        img = Image.fromarray(img)

        self.seed = random.randint(0, 65535)
        seed_everything(self.seed)

        prompt = instruct_text
        prompt = prompt + ', ' + self.a_prompt
        image = \
            self.pipe(prompt, img, num_inference_steps=self.num_inference_steps, eta=0.0, negative_prompt=self.n_prompt,
                      guidance_scale=self.unconditional_guidance_scale).images[0]
        updated_image_path = get_new_image_name(image_path, func_name="normal2image")
        image.save(updated_image_path)
        return updated_image_path

# class image2normal:
#     def __init__(self):
#         print("Direct normal estimation.")
#         self.detector = MidasDetector()
#         self.resolution = 512
#         self.bg_threshold = 0.4
#
#     def inference(self, inputs):
#         print("===>Starting image2 normal Inference")
#         image = Image.open(inputs)
#         image = np.array(image)
#         image = HWC3(image)
#         _, detected_map = self.detector(resize_image(image, self.resolution), bg_th=self.bg_threshold)
#         detected_map = HWC3(detected_map)
#         image = resize_image(image, self.resolution)
#         H, W, C = image.shape
#         detected_map = cv2.resize(detected_map, (W, H), interpolation=cv2.INTER_LINEAR)
#         updated_image_path = get_new_image_name(inputs, func_name="normal-map")
#         image = Image.fromarray(detected_map)
#         image.save(updated_image_path)
#         return updated_image_path
#
# class normal2image:
#     def __init__(self, device):
#         print("Initialize normal2image model...")
#         model = create_model('ControlNet/models/cldm_v15.yaml', device=device).to(device)
#         model.load_state_dict(load_state_dict('ControlNet/models/control_sd15_normal.pth', location='cpu'))
#         self.model = model.to(device)
#         self.device = device
#         self.ddim_sampler = DDIMSampler(self.model)
#         self.ddim_steps = 20
#         self.image_resolution = 512
#         self.num_samples = 1
#         self.save_memory = False
#         self.strength = 1.0
#         self.guess_mode = False
#         self.scale = 9.0
#         self.seed = -1
#         self.a_prompt = 'best quality, extremely detailed'
#         self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
#
#     def inference(self, inputs):
#         print("===>Starting normal2image Inference")
#         image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
#         image = Image.open(image_path)
#         image = np.array(image)
#         prompt = instruct_text
#         img = image[:, :, ::-1].copy()
#         img = resize_image(HWC3(img), self.image_resolution)
#         H, W, C = img.shape
#         img = cv2.resize(img, (W, H), interpolation=cv2.INTER_NEAREST)
#         control = torch.from_numpy(img.copy()).float().to(device=self.device) / 255.0
#         control = torch.stack([control for _ in range(self.num_samples)], dim=0)
#         control = einops.rearrange(control, 'b h w c -> b c h w').clone()
#         self.seed = random.randint(0, 65535)
#         seed_everything(self.seed)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         cond = {"c_concat": [control], "c_crossattn": [self.model.get_learned_conditioning([prompt + ', ' + self.a_prompt] * self.num_samples)]}
#         un_cond = {"c_concat": None if self.guess_mode else [control], "c_crossattn": [self.model.get_learned_conditioning([self.n_prompt] * self.num_samples)]}
#         shape = (4, H // 8, W // 8)
#         self.model.control_scales = [self.strength * (0.825 ** float(12 - i)) for i in range(13)] if self.guess_mode else ([self.strength] * 13)
#         samples, intermediates = self.ddim_sampler.sample(self.ddim_steps, self.num_samples, shape, cond, verbose=False, eta=0., unconditional_guidance_scale=self.scale, unconditional_conditioning=un_cond)
#         if self.save_memory:
#             self.model.low_vram_shift(is_diffusing=False)
#         x_samples = self.model.decode_first_stage(samples)
#         x_samples = (einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8)
#         updated_image_path = get_new_image_name(image_path, func_name="normal2image")
#         real_image = Image.fromarray(x_samples[0])  # default the index0 image
#         real_image.save(updated_image_path)
#         return updated_image_path

class BLIPVQA:
    def __init__(self, device):
        print("Initializing BLIP VQA to %s" % device)
        self.device = device
        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
        self.model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(self.device)

    def get_answer_from_question_and_image(self, inputs):
        image_path, question = inputs.split(",")
        raw_image = Image.open(image_path).convert('RGB')
        print(F'BLIPVQA :question :{question}')
        inputs = self.processor(raw_image, question, return_tensors="pt").to(self.device)
        out = self.model.generate(**inputs)
        answer = self.processor.decode(out[0], skip_special_tokens=True)
        return answer