make the app runable

.gitignore

@@ -160,3 +160,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+my_*

.vscode/settings.json

@@ -3,7 +3,7 @@
         "editor.defaultFormatter": "ms-python.black-formatter",
         "editor.formatOnType": true,
         "editor.codeActionsOnSave": {
-            "source.organizeImports": true
+            "source.organizeImports": "explicit"
         }
     },
     "black-formatter.args": [

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 hysts
+Copyright (c) 2024 wuhecong
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

LICENSE.hysts

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 hysts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 from __future__ import annotations
 
 import spaces
@@ -15,6 +14,7 @@ from app_ip2p import create_demo as create_demo_ip2p
 from app_lineart import create_demo as create_demo_lineart
 from app_mlsd import create_demo as create_demo_mlsd
 from app_normal import create_demo as create_demo_normal
+from app_normal_old import create_demo as create_demo_normal_old
 from app_openpose import create_demo as create_demo_openpose
 from app_scribble import create_demo as create_demo_scribble
 from app_scribble_interactive import create_demo as create_demo_scribble_interactive
@@ -26,12 +26,17 @@ from model import Model
 from settings import ALLOW_CHANGING_BASE_MODEL, DEFAULT_MODEL_ID, SHOW_DUPLICATE_BUTTON
 
 DESCRIPTION = r"""
-# ControlLoRA Version 3: LoRA Is All You Need to Control the Spatial Information of Stable Diffusion
-
 <center>
+<h1>ControlLoRA Version 3: LoRA Is All You Need to Control the Spatial Information of Stable Diffusion</h1>
+
 <a href="https://huggingface.co/HighCWu/control-lora-v3">[Models]</a>
 <a href="https://github.com/HighCWu/control-lora-v3">[Github]</a>
 </center>
+
+***Note:*** I used a high learning rate and a short number of steps for training, and the dataset was also generated, 
+so the generation results may not be very good. 
+It is recommended that researchers use real data, lower learning and longer training steps to train to achieve better generation results.
+
 """
 
 model = Model(base_model_id=DEFAULT_MODEL_ID, task_name="Canny")
@@ -61,8 +66,10 @@ with gr.Blocks(css="style.css") as demo:
             create_demo_segmentation(spaces.GPU(model.process_segmentation))
         with gr.TabItem("Depth"):
             create_demo_depth(spaces.GPU(model.process_depth))
-        with gr.TabItem("Normal map"):
-            create_demo_normal(spaces.GPU(model.process_normal))
+        # with gr.TabItem("Normal map"):
+        #     create_demo_normal(spaces.GPU(model.process_normal))
+        with gr.TabItem("Normal map (old)"):
+            create_demo_normal_old(spaces.GPU(model.process_normal_old))
         # with gr.TabItem("Lineart"):
         #     create_demo_lineart(spaces.GPU(model.process_lineart))
         # with gr.TabItem("Content Shuffle"):

app_canny.py

@@ -37,7 +37,7 @@ def create_demo(process):
                         label="Canny high threshold", minimum=1, maximum=255, value=200, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_depth.py

@@ -37,7 +37,7 @@ def create_demo(process):
                         label="Preprocess resolution", minimum=128, maximum=512, value=384, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_ip2p.py

@@ -31,7 +31,7 @@ def create_demo(process):
                         step=256,
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_lineart.py

@@ -47,7 +47,7 @@ def create_demo(process):
                         label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_mlsd.py

@@ -40,7 +40,7 @@ def create_demo(process):
                         label="Hough distance threshold (MLSD)", minimum=0.01, maximum=20.0, value=0.1, step=0.01
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_normal.py

@@ -37,7 +37,7 @@ def create_demo(process):
                         label="Preprocess resolution", minimum=128, maximum=512, value=384, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_normal_old.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+import gradio as gr
+
+from settings import (
+    DEFAULT_IMAGE_RESOLUTION,
+    DEFAULT_NUM_IMAGES,
+    MAX_IMAGE_RESOLUTION,
+    MAX_NUM_IMAGES,
+    MAX_SEED,
+)
+from utils import randomize_seed_fn
+
+
+def create_demo(process):
+    with gr.Blocks() as demo:
+        with gr.Row():
+            with gr.Column():
+                image = gr.Image()
+                prompt = gr.Textbox(label="Prompt")
+                run_button = gr.Button("Run")
+                with gr.Accordion("Advanced options", open=False):
+                    preprocessor_name = gr.Radio(
+                        label="Preprocessor", choices=["Normal", "None"], type="value", value="Normal"
+                    )
+                    num_samples = gr.Slider(
+                        label="Images", minimum=1, maximum=MAX_NUM_IMAGES, value=DEFAULT_NUM_IMAGES, step=1
+                    )
+                    image_resolution = gr.Slider(
+                        label="Image resolution",
+                        minimum=256,
+                        maximum=MAX_IMAGE_RESOLUTION,
+                        value=DEFAULT_IMAGE_RESOLUTION,
+                        step=256,
+                    )
+                    preprocess_resolution = gr.Slider(
+                        label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
+                    )
+                    num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
+                    seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
+                    randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+                    a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
+                    n_prompt = gr.Textbox(
+                        label="Negative prompt",
+                        value="longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality",
+                    )
+            with gr.Column():
+                result = gr.Gallery(label="Output", show_label=False, columns=2, object_fit="scale-down")
+        inputs = [
+            image,
+            prompt,
+            a_prompt,
+            n_prompt,
+            num_samples,
+            image_resolution,
+            preprocess_resolution,
+            num_steps,
+            guidance_scale,
+            seed,
+            preprocessor_name,
+        ]
+        prompt.submit(
+            fn=randomize_seed_fn,
+            inputs=[seed, randomize_seed],
+            outputs=seed,
+            queue=False,
+            api_name=False,
+        ).then(
+            fn=process,
+            inputs=inputs,
+            outputs=result,
+            api_name=False,
+        )
+        run_button.click(
+            fn=randomize_seed_fn,
+            inputs=[seed, randomize_seed],
+            outputs=seed,
+            queue=False,
+            api_name=False,
+        ).then(
+            fn=process,
+            inputs=inputs,
+            outputs=result,
+            api_name="normal",
+        )
+    return demo
+
+
+if __name__ == "__main__":
+    from model import Model
+
+    model = Model(task_name="NormalBae")
+    demo = create_demo(model.process_normal)
+    demo.queue().launch()

app_openpose.py

@@ -37,7 +37,7 @@ def create_demo(process):
                         label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_scribble.py

@@ -37,7 +37,7 @@ def create_demo(process):
                         label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_scribble_interactive.py

@@ -51,7 +51,7 @@ def create_demo(process):
                         step=256,
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_segmentation.py

@@ -21,7 +21,7 @@ def create_demo(process):
                 run_button = gr.Button("Run")
                 with gr.Accordion("Advanced options", open=False):
                     preprocessor_name = gr.Radio(
-                        label="Preprocessor", choices=["UPerNet", "None"], type="value", value="UPerNet"
+                        label="Preprocessor", choices=["OneFormer", "UPerNet", "None"], type="value", value="OneFormer"
                     )
                     num_samples = gr.Slider(
                         label="Number of images", minimum=1, maximum=MAX_NUM_IMAGES, value=DEFAULT_NUM_IMAGES, step=1
@@ -37,7 +37,7 @@ def create_demo(process):
                         label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_shuffle.py

@@ -34,7 +34,7 @@ def create_demo(process):
                         step=256,
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_softedge.py

@@ -46,7 +46,7 @@ def create_demo(process):
                         label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

app_tile.py

@@ -31,7 +31,7 @@ def create_demo(process):
                         step=256,
                     )
                     num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
-                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
+                    guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
                     seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
                     randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                     a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")

depth_estimator.py

@@ -1,3 +1,4 @@
+import torch
 import numpy as np
 import PIL.Image
 from controlnet_aux.util import HWC3
@@ -8,7 +9,8 @@ from cv_utils import resize_image
 
 class DepthEstimator:
     def __init__(self):
-        self.model = pipeline("depth-estimation")
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+        self.model = pipeline("depth-estimation", device=self.device)
 
     def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
         detect_resolution = kwargs.pop("detect_resolution", 512)

image_segmentor.py

@@ -3,15 +3,17 @@ import numpy as np
 import PIL.Image
 import torch
 from controlnet_aux.util import HWC3, ade_palette
-from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
+from transformers import AutoImageProcessor, UperNetForSemanticSegmentation, OneFormerProcessor, OneFormerForUniversalSegmentation
 
 from cv_utils import resize_image
 
 
 class ImageSegmentor:
     def __init__(self):
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
         self.image_processor = AutoImageProcessor.from_pretrained("openmmlab/upernet-convnext-small")
         self.image_segmentor = UperNetForSemanticSegmentation.from_pretrained("openmmlab/upernet-convnext-small")
+        self.image_segmentor.to(self.device)
 
     @torch.inference_mode()
     def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
@@ -22,8 +24,36 @@ class ImageSegmentor:
         image = PIL.Image.fromarray(image)
 
         pixel_values = self.image_processor(image, return_tensors="pt").pixel_values
-        outputs = self.image_segmentor(pixel_values)
-        seg = self.image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0]
+        outputs = self.image_segmentor(pixel_values.to(self.device))
+        seg = self.image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0].cpu()
+        color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
+        for label, color in enumerate(ade_palette()):
+            color_seg[seg == label, :] = color
+        color_seg = color_seg.astype(np.uint8)
+
+        color_seg = resize_image(color_seg, resolution=image_resolution, interpolation=cv2.INTER_NEAREST)
+        return PIL.Image.fromarray(color_seg)
+
+
+class ImageSegmentorOneFormer:
+    def __init__(self):
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+        self.image_processor = OneFormerProcessor.from_pretrained("shi-labs/oneformer_ade20k_swin_tiny")
+        self.image_segmentor = OneFormerForUniversalSegmentation.from_pretrained("shi-labs/oneformer_ade20k_swin_tiny")
+        self.image_segmentor.to(self.device)
+
+    @torch.inference_mode()
+    def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
+        detect_resolution = kwargs.pop("detect_resolution", 512)
+        image_resolution = kwargs.pop("image_resolution", 512)
+        image = HWC3(image)
+        image = resize_image(image, resolution=detect_resolution)
+        image = PIL.Image.fromarray(image)
+
+        inputs = self.image_processor(image, ["semantic"], return_tensors="pt")
+        inputs = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()}
+        outputs = self.image_segmentor(**inputs)
+        seg = self.image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0].cpu()
         color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
         for label, color in enumerate(ade_palette()):
             color_seg[seg == label, :] = color

model.py

@@ -22,9 +22,8 @@ CONTROL_LORA_V3_MODEL_IDS = OrderedDict([
     ("Openpose", "sd-control-lora-v3-pose-half-rank128-conv_in-rank128"),
     ("Canny", "sd-control-lora-v3-canny-half_skip_attn-rank16-conv_in-rank64"),
     ("segmentation", "sd-control-lora-v3-segmentation-half_skip_attn-rank128-conv_in-rank128"),
-    ("depth", "lllyasviel/control_v11f1p_sd15_depth"),
-    ("NormalBae", "sd-control-lora-v3-normal-half-rank32-conv_in-rank128"),
     ("depth", "sd-control-lora-v3-depth-half-rank8-conv_in-rank128"),
+    ("Normal", "sd-control-lora-v3-normal-half-rank32-conv_in-rank128"),
     ("Tile", "sd-control-lora-v3-tile-half_skip_attn-rank16-conv_in-rank64"),
 ])
 
@@ -37,6 +36,14 @@ class Model:
         self.pipe: StableDiffusionControlLoraV3Pipeline = self.load_pipe(base_model_id, task_name)
         self.preprocessor = Preprocessor()
 
+        # preload
+        preprocessor = self.preprocessor
+        preprocessor.load("Openpose")
+        preprocessor.load("Canny")
+        preprocessor.load("OneFormer"); preprocessor.load("UPerNet") # segmentation
+        preprocessor.load("DPT") # depth
+        preprocessor.load("Midas") # normal (old)
+
     def load_pipe(self, base_model_id: str, task_name) -> StableDiffusionControlLoraV3Pipeline:
         if (
             base_model_id == self.base_model_id
@@ -55,6 +62,7 @@ class Model:
         )
         for _task_name, subfolder in CONTROL_LORA_V3_MODEL_IDS.items():
             pipe.load_lora_weights("HighCWu/control-lora-v3", adapter_name=_task_name, subfolder=subfolder)
+        pipe.unet.set_adapter(task_name)
         pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
         if self.device.type == "cuda":
             pipe.enable_xformers_memory_efficient_attention()
@@ -484,6 +492,52 @@ class Model:
         )
         return [control_image] + results
 
+    @torch.inference_mode()
+    def process_normal_old(
+        self,
+        image: np.ndarray,
+        prompt: str,
+        additional_prompt: str,
+        negative_prompt: str,
+        num_images: int,
+        image_resolution: int,
+        preprocess_resolution: int,
+        num_steps: int,
+        guidance_scale: float,
+        seed: int,
+        preprocessor_name: str,
+    ) -> list[PIL.Image.Image]:
+        if image is None:
+            raise ValueError
+        if image_resolution > MAX_IMAGE_RESOLUTION:
+            raise ValueError
+        if num_images > MAX_NUM_IMAGES:
+            raise ValueError
+
+        if preprocessor_name == "None":
+            image = HWC3(image)
+            image = resize_image(image, resolution=image_resolution)
+            control_image = PIL.Image.fromarray(image)
+        else:
+            self.preprocessor.load("Midas")
+            control_image = self.preprocessor(
+                image=image,
+                image_resolution=image_resolution,
+                detect_resolution=preprocess_resolution,
+                depth_and_normal=True
+            )
+        self.load_controlnet_weight("Normal")
+        results = self.run_pipe(
+            prompt=self.get_prompt(prompt, additional_prompt),
+            negative_prompt=negative_prompt,
+            control_image=control_image,
+            num_images=num_images,
+            num_steps=num_steps,
+            guidance_scale=guidance_scale,
+            seed=seed,
+        )
+        return [control_image] + results
+    
     @torch.inference_mode()
     def process_normal(
         self,

pipeline.py

@@ -896,7 +896,10 @@ class StableDiffusionControlLoraV3Pipeline(
         kwargs["weight_name"] = kwargs.pop("weight_name", "pytorch_lora_weights.safetensors")
 
         if adapter_name is not None and adapter_name not in unet.extra_condition_names:
-            return super().load_lora_weights(pretrained_model_name_or_path_or_dict, adapter_name, **kwargs)
+            unet._hf_peft_config_loaded = True
+            super().load_lora_weights(pretrained_model_name_or_path_or_dict, adapter_name, **kwargs)
+            unet.set_adapter(adapter_name)
+            return
 
         if not isinstance(pretrained_model_name_or_path_or_dict, list):
             pretrained_model_name_or_path_or_dict = [pretrained_model_name_or_path_or_dict] * num_condition_names

preprocessor.py

@@ -19,7 +19,7 @@ from controlnet_aux.util import HWC3
 
 from cv_utils import resize_image
 from depth_estimator import DepthEstimator
-from image_segmentor import ImageSegmentor
+from image_segmentor import ImageSegmentor, ImageSegmentorOneFormer
 
 
 class Preprocessor:
@@ -27,11 +27,16 @@ class Preprocessor:
 
     def __init__(self):
         self.model = None
+        self.models = {}
         self.name = ""
 
     def load(self, name: str) -> None:
         if name == self.name:
             return
+        if name in self.models:
+            self.name = name
+            self.model = self.models[name]
+            return
         if name == "HED":
             self.model = HEDdetector.from_pretrained(self.MODEL_ID)
         elif name == "Midas":
@@ -56,12 +61,15 @@ class Preprocessor:
             self.model = DepthEstimator()
         elif name == "UPerNet":
             self.model = ImageSegmentor()
+        elif name == "OneFormer":
+            self.model = ImageSegmentorOneFormer()
         else:
             raise ValueError
-        if torch.cuda.is_available():
-            torch.cuda.empty_cache()
-        gc.collect()
+        # if torch.cuda.is_available():
+        #     torch.cuda.empty_cache()
+        # gc.collect()
         self.name = name
+        self.models[name] = self.model
 
     def __call__(self, image: PIL.Image.Image, **kwargs) -> PIL.Image.Image:
         if self.name == "Canny":
@@ -79,6 +87,8 @@ class Preprocessor:
             image = HWC3(image)
             image = resize_image(image, resolution=detect_resolution)
             image = self.model(image, **kwargs)
+            if isinstance(image, tuple):
+                image = image[-1][...,::-1] # normal old
             image = HWC3(image)
             image = resize_image(image, resolution=image_resolution)
             return PIL.Image.fromarray(image)

settings.py

@@ -7,7 +7,7 @@ DEFAULT_MODEL_ID = os.getenv("DEFAULT_MODEL_ID", "SG161222/Realistic_Vision_V4.0
 MAX_NUM_IMAGES = int(os.getenv("MAX_NUM_IMAGES", "3"))
 DEFAULT_NUM_IMAGES = min(MAX_NUM_IMAGES, int(os.getenv("DEFAULT_NUM_IMAGES", "3")))
 MAX_IMAGE_RESOLUTION = int(os.getenv("MAX_IMAGE_RESOLUTION", "768"))
-DEFAULT_IMAGE_RESOLUTION = min(MAX_IMAGE_RESOLUTION, int(os.getenv("DEFAULT_IMAGE_RESOLUTION", "768")))
+DEFAULT_IMAGE_RESOLUTION = min(MAX_IMAGE_RESOLUTION, int(os.getenv("DEFAULT_IMAGE_RESOLUTION", "512")))
 
 ALLOW_CHANGING_BASE_MODEL = os.getenv("SPACE_ID") != "HighCWu/control-lora-v3"
 SHOW_DUPLICATE_BUTTON = os.getenv("SHOW_DUPLICATE_BUTTON") == "1"

unet.py

@@ -147,6 +147,8 @@ class UNet2DConditionModelEx(UNet2DConditionModel):
     
     def activate_extra_condition_adapters(self):
         lora_layers = [layer for layer in self.modules() if isinstance(layer, LoraLayer)]
+        if len(lora_layers) > 0:
+            self._hf_peft_config_loaded = True
         for lora_layer in lora_layers:
             adapter_names = [k for k in lora_layer.scaling.keys() if k in self.config.extra_condition_names] 
             adapter_names += lora_layer.active_adapters