Add files

.pre-commit-config.yaml

@@ -0,0 +1,50 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-executables-have-shebangs
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-shebang-scripts-are-executable
+  - id: check-toml
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+    args: ["--fix=lf"]
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/myint/docformatter
+  rev: v1.7.5
+  hooks:
+  - id: docformatter
+    args: ["--in-place"]
+- repo: https://github.com/pycqa/isort
+  rev: 5.12.0
+  hooks:
+    - id: isort
+      args: ["--profile", "black"]
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.5.1
+  hooks:
+    - id: mypy
+      args: ["--ignore-missing-imports"]
+      additional_dependencies: ["types-python-slugify", "types-requests", "types-PyYAML"]
+- repo: https://github.com/psf/black
+  rev: 23.7.0
+  hooks:
+    - id: black
+      language_version: python3.10
+      args: ["--line-length", "119"]
+- repo: https://github.com/kynan/nbstripout
+  rev: 0.6.1
+  hooks:
+    - id: nbstripout
+      args: ["--extra-keys", "metadata.interpreter metadata.kernelspec cell.metadata.pycharm"]
+- repo: https://github.com/nbQA-dev/nbQA
+  rev: 1.7.0
+  hooks:
+    - id: nbqa-black
+    - id: nbqa-pyupgrade
+      args: ["--py37-plus"]
+    - id: nbqa-isort
+      args: ["--float-to-top"]

.vscode/settings.json

@@ -0,0 +1,21 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnType": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": true
+        }
+    },
+    "black-formatter.args": [
+        "--line-length=119"
+    ],
+    "isort.args": ["--profile", "black"],
+    "flake8.args": [
+        "--max-line-length=119"
+    ],
+    "ruff.args": [
+        "--line-length=119"
+    ],
+    "editor.formatOnSave": true,
+    "files.insertFinalNewline": true
+}

Dockerfile

@@ -0,0 +1,57 @@
+FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    git \
+    git-lfs \
+    wget \
+    curl \
+    # python build dependencies \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    # gradio dependencies \
+    ffmpeg && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:${PATH}
+WORKDIR ${HOME}/app
+
+RUN curl https://pyenv.run | bash
+ENV PATH=${HOME}/.pyenv/shims:${HOME}/.pyenv/bin:${PATH}
+ARG PYTHON_VERSION=3.10.12
+RUN pyenv install ${PYTHON_VERSION} && \
+    pyenv global ${PYTHON_VERSION} && \
+    pyenv rehash && \
+    pip install --no-cache-dir -U pip setuptools wheel
+
+COPY --chown=1000 ./requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt && \
+    mim install mmcv==2.0.1 && \
+    mim install mmdet==3.1.0 && \
+    mim install mmpose==1.1.0
+
+COPY --chown=1000 . ${HOME}/app
+ENV PYTHONPATH=${HOME}/app \
+    PYTHONUNBUFFERED=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_THEME=huggingface \
+    SYSTEM=spaces
+CMD ["python", "app.py"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 hysts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,12 @@
 ---
-title: T2I Adapter SDXL
-emoji: 👀
+title: T2I-Adapter-SDXL
+emoji: 🚀
 colorFrom: purple
 colorTo: yellow
-sdk: gradio
-sdk_version: 3.42.0
-app_file: app.py
+sdk: docker
 pinned: false
+license: mit
+suggested_hardware: t4-small
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+
+import os
+import random
+
+import gradio as gr
+import numpy as np
+import torch
+
+from model import ADAPTER_NAMES, Model
+
+DESCRIPTION = "# T2I-Adapter-SDXL"
+
+if not torch.cuda.is_available():
+    DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
+
+MAX_SEED = np.iinfo(np.int32).max
+
+
+def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
+    if randomize_seed:
+        seed = random.randint(0, MAX_SEED)
+    return seed
+
+
+model = Model(ADAPTER_NAMES[0])
+
+with gr.Blocks(css="style.css") as demo:
+    gr.Markdown(DESCRIPTION)
+    gr.DuplicateButton(
+        value="Duplicate Space for private use",
+        elem_id="duplicate-button",
+        visible=os.getenv("SHOW_DUPLICATE_BUTTON") == "1",
+    )
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Group():
+                image = gr.Image(label="Input image", type="pil", height=600)
+                prompt = gr.Textbox(label="Prompt")
+                adapter_name = gr.Dropdown(label="Adapter", choices=ADAPTER_NAMES, value=ADAPTER_NAMES[0])
+                run_button = gr.Button("Run")
+            with gr.Accordion("Advanced options", open=False):
+                apply_preprocess = gr.Checkbox(label="Apply preprocess", value=True)
+                negative_prompt = gr.Textbox(
+                    label="Negative prompt",
+                    value="anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured",
+                )
+                num_inference_steps = gr.Slider(
+                    label="Number of steps",
+                    minimum=1,
+                    maximum=Model.MAX_NUM_INFERENCE_STEPS,
+                    step=1,
+                    value=30,
+                )
+                guidance_scale = gr.Slider(
+                    label="Guidance scale",
+                    minimum=0.1,
+                    maximum=30.0,
+                    step=0.1,
+                    value=7.5,
+                )
+                adapter_conditioning_scale = gr.Slider(
+                    label="Adapter Conditioning Scale",
+                    minimum=0.5,
+                    maximum=1,
+                    step=0.1,
+                    value=0.8,
+                )
+                cond_tau = gr.Slider(
+                    label="Fraction of timesteps for which adapter should be applied",
+                    minimum=0.1,
+                    maximum=1.0,
+                    step=0.1,
+                    value=0.8,
+                )
+                seed = gr.Slider(
+                    label="Seed",
+                    minimum=0,
+                    maximum=MAX_SEED,
+                    step=1,
+                    value=0,
+                )
+                randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+        with gr.Column():
+            result = gr.Gallery(label="Result", columns=2, height=600, object_fit="scale-down", show_label=False)
+
+    inputs = [
+        image,
+        prompt,
+        negative_prompt,
+        num_inference_steps,
+        guidance_scale,
+        adapter_conditioning_scale,
+        cond_tau,
+        seed,
+        apply_preprocess,
+    ]
+    prompt.submit(
+        fn=randomize_seed_fn,
+        inputs=[seed, randomize_seed],
+        outputs=seed,
+        queue=False,
+        api_name=False,
+    ).then(
+        fn=model.change_adapter,
+        inputs=adapter_name,
+        api_name=False,
+    ).success(
+        fn=model.run,
+        inputs=inputs,
+        outputs=result,
+        api_name=False,
+    )
+    negative_prompt.submit(
+        fn=randomize_seed_fn,
+        inputs=[seed, randomize_seed],
+        outputs=seed,
+        queue=False,
+        api_name=False,
+    ).then(
+        fn=model.change_adapter,
+        inputs=adapter_name,
+        api_name=False,
+    ).success(
+        fn=model.run,
+        inputs=inputs,
+        outputs=result,
+        api_name=False,
+    )
+    run_button.click(
+        fn=randomize_seed_fn,
+        inputs=[seed, randomize_seed],
+        outputs=seed,
+        queue=False,
+        api_name=False,
+    ).then(
+        fn=model.change_adapter,
+        inputs=adapter_name,
+        api_name=False,
+    ).success(
+        fn=model.run,
+        inputs=inputs,
+        outputs=result,
+        api_name="run",
+    )
+
+if __name__ == "__main__":
+    demo.queue(max_size=20).launch()

model.py

@@ -0,0 +1,191 @@
+from typing import Callable
+
+import PIL.Image
+import torch
+from controlnet_aux import (
+    CannyDetector,
+    LineartDetector,
+    MidasDetector,
+    PidiNetDetector,
+    ZoeDetector,
+)
+from diffusers import (
+    AutoencoderKL,
+    EulerAncestralDiscreteScheduler,
+    StableDiffusionXLAdapterPipeline,
+    T2IAdapter,
+)
+
+ADAPTER_NAMES = [
+    "TencentARC/t2i-adapter-canny-sdxl-1.0",
+    "TencentARC/t2i-adapter-sketch-sdxl-1.0",
+    "TencentARC/t2i-adapter-lineart-sdxl-1.0",
+    "TencentARC/t2i-adapter-depth-midas-sdxl-1.0",
+    "TencentARC/t2i-adapter-depth-zoe-sdxl-1.0",
+    "TencentARC/t2i-adapter-recolor-sdxl-1.0",
+]
+
+
+class CannyPreprocessor:
+    def __init__(self):
+        self.model = CannyDetector()
+
+    def __call__(self, image: PIL.Image.Image) -> PIL.Image.Image:
+        return self.model(image, detect_resolution=384, image_resolution=1024)
+
+
+class LineartPreprocessor:
+    def __init__(self):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model = LineartDetector.from_pretrained("lllyasviel/Annotators").to(device)
+
+    def __call__(self, image: PIL.Image.Image) -> PIL.Image.Image:
+        return self.model(image, detect_resolution=384, image_resolution=1024)
+
+
+class MidasPreprocessor:
+    def __init__(self):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model = MidasDetector.from_pretrained(
+            "valhalla/t2iadapter-aux-models", filename="dpt_large_384.pt", model_type="dpt_large"
+        ).to(device)
+
+    def __call__(self, image: PIL.Image.Image) -> PIL.Image.Image:
+        return self.model(image, detect_resolution=512, image_resolution=1024)
+
+
+class PidiNetPreprocessor:
+    def __init__(self):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model = PidiNetDetector.from_pretrained("lllyasviel/Annotators").to(device)
+
+    def __call__(self, image: PIL.Image.Image) -> PIL.Image.Image:
+        return self.model(image, detect_resolution=512, image_resolution=1024, apply_filter=True)
+
+
+class RecolorPreprocessor:
+    def __call__(self, image: PIL.Image.Image) -> PIL.Image.Image:
+        return image.convert("L").convert("RGB")
+
+
+class ZoePreprocessor:
+    def __init__(self):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model = ZoeDetector.from_pretrained(
+            "valhalla/t2iadapter-aux-models", filename="zoed_nk.pth", model_type="zoedepth_nk"
+        ).to(device)
+
+    def __call__(self, image: PIL.Image.Image) -> PIL.Image.Image:
+        return self.model(image, gamma_corrected=True)
+
+
+def get_preprocessor(adapter_name: str) -> Callable[[PIL.Image.Image], PIL.Image.Image]:
+    if adapter_name == "TencentARC/t2i-adapter-canny-sdxl-1.0":
+        return CannyPreprocessor()
+    elif adapter_name == "TencentARC/t2i-adapter-sketch-sdxl-1.0":
+        return PidiNetPreprocessor()
+    elif adapter_name == "TencentARC/t2i-adapter-lineart-sdxl-1.0":
+        return LineartPreprocessor()
+    elif adapter_name == "TencentARC/t2i-adapter-depth-midas-sdxl-1.0":
+        return MidasPreprocessor()
+    elif adapter_name == "TencentARC/t2i-adapter-depth-zoe-sdxl-1.0":
+        return ZoePreprocessor()
+    elif adapter_name == "TencentARC/t2i-adapter-recolor-sdxl-1.0":
+        return RecolorPreprocessor()
+    else:
+        raise ValueError(f"Adapter name must be one of {ADAPTER_NAMES}")
+
+
+class Model:
+    MAX_NUM_INFERENCE_STEPS = 50
+
+    def __init__(self, adapter_name: str):
+        if adapter_name not in ADAPTER_NAMES:
+            raise ValueError(f"Adapter name must be one of {ADAPTER_NAMES}")
+
+        self.adapter_name = adapter_name
+
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        if torch.cuda.is_available():
+            self.preprocessor = get_preprocessor(adapter_name)
+
+            model_id = "stabilityai/stable-diffusion-xl-base-1.0"
+            adapter = T2IAdapter.from_pretrained(
+                adapter_name,
+                torch_dtype=torch.float16,
+                varient="fp16",
+            ).to(self.device)
+            euler_a = EulerAncestralDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
+            vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
+            self.pipe = StableDiffusionXLAdapterPipeline.from_pretrained(
+                model_id,
+                vae=vae,
+                adapter=adapter,
+                scheduler=euler_a,
+                torch_dtype=torch.float16,
+                variant="fp16",
+            ).to(self.device)
+            self.pipe.enable_xformers_memory_efficient_attention()
+        else:
+            self.pipe = None
+
+    def change_adapter(self, adapter_name: str) -> None:
+        if not torch.cuda.is_available():
+            raise RuntimeError("This demo does not work on CPU.")
+        if adapter_name not in ADAPTER_NAMES:
+            raise ValueError(f"Adapter name must be one of {ADAPTER_NAMES}")
+        if adapter_name == self.adapter_name:
+            return
+
+        self.preprocessor = None  # type: ignore
+        torch.cuda.empty_cache()
+        self.preprocessor = get_preprocessor(adapter_name)
+
+        self.pipe.adapter = None
+        torch.cuda.empty_cache()
+        self.pipe.adapter = T2IAdapter.from_pretrained(
+            adapter_name,
+            torch_dtype=torch.float16,
+            varient="fp16",
+        ).to(self.device)
+
+    def resize_image(self, image: PIL.Image.Image) -> PIL.Image.Image:
+        w, h = image.size
+        scale = 1024 / max(w, h)
+        new_w = int(w * scale)
+        new_h = int(h * scale)
+        return image.resize((new_w, new_h), PIL.Image.LANCZOS)
+
+    def run(
+        self,
+        image: PIL.Image.Image,
+        prompt: str,
+        negative_prompt: str,
+        num_inference_steps: int = 30,
+        guidance_scale: float = 7.5,
+        adapter_conditioning_scale: float = 0.8,
+        cond_tau: float = 0.8,
+        seed: int = 0,
+        apply_preprocess: bool = True,
+    ) -> list[PIL.Image.Image]:
+        if num_inference_steps > self.MAX_NUM_INFERENCE_STEPS:
+            raise ValueError(f"Number of steps must be less than {self.MAX_NUM_INFERENCE_STEPS}")
+
+        # Resize image to avoid OOM
+        image = self.resize_image(image)
+
+        if apply_preprocess:
+            image = self.preprocessor(image)
+
+        generator = torch.Generator(device=self.device).manual_seed(seed)
+        out = self.pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image=image,
+            num_inference_steps=num_inference_steps,
+            adapter_conditioning_scale=adapter_conditioning_scale,
+            cond_tau=cond_tau,
+            generator=generator,
+            guidance_scale=guidance_scale,
+        ).images[0]
+        return [image, out]

requirements.txt

@@ -0,0 +1,12 @@
+accelerate==0.22.0
+controlnet_aux==0.0.7
+git+https://github.com/huggingface/diffusers@t2iadapterxl
+gradio==3.42.0
+openmim==0.3.9
+Pillow==10.0.0
+safetensors==0.3.3
+timm==0.6.12
+torch==2.0.1
+torchvision==0.15.2
+transformers==4.33.0
+xformers==0.0.20

style.css

@@ -0,0 +1,10 @@
+h1 {
+  text-align: center;
+}
+
+#duplicate-button {
+  margin: auto;
+  color: #fff;
+  background: #1565c0;
+  border-radius: 100vh;
+}