SD3 Controlnet

raw	control image	output

Install Diffusers-SD3-Controlnet

The current diffusers have not been merged into the official code yet.

git clone -b sd3_control https://github.com/instantX-research/diffusers_sd3_control.git
cd diffusers
pip install -e .

Demo

import torch
from diffusers import StableDiffusion3Pipeline
from diffusers.models.controlnet_sd3 import ControlNetSD3Model
from diffusers.utils.torch_utils import randn_tensor
import sys, os
sys.path.append('/path/diffusers/examples/community')
from pipeline_stable_diffusion_3_controlnet import StableDiffusion3CommonPipeline
# load pipeline
base_model = 'stabilityai/stable-diffusion-3-medium-diffusers'
pipe = StableDiffusion3CommonPipeline.from_pretrained(
    base_model, 
    controlnet_list=['InstantX/SD3-Controlnet-Canny_alpha_512']
)
pipe.to('cuda:0', torch.float16)
prompt = 'Anime style illustration of a girl wearing a suit. A moon in sky. In the background we see a big rain approaching. text "InstantX" on image'
n_prompt = 'NSFW, nude, naked, porn, ugly'
# controlnet config
controlnet_conditioning = [
    dict(
        control_index=0,
        control_image=load_image('https://huggingface.co/InstantX/SD3-Controlnet-Canny/resolve/main/canny.jpg'),
        control_weight=0.7,
        control_pooled_projections='zeros'
    )
]
# infer
image = pipe(
    prompt=prompt,
    negative_prompt=n_prompt,
    controlnet_conditioning=controlnet_conditioning,
    num_inference_steps=28,
    guidance_scale=7.0,
    height=1024,
    width=1024,
    latents=latents,
).images[0]