Commit
·
ee24263
1
Parent(s):
3863db3
up
Browse files- combined_pipe.py +4 -0
- control_net_canny.py +59 -27
- run_xl_ediffi.py +10 -19
combined_pipe.py
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
from diffusers import KandinskyV22CombinedPipeline
|
3 |
+
|
4 |
+
pipe = KandinskyV22CombinedPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder")
|
control_net_canny.py
CHANGED
@@ -12,43 +12,75 @@ from diffusers import (
|
|
12 |
ControlNetModel,
|
13 |
EulerDiscreteScheduler,
|
14 |
StableDiffusionControlNetPipeline,
|
|
|
15 |
UniPCMultistepScheduler,
|
16 |
)
|
17 |
import sys
|
18 |
|
19 |
checkpoint = sys.argv[1]
|
20 |
|
21 |
-
|
22 |
-
"
|
23 |
-
|
24 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
25 |
|
26 |
-
|
27 |
-
high_threshold = 200
|
28 |
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
33 |
|
34 |
-
|
35 |
-
pipe
|
36 |
-
"stabilityai/stable-diffusion-xl-base-0.9", controlnet=[controlnet, controlnet], torch_dtype=torch.float16
|
37 |
-
)
|
38 |
-
pipe.enable_model_cpu_offload()
|
39 |
|
40 |
-
generator = torch.manual_seed(33)
|
41 |
-
out_image = pipe("a blue paradise bird in the jungle", generator=generator, image=[canny_image, canny_image]).images[0]
|
42 |
|
43 |
-
|
44 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
45 |
|
46 |
-
|
|
|
47 |
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
)
|
54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
ControlNetModel,
|
13 |
EulerDiscreteScheduler,
|
14 |
StableDiffusionControlNetPipeline,
|
15 |
+
StableDiffusionXLControlNetPipeline,
|
16 |
UniPCMultistepScheduler,
|
17 |
)
|
18 |
import sys
|
19 |
|
20 |
checkpoint = sys.argv[1]
|
21 |
|
22 |
+
prompts = [
|
23 |
+
"beautiful room",
|
24 |
+
"a photo-realistic image of two paradise birds",
|
25 |
+
"a snowy house behind a forest",
|
26 |
+
"a couple watching a romantic sunset",
|
27 |
+
"boats in the Amazonas",
|
28 |
+
"a photo of a beautiful face of a woman",
|
29 |
+
"a skater in Brooklyn",
|
30 |
+
"a tornado in Iowa"
|
31 |
+
]
|
32 |
|
33 |
+
sd_xl = "control_v11p" not in checkpoint
|
|
|
34 |
|
35 |
+
if sd_xl:
|
36 |
+
base_ckpt = "stabilityai/stable-diffusion-xl-base-0.9"
|
37 |
+
controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
|
38 |
+
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
|
39 |
+
base_ckpt, controlnet=controlnet, torch_dtype=torch.float16
|
40 |
+
)
|
41 |
+
size = 1024
|
42 |
+
else:
|
43 |
+
base_ckpt = "runwayml/stable-diffusion-v1-5"
|
44 |
+
controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
|
45 |
+
pipe = StableDiffusionControlNetPipeline.from_pretrained(
|
46 |
+
base_ckpt, controlnet=controlnet, torch_dtype=torch.float16
|
47 |
+
)
|
48 |
+
size = 512
|
49 |
|
50 |
+
# pipe.enable_model_cpu_offload()
|
51 |
+
pipe.to("cuda")
|
|
|
|
|
|
|
52 |
|
|
|
|
|
53 |
|
54 |
+
for i in range(8):
|
55 |
+
for seed in range(4):
|
56 |
+
image = load_image(
|
57 |
+
f"https://huggingface.co/datasets/patrickvonplaten/webdatasets_images/resolve/main/image_{i}.png"
|
58 |
+
)
|
59 |
+
image = image.resize((size, size))
|
60 |
+
prompt = prompts[i]
|
61 |
+
image = np.array(image)
|
62 |
|
63 |
+
low_threshold = 100
|
64 |
+
high_threshold = 200
|
65 |
|
66 |
+
image = cv2.Canny(image, low_threshold, high_threshold)
|
67 |
+
image = image[:, :, None]
|
68 |
+
image = np.concatenate([image, image, image], axis=2)
|
69 |
+
canny_image = Image.fromarray(image)
|
70 |
+
|
71 |
+
generator = torch.manual_seed(seed)
|
72 |
+
out_image = pipe(prompt, generator=generator, num_inference_steps=20, image=canny_image, controlnet_conditioning_scale=1.0).images[0]
|
73 |
+
|
74 |
+
path = os.path.join(Path.home(), "images", "control_sdxl", f"{i}_{seed}.png")
|
75 |
+
path_in_repo = "/".join(path.split("/")[-2:])
|
76 |
+
out_image.save(path)
|
77 |
+
|
78 |
+
api = HfApi()
|
79 |
+
|
80 |
+
api.upload_file(
|
81 |
+
path_or_fileobj=path,
|
82 |
+
path_in_repo=path_in_repo,
|
83 |
+
repo_id="patrickvonplaten/images",
|
84 |
+
repo_type="dataset",
|
85 |
+
)
|
86 |
+
print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/control_sdxl/{i}_{seed}.png")
|
run_xl_ediffi.py
CHANGED
@@ -18,40 +18,31 @@ from torch.nn.functional import fractional_max_pool2d_with_indices
|
|
18 |
api = HfApi()
|
19 |
start_time = time.time()
|
20 |
|
21 |
-
|
22 |
-
|
23 |
-
beta_end=0.012,
|
24 |
-
beta_schedule="scaled_linear",
|
25 |
-
prediction_type="epsilon",
|
26 |
-
num_train_timesteps=1000,
|
27 |
-
trained_betas=None,
|
28 |
-
thresholding=False,
|
29 |
-
algorithm_type="dpmsolver++",
|
30 |
-
solver_type="midpoint",
|
31 |
-
lower_order_final=True,
|
32 |
-
use_karras_sigmas=True,
|
33 |
-
)
|
34 |
|
35 |
model_id = "stabilityai/stable-diffusion-xl-base-0.9"
|
36 |
pipe_high_noise = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16, variant="fp16", use_safetensors=True, local_files_only=True)
|
37 |
-
pipe_high_noise.scheduler = scheduler
|
38 |
pipe_high_noise.to("cuda")
|
39 |
|
40 |
pipe_low_noise = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-0.9", torch_dtype=torch.float16, use_safetensors=True, variant="fp16")
|
41 |
-
pipe_low_noise.scheduler = scheduler
|
42 |
pipe_low_noise.to("cuda")
|
43 |
|
44 |
prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"
|
45 |
|
46 |
-
|
47 |
random_generator = torch.Generator()
|
48 |
random_generator.manual_seed(0)
|
49 |
|
50 |
-
num_inference_steps =
|
51 |
high_noise_frac = 0.8
|
52 |
|
53 |
-
image = pipe_high_noise(prompt=prompt, num_inference_steps=num_inference_steps, denoising_end=high_noise_frac, output_type="latent").images
|
54 |
-
|
|
|
|
|
|
|
55 |
|
56 |
file_name = f"aaa_1"
|
57 |
path = os.path.join(Path.home(), "images", "ediffi_sdxl", f"{file_name}.png")
|
|
|
18 |
api = HfApi()
|
19 |
start_time = time.time()
|
20 |
|
21 |
+
model_id = "stabilityai/stable-diffusion-xl-base-0.9"
|
22 |
+
scheduler = DPMSolverMultistepScheduler.from_pretrained(model_id, subfolder="scheduler")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
23 |
|
24 |
model_id = "stabilityai/stable-diffusion-xl-base-0.9"
|
25 |
pipe_high_noise = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16, variant="fp16", use_safetensors=True, local_files_only=True)
|
26 |
+
# pipe_high_noise.scheduler = scheduler
|
27 |
pipe_high_noise.to("cuda")
|
28 |
|
29 |
pipe_low_noise = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-0.9", torch_dtype=torch.float16, use_safetensors=True, variant="fp16")
|
30 |
+
# pipe_low_noise.scheduler = scheduler
|
31 |
pipe_low_noise.to("cuda")
|
32 |
|
33 |
prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"
|
34 |
|
|
|
35 |
random_generator = torch.Generator()
|
36 |
random_generator.manual_seed(0)
|
37 |
|
38 |
+
num_inference_steps = 40
|
39 |
high_noise_frac = 0.8
|
40 |
|
41 |
+
image = pipe_high_noise(prompt=prompt, num_inference_steps=num_inference_steps, num_images_per_prompt=2, denoising_end=high_noise_frac, output_type="latent").images
|
42 |
+
images = pipe_low_noise(prompt=prompt, num_inference_steps=num_inference_steps, num_images_per_prompt=2, denoising_start=high_noise_frac, image=image).images
|
43 |
+
|
44 |
+
print(len(images))
|
45 |
+
image = images[1]
|
46 |
|
47 |
file_name = f"aaa_1"
|
48 |
path = os.path.join(Path.home(), "images", "ediffi_sdxl", f"{file_name}.png")
|