Update app.py

app.py

@@ -24,16 +24,6 @@ import cv2
 
 vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
 
-controlnet = ControlNetModel.from_pretrained(
-    "diffusers/controlnet-canny-sdxl-1.0",
-    torch_dtype=torch.float16
-)
-
-controlnet_lineart = ControlNetModel.from_pretrained(
-    "fffiloni/cn_malgras_test_11",
-    torch_dtype=torch.float16,
-    token=hf_token
-)
 
 def check_use_custom_or_no(value):
     if value is True:
@@ -114,17 +104,6 @@ def resize_image(input_path, output_path, target_height):
 
 @spaces.GPU
 def infer(use_custom_model, model_name, weight_name, custom_lora_weight, image_in, prompt, negative_prompt, preprocessor, controlnet_conditioning_scale, guidance_scale, inf_steps, seed, progress=gr.Progress(track_tqdm=True)):
-
-    pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
-        "stabilityai/stable-diffusion-xl-base-1.0",
-        controlnet=controlnet_lineart,
-        vae=vae,
-        torch_dtype=torch.float16, 
-        variant="fp16",
-        use_safetensors=True
-    )
-    
-    pipe.to(device)
     
     prompt = prompt
     negative_prompt = negative_prompt
@@ -149,12 +128,34 @@ def infer(use_custom_model, model_name, weight_name, custom_lora_weight, image_i
         image = np.concatenate([image, image, image], axis=2)
         image = Image.fromarray(image)
 
+        controlnet = ControlNetModel.from_pretrained(
+            "diffusers/controlnet-canny-sdxl-1.0",
+            torch_dtype=torch.float16
+        )
+
     if preprocessor == "lineart":
         image = Image.open(image_in)
         image = image.convert("RGB")
         image = np.array(image)
         #image = 255 - image
         image = Image.fromarray(image)
+
+        controlnet = ControlNetModel.from_pretrained(
+            "fffiloni/cn_malgras_test_11",
+            torch_dtype=torch.float16,
+            token=hf_token
+        )
+
+    pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+        "stabilityai/stable-diffusion-xl-base-1.0",
+        controlnet=controlnet_lineart,
+        vae=vae,
+        torch_dtype=torch.float16, 
+        variant="fp16",
+        use_safetensors=True
+    )
+    
+    pipe.to(device)
     
     if use_custom_model: