Update README.md

README.md

@@ -17,7 +17,7 @@ inference: true
 should probably proofread and complete it, then remove this comment. -->
 
 
-# sd-control-lora-v3-HighCWu/sdxl-control-lora-v3-canny-half_skip_attn-rank16-conv_in-rank64
+# sdxl-control-lora-v3-canny
 
 These are control-lora-v3 weights trained on stabilityai/stable-diffusion-xl-base-1.0 with new type of conditioning.
 You can find some example images below.
@@ -37,8 +37,58 @@ prompt: portrait of a dancing eagle woman, beautiful blonde haired lakota sioux
 
 #### How to use
 
-```python
-# TODO: add an example code snippet for running this diffusion pipeline
+First clone the [control-lora-v3](https://github.com/HighCWu/control-lora-v3) and `cd` in the directory:
+```sh
+git clone https://github.com/HighCWu/control-lora-v3
+cd control-lora-v3
+```
+
+Then run the python code:
+```py
+# !pip install opencv-python transformers accelerate
+from diffusers import AutoencoderKL
+from diffusers.utils import load_image
+from model import UNet2DConditionModelEx
+from pipeline_sdxl import StableDiffusionXLControlLoraV3Pipeline
+import numpy as np
+import torch
+
+import cv2
+from PIL import Image
+
+prompt = "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting"
+negative_prompt = "low quality, bad quality, sketches"
+
+# download an image
+image = load_image(
+    "https://hf.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png"
+)
+
+# initialize the models and pipeline
+unet: UNet2DConditionModelEx = UNet2DConditionModelEx.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0", subfolder="unet", torch_dtype=torch.float16
+)
+unet = unet.add_extra_conditions(["canny"])
+vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
+pipe = StableDiffusionXLControlLoraV3Pipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0", unet=unet, vae=vae, torch_dtype=torch.float16
+)
+# load attention processors
+pipe.load_lora_weights("HighCWu/sdxl-control-lora-v3-canny")
+pipe.enable_model_cpu_offload()
+
+# get canny image
+image = np.array(image)
+image = cv2.Canny(image, 100, 200)
+image = image[:, :, None]
+image = np.concatenate([image, image, image], axis=2)
+canny_image = Image.fromarray(image)
+
+# generate image
+image = pipe(
+    prompt, image=canny_image
+).images[0]
+image.show()
 ```
 
 #### Limitations and bias