Spaces:
Sleeping
Sleeping
| <html> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <title>Real-Time Latent Consistency Model ControlNet Lora</title> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <script | |
| src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.1/iframeResizer.contentWindow.min.js"></script> | |
| <script src="https://cdn.jsdelivr.net/npm/piexifjs@1.0.6/piexif.min.js"></script> | |
| <script src="https://cdn.tailwindcss.com"></script> | |
| <style type="text/tailwindcss"> | |
| .button { | |
| @apply bg-gray-700 hover:bg-gray-800 text-white font-normal p-2 rounded disabled:bg-gray-300 dark:disabled:bg-gray-700 disabled:cursor-not-allowed dark:disabled:text-black | |
| } | |
| </style> | |
| <script type="module"> | |
| const getValue = (id) => { | |
| const el = document.querySelector(`${id}`) | |
| if (el.type === "checkbox") | |
| return el.checked; | |
| return el.value; | |
| } | |
| const startBtn = document.querySelector("#start"); | |
| const stopBtn = document.querySelector("#stop"); | |
| const videoEl = document.querySelector("#webcam"); | |
| const imageEl = document.querySelector("#player"); | |
| const queueSizeEl = document.querySelector("#queue_size"); | |
| const errorEl = document.querySelector("#error"); | |
| const snapBtn = document.querySelector("#snap"); | |
| const webcamsEl = document.querySelector("#webcams"); | |
| function LCMLive(webcamVideo, liveImage) { | |
| let websocket; | |
| async function start() { | |
| return new Promise((resolve, reject) => { | |
| const websocketURL = `${window.location.protocol === "https:" ? "wss" : "ws" | |
| }:${window.location.host}/ws`; | |
| const socket = new WebSocket(websocketURL); | |
| socket.onopen = () => { | |
| console.log("Connected to websocket"); | |
| }; | |
| socket.onclose = () => { | |
| console.log("Disconnected from websocket"); | |
| stop(); | |
| resolve({ "status": "disconnected" }); | |
| }; | |
| socket.onerror = (err) => { | |
| console.error(err); | |
| reject(err); | |
| }; | |
| socket.onmessage = (event) => { | |
| const data = JSON.parse(event.data); | |
| switch (data.status) { | |
| case "success": | |
| break; | |
| case "start": | |
| const userId = data.userId; | |
| initVideoStream(userId); | |
| break; | |
| case "timeout": | |
| stop(); | |
| resolve({ "status": "timeout" }); | |
| case "error": | |
| stop(); | |
| reject(data.message); | |
| } | |
| }; | |
| websocket = socket; | |
| }) | |
| } | |
| function switchCamera() { | |
| const constraints = { | |
| audio: false, | |
| video: { width: 1024, height: 1024, deviceId: mediaDevices[webcamsEl.value].deviceId } | |
| }; | |
| navigator.mediaDevices | |
| .getUserMedia(constraints) | |
| .then((mediaStream) => { | |
| webcamVideo.removeEventListener("timeupdate", videoTimeUpdateHandler); | |
| webcamVideo.srcObject = mediaStream; | |
| webcamVideo.onloadedmetadata = () => { | |
| webcamVideo.play(); | |
| webcamVideo.addEventListener("timeupdate", videoTimeUpdateHandler); | |
| }; | |
| }) | |
| .catch((err) => { | |
| console.error(`${err.name}: ${err.message}`); | |
| }); | |
| } | |
| async function videoTimeUpdateHandler() { | |
| const model_id = getValue("input[name=base_model]:checked"); | |
| const [WIDTH, HEIGHT] = [512, 512]; | |
| const canvas = new OffscreenCanvas(WIDTH, HEIGHT); | |
| const videoW = webcamVideo.videoWidth; | |
| const videoH = webcamVideo.videoHeight; | |
| const aspectRatio = WIDTH / HEIGHT; | |
| const ctx = canvas.getContext("2d"); | |
| ctx.drawImage(webcamVideo, videoW / 2 - videoH * aspectRatio / 2, 0, videoH * aspectRatio, videoH, 0, 0, WIDTH, HEIGHT) | |
| const blob = await canvas.convertToBlob({ type: "image/jpeg", quality: 1 }); | |
| websocket.send(blob); | |
| websocket.send(JSON.stringify({ | |
| "seed": getValue("#seed"), | |
| "prompt": getValue("#prompt"), | |
| "guidance_scale": getValue("#guidance-scale"), | |
| "strength": getValue("#strength"), | |
| "steps": getValue("#steps"), | |
| "width": WIDTH, | |
| "height": HEIGHT, | |
| "controlnet_scale": getValue("#controlnet_scale"), | |
| "controlnet_start": getValue("#controlnet_start"), | |
| "controlnet_end": getValue("#controlnet_end"), | |
| "canny_low_threshold": getValue("#canny_low_threshold"), | |
| "canny_high_threshold": getValue("#canny_high_threshold"), | |
| "debug_canny": getValue("#debug_canny"), | |
| "model_id": model_id | |
| })); | |
| } | |
| let mediaDevices = []; | |
| async function initVideoStream(userId) { | |
| liveImage.src = `/stream/${userId}`; | |
| await navigator.mediaDevices.enumerateDevices() | |
| .then(devices => { | |
| const cameras = devices.filter(device => device.kind === 'videoinput'); | |
| mediaDevices = cameras; | |
| webcamsEl.innerHTML = ""; | |
| cameras.forEach((camera, index) => { | |
| const option = document.createElement("option"); | |
| option.value = index; | |
| option.innerText = camera.label; | |
| webcamsEl.appendChild(option); | |
| option.selected = index === 0; | |
| }); | |
| webcamsEl.addEventListener("change", switchCamera); | |
| }) | |
| .catch(err => { | |
| console.error(err); | |
| }); | |
| const constraints = { | |
| audio: false, | |
| video: { width: 1024, height: 1024, deviceId: mediaDevices[0].deviceId } | |
| }; | |
| navigator.mediaDevices | |
| .getUserMedia(constraints) | |
| .then((mediaStream) => { | |
| webcamVideo.srcObject = mediaStream; | |
| webcamVideo.onloadedmetadata = () => { | |
| webcamVideo.play(); | |
| webcamVideo.addEventListener("timeupdate", videoTimeUpdateHandler); | |
| }; | |
| }) | |
| .catch((err) => { | |
| console.error(`${err.name}: ${err.message}`); | |
| }); | |
| } | |
| async function stop() { | |
| websocket.close(); | |
| navigator.mediaDevices.getUserMedia({ video: true }).then((mediaStream) => { | |
| mediaStream.getTracks().forEach((track) => track.stop()); | |
| }); | |
| webcamVideo.removeEventListener("timeupdate", videoTimeUpdateHandler); | |
| webcamsEl.removeEventListener("change", switchCamera); | |
| webcamVideo.srcObject = null; | |
| } | |
| return { | |
| start, | |
| stop | |
| } | |
| } | |
| function toggleMessage(type) { | |
| errorEl.hidden = false; | |
| errorEl.scrollIntoView(); | |
| switch (type) { | |
| case "error": | |
| errorEl.innerText = "To many users are using the same GPU, please try again later."; | |
| errorEl.classList.toggle("bg-red-300", "text-red-900"); | |
| break; | |
| case "success": | |
| errorEl.innerText = "Your session has ended, please start a new one."; | |
| errorEl.classList.toggle("bg-green-300", "text-green-900"); | |
| break; | |
| } | |
| setTimeout(() => { | |
| errorEl.hidden = true; | |
| }, 2000); | |
| } | |
| function snapImage() { | |
| try { | |
| const zeroth = {}; | |
| const exif = {}; | |
| const gps = {}; | |
| zeroth[piexif.ImageIFD.Make] = "LCM Image-to-Image ControNet"; | |
| zeroth[piexif.ImageIFD.ImageDescription] = `prompt: ${getValue("#prompt")} | seed: ${getValue("#seed")} | guidance_scale: ${getValue("#guidance-scale")} | strength: ${getValue("#strength")} | controlnet_start: ${getValue("#controlnet_start")} | controlnet_end: ${getValue("#controlnet_end")} | steps: ${getValue("#steps")}`; | |
| zeroth[piexif.ImageIFD.Software] = "https://github.com/radames/Real-Time-Latent-Consistency-Model"; | |
| exif[piexif.ExifIFD.DateTimeOriginal] = new Date().toISOString(); | |
| const exifObj = { "0th": zeroth, "Exif": exif, "GPS": gps }; | |
| const exifBytes = piexif.dump(exifObj); | |
| const canvas = document.createElement("canvas"); | |
| canvas.width = imageEl.naturalWidth; | |
| canvas.height = imageEl.naturalHeight; | |
| const ctx = canvas.getContext("2d"); | |
| ctx.drawImage(imageEl, 0, 0); | |
| const dataURL = canvas.toDataURL("image/jpeg"); | |
| const withExif = piexif.insert(exifBytes, dataURL); | |
| const a = document.createElement("a"); | |
| a.href = withExif; | |
| a.download = `lcm_txt_2_img${Date.now()}.png`; | |
| a.click(); | |
| } catch (err) { | |
| console.log(err); | |
| } | |
| } | |
| const models_id = { | |
| "nitrosocke/Ghibli-Diffusion": "ghibli style", | |
| "nitrosocke/mo-di-diffusion": "modern disney style", | |
| "plasmo/woolitize": "woolitize" | |
| } | |
| document.addEventListener("DOMContentLoaded", () => { | |
| const models_options = document.querySelector("#models_options"); | |
| Object.entries(models_id).forEach(([model, activation], i) => { | |
| const modelEl = document.createElement("div"); | |
| modelEl.innerHTML = ` | |
| <input type="radio" id="${model}" name="base_model" value="${model}" class="cursor-pointer" ${i === 0 ? "checked" : ""}> | |
| <label for="${model}" class="text-sm cursor-pointer" title="Use the keyword on your prompt: ${activation}">${model}: <b>${activation}</b> | |
| <a href="https://hf.co/${model}" title="Model link on Hugging Face" target="_blank" class="text-sm text-blue-500 underline hover:no-underline">⤴️</a></label> | |
| `; | |
| models_options.appendChild(modelEl); | |
| }) | |
| models_options.addEventListener("change", () => { | |
| const model = getValue("input[name=base_model]:checked"); | |
| const prompt = getValue("#prompt"); | |
| const activation = models_id[model]; | |
| if (prompt.includes(activation)) | |
| return; | |
| document.querySelector("#prompt").value = `${activation} portrait of a person`; | |
| }) | |
| }) | |
| const lcmLive = LCMLive(videoEl, imageEl); | |
| startBtn.addEventListener("click", async () => { | |
| try { | |
| startBtn.disabled = true; | |
| snapBtn.disabled = false; | |
| const res = await lcmLive.start(); | |
| startBtn.disabled = false; | |
| if (res.status === "timeout") | |
| toggleMessage("success") | |
| } catch (err) { | |
| console.log(err); | |
| toggleMessage("error") | |
| startBtn.disabled = false; | |
| } | |
| }); | |
| stopBtn.addEventListener("click", () => { | |
| lcmLive.stop(); | |
| }); | |
| window.addEventListener("beforeunload", () => { | |
| lcmLive.stop(); | |
| }); | |
| snapBtn.addEventListener("click", snapImage); | |
| setInterval(() => | |
| fetch("/queue_size") | |
| .then((res) => res.json()) | |
| .then((data) => { | |
| queueSizeEl.innerText = data.queue_size; | |
| }) | |
| .catch((err) => { | |
| console.log(err); | |
| }) | |
| , 5000); | |
| </script> | |
| </head> | |
| <body class="text-black dark:bg-gray-900 dark:text-white"> | |
| <div class="fixed right-2 top-2 p-4 font-bold text-sm rounded-lg max-w-xs text-center" id="error"> | |
| </div> | |
| <main class="container mx-auto px-4 py-4 max-w-4xl flex flex-col gap-4"> | |
| <article class="text-center max-w-xl mx-auto"> | |
| <h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1> | |
| <h2 class="text-2xl font-bold mb-4">ControlNet LoRa</h2> | |
| <p class="text-sm"> | |
| This demo showcases | |
| <a href="https://huggingface.co/blog/lcm_lora" target="_blank" | |
| class="text-blue-500 underline hover:no-underline">LCM LoRa</a> ControlNet pipeline | |
| using <a | |
| href="https://huggingface.co/docs/diffusers/api/pipelines/latent_consistency_models#latent-consistency-models" | |
| target="_blank" class="text-blue-500 underline hover:no-underline">Diffusers</a> with a MJPEG | |
| stream server. | |
| </p> | |
| <p class="text-sm"> | |
| There are <span id="queue_size" class="font-bold">0</span> user(s) sharing the same GPU. | |
| </p> | |
| </article> | |
| <div> | |
| <h2 class="font-medium">Prompt</h2> | |
| <p class="text-sm text-gray-500"> | |
| Change the prompt to generate different images, accepts <a | |
| href="https://github.com/damian0815/compel/blob/main/doc/syntax.md" target="_blank" | |
| class="text-blue-500 underline hover:no-underline">Compel</a> syntax. | |
| </p> | |
| <div class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center"> | |
| <textarea type="text" id="prompt" class="font-light w-full px-3 py-2 mx-1 outline-none dark:text-black" | |
| title="Prompt, this is an example, feel free to modify" | |
| placeholder="Add your prompt here...">ghibli style portrait of a person</textarea> | |
| </div> | |
| </div> | |
| <!-- --> | |
| <label class="font-medium" for="base_model">Base Model</label> | |
| <fieldset class="flex flex-col gap-2" id="models_options"> | |
| </fieldset> | |
| <!-- --> | |
| <div class=""> | |
| <details> | |
| <summary class="font-medium cursor-pointer">Advanced Options</summary> | |
| <div class="grid grid-cols-3 sm:grid-cols-6 items-center gap-3 py-3"> | |
| <label for="webcams" class="text-sm font-medium">Camera Options: </label> | |
| <select id="webcams" class="text-sm border-2 border-gray-500 rounded-md font-light dark:text-black"> | |
| </select> | |
| <div></div> | |
| <label class="text-sm font-medium " for="steps">Inference Steps | |
| </label> | |
| <input type="range" id="steps" name="steps" min="2" max="10" value="4" | |
| oninput="this.nextElementSibling.value = Number(this.value)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 4</output> | |
| <label class="text-sm font-medium" for="guidance-scale">Guidance Scale | |
| </label> | |
| <input type="range" id="guidance-scale" name="guidance-scale" min="0" max="5" step="0.001" | |
| value="0.3" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.3</output> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="strength">Strength</label> | |
| <input type="range" id="strength" name="strength" min="0.1" max="1" step="0.0001" value="0.50" | |
| oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.5</output> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="controlnet_scale">ControlNet Condition Scale</label> | |
| <input type="range" id="controlnet_scale" name="controlnet_scale" min="0.0" max="1" step="0.001" | |
| value="0.80" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.8</output> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="controlnet_start">ControlNet Guidance Start</label> | |
| <input type="range" id="controlnet_start" name="controlnet_start" min="0.0" max="1.0" step="0.001" | |
| value="0.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.0</output> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="controlnet_end">ControlNet Guidance End</label> | |
| <input type="range" id="controlnet_end" name="controlnet_end" min="0.0" max="1.0" step="0.001" | |
| value="0.8" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.8</output> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="canny_low_threshold">Canny Low Threshold</label> | |
| <input type="range" id="canny_low_threshold" name="canny_low_threshold" min="0.0" max="1.0" | |
| step="0.001" value="0.1" | |
| oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.1</output> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="canny_high_threshold">Canny High Threshold</label> | |
| <input type="range" id="canny_high_threshold" name="canny_high_threshold" min="0.0" max="1.0" | |
| step="0.001" value="0.2" | |
| oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> | |
| <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.2</output> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="seed">Seed</label> | |
| <input type="number" id="seed" name="seed" value="299792458" | |
| class="font-light border border-gray-700 text-right rounded-md p-2 dark:text-black"> | |
| <button | |
| onclick="document.querySelector('#seed').value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)" | |
| class="button"> | |
| Rand | |
| </button> | |
| <!-- --> | |
| <!-- --> | |
| <!-- <label class="text-sm font-medium" for="dimension">Image Dimensions</label> | |
| <div class="col-span-2 flex gap-2"> | |
| <div class="flex gap-1"> | |
| <input type="radio" id="dimension512" name="dimension" value="[512,512]" checked | |
| class="cursor-pointer"> | |
| <label for="dimension512" class="text-sm cursor-pointer">512x512</label> | |
| </div> | |
| <div class="flex gap-1"> | |
| <input type="radio" id="dimension768" name="dimension" value="[768,768]" | |
| lass="cursor-pointer"> | |
| <label for="dimension768" class="text-sm cursor-pointer">768x768</label> | |
| </div> | |
| </div> --> | |
| <!-- --> | |
| <!-- --> | |
| <label class="text-sm font-medium" for="debug_canny">Debug Canny</label> | |
| <div class="col-span-2 flex gap-2"> | |
| <input type="checkbox" id="debug_canny" name="debug_canny" class="cursor-pointer"> | |
| <label for="debug_canny" class="text-sm cursor-pointer"></label> | |
| </div> | |
| <div></div> | |
| <!-- --> | |
| </div> | |
| </details> | |
| </div> | |
| <div class="flex gap-3"> | |
| <button id="start" class="button"> | |
| Start | |
| </button> | |
| <button id="stop" class="button"> | |
| Stop | |
| </button> | |
| <button id="snap" disabled class="button ml-auto"> | |
| Snapshot | |
| </button> | |
| </div> | |
| <div class="relative rounded-lg border border-slate-300 overflow-hidden"> | |
| <img id="player" class="w-full aspect-square rounded-lg" | |
| src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="> | |
| <div class="absolute top-0 left-0 w-1/4 aspect-square"> | |
| <video id="webcam" class="w-full aspect-square relative z-10 object-cover" playsinline autoplay muted | |
| loop></video> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 448" width="100" | |
| class="w-full p-4 absolute top-0 opacity-20 z-0"> | |
| <path fill="currentColor" | |
| d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z" /> | |
| </svg> | |
| </div> | |
| </div> | |
| </main> | |
| </body> | |
| </html> |