Initial commit

.gitignore

@@ -0,0 +1,3 @@
+venv
+__pycache__
+*.pyc

README.md

@@ -5,6 +5,7 @@ colorFrom: red
 colorTo: green
 sdk: gradio
 sdk_version: 3.13.0
+python_version: 3.10.4
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -0,0 +1,58 @@
+import flask
+import os
+
+app = flask.Flask(__name__, template_folder="./templates/")
+
+import numpy as np
+import torch
+
+from diffusers import LDMSuperResolutionPipeline
+from diffusers.utils import PIL_INTERPOLATION, load_image, torch_device
+
+from pkg.util import img_binary_data_to_pil, resizePilToMaxSide, pil_to_base64
+
+
+torch.backends.cuda.matmul.allow_tf32 = False
+ldm = LDMSuperResolutionPipeline.from_pretrained("duongna/ldm-super-resolution", device_map="auto")
+ldm.to(torch_device)
+ldm.set_progress_bar_config(disable=None)
+generator = torch.Generator(device=torch_device).manual_seed(0)
+
+@app.route('/')
+def index():
+    print('Route: /')
+    return flask.render_template('index.html')
+
+@app.route('/superres', methods=['POST'])
+def superres():
+    print('Route: /superres')
+    if flask.request.method != 'POST':
+        return flask.jsonify(
+            isError=True,
+            message=f"This route doesn't support {flask.request.method} method."
+        )
+    imgBinary = flask.request.data
+    maxSideLength = flask.request.args.get('maxSideLength', default=100, type=int)
+    numIterations = flask.request.args.get('numIterations', default=20, type=int)
+
+
+    img = img_binary_data_to_pil(imgBinary)
+    # img.show()
+    # arr = np.asarray(img)
+
+    img = resizePilToMaxSide(img, maxSideLength=maxSideLength)
+    # img.show()
+
+    result = ldm(image=img, generator=generator, num_inference_steps=numIterations, output_type="pil").images[0]
+    # result.show()
+    resultBinary = pil_to_base64(result)
+
+    return flask.jsonify(
+        isError=False,
+        message='Success',
+        statusCode=200,
+        data=resultBinary
+    )
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0',  port=int(os.environ.get('PORT', 7860)))

pkg/util.py

@@ -0,0 +1,37 @@
+import io
+import numpy as np
+import numpy.typing as npt
+from PIL import Image
+from PIL.Image import Image as pilImage
+import base64
+import re
+from diffusers.utils import PIL_INTERPOLATION
+
+def img_binary_data_to_pil(imgBinary: bytes) -> pilImage:
+    image_data = re.sub('^data:image/.+;base64,', '', imgBinary.decode())
+    decoded = base64.b64decode(image_data)
+    img = Image.open(io.BytesIO(decoded))
+    return img
+
+def resizePilToMaxSide(img: pilImage, maxSideLength: int) -> pilImage:
+    w, h = img.size
+    maxSide = max(w, h, maxSideLength)
+    scale = maxSideLength / maxSide
+    targetSize = (int(scale * w), int(scale * h))
+    img = img.resize(targetSize, resample=PIL_INTERPOLATION["lanczos"])
+    return img
+
+def pil_to_base64(img: pilImage | npt.NDArray[np.uint8]) -> str:
+    if type(img) is pilImage:
+        pass
+    elif type(img) is np.ndarray:
+        img = pilImage.fromarray(img.astype('uint8'))
+    else:
+        raise TypeError
+    
+    # https://stackoverflow.com/a/59583262/13797085
+    file_object = io.BytesIO()
+    img.save(file_object, 'PNG')
+    base64img = "data:image/png;base64," \
+        + base64.b64encode(file_object.getvalue()).decode('ascii')
+    return base64img

requirements.txt

@@ -0,0 +1,28 @@
+accelerate==0.15.0
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.3
+diffusers @ git+https://github.com/huggingface/diffusers@69de9b2eaa3a9047e0074b49753eb7a71aec4a5d
+filelock==3.8.2
+Flask==2.2.2
+huggingface-hub==0.11.1
+idna==3.4
+importlib-metadata==5.1.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+numpy==1.23.5
+packaging==22.0
+Pillow==9.3.0
+psutil==5.9.4
+PyYAML==6.0
+regex==2022.10.31
+requests==2.28.1
+torch==1.12.1+cu102
+torchaudio==0.12.1+cu102
+torchvision==0.13.1+cu102
+tqdm==4.64.1
+typing_extensions==4.4.0
+urllib3==1.26.13
+Werkzeug==2.2.2
+zipp==3.11.0

static/css/main.css

@@ -0,0 +1,159 @@
+body {
+    padding: 2rem;
+    font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
+    overflow: auto;
+}
+
+.card {
+    max-width: 620px;
+    margin: 0 auto;
+    padding: 16px;
+    border: 1px solid lightgray;
+    border-radius: 16px;
+}
+
+.card p {
+    color: rgb(107, 114, 128);
+    font-size: 15px;
+    margin-bottom: 10px;
+    margin-top: 5px;
+}
+
+.card h1 {
+    font-size: 16px;
+    margin-top: 0;
+}
+
+.card p:last-child {
+    margin-bottom: 0;
+}
+
+#mainContainer {
+    display: flex;
+    /* width: fit-content; */
+    width: 800px;
+    /* height: fit-content; */
+    height: 600px;
+    background-color: gray;
+}
+
+#resultContainer {
+    width: 65%;
+    height: 50%;
+    margin-left: 5%;
+    background-color: red;
+}
+
+#resultContainer img {
+    display: flex;
+    max-width: 100%;
+    height: 100%;
+    justify-content: center;
+    align-items: center;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+#sidebarContainer {
+    width: 35%;
+    height: 50%;
+    margin-right: 5%;
+    background-color: green;
+}
+
+#scDropZone {
+    display: flex;
+    border: 5px solid blue;
+    width: fit-content;
+    height: calc(20% - 5px * 2);
+    margin-left: auto;
+    margin-right: auto;
+    padding: 5px;
+    color: blue;
+    background-color: rgba(0.0, 255, 255, 0.2);
+}
+
+#scDropZone * {
+    text-align: middle;
+    margin-top: auto;
+    margin-bottom: auto;
+}
+
+#scDropImages {
+    display: flex;
+    max-width: 100%;
+    height: calc(20% - 5px * 2);
+    justify-content: center;
+    margin-left: auto;
+    margin-right: auto;
+    border: 5px solid rgba(0, 255, 255, 0);
+}
+
+#scDropImages .hoverDropImage {
+    border: 5px solid white;
+}
+
+#scDropImages .selectedDropImage {
+    border: 5px solid greenyellow;
+}
+
+.centeredBox {
+    border: 5px solid rgba(0, 0, 0, 0);
+}
+
+.centeredBox * {
+    text-align: middle;
+    margin-top: auto;
+    margin-bottom: auto;
+}
+
+.centeredBox p {
+    color: greenyellow;
+    margin-left: 10px;
+}
+
+.centeredBox input {
+    background-color: white;
+    width: 75%;
+    margin-right: 10px;
+}
+
+#scPromptsContainer {
+    /* width: fit-content; */
+    height: calc(60% - 5px * 2 * 2);
+    border: 5px solid rgba(128, 0, 128, 0);
+}
+
+#scMaxSidePrompt {
+    display: flex;
+    height: calc(33% - 5px * 2);
+    margin-top: auto;
+    margin-bottom: auto;
+    padding-left: 10px;
+}
+
+#scNumIters {
+    display: flex;
+    height: calc(33% - 5px * 2);
+}
+
+#scBtnBox {
+    display: flex;
+    height: calc(34% - 5px * 2);
+}
+
+#scBtnBox button {
+    margin-left: auto;
+    margin-right: auto;
+    width: 40%;
+    height: 75%;
+    font-size: 12px;
+    border: 2px solid greenyellow;
+    color: greenyellow;
+    background-color: blue;
+    border-radius: 3px;
+}
+
+#scBtnBox button.hoverScBtn {
+    border: 2px solid red;
+}

static/js/main.js

@@ -0,0 +1,220 @@
+class ImageHandler {
+    constructor(
+        maxImageSize = 10e6, //10MB
+        clearOnInit = true
+    ) {
+        this.resultImg = document.getElementById('resultImg');
+        this.maxImageSize = maxImageSize;
+        this.validTypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
+
+        this.scDropZone = document.getElementById('scDropZone');
+        this.scDropImages = document.getElementById('scDropImages');
+
+        this.scRunBtn = document.getElementById('scRunBtn');
+        this.scDeleteBtn = document.getElementById('scDeleteBtn');
+
+        if (clearOnInit)
+            this.clearImages();
+        else {
+            for (var i = 0; i < this.scDropImages.children.length; i++) {
+                let child = this.scDropImages.children[i];
+                if (child.tagName == 'IMG') {
+                    if (child.className == "hoverDropImage")
+                        child.className = "";
+                    this.processNewImage(child);
+                }
+            }
+        }
+
+        this.initButtons();
+
+        this.scDropZone.addEventListener('drop', e => this.processDrop(e));
+        this.scDropZone.addEventListener('dragover', e => this.processDragOver(e));
+    }
+
+    initButtons() {
+        var scBtnBox = document.getElementById('scBtnBox');
+        for (var i = 0; i < scBtnBox.children.length; i++) {
+            let child = scBtnBox.children[i];
+            if (child.tagName == "BUTTON") {
+                child.addEventListener(
+                    'mouseover',
+                    function () {
+                        child.className = 'hoverScBtn';
+                    }
+                );
+                child.addEventListener(
+                    'mouseout',
+                    function () {
+                        child.className = '';
+                    }
+                );
+            }
+        }
+
+        // TODO: scRunBtn
+        this.scRunBtn.addEventListener(
+            'mouseup',
+            (e) => this.run()
+        );
+
+        // scDeleteBtn
+        this.scDeleteBtn.addEventListener(
+            'mouseup',
+            (e) => this.deleteSelectedImage()
+        );
+
+    }
+
+    deleteSelectedImage() {
+        for (var i = this.scDropImages.children.length - 1; i >= 0; i--) {
+            let child = this.scDropImages.children[i];
+            if (child.tagName == 'IMG') {
+                if (child.className == "selectedDropImage") {
+                    this.scDropImages.removeChild(child);
+                    break;
+                }
+            }
+        }
+    }
+
+    clearImages() {
+        this.scDropImages.innerHTML = "";
+    }
+
+    validateDroppedImage(image) {
+        if (this.validTypes.indexOf(image.type) == -1) {
+            console.log('Invalid File Type: ' + image.type);
+            return false;
+        }
+
+        if (image.size > this.maxImageSize) {
+            console.log('File is too large: ' + image.size);
+            return false
+        }
+
+        return true;
+    }
+
+    processNewImage(img) {
+        img.addEventListener(
+            'mouseover',
+            function () {
+                if (img.className != "selectedDropImage")
+                    img.className = "hoverDropImage";
+            }
+        );
+        img.addEventListener(
+            'mouseout',
+            function () {
+                if (img.className != "selectedDropImage")
+                    img.className = "";
+            }
+        );
+        img.addEventListener(
+            'mouseup',
+            function () {
+                if (img.className == "hoverDropImage") {
+                    var existingSelectedImages = document.getElementsByClassName("selectedDropImage");
+                    for (var i = 0; i < existingSelectedImages.length; i++)
+                        existingSelectedImages[i].className = "";
+                    img.className = "selectedDropImage";
+                }
+            }
+        )
+    }
+
+    /**
+     * Refer to https://soshace.com/the-ultimate-guide-to-drag-and-drop-image-uploading-with-pure-javascript/
+     * @param {object} event 
+     */
+    processDrop(event) {
+        // Prevent default behavior (Prevent file from being opened)
+        event.preventDefault();
+
+        var dt = event.dataTransfer;
+        var files = dt.files;
+        if (files.length > 0) {
+            for (var i = 0; i < files.length; i++) {
+                if (this.validateDroppedImage(files[i])) {
+                    var img = document.createElement('img');
+                    var reader = new FileReader();
+                    reader.onload = function (e) {
+                        img.src = e.target.result;
+                        // imageNav.sources.push(e.target.result);
+                    }
+                    this.processNewImage(img);
+                    reader.readAsDataURL(files[i]);
+                    this.scDropImages.appendChild(img);
+                }
+            }
+        }
+        else {
+            // console.log('Dropped ' + files.length + ' files.');
+        }
+    }
+
+    processDragOver(event) {
+        // console.log('File(s) in drop zone');
+
+        // Prevent default behavior (Prevent file from being opened)
+        event.preventDefault();
+    }
+
+    get maxSideLength() {
+        var result = document.getElementById('scMaxSidePromptInput').value;
+        return Number(result);
+    }
+
+    get numIterations() {
+        var result = document.getElementById('scNumItersInput').value;
+        return Number(result);
+    }
+
+    async infer(imgData) {
+        try {
+            let url = '../../superres?maxSideLength='
+                + this.maxSideLength
+                + '&&numIterations='
+                + this.numIterations;
+            let res = await fetch(
+                url,
+                {
+                    method: 'POST',
+                    body: imgData
+                }
+            );
+            return await res.json();
+        } catch (error) {
+            console.log(error);
+        }
+    }
+
+    async run() {
+        let imgData = null;
+        for (var i = this.scDropImages.children.length - 1; i >= 0; i--) {
+            let child = this.scDropImages.children[i];
+            if (child.tagName == 'IMG') {
+                if (child.className == "selectedDropImage") {
+                    imgData = child.src;
+                    break;
+                }
+            }
+        }
+        if (imgData != null) {
+            this.resultImg.src = '../static/img/loadingScreen.webp';
+            let result = await this.infer(imgData);
+            // result['isError']
+            if (result == null) {
+                this.resultImg.src = 'https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcR_0fzX1q44MiXQupdeYQTpGd_vAHs9zvTTVLxkfJu6g0RL9oawDeC5Edf-kRt27ygD4ZQ&usqp=CAU';
+            }
+            else
+                this.resultImg.src = result.data;
+        }
+    }
+}
+
+var imageHandler = new ImageHandler(
+    maxImageSize = 10e6,
+    clearOnInit = true
+);

templates/index.html

@@ -0,0 +1,63 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>Latent Defusion Super Resolution</title>
+    <link rel="stylesheet" href="../static/css/main.css" />
+</head>
+
+<body>
+    <h1>Latent Defusion Super Resolution</h1>
+    <div class="card">
+        <h1>Disclaimer</h1>
+        <p>
+            This page is still under testing. If you are reading this, don't expect the model to work as expected right
+            now.
+        </p>
+        <p>
+            I initially plan on using a model that was trained by someone else.
+            I can think about how to train a custom model afterwards.
+        </p>
+    </div>
+    <div id="mainContainer">
+        <div id="resultContainer">
+            <img id="resultImg" src="">
+        </div>
+        <div id="sidebarContainer">
+            <div id="scDropZone">
+                <p>Drop An Image In Here</p>
+            </div>
+            <div id="scDropImages">
+                <img class="selectedDropImage" src="../static/img/loadingScreen.webp">
+                <img class="hoverDropImage" src="../static/img/loadingScreen.webp">
+                <img src="../static/img/loadingScreen.webp">
+            </div>
+            <div id="scPromptsContainer">
+                <div id="scMaxSidePrompt" class="centeredBox">
+                    <p>Max Side Length:</p>
+                    <input id="scMaxSidePromptInput" type="text" class="textbox" value="100" />
+                </div>
+                <div id="scNumIters" class="centeredBox">
+                    <p>Number of Iterations:</p>
+                    <input id="scNumItersInput" type="text" class="textbox" value="20" />
+                </div>
+                <div id="scBtnBox" class="centeredBox">
+                    <button id="scRunBtn">
+                        <b>Run</b>
+                    </button>
+                    <button id="scDeleteBtn">
+                        <b>Delete</b>
+                    </button>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <script type="text/javascript" src="../static/js/main.js"></script>
+    <script
+        src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.2/iframeResizer.contentWindow.min.js"></script>
+</body>
+
+</html>