Go

app.py

@@ -0,0 +1,185 @@
+import gradio as gr
+from core import Ladeco
+from matplotlib.figure import Figure
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+import spaces
+from PIL import Image
+
+
+mpl.style.use("dark_background")
+ladeco = Ladeco()
+
+
+@spaces.GPU
+def infer(img: str) -> tuple[Figure, Figure]:
+    out = ladeco.predict(img)
+
+    seg = out.visualize(level=2)[0].image
+    colormap = out.color_map(level=2)
+
+    area = out.area()[0]
+
+    # match the color of segmentation image and pie chart
+    colors = []
+    l2_area = {}
+    for labelname, area_ratio in area.items():
+        if labelname.startswith("l2") and area_ratio > 0:
+            colors.append(colormap[labelname])
+            labelname = labelname.replace("l2_", "").capitalize()
+            l2_area[labelname] = area_ratio
+
+    pie = plot_pie(l2_area, colors=colors)
+
+    return seg, pie
+
+
+def plot_pie(data: dict[str, float], colors=None) -> Figure:
+    fig, ax = plt.subplots()
+
+    labels = list(data.keys())
+    sizes = list(data.values())
+
+    *_, autotexts = ax.pie(sizes, labels=labels, autopct="%1.1f%%", colors=colors)
+
+    for percent_text in autotexts:
+        percent_text.set_color("k")
+
+    ax.axis("equal")
+
+    return fig
+
+
+def choose_example(imgpath: str) -> gr.Image:
+    img = Image.open(imgpath)
+    width, height = img.size
+    ratio = 512 / max(width, height)
+    img = img.resize((int(width * ratio), int(height * ratio)))
+    return gr.Image(value=img, label="輸入影像（不支援 SVG 格式）", type="filepath")
+
+
+css = """
+.reference {
+    text-align: center;
+    font-size: 1.2em;
+    color: #d1d5db;
+    margin-bottom: 20px;
+}
+.reference a {
+    color: #FB923C;
+    text-decoration: none;
+}
+.reference a:hover {
+    text-decoration: underline;
+    color: #FB923C;
+}
+.description {
+    text-align: center;
+    font-size: 1.1em;
+    color: #d1d5db;
+    margin-bottom: 25px;
+}
+.footer {
+    text-align: center;
+    margin-top: 30px;
+    padding-top: 20px;
+    border-top: 1px solid #ddd;
+    color: #d1d5db;
+    font-size: 14px;
+}
+.main-title {
+    font-size: 24px;
+    font-weight: bold;
+    text-align: center;
+    margin-bottom: 20px;
+}
+.selected-image {
+    height: 756px;
+}
+.example-image {
+    height: 220px;
+}
+""".strip()
+theme = gr.themes.Base(
+    primary_hue="orange",
+    secondary_hue="orange",
+    neutral_hue="gray",
+    font=gr.themes.GoogleFont("Source Sans Pro"),
+).set(
+    background_fill_primary="*neutral_950",  # 主背景色（深黑）
+    button_primary_background_fill="*primary_500",  # 按鈕顏色（橘色）
+    body_text_color="*neutral_200",  # 文字顏色（淺色）
+)
+with gr.Blocks(css=css, theme=theme) as demo:
+    gr.HTML(
+        """
+        <div class="main-title">LaDeco 景觀環境影像語意分析模型</div>
+        <div class="reference">
+          引用資料：
+          <a href="https://www.sciencedirect.com/science/article/pii/S1574954123003187" target="_blank">
+            Li-Chih Ho (2023), LaDeco: A Tool to Analyze Visual Landscape Elements, Ecological Informatics, vol. 78.
+          </a>
+        </div>
+    """.strip()
+    )
+    with gr.Row(equal_height=True):
+        with gr.Group():
+            img = gr.Image(
+                label="輸入影像（不支援 SVG 格式）",
+                type="filepath",
+                elem_classes="selected-image",
+            )
+            gr.Label("範例影像", show_label=False)
+            with gr.Row():
+                ex1 = gr.Image(
+                    value="examples/beach.jpg",
+                    show_label=False,
+                    type="filepath",
+                    elem_classes="example-image",
+                    interactive=False,
+                    show_download_button=False,
+                    show_fullscreen_button=False,
+                )
+                ex2 = gr.Image(
+                    value="examples/field.jpg",
+                    show_label=False,
+                    type="filepath",
+                    elem_classes="example-image",
+                    interactive=False,
+                    show_download_button=False,
+                    show_fullscreen_button=False,
+                )
+                ex3 = gr.Image(
+                    value="examples/sky.jpg",
+                    show_label=False,
+                    type="filepath",
+                    elem_classes="example-image",
+                    interactive=False,
+                    show_download_button=False,
+                    show_fullscreen_button=False,
+                )
+
+        with gr.Column():
+            seg = gr.Plot(label="語意分割")
+            pie = gr.Plot(label="元素面積比例")
+
+    start = gr.Button("開始分析", variant="primary")
+
+    gr.HTML(
+        """
+        <div class="footer">
+            © 2024 LaDeco 版權所���<br>
+            開發者：何立智、楊哲睿
+        </div>
+    """.strip()
+    )
+
+    start.click(fn=infer, inputs=img, outputs=[seg, pie])
+
+    ex1.select(fn=choose_example, inputs=ex1, outputs=img)
+    ex2.select(fn=choose_example, inputs=ex2, outputs=img)
+    ex3.select(fn=choose_example, inputs=ex3, outputs=img)
+
+
+if __name__ == "__main__":
+    demo.launch()

core.py

@@ -0,0 +1,356 @@
+"""Core part of LaDeco v2
+
+Example usage:
+>>> from core import Ladeco
+>>> from PIL import Image
+>>> from pathlib import Path
+>>>
+>>> # predict
+>>> ldc = Ladeco()
+>>> imgs = (thing for thing in Path("example").glob("*.jpg"))
+>>> out = ldc.predict(imgs)
+>>>
+>>> # output - visualization
+>>> segs = out.visualize(level=2)
+>>> segs[0].image.show()
+>>>
+>>> # output - element area
+>>> area = out.area()
+>>> area[0]
+{"fid": "example/.jpg", "l1_nature": 0.673, "l1_man_made": 0.241, ...}
+"""
+from matplotlib.patches import Rectangle
+from pathlib import Path
+from PIL import Image
+from transformers import AutoModelForUniversalSegmentation, AutoProcessor
+import math
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+from functools import lru_cache
+from matplotlib.figure import Figure
+import numpy.typing as npt
+from typing import Iterable, NamedTuple, Generator
+from tqdm import tqdm
+
+
+class LadecoVisualization(NamedTuple):
+    filename: str
+    image: Figure
+
+
+class Ladeco:
+
+    def __init__(self,
+        model_name: str = "shi-labs/oneformer_ade20k_swin_large",
+        area_threshold: float = 0.01,
+        device: str | None = None,
+    ):
+        if device is None:
+            self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        else:
+            self.device = device
+
+        self.processor = AutoProcessor.from_pretrained(model_name)
+        self.model = AutoModelForUniversalSegmentation.from_pretrained(model_name).to(self.device)
+
+        self.area_threshold = area_threshold
+
+        self.ade20k_labels = {
+            name.strip(): int(idx)
+            for name, idx in self.model.config.label2id.items()
+        }
+        self.ladeco2ade20k: dict[str, tuple[int]] = _get_ladeco_labels(self.ade20k_labels)
+
+    def predict(
+            self, image_paths: str | Path | Iterable[str | Path], show_progress: bool = False
+        ) -> "LadecoOutput":
+        if isinstance(image_paths, (str, Path)):
+            imgpaths = [image_paths]
+        else:
+            imgpaths = list(image_paths)
+
+        images = (
+            Image.open(img_path).convert("RGB")
+            for img_path in imgpaths
+        )
+
+        # batch inference functionality of OneFormer is broken
+        masks: list[torch.Tensor] = []
+        for img in tqdm(images, total=len(imgpaths), desc="Segmenting", disable=not show_progress):
+            samples = self.processor(
+                images=img, task_inputs=["semantic"], return_tensors="pt"
+            ).to(self.device)
+
+            with torch.no_grad():
+                outputs = self.model(**samples)
+
+            masks.append(
+                self.processor.post_process_semantic_segmentation(outputs)[0]
+            )
+
+        return LadecoOutput(imgpaths, masks, self.ladeco2ade20k, self.area_threshold)
+
+
+class LadecoOutput:
+
+    def __init__(
+        self,
+        filenames: list[str | Path],
+        masks: torch.Tensor,
+        ladeco2ade: dict[str, tuple[int]],
+        threshold: float,
+    ):
+        self.filenames = filenames
+        self.masks = masks
+        self.ladeco2ade: dict[str, tuple[int]] = ladeco2ade
+        self.ade2ladeco: dict[int, str] = {
+            idx: label
+            for label, indices in self.ladeco2ade.items()
+            for idx in indices
+        }
+        self.threshold = threshold
+
+    def visualize(self, level: int) -> list[LadecoVisualization]:
+        return list(self.ivisualize(level))
+
+    def ivisualize(self, level: int) -> Generator[LadecoVisualization, None, None]:
+        colormaps = self.color_map(level)
+        labelnames = [name for name in self.ladeco2ade if name.startswith(f"l{level}")]
+
+        for fname, mask in zip(self.filenames, self.masks):
+            size = mask.shape + (3,)  # (H, W, RGB)
+            vis = torch.zeros(size, dtype=torch.uint8)
+            for name in labelnames:
+                for idx in self.ladeco2ade[name]:
+                    color = torch.tensor(colormaps[name] * 255, dtype=torch.uint8)
+                    vis[mask == idx] = color
+
+            with Image.open(fname) as img:
+                target_size = img.size
+            vis = Image.fromarray(vis.numpy(), mode="RGB").resize(target_size)
+
+            fig, ax = plt.subplots()
+            ax.imshow(vis)
+            ax.axis('off')
+
+            yield LadecoVisualization(filename=str(fname), image=fig)
+
+    def area(self) -> list[dict[str, float | str]]:
+        return list(self.iarea())
+
+    def iarea(self) -> Generator[dict[str, float | str], None, None]:
+        n_label_ADE20k = 150
+        for filename, mask in zip(self.filenames, self.masks):
+            ade_ratios = torch.tensor([(mask == i).count_nonzero() / mask.numel() for i in range(n_label_ADE20k)])
+            #breakpoint()
+            ldc_ratios: dict[str, float] = {
+                label: round(ade_ratios[list(ade_indices)].sum().item(), 4)
+                for label, ade_indices in self.ladeco2ade.items()
+            }
+            ldc_ratios: dict[str, float] = {
+                label: 0 if ratio < self.threshold else ratio
+                for label, ratio in ldc_ratios.items()
+            }
+            others = round(1 - ldc_ratios["l1_nature"] - ldc_ratios["l1_man_made"], 4)
+            nfi = round(ldc_ratios["l1_nature"]/ (ldc_ratios["l1_nature"] + ldc_ratios.get("l1_man_made", 0) + 1e-6), 4)
+
+            yield {
+                "fid": str(filename), **ldc_ratios, "others": others, "LC_NFI": nfi,
+            }
+
+    def color_map(self, level: int) -> dict[str, npt.NDArray[np.float64]]:
+        "returns {'label_name': (R, G, B), ...}, where (R, G, B) in range [0, 1]"
+        labels = [
+            name for name in self.ladeco2ade.keys() if name.startswith(f"l{level}")
+        ]
+        if len(labels) == 0:
+            raise RuntimeError(
+                f"LaDeco only has 4 levels in 1, 2, 3, 4. You assigned {level}."
+            )
+        colormap = mpl.colormaps["viridis"].resampled(len(labels)).colors[:, :-1]
+        # [:, :-1]: discard alpha channel
+        return {name: color for name, color in zip(labels, colormap)}
+
+    def color_legend(self, level: int) -> Figure:
+        colors = self.color_map(level)
+
+        match level:
+            case 1:
+                ncols = 1
+            case 2:
+                ncols = 1
+            case 3:
+                ncols = 2
+            case 4:
+                ncols = 5
+
+        cell_width = 212
+        cell_height = 22
+        swatch_width = 48
+        margin = 12
+
+        nrows = math.ceil(len(colors) / ncols)
+
+        width = cell_width * ncols + 2 * margin
+        height = cell_height * nrows + 2 * margin
+        dpi = 72
+
+        fig, ax = plt.subplots(figsize=(width / dpi, height / dpi), dpi=dpi)
+        fig.subplots_adjust(margin/width, margin/height,
+                            (width-margin)/width, (height-margin*2)/height)
+        ax.set_xlim(0, cell_width * ncols)
+        ax.set_ylim(cell_height * (nrows-0.5), -cell_height/2.)
+        ax.yaxis.set_visible(False)
+        ax.xaxis.set_visible(False)
+        ax.set_axis_off()
+
+        for i, name in enumerate(colors):
+            row = i % nrows
+            col = i // nrows
+            y = row * cell_height
+
+            swatch_start_x = cell_width * col
+            text_pos_x = cell_width * col + swatch_width + 7
+
+            ax.text(text_pos_x, y, name, fontsize=14,
+                    horizontalalignment='left',
+                    verticalalignment='center')
+
+            ax.add_patch(
+                Rectangle(xy=(swatch_start_x, y-9), width=swatch_width,
+                            height=18, facecolor=colors[name], edgecolor='0.7')
+            )
+
+            ax.set_title(f"LaDeco Color Legend - Level {level}")
+
+        return fig
+    
+
+def _get_ladeco_labels(ade20k: dict[str, int]) -> dict[str, tuple[int]]:
+    labels =  {
+        # level 4 labels
+        # under l3_architecture
+        "l4_hovel": (ade20k["hovel, hut, hutch, shack, shanty"],),
+        "l4_building": (ade20k["building"], ade20k["house"]),
+        "l4_skyscraper": (ade20k["skyscraper"],),
+        "l4_tower": (ade20k["tower"],),
+        # under l3_archi_parts
+        "l4_step": (ade20k["step, stair"],),
+        "l4_canopy": (ade20k["awning, sunshade, sunblind"], ade20k["canopy"]),
+        "l4_arcade": (ade20k["arcade machine"],),
+        "l4_door": (ade20k["door"],),
+        "l4_window": (ade20k["window"],),
+        "l4_wall": (ade20k["wall"],),
+        # under l3_roadway
+        "l4_stairway": (ade20k["stairway, staircase"],),
+        "l4_sidewalk": (ade20k["sidewalk, pavement"],),
+        "l4_road": (ade20k["road, route"],),
+        # under l3_furniture
+        "l4_sculpture": (ade20k["sculpture"],),
+        "l4_flag": (ade20k["flag"],),
+        "l4_can": (ade20k["trash can"],),
+        "l4_chair": (ade20k["chair"],),
+        "l4_pot": (ade20k["pot"],),
+        "l4_booth": (ade20k["booth"],),
+        "l4_streetlight": (ade20k["street lamp"],),
+        "l4_bench": (ade20k["bench"],),
+        "l4_fence": (ade20k["fence"],),
+        "l4_table": (ade20k["table"],),
+        # under l3_vehicle
+        "l4_bike": (ade20k["bicycle"],),
+        "l4_motorbike": (ade20k["minibike, motorbike"],),
+        "l4_van": (ade20k["van"],),
+        "l4_truck": (ade20k["truck"],),
+        "l4_bus": (ade20k["bus"],),
+        "l4_car": (ade20k["car"],),
+        # under l3_sign
+        "l4_traffic_sign": (ade20k["traffic light"],),
+        "l4_poster": (ade20k["poster, posting, placard, notice, bill, card"],),
+        "l4_signboard": (ade20k["signboard, sign"],),
+        # under l3_vert_land
+        "l4_rock": (ade20k["rock, stone"],),
+        "l4_hill": (ade20k["hill"],),
+        "l4_mountain": (ade20k["mountain, mount"],),
+        # under l3_hori_land
+        "l4_ground": (ade20k["earth, ground"], ade20k["land, ground, soil"]),
+        "l4_field": (ade20k["field"],),
+        "l4_sand": (ade20k["sand"],),
+        "l4_dirt": (ade20k["dirt track"],),
+        "l4_path": (ade20k["path"],),
+        # under l3_flower
+        "l4_flower": (ade20k["flower"],),
+        # under l3_grass
+        "l4_grass": (ade20k["grass"],),
+        # under l3_shrub
+        "l4_flora": (ade20k["plant"],),
+        # under l3_arbor
+        "l4_tree": (ade20k["tree"],),
+        "l4_palm": (ade20k["palm, palm tree"],),
+        # under l3_hori_water
+        "l4_lake": (ade20k["lake"],),
+        "l4_pool": (ade20k["pool"],),
+        "l4_river": (ade20k["river"],),
+        "l4_sea": (ade20k["sea"],),
+        "l4_water": (ade20k["water"],),
+        # under l3_vert_water
+        "l4_fountain": (ade20k["fountain"],),
+        "l4_waterfall": (ade20k["falls"],),
+        # under l3_human
+        "l4_person": (ade20k["person"],),
+        # under l3_animal
+        "l4_animal": (ade20k["animal"],),
+        # under l3_sky
+        "l4_sky": (ade20k["sky"],),
+    }
+    labels = labels | {
+        # level 3 labels
+        # under l2_landform
+        "l3_hori_land": labels["l4_ground"] + labels["l4_field"] + labels["l4_sand"] + labels["l4_dirt"] + labels["l4_path"],
+        "l3_vert_land": labels["l4_mountain"] + labels["l4_hill"] + labels["l4_rock"],
+        # under l2_vegetation
+        "l3_woody_plant": labels["l4_tree"] + labels["l4_palm"] + labels["l4_flora"],
+        "l3_herb_plant": labels["l4_grass"],
+        "l3_flower": labels["l4_flower"],
+        # under l2_water
+        "l3_hori_water": labels["l4_water"] + labels["l4_sea"] + labels["l4_river"] + labels["l4_pool"] + labels["l4_lake"],
+        "l3_vert_water": labels["l4_fountain"] + labels["l4_waterfall"],
+        # under l2_bio
+        "l3_human": labels["l4_person"],
+        "l3_animal": labels["l4_animal"],
+        # under l2_sky
+        "l3_sky": labels["l4_sky"],
+        # under l2_archi
+        "l3_architecture": labels["l4_building"] + labels["l4_hovel"] + labels["l4_tower"] + labels["l4_skyscraper"],
+        "l3_archi_parts": labels["l4_wall"] + labels["l4_window"] + labels["l4_door"] + labels["l4_arcade"] + labels["l4_canopy"] + labels["l4_step"],
+        # under l2_street
+        "l3_roadway": labels["l4_road"] + labels["l4_sidewalk"] + labels["l4_stairway"],
+        "l3_furniture": labels["l4_table"] + labels["l4_chair"] + labels["l4_fence"] + labels["l4_bench"] + labels["l4_streetlight"] + labels["l4_booth"] + labels["l4_pot"] + labels["l4_can"] + labels["l4_flag"] + labels["l4_sculpture"],
+        "l3_vehicle": labels["l4_car"] + labels["l4_bus"] + labels["l4_truck"] + labels["l4_van"] + labels["l4_motorbike"] + labels["l4_bike"],
+        "l3_sign": labels["l4_signboard"] + labels["l4_poster"] + labels["l4_traffic_sign"],
+    }
+    labels = labels | {
+        # level 2 labels
+        # under l1_nature
+        "l2_landform": labels["l3_hori_land"] + labels["l3_vert_land"],
+        "l2_vegetation": labels["l3_woody_plant"] + labels["l3_herb_plant"] + labels["l3_flower"],
+        "l2_water": labels["l3_hori_water"] + labels["l3_vert_water"],
+        "l2_bio": labels["l3_human"] + labels["l3_animal"],
+        "l2_sky": labels["l3_sky"],
+        # under l1_man_made
+        "l2_archi": labels["l3_architecture"] + labels["l3_archi_parts"],
+        "l2_street": labels["l3_roadway"] + labels["l3_furniture"] + labels["l3_vehicle"] + labels["l3_sign"],
+    }
+    labels = labels | {
+        # level 1 labels
+        "l1_nature": labels["l2_landform"] + labels["l2_vegetation"] + labels["l2_water"] + labels["l2_bio"] + labels["l2_sky"],
+        "l1_man_made": labels["l2_archi"] + labels["l2_street"],
+    }
+    return labels
+
+
+if __name__ == "__main__":
+    ldc = Ladeco()
+    image = Path("images") / "canyon_3011_00002354.jpg"
+    out = ldc.predict(image)

requirements.txt

@@ -0,0 +1,10 @@
+--extra-index-url https://download.pytorch.org/whl/cu121
+torch==2.5.1
+torchvision
+tokenizers==0.20.3
+transformers==4.46.2
+tqdm
+matplotlib
+pillow==10.4.0
+scipy
+gradio