Upload 338 files

.gitattributes

@@ -1,35 +1,14 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+a.png filter=lfs diff=lfs merge=lfs -text
+cam_result.png filter=lfs diff=lfs merge=lfs -text
+face_recognition_dynamic.gif filter=lfs diff=lfs merge=lfs -text
+medicalDocuments/chan-doan-va-dieu-tri-benh-phoi-tac-nghen-man-tinh-copd-2018.pdf filter=lfs diff=lfs merge=lfs -text
+model.ckpt filter=lfs diff=lfs merge=lfs -text
+pages/images/segment_result.png filter=lfs diff=lfs merge=lfs -text
+pages/output_yolov9/temp_image.png filter=lfs diff=lfs merge=lfs -text
+segment_result.png filter=lfs diff=lfs merge=lfs -text
+temp_image_3d.stl filter=lfs diff=lfs merge=lfs -text
+temp_image.png filter=lfs diff=lfs merge=lfs -text
+visualize_x3d/Eiffel_tower_sample.stl filter=lfs diff=lfs merge=lfs -text
+yolov9/figure/multitask.png filter=lfs diff=lfs merge=lfs -text
+yolov9/yolov9_vinbigData.pt filter=lfs diff=lfs merge=lfs -text

Database/test_db.py

@@ -0,0 +1,24 @@
+import sqlite3
+
+# Đường dẫn đến file cơ sở dữ liệu
+db_path = 'student_attendance.db'
+
+# Kết nối tới cơ sở dữ liệu
+conn = sqlite3.connect(db_path)
+
+# Tạo con trỏ để thực hiện các truy vấn SQL
+cursor = conn.cursor()
+
+# Thực hiện truy vấn SQL, ví dụ để lấy tất cả các bảng trong cơ sở dữ liệu
+cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+
+# Lấy kết quả
+tables = cursor.fetchall()
+
+# In danh sách các bảng
+print("Danh sách các bảng trong cơ sở dữ liệu:")
+for table in tables:
+    print(table[0])
+
+# Đóng kết nối khi không còn sử dụng
+conn.close()

Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.8.9
+
+WORKDIR /app
+
+COPY ./requirements.txt /app/requirements.txt
+COPY ./packages.txt /app/packages.txt
+
+RUN apt-get update && xargs -r -a /app/packages.txt apt-get install -y && rm -rf /var/lib/apt/lists/*
+RUN pip3 install --no-cache-dir -r /app/requirements.txt
+
+# User
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME /home/user
+ENV PATH $HOME/.local/bin:$PATH
+
+WORKDIR $HOME
+RUN mkdir app
+WORKDIR $HOME/app
+COPY . $HOME/app
+
+EXPOSE 8501
+CMD streamlit run app.py \
+    --server.headless true \
+    --server.enableCORS false \
+    --server.enableXsrfProtection false \
+    --server.fileWatcherType none

README.md

@@ -1,10 +1,12 @@
----
-title: KN2024DockerFinal
-emoji: 🐨
-colorFrom: green
-colorTo: purple
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: KN 2024 Final
+emoji: 📊
+colorFrom: indigo
+colorTo: gray
+sdk: streamlit
+sdk_version: 1.38.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

a.txt

@@ -0,0 +1,30 @@
+Folder PATH listing for volume Data2
+Volume serial number is 6AB0-11CA
+E:.
++---chestXray14
+�   +---imges
+�   +---models
+�   �   +---__pycache__
+�   +---__pycache__
++---image_to_3D
+�   +---tsr
+�   �   +---models
+�   �   �   +---tokenizers
+�   �   �   �   +---__pycache__
+�   �   �   +---transformer
+�   �   �   �   +---__pycache__
+�   �   �   +---__pycache__
+�   �   +---__pycache__
+�   +---__pycache__
++---pages
+�   +---images
++---resources_img
++---visualize_x3d
+�   +---__pycache__
++---youtube-streamlit-image-grid
+    +---data
+    +---images
+        +---cea_FaZellweger-90A-01-2
+        �   +---.ipynb_checkpoints
+        +---cjbg_fbg0006
+        +---csg_1276

app.py

@@ -0,0 +1,74 @@
+import os
+import subprocess
+import time
+import streamlit as st
+
+# Install torchmcubes from GitHub
+st.set_page_config(layout='wide')
+
+with st.spinner('Lung Cancer  is in force...'):
+    time.sleep(1.8)
+
+col_1, col_2 = st.columns([2.6, 5])
+
+with col_1:
+    st.image(f'resources_img/vita.jpg', caption='Hệ thống chẩn đoán bệnh phổi sử dụng trí tuệ nhân tạo')
+
+with col_2:
+    st.markdown("<h1 style='text-align: center;'>HƯỚNG DẪN SỬ DỤNG</h1>", unsafe_allow_html=True)
+    st.info('**1.Chức năng Scan**')
+    st.write("Để sử dụng tính năng Scan, bạn hãy chọn mục **🔬 Scan** trên thanh chức năng bên trái của màn hình.")
+    st.image('./resources_img/guide1.jpg', caption='Hình 1: Hướng dẫn sử dụng tiện ích Scan ảnh phổi (1)')
+    st.write("Tiếp đến, ở phía dưới bên trái màn hình có phần **Upload your scan**, tại đây, bạn có thể chọn đầu vào tiện ích Scan. "
+             "Đầu vào được cho phép gồm: **Ảnh**, được thể hiện trong kí hiệu (1) của hình 2 dưới đây.")
+    st.image('./resources_img/guide2.png', caption='Hình 2: Hướng dẫn sử dụng tiện ích Scan ảnh phổi (2)')
+    st.write("Đường bao màu (tím) trong hình 2 được sử dụng để tải lên ảnh Scan phổi của bệnh nhân, khi tải lên thành công ảnh sẽ hiển thị ở phần đường bao màu (xanh lá cây)")
+    st.write("Nếu còn chưa hiểu cách sử dụng, hãy chọn '**Hướng dẫn**' trong hộp đánh dấu màu (cam) của hình 2. Trong phần này sẽ mô tả các bước có thể làm việc với phần mềm.")
+
+    st.info('**2. Phân loại các loại bệnh phổi**')
+    st.write("Để sử dụng chức năng phân loại các loại bệnh phổi, bạn hãy chọn vào mục '**⭐ Thoracic Classification**' trên thanh chức năng phía bên trái màn hình. ")
+    st.image('./resources_img/guide3.png', caption='Hình 3: Hướng dẫn sử dụng chức năng phân loại các loại bệnh phổi')
+    st.write("Tiếp đến, ở phía dưới bên trái màn hình có phần **Upload your scan**, tại đây, bạn có thể chọn đầu vào tiện ích Scan. "
+        "Đầu vào được cho phép gồm: **Ảnh**, được thể hiện trong kí hiệu (1) của hình 2 dưới đây.")
+    st.write("Sau khi duyệt và chọn ảnh cần hiển thị. Nó sẽ hiển thị hiển thị cho chúng ta ảnh ở góc trái màn hình.")
+    st.image('./resources_img/guide3_1.png', caption='Hình 3.1: Các chức năng phân loại các loại bệnh phổi')
+
+    st.write("**2.1. Dự đoán bệnh phổi**")
+    st.write("Trong giao diện chức năng. Sẽ có 5 chức năng để bạn làm việc bao gồm **Predict All Scans** (1), **Cam Visualization(2), **Segmenation Visualization for Lung** (3), **View Report** (4), **Download Report** (5).")
+    st.write("_**Lưu ý: Phải chọn Predict All Scan trước khi dùng các chức năng khác**_")
+    st.write("Bước tiếp theo, bạn cần chọn Predict All Scans. Sau khi chọn **Predict All Scans**, hệ thống sẽ tự động tính toán và lấy các kết quả đầu ra.")
+    st.image('./resources_img/guide4.png', caption='Hình 4: Hướng dẫn sử dụng dự đoán bệnh phổi')
+    st.write("Sau khi chương trình chạy thành công, nó sẽ hiện thông báo màu xanh để chúng ta biết" )
+
+    st.write("**2.2. CAM Visualization**")
+    st.write("Để xem ảnh CAM( Class Activation Map), bạn chọn chức năng **CAM Visualization**")
+    st.write("Sau đó bạn có thể lướt xuống dưới để xem ảnh")
+    st.image('./resources_img/guide5.png', caption='Hình 5: Hướng dẫn sử dụng chức năng Sửa thông tin định danh')
+    st.write("Bạn có thể phóng to và thu nhỏ ảnh bằng cách di chuột vào ảnh và cuộn chuột")
+
+    st.write("**2.3. Segmentation Visualization for Lung**")
+    st.write("Để xem ảnh Segmentation, bạn chọn chức năng **Segmentation Visualization for Lung**")
+    st.write("Sau đó bạn có thể lướt xuống dưới để xem ảnh")
+    st.image('./resources_img/guide6.png', caption='Hình 6: Hướng dẫn sử dụng chức năng Segmentation Visualization for Lung')
+    st.write("Bạn có thể phóng to và thu nhỏ ảnh bằng cách di chuột vào ảnh và cuộn chuột")
+
+    st.write("**2.4. Xem báo cáo**")
+    st.write("Để xem ảnh báo cáo về khả năng nhiễm bệnh, bạn chọn chức năng **View Report**")
+    st.write("Sau đó bạn có thể lướt xuống dưới để xem bảng thông tin về khả năng nhiễm bệnh.")
+    st.image('./resources_img/guide7.png',
+             caption='Hình 7: Hướng dẫn sử dụng chức năng xem báo cáo')
+
+    st.write("**2.5. Tải báo cáo**")
+    st.write("Để tải báo cáo dưới dạng PDF về khả năng nhiễm bệnh, bạn chọn chức năng **View Report**")
+    st.write("Sau đó bạn có thể lướt xuống dưới để điền một số trường thông tin liên quan tới bệnh nhân. Sau đó chọn Submit báo cáo sẽ tự tải về máy dưới dạng PDF")
+    st.image('./resources_img/guide8.png',
+             caption='Hình 7: Hướng dẫn sử dụng chức năng tải và in báo cáo')
+
+    st.info('**4. Chức năng x3D Lung Viewer**')
+    st.write("There are many variations of passages of Lorem Ipsum available, but the majority have suffered alteration "
+             "in some form, by injected humour, or randomised words which don't look even slightly believable. If you are"
+             " going to use a passage of Lorem Ipsum, you need to be sure there isn't anything embarrassing hidden in the"
+             " middle of text. All the Lorem Ipsum generators on the Internet tend to repeat predefined chunks as necessary, "
+             "making this the first true generator on the Internet. It uses a dictionary of over 200 Latin words, combined "
+             "with a handful of model sentence structures, to generate Lorem Ipsum which looks reasonable. The generated Lorem "
+             "Ipsum is therefore always free from repetition, injected humour, or non-characteristic words etc.")

chestXray14/__init__.py

@@ -0,0 +1,7 @@
+from test import *
+from chexnet import *
+from constant import *
+from heatmap import *
+from unet import *
+from chestXray_utils import *
+from layers import *

chestXray14/chestXray_utils.py

@@ -0,0 +1,35 @@
+import pandas as pd
+import numpy as np
+from skimage import color, morphology
+from constant import PATH, TRAIN_CSV, VAL_CSV, TEST_CSV
+
+def get_chestxray_from_csv():
+    result = []
+    for f in [PATH/TRAIN_CSV, PATH/VAL_CSV, PATH/TEST_CSV]:
+        df = pd.read_csv(f, sep=' ', header=None)
+        images = df.iloc[:, 0].values
+        labels = df.iloc[:, 1:].values
+        result.append((images, labels))
+    return result
+
+def sigmoid_np(x):
+    return 1. / (1. + np.exp(-x))
+
+def blend_segmentation(image, mask, gt_mask=None, boundary=False, alpha=1):
+    image = np.array(image)  # Convert PIL Image to NumPy array
+    w, h = image.shape[1], image.shape[0]
+    color_mask = np.zeros((h, w, 3)) # PIL Image
+    if boundary: mask = morphology.dilation(mask, morphology.disk(3)) - mask
+    color_mask[mask==1] = [1, 0, 0] # RGB
+
+    if gt_mask is not None:
+        gt_boundary = morphology.dilation(gt_mask, morphology.disk(3)) - gt_mask
+        color_mask[gt_boundary==1] = [0, 1, 0] # RGB
+
+    image_hsv = color.rgb2hsv(image)
+    color_mask_hsv = color.rgb2hsv(color_mask)
+
+    image_hsv[..., 0] = color_mask_hsv[..., 0]
+    image_hsv[..., 1] = color_mask_hsv[..., 1] * alpha
+
+    return color.hsv2rgb(image_hsv)

chestXray14/chexnet.py

@@ -0,0 +1,67 @@
+import torch.nn as nn
+import pretrainedmodels
+from torchvision.models import densenet121
+from layers import Flatten
+import torch
+import torchvision.transforms as transforms
+from pathlib import Path
+from constant import IMAGENET_MEAN, IMAGENET_STD
+import os
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+yolov9 = os.path.join(script_dir, '..', 'chestXray14')
+sys.path.append(yolov9)
+
+class ChexNet(nn.Module):
+    tfm = transforms.Compose([
+        transforms.Resize((224, 224)),
+        transforms.ToTensor(),
+        transforms.Normalize(IMAGENET_MEAN, IMAGENET_STD)
+    ])
+
+    def __init__(self, trained=False, model_name='20180525-222635'):
+        super().__init__()
+        # chexnet.parameters() is freezed except head
+        if trained:
+            self.load_model(model_name)
+        else:
+            self.load_pretrained()
+
+    def load_model(self, model_name):
+        self.backbone = densenet121(False).features
+        self.head = nn.Sequential(
+            nn.AdaptiveAvgPool2d(1),
+            Flatten(),
+            nn.Linear(1024, 14)
+        )
+        path = Path('chestX-ray-14')
+        state_dict = torch.load('chexnet.h5')
+        self.load_state_dict(state_dict)
+
+    def load_pretrained(self, torch=False):
+        if torch:
+            self.backbone = densenet121(True).features
+        else:
+            self.backbone = pretrainedmodels.__dict__['densenet121']().features
+
+        self.head = nn.Sequential(
+            nn.AdaptiveAvgPool2d(1),
+            Flatten(),
+            nn.Linear(1024, 14)
+        )
+
+    def forward(self, x):
+        return self.head(self.backbone(x))
+
+    def predict(self, image):
+        """
+        input: PIL image (w, h, c)
+        output: prob np.array
+        """
+        image_tensor = self.tfm(image).unsqueeze(0)  # Add batch dimension
+        image_tensor = image_tensor.to(next(self.parameters()).device)  # Move to the same device as the model
+        with torch.no_grad():
+            py = torch.sigmoid(self(image_tensor))
+        prob = py.cpu().numpy()[0]
+        return prob

chestXray14/constant.py

@@ -0,0 +1,58 @@
+import os
+from pathlib import Path
+import numpy as np
+
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(current_dir, os.path.pardir))
+
+N_CLASSES = 14
+CLASS_NAMES = ['Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass', 'Nodule', 'Pneumonia',
+                'Pneumothorax', 'Consolidation', 'Edema', 'Emphysema',
+                'Fibrosis', 'Pleural Thickening', 'Hernia']
+
+IMAGENET_MEAN = np.array([0.485, 0.456, 0.406])
+IMAGENET_STD = np.array([0.229, 0.224, 0.225])
+
+PATH = Path('/home/dattran/data/xray-thesis/chestX-ray14')
+ATTENTION_DN = 'tmp/attention'
+IMAGE_DN = 'images'
+TRAIN_CSV = 'train.csv'
+VAL_CSV = 'val.csv'
+TEST_CSV = 'test.csv'
+
+"""
+Below may not need any more
+"""
+# EPOCHS = 2# 100
+# # BATCHES = 500 # 500
+# BATCHSIZE = 32
+# VALIDATE_EVERY_N_EPOCHS = 5
+SCALE_FACTOR = .875
+DATA_DIR = '/mnt/data/xray-thesis/data/chestX-ray14/images/'
+PERCENTAGE = 0.01 # percentage of data use for quick run
+TEST_AGUMENTED = False
+DISEASE_THRESHOLD = 0.5
+
+MODEL_DIR = '/mnt/data/xray-thesis/models'
+LOG_DIR = 'mnt/data/xray-thesis/logs'
+CSV_DIR = '%s/csv' % ROOT
+STAT_DIR = '%s/stats' % ROOT
+
+# chexnet file
+CHEXNET_MODEL_NAME = '%s/chexnet_densenet.pth.tar' % MODEL_DIR
+CHEXNET_TRAIN_CSV = '%s/chexnet_train_list.csv' % CSV_DIR
+CHEXNET_VAL_CSV = '%s/chexnet_val_list.csv' % CSV_DIR
+CHEXNET_TEST_CSV = '%s/chexnet_test_list.csv' % CSV_DIR
+TRAIN_CSV = '%s/train_list.csv' % CSV_DIR
+VAL_CSV = '%s/val_list.csv' % CSV_DIR
+TEST_CSV = '%s/test_list.csv' % CSV_DIR
+
+# different model
+DENSENET121_DIR = '%s/densenet121' % MODEL_DIR
+
+# stat
+TRAIN_STAT = '%s/train.csv' % STAT_DIR
+TEST_STAT = '%s/test.csv' % STAT_DIR
+
+PREPROCESS = False

chestXray14/heatmap.py

@@ -0,0 +1,38 @@
+import torch
+import numpy as np
+import cv2
+from chexnet import ChexNet
+from layers import SaveFeature
+from constant import CLASS_NAMES
+
+
+class HeatmapGenerator:
+
+    # def __init__(self, model_name='20180429-130928', mode=None):
+    def __init__(self, chexnet, mode=None):
+        self.chexnet = chexnet
+        self.sf = SaveFeature(chexnet.backbone)
+        self.weight = list(list(self.chexnet.head.children())[-1].parameters())[0]
+        self.mapping = self.cam if mode == 'cam' else self.default
+
+    def cam(self, pred_y):
+        heatmap = self.sf.features[0].permute(1, 2, 0).detach().numpy() @ self.weight[pred_y].detach().numpy()
+        return heatmap
+
+    # def default(self, pred_ys):
+    #     return torch.max(torch.abs(self.sf.features), dim=1)[0]
+
+    def generate(self, image):
+        prob = self.chexnet.predict(image)
+        w, h = image.size
+        return self.from_prob(prob, w, h)
+
+    def from_prob(self, prob, w, h):
+        pred_y = np.argmax(prob)
+        heatmap = self.mapping(pred_y)
+
+        heatmap = heatmap - np.min(heatmap)
+        heatmap = heatmap / np.max(heatmap)
+        heatmap = cv2.resize(heatmap, (w, h))
+
+        return heatmap, CLASS_NAMES[pred_y]

chestXray14/layers.py

@@ -0,0 +1,104 @@
+from torch import nn
+import torch
+import torch.nn.functional as F
+import numpy as np
+
+
+class Flatten(nn.Module):
+    def forward(self, x):
+        x = x.view(x.size()[0], -1)
+        return x
+
+class LSEPool2d(nn.Module):
+
+    def __init__(self, r=3):
+        super().__init__()
+        self.r =r
+
+    def forward(self, x):
+        s = x.size()[3]  # x: bs*2048*7*7
+        r = self.r
+        x_max = F.adaptive_max_pool2d(x, 1) # x_max: bs*2048*1*1
+        p = ((1/r) * torch.log((1 / (s*s)) * torch.exp(r*(x - x_max)).sum(3).sum(2)))
+        x_max = x_max.view(x.size(0), -1) # bs*2048
+        return x_max+p
+
+
+class WeightedBCEWithLogitsLoss(nn.Module):
+
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, input, target):
+        w = self.get_weight(input, target)
+        return F.binary_cross_entropy_with_logits(input, target, w, reduction='mean')
+
+    def get_weight(self, input, target):
+        y = target.cpu().data.numpy()
+        y_hat = input.cpu().data.numpy()
+        P = np.count_nonzero(y == 1)
+        N = np.count_nonzero(y == 0)
+        beta_p = (P + N) / (P + 1) # may not contain disease
+        beta_n = (P + N) / N
+        w = np.empty(y.shape)
+        w[y==0] = beta_n
+        w[y==1] = beta_p
+        w = torch.FloatTensor(w).cuda()
+        return w
+
+class SaveFeature:
+    features = None
+
+    def __init__(self, m):
+        self.hook = m.register_forward_hook(self.hook_fn)
+
+    def hook_fn(self, module, input, output):
+        self.features = output
+
+    def remove(self):
+        self.hook.remove()
+
+# class FocalLoss(WeightedBCELoss):
+
+#     def __init__(self, theta=2):
+#         super().__init__()
+#         self.theta = theta
+
+#     def forward(self, input, target):
+# #         pt = target*input + (1-target)*(1-input)
+# #         target *= (1-pt)**self.theta
+#         w = self.get_weight(input, target)
+#         return F.binary_cross_entropy_with_logits(input, target, w)
+
+
+
+# class FocalLoss(nn.Module):
+#     def __init__(self, gamma=0, alpha=None, size_average=True):
+#         super(FocalLoss, self).__init__()
+#         self.gamma = gamma
+#         self.alpha = alpha
+#         if isinstance(alpha,(float,int,long)): self.alpha = torch.Tensor([alpha,1-alpha])
+#         if isinstance(alpha,list): self.alpha = torch.Tensor(alpha)
+#         self.size_average = size_average
+
+#     def forward(self, input, target):
+#         if input.dim()>2:
+#             input = input.view(input.size(0),input.size(1),-1)  # N,C,H,W => N,C,H*W
+#             input = input.transpose(1,2)    # N,C,H*W => N,H*W,C
+#             input = input.contiguous().view(-1,input.size(2))   # N,H*W,C => N*H*W,C
+#         target = target.view(-1,1)
+
+#         logpt = F.log_softmax(input)
+#         logpt = logpt.gather(1,target)
+#         logpt = logpt.view(-1)
+#         pt = Variable(logpt.data.exp())
+
+#         if self.alpha is not None:
+#             if self.alpha.type()!=input.data.type():
+#                 self.alpha = self.alpha.type_as(input.data)
+#             at = self.alpha.gather(0,target.data.view(-1))
+#             logpt = logpt * Variable(at)
+
+#         loss = -1 * (1-pt)**self.gamma * logpt
+#         if self.size_average: return loss.mean()
+#         else: return loss.sum()

chestXray14/models/Model.ipynb

@@ -0,0 +1,287 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torchvision\n",
+    "import pretrainedmodels\n",
+    "import torch\n",
+    "import pretrainedmodels.utils as utils\n",
+    "import torchvision.transforms as transforms\n",
+    "import torchvision.models as models\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.functional as F\n",
+    "from PIL import Image\n",
+    "from collections import OrderedDict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'imagenet': {'url': 'http://data.lip6.fr/cadene/pretrainedmodels/nasnetalarge-a1897284.pth', 'input_space': 'RGB', 'input_size': [3, 331, 331], 'input_range': [0, 1], 'mean': [0.5, 0.5, 0.5], 'std': [0.5, 0.5, 0.5], 'num_classes': 1000}, 'imagenet+background': {'url': 'http://data.lip6.fr/cadene/pretrainedmodels/nasnetalarge-a1897284.pth', 'input_space': 'RGB', 'input_size': [3, 331, 331], 'input_range': [0, 1], 'mean': [0.5, 0.5, 0.5], 'std': [0.5, 0.5, 0.5], 'num_classes': 1001}}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Downloading: \"http://data.lip6.fr/cadene/pretrainedmodels/nasnetalarge-a1897284.pth\" to /home/dattran/.torch/models/nasnetalarge-a1897284.pth\n",
+      "100%|██████████| 356056626/356056626 [07:27<00:00, 795221.08it/s] \n"
+     ]
+    }
+   ],
+   "source": [
+    "print(pretrainedmodels.pretrained_settings['nasnetalarge'])\n",
+    "model = pretrainedmodels.__dict__['nasnetalarge'](num_classes=1000, pretrained='imagenet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Variable containing:\n",
+       "( 0  , 0  ,.,.) = \n",
+       "  0.1121  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.4418  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.0000  0.0000  0.0000  ...   0.8554  0.0000  0.0000\n",
+       "           ...             ⋱             ...          \n",
+       "  0.0000  0.5682  0.5166  ...   0.0301  0.0000  0.0000\n",
+       "  0.0000  0.0921  0.2531  ...   2.0754  0.4212  0.0000\n",
+       "  0.0000  0.0000  0.0000  ...   1.8003  0.5220  0.0000\n",
+       "\n",
+       "( 0  , 1  ,.,.) = \n",
+       "  0.1536  0.1211  0.0000  ...   1.3220  0.1388  0.0000\n",
+       "  0.4111  0.2736  0.1038  ...   0.7770  0.0000  0.0000\n",
+       "  0.0000  0.0000  0.0000  ...   1.5591  0.0000  0.0000\n",
+       "           ...             ⋱             ...          \n",
+       "  0.0000  0.0000  0.3120  ...   0.0000  0.0000  0.0000\n",
+       "  1.2059  0.9648  0.7537  ...   0.0000  0.0000  0.0000\n",
+       "  1.8497  0.5725  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "\n",
+       "( 0  , 2  ,.,.) = \n",
+       "  0.3311  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.3876  0.2232  0.1740  ...   0.0000  0.0000  0.0000\n",
+       "           ...             ⋱             ...          \n",
+       "  0.0000  0.0000  0.3968  ...   0.0000  0.2945  0.0000\n",
+       "  0.4885  0.8537  1.2278  ...   0.2380  0.6205  1.0980\n",
+       "  2.1136  1.3682  1.5039  ...   0.9723  0.9817  0.0760\n",
+       "      ... \n",
+       "\n",
+       "( 0  ,1533,.,.) = \n",
+       "  0.8787  0.9274  0.5682  ...   0.4111  0.5084  0.5470\n",
+       "  0.5436  0.6317  0.5634  ...   0.5417  0.3694  0.4478\n",
+       "  0.0000  0.3396  0.5478  ...   0.8584  0.5552  0.5867\n",
+       "           ...             ⋱             ...          \n",
+       "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.4349  0.2017  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.3743  0.3398  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "\n",
+       "( 0  ,1534,.,.) = \n",
+       "  0.1190  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.0259  0.0000  0.0000  ...   0.3597  0.0000  0.0000\n",
+       "  0.0000  0.0000  0.0000  ...   1.4928  0.0000  0.0000\n",
+       "           ...             ⋱             ...          \n",
+       "  0.0000  0.0000  0.0000  ...   1.4441  1.2858  0.6525\n",
+       "  0.0000  0.1889  0.5281  ...   0.7508  0.9813  0.5251\n",
+       "  0.0000  0.9832  1.3777  ...   0.0639  0.2403  0.0000\n",
+       "\n",
+       "( 0  ,1535,.,.) = \n",
+       "  0.0000  0.1068  0.3845  ...   0.0000  0.0000  0.0000\n",
+       "  0.0000  0.2751  0.7059  ...   0.0000  0.0000  0.0000\n",
+       "  0.0000  0.0711  0.4025  ...   1.2069  1.4548  1.1041\n",
+       "           ...             ⋱             ...          \n",
+       "  0.0000  0.0000  0.0000  ...   0.7915  0.3439  0.1936\n",
+       "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "  0.0275  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
+       "[torch.FloatTensor of size 1x1536x8x8]"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "load_img = utils.LoadImage()\n",
+    "# transformations depending on the model\n",
+    "# rescale, center crop, normalize, and others (ex: ToBGR, ToRange255)\n",
+    "tf_img = utils.TransformImage(model) \n",
+    "\n",
+    "path_img = '/home/dattran/data/xray/00000013_029.png'\n",
+    "\n",
+    "input_img = load_img(path_img)\n",
+    "input_tensor = tf_img(input_img)         # 3x400x225 -> 3x299x299 size may differ\n",
+    "input_tensor = input_tensor.unsqueeze(0) # 3x299x299 -> 1x3x299x299\n",
+    "input = torch.autograd.Variable(input_tensor,\n",
+    "    requires_grad=False)\n",
+    "model.features(input)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001B[0;31mInit signature:\u001B[0m \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnn\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mMaxPool2d\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mkernel_size\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstride\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mNone\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mpadding\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdilation\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mreturn_indices\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mceil_mode\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
+       "\u001B[0;31mDocstring:\u001B[0m     \n",
+       "Applies a 2D max pooling over an input signal composed of several input\n",
+       "planes.\n",
+       "\n",
+       "In the simplest case, the output value of the layer with input size :math:`(N, C, H, W)`,\n",
+       "output :math:`(N, C, H_{out}, W_{out})` and :attr:`kernel_size` :math:`(kH, kW)`\n",
+       "can be precisely described as:\n",
+       "\n",
+       ".. math::\n",
+       "\n",
+       "    \\begin{array}{ll}\n",
+       "    out(N_i, C_j, h, w)  = \\max_{{m}=0}^{kH-1} \\max_{{n}=0}^{kW-1}\n",
+       "                           input(N_i, C_j, stride[0] * h + m, stride[1] * w + n)\n",
+       "    \\end{array}\n",
+       "\n",
+       "| If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides\n",
+       "  for :attr:`padding` number of points\n",
+       "| :attr:`dilation` controls the spacing between the kernel points. It is harder to describe,\n",
+       "  but this `link`_ has a nice visualization of what :attr:`dilation` does.\n",
+       "\n",
+       "The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`dilation` can either be:\n",
+       "\n",
+       "    - a single ``int`` -- in which case the same value is used for the height and width dimension\n",
+       "    - a ``tuple`` of two ints -- in which case, the first `int` is used for the height dimension,\n",
+       "      and the second `int` for the width dimension\n",
+       "\n",
+       "Args:\n",
+       "    kernel_size: the size of the window to take a max over\n",
+       "    stride: the stride of the window. Default value is :attr:`kernel_size`\n",
+       "    padding: implicit zero padding to be added on both sides\n",
+       "    dilation: a parameter that controls the stride of elements in the window\n",
+       "    return_indices: if ``True``, will return the max indices along with the outputs.\n",
+       "                    Useful when Unpooling later\n",
+       "    ceil_mode: when True, will use `ceil` instead of `floor` to compute the output shape\n",
+       "\n",
+       "Shape:\n",
+       "    - Input: :math:`(N, C, H_{in}, W_{in})`\n",
+       "    - Output: :math:`(N, C, H_{out}, W_{out})` where\n",
+       "      :math:`H_{out} = floor((H_{in}  + 2 * padding[0] - dilation[0] * (kernel\\_size[0] - 1) - 1) / stride[0] + 1)`\n",
+       "      :math:`W_{out} = floor((W_{in}  + 2 * padding[1] - dilation[1] * (kernel\\_size[1] - 1) - 1) / stride[1] + 1)`\n",
+       "\n",
+       "Examples::\n",
+       "\n",
+       "    >>> # pool of square window of size=3, stride=2\n",
+       "    >>> m = nn.MaxPool2d(3, stride=2)\n",
+       "    >>> # pool of non-square window\n",
+       "    >>> m = nn.MaxPool2d((3, 2), stride=(2, 1))\n",
+       "    >>> input = autograd.Variable(torch.randn(20, 16, 50, 32))\n",
+       "    >>> output = m(input)\n",
+       "\n",
+       ".. _link:\n",
+       "    https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md\n",
+       "\u001B[0;31mFile:\u001B[0m           ~/miniconda2/envs/dat/lib/python3.6/site-packages/torch/nn/modules/pooling.py\n",
+       "\u001B[0;31mType:\u001B[0m           type\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "torch.nn.MaxPool2d?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pretrainedmodels.models.dpn import adaptive_avgmax_pool2d"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1000"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x = nn.Conv2d(1000, 14,kernel_size=1, bias=True)\n",
+    "x.in_channels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "False"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.eval()\n",
+    "model.training"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

chestXray14/models/densenet.py

@@ -0,0 +1,71 @@
+import torchvision
+import torch.nn as nn
+import pretrainedmodels
+import torch.nn.functional as F
+import torch
+from constant import SCALE_FACTOR
+import math
+import pdb
+
+class DenseNet(nn.Module):
+
+    def __init__(self, variant):
+        super(DenseNet, self).__init__()
+        assert variant in ['densenet121', 'densenet161', 'densenet201']
+
+        # load retrain model
+        model = pretrainedmodels.__dict__[variant](num_classes=1000, pretrained='imagenet')
+        self.features = model.features
+        num_ftrs = model.last_linear.in_features
+        self.classifier = nn.Sequential(
+            nn.Linear(num_ftrs, 14),
+            nn.Sigmoid()
+        )
+        # TODO: BCELoss with logit for numeric stable
+        # self.classifier = nn.Linear(num_ftrs, 14)
+
+        # load other info
+        self.mean = model.mean
+        self.std = model.std
+        self.input_size = model.input_size[1] # assume every input is a square image
+        self.input_range = model.input_range
+        self.input_space = model.input_space
+        self.resize_size = int(math.floor(self.input_size / SCALE_FACTOR))
+
+    def forward(self, x, **kwargs):
+        x = self.features(x) # 1x1024x7x7
+        s = x.size()[3] # 7 if input image is 224x224, 16 if input image is 512x512
+        x = F.relu(x, inplace=True) # 1x1024x7x7
+
+        pooling = kwargs['pooling']
+        if pooling == 'MAX':
+            x = F.max_pool2d(x, kernel_size=s, stride=1)
+            x = x.view(x.size(0), -1) # 1x1024
+        elif pooling == 'AVG':
+            x = F.avg_pool2d(x, kernel_size=s, stride=1) # 1x1024x1x1
+            x = x.view(x.size(0), -1) # 1x1024
+        elif pooling == 'LSE':
+            r = kwargs.lse_r
+            x_max = F.max_pool2d(x, kernel_size=s, stride=1)
+            p = ((1/r) * torch.log((1 / (s*s)) * torch.exp(r*(x - x_max)).sum(3).sum(2)))
+            x_max = x_max.view(x.size(0), -1)
+            x = x_max + p
+        else:
+            raise ValueError('Invalid pooling')
+
+        x = self.classifier(x) # 1x1000
+        return x
+
+    def extract(self, x):
+        return self.features(x)
+
+    # def count_params(self):
+    #     return sum(p.numel() for p in self.parameters() if p.requires_grad)
+
+def build(variant):
+    net = DenseNet(variant).cuda()
+    return net
+
+architect='densenet'
+
+

chestXray14/models/dpn.py

@@ -0,0 +1,52 @@
+import torchvision
+import torch.nn as nn
+import pretrainedmodels
+import torch.nn.functional as F
+from constant import SCALE_FACTOR
+import math
+from pretrainedmodels.models.dpn import adaptive_avgmax_pool2d
+
+class DPN(nn.Module):
+    
+    def __init__(self, variant):
+        super(DPN, self).__init__()
+        assert variant in ['dpn68', 'dpn68b', 'dpn92', 'dpn98', 'dpn131', 'dpn107']
+        
+        # load retrain model 
+        model = pretrainedmodels.__dict__[variant](num_classes=1000, pretrained='imagenet')
+        self.features = model.features
+        num_ftrs = model.classifier.in_channels
+        self.classifier = nn.Sequential(
+            nn.Conv2d(num_ftrs, 14, kernel_size=1, bias=True), # something wrong here abt dimension
+            nn.Sigmoid()
+        )
+        
+        # load other info
+        self.mean = model.mean
+        self.std = model.std
+        self.input_size = model.input_size[1] # assume every input is a square image
+        self.input_range = model.input_range
+        self.input_space = model.input_space
+        self.resize_size = int(math.floor(self.input_size / SCALE_FACTOR))
+         
+    def forward(self, x):
+        x = self.features(x) # 1x1024x7x7
+        if not self.training and self.test_time_tool:
+            x = F.avg_pool2d(x, kernel_size=7, stride=1)
+            x = self.classifier(x)
+            x = adaptive_avgmax_pool2d(out, pool_type='avgmax') # something wrong here abt dimension
+        else:
+            x = adaptive_avgmax_pool2d(x, pool_type='avg')
+            x = self.classifier(x)
+        return x
+        
+    def extract(self, x):
+        return self.features(x)
+
+def build(variant):
+    net = DPN(variant).cuda()
+    return net
+
+architect='dpn'
+    
+    

chestXray14/models/inception.py

@@ -0,0 +1,91 @@
+import torchvision
+import torch.nn as nn
+import pretrainedmodels
+import torch.nn.functional as F
+from collections import OrderedDict
+from constant import SCALE_FACTOR
+import math
+
+class InceptionNet(nn.Module):
+    
+    def __init__(self, variant):
+        super(InceptionNet, self).__init__()
+        assert variant in ['inceptionv4', 'inceptionv3', 'inceptionresnetv2']
+        
+        # load pretrain model
+        model = pretrainedmodels.__dict__[variant](num_classes=1000, pretrained='imagenet')
+        self.features = _get_features(model, variant)
+        num_ftrs = model.last_linear.in_features
+        self.classifier = nn.Sequential(
+            nn.Linear(num_ftrs, 14),
+            nn.Sigmoid()
+        )
+        
+        # load other info
+        self.mean = model.mean
+        self.std = model.std
+        self.input_size = model.input_size[1] # assume every input is a square image
+        self.input_range = model.input_range
+        self.input_space = model.input_space
+        self.resize_size = int(math.floor(self.input_size / SCALE_FACTOR))
+        
+    def forward(self, x):
+        x = self.features(x) # 1x1536x8x8
+        s = x.size()[3] # 8 if input image is 224x224
+        x = F.avg_pool2d(x, kernel_size=s, count_include_pad=False) # 1x1536x1x1, same for inceptionv4 and inceptionresnetv2
+        x = x.view(x.size(0), -1) # 1x1536
+        x = self.classifier(x) # 1x1000
+        return x
+        
+    def extract(self, x):
+        return self.features(x) # 1x1536x8x8
+    
+def build(variant):
+    net = InceptionNet(variant).cuda()
+    return net
+
+def _get_features(model, variant):
+    if variant == 'inceptionv4':
+        features =  model.features
+    elif variant == 'inceptionv3':
+        # TODO: Take a look on this
+        features = nn.Sequential(OrderedDict([
+            ('Conv2d_1a_3x3', model.Conv2d_1a_3x3),
+            ('Conv2d_2a_3x3', model.Conv2d_2a_3x3),
+            ('Conv2d_2b_3x3', model.Conv2d_2b_3x3),
+            ('max_pool2d_1', torch.nn.MaxPool2d(3, stride=2)),
+            ('Conv2d_3b_1x1', model.Conv2d_3b_1x1),
+            ('Conv2d_4a_3x3', model.Conv2d_4a_3x3),
+            ('max_pool2d_2', torch.nn.MaxPool2d(3, stride=2)),
+            ('Mixed_5b', model.Mixed_5b),
+            ('Mixed_5c', model.Mixed_5c),
+            ('Mixed_5d', model.Mixed_5d),
+            ('Mixed_6a', model.Mixed_6a),
+            ('Mixed_6b', model.Mixed_6b),
+            ('Mixed_6c', model.Mixed_6c),
+            ('Mixed_6d', model.Mixed_6b),
+            # ('Mixed_6c', model.Mixed_6c),
+        ]))
+    elif variant == 'inceptionresnetv2':
+        features = nn.Sequential(OrderedDict([
+            ('conv2d_1a', model.conv2d_1a),
+            ('conv2d_2a', model.conv2d_2a),
+            ('conv2d_2b', model.conv2d_2b),
+            ('maxpool_3a', model.maxpool_3a),
+            ('conv2d_3b', model.conv2d_3b),
+            ('conv2d_4a', model.conv2d_4a),
+            ('maxpool_5a', model.maxpool_5a),
+            ('mixed_5b', model.mixed_5b),
+            ('repeat', model.repeat),
+            ('mixed_6a', model.mixed_6a),
+            ('repeat_1', model.repeat_1),
+            ('mixed_7a', model.mixed_7a),
+            ('repeat_2', model.repeat_2),
+            ('block8', model.block8),
+            ('conv2d_7b', model.conv2d_7b)
+        ]))
+    else:
+        raise "Unknown variant"
+    return features
+
+architect='inception'

chestXray14/models/nasnet.py

@@ -0,0 +1,73 @@
+import torchvision
+import torch.nn as nn
+import pretrainedmodels
+import torch.nn.functional as F
+from collections import OrderedDict
+
+class Nasnet(nn.Module):
+    
+    def __init__(self, variant):
+        super(Nasnet, self).__init__()
+        assert variant in ['nasnetalarge']
+        
+        # load retrain model
+        self.model = pretrainedmodels.__dict__[variant](num_classes=1000, pretrained='imagenet')
+#         self.features = nn.Sequential(OrderedDict([
+#             ('conv0', model.conv0),
+#             ('cell_stem_0', model.cell_stem_0),
+#             ('cell_stem_1', model.cell_stem_1),
+#             ('cell_0', model.cell_0),
+#             ('cell_1', model.cell_1),
+#             ('cell_2', model.cell_2),
+#             ('cell_3', model.cell_3),
+#             ('cell_4', model.cell_4),
+#             ('cell_5', model.cell_5),
+#             ('reduction_cell_0', model.reduction_cell_0),
+#             ('cell_6', model.cell_6),
+#             ('cell_7', model.cell_7),
+#             ('cell_8', model.cell_8),
+#             ('cell_9', model.cell_9),
+#             ('cell_10', model.cell_10),
+#             ('cell_11', model.cell_11),
+#             ('reduction_cell_1', model.reduction_cell_1),
+#             ('cell_12', model.cell_6),
+#             ('cell_13', model.cell_7),
+#             ('cell_14', model.cell_8),
+#             ('cell_15', model.cell_9),
+#             ('cell_16', model.cell_10),
+#             ('cell_17', model.cell_11)
+
+#         ]))
+        num_ftrs = self.model.last_linear.in_features
+        self.model.last_linear = nn.Sequential(
+            nn.Linear(num_ftrs, 14),
+            nn.Sigmoid()
+        )
+        
+        # load other info
+        # load other info
+        self.mean = self.model.mean
+        self.std = self.model.std
+        self.input_size = self.model.input_size[1] # assume every input is a square image
+        self.input_range = self.model.input_range
+        self.input_space = self.model.input_space
+        self.resize_size = 354 # as in pretrainmodels repo
+        
+    def forward(self, x):
+        # x = self.features(x)  
+        # x = F.avg_pool2d(x, kernel_size=11, stride=1, padding=0)
+        # x = x.view(x.size(0), -1) 
+        # x = x.dropout(training=self.training)
+        # x = self.classifier(x) # 1x1000
+        # return x
+        return self.model.forward(x)
+        
+    def extract(self, x):
+        # return self.features(x)
+        return self.model.features(x)
+    
+def build(variant):
+    net = Nasnet(variant).cuda()
+    return net
+
+architect='nasnet'

chestXray14/models/resnet.py

@@ -0,0 +1,57 @@
+import torchvision
+import torch.nn as nn
+import pretrainedmodels
+import torch.nn.functional as F
+from collections import OrderedDict
+from constant import SCALE_FACTOR
+import math
+
+class Resnet(nn.Module):
+    
+    def __init__(self, variant):
+        super(Resnet, self).__init__()
+        assert variant in ['resnet50', 'resnet101', 'resnet152']
+        
+        # load retrain model
+        model = pretrainedmodels.__dict__[variant](num_classes=1000, pretrained='imagenet')
+        self.features = nn.Sequential(OrderedDict([
+            ('conv1', model.conv1),
+            ('bn1', model.bn1),
+            ('relu', model.relu),
+            ('maxpool', model.maxpool),
+            ('layer1', model.layer1),
+            ('layer2', model.layer2),
+            ('layer3', model.layer3),
+            ('layer4', model.layer4)
+        ]))
+        num_ftrs = model.last_linear.in_features
+        self.classifier = nn.Sequential(
+            nn.Linear(num_ftrs, 14),
+            nn.Sigmoid()
+        )
+        
+        # load other info
+        # load other info
+        self.mean = model.mean
+        self.std = model.std
+        self.input_size = model.input_size[1] # assume every input is a square image
+        self.input_range = model.input_range
+        self.input_space = model.input_space
+        self.resize_size = int(math.floor(self.input_size / SCALE_FACTOR))
+        
+    def forward(self, x):
+        x = self.features(x) # 1x2048x7x7
+        s = x.size()[3] # 7 if input image is 224x224, 16 if input image is 512x512
+        x = F.avg_pool2d(x, kernel_size=s, stride=1) # 1x2048x1x1
+        x = x.view(x.size(0), -1) # 1x2048
+        x = self.classifier(x) # 1x1000
+        return x
+        
+    def extract(self, x):
+        return self.features(x)
+    
+def build(variant):
+    net = Resnet(variant).cuda()
+    return net
+
+architect='resnet'

chestXray14/models/resnext.py

@@ -0,0 +1,48 @@
+import torchvision
+import torch.nn as nn
+import pretrainedmodels
+import torch.nn.functional as F
+from constant import SCALE_FACTOR
+import math
+
+class Resnext(nn.Module):
+    
+    def __init__(self, variant):
+        super(Resnext, self).__init__()
+        assert variant in ['resnext101_32x4d', 'resnext101_64x4d']
+        
+        # load retrain model 
+        model = pretrainedmodels.__dict__[variant](num_classes=1000, pretrained='imagenet')
+        self.features = model.features
+        num_ftrs = model.last_linear.in_features
+        self.classifier = nn.Sequential(
+            nn.Linear(num_ftrs, 14),
+            nn.Sigmoid()
+        )
+        
+        # load other info
+        self.mean = model.mean
+        self.std = model.std
+        self.input_size = model.input_size[1] # assume every input is a square image
+        self.input_range = model.input_range
+        self.input_space = model.input_space
+        self.resize_size = int(math.floor(self.input_size / SCALE_FACTOR))
+         
+    def forward(self, x):
+        x = self.features(x) # 
+        s = x.size()[3] # 7 if input image is 224x224, 16 if input image is 512x512
+        x = F.avg_pool2d(x, kernel_size=(7, 7), stride=(1, 1)) # 1x1024x1x1
+        x = x.view(x.size(0), -1) # 1x1024
+        x = self.classifier(x) # 1x1000
+        return x
+        
+    def extract(self, x):
+        return self.features(x)
+
+def build(variant):
+    net = Resnext(variant).cuda()
+    return net
+
+architect='resnext'
+    
+    

chestXray14/models/senet.py

@@ -0,0 +1,61 @@
+import torchvision
+import torch.nn as nn
+import pretrainedmodels
+import torch.nn.functional as F
+from collections import OrderedDict
+from constant import SCALE_FACTOR
+import math
+
+class Resnet(nn.Module):
+    
+    def __init__(self, variant):
+        super(Resnet, self).__init__()
+        assert variant in ['senet154', 'se_resnext101_32x4d', 'se_resnext50_32x4d', 'se_resnet152', 'se_resnet101', 'se_resnet50']
+        
+        # load retrain model
+        model = pretrainedmodels.__dict__[variant](num_classes=1000, pretrained='imagenet')
+        self.features = nn.Sequential(OrderedDict([
+            ('layer0', model.layer0),
+            ('layer1', model.layer1),
+            ('layer2', model.layer2),
+            ('layer3', model.layer3),
+            ('layer4', model.layer4)
+        ]))
+        '''
+        Dropout
+        - For SENet154: 0.2
+        - For SE-ResNet models: None
+        - For SE-ResNeXt models: None
+        '''
+        self.dropout = model.dropout
+        num_ftrs = model.last_linear.in_features
+        self.classifier = nn.Sequential(
+            nn.Linear(num_ftrs, 14),
+            nn.Sigmoid()
+        )
+        
+        # load other info
+        # load other info
+        self.mean = model.mean
+        self.std = model.std
+        self.input_size = model.input_size[1] # assume every input is a square image
+        self.input_range = model.input_range
+        self.input_space = model.input_space
+        self.resize_size = int(math.floor(self.input_size / SCALE_FACTOR))
+        
+    def forward(self, x):
+        x = self.features(x) # 1x2048x7x7
+        s = x.size()[3] # 7 if input image is 224x224, 16 if input image is 512x512
+        x = F.avg_pool2d(x, kernel_size=s, stride=1) # 1x2048x1x1
+        x = x.view(x.size(0), -1) # 1x2048
+        x = self.classifier(x) # 1x1000
+        return x
+        
+    def extract(self, x):
+        return self.features(x)
+    
+def build(variant):
+    net = Resnet(variant).cuda()
+    return net
+
+architect='senet'

chestXray14/test.py

@@ -0,0 +1,93 @@
+from PIL import Image
+import numpy as np
+import matplotlib.pyplot as plt
+import cv2
+import os
+
+from chexnet import ChexNet
+from unet import Unet
+from heatmap import HeatmapGenerator
+from constant import IMAGENET_MEAN, IMAGENET_STD, CLASS_NAMES
+
+import sys
+script_dir = os.path.dirname(os.path.abspath(__file__))
+imgto3d_path = os.path.join(script_dir, '.')
+sys.path.append(imgto3d_path)
+
+from chestXray_utils import blend_segmentation
+import torch
+import pandas as pd
+
+
+output_dir = "pages/images"
+os.makedirs(output_dir, exist_ok=True)
+
+unet_model = '20190211-101020'
+chexnet_model = '20180429-130928'
+DISEASES = np.array(CLASS_NAMES)
+
+
+# Initialize models
+unet = Unet(trained=True, model_name=unet_model)
+chexnet = ChexNet(trained=True, model_name=chexnet_model)
+heatmap_generator = HeatmapGenerator(chexnet, mode='cam')
+unet.eval()
+chexnet.eval()
+
+
+def process_image(image_path):
+    image = Image.open(image_path).convert('RGB')
+
+    # Run through net
+    (t, l, b, r), mask = unet.segment(image)
+    cropped_image = image.crop((l, t, r, b))
+    prob = chexnet.predict(cropped_image)
+
+    # Save segmentation result
+    blended = blend_segmentation(image, mask)
+    blended = (blended - blended.min()) / (blended.max() - blended.min())  # Normalize to [0, 1]
+    blended = (blended * 255).astype(np.uint8)  # Convert to 0-255 range for cv2
+    cv2.rectangle(blended, (l, t), (r, b), (255, 0, 0), 5)  # Color in BGR format for cv2
+    segment_result_path = os.path.join(output_dir, 'segment_result.png')
+    plt.imsave(segment_result_path, blended)
+
+    # Save CAM result
+    w, h = cropped_image.size
+    heatmap, _ = heatmap_generator.from_prob(prob, w, h)
+
+    # Resize the heatmap to match the original image dimensions
+    heatmap_resized = cv2.resize(heatmap, (image.width, image.height))
+    heatmap_resized = np.repeat(heatmap_resized[:, :, np.newaxis], 3, axis=2)  # Ensure it has 3 channels
+
+    heatmap_resized = ((heatmap_resized - heatmap_resized.min()) * (
+                1 / (heatmap_resized.max() - heatmap_resized.min())) * 255).astype(np.uint8)
+
+    cam = cv2.applyColorMap(heatmap_resized, cv2.COLORMAP_JET)
+    cam = cv2.resize(cam, (image.width, image.height))  # Ensure cam has same dimensions as image
+    cam = cv2.addWeighted(cam, 0.4, np.array(image), 0.6, 0)  # Combine heatmap with the original image
+    cam_result_path = os.path.join(output_dir, 'cam_result.png')
+    print("a",cam_result_path)
+    cv2.imwrite(cam_result_path, cam)
+
+    # Top-10 diseases
+    idx = np.argsort(-prob)
+    top_prob = prob[idx[:10]]
+    top_prob = [f'{x:.3}' for x in top_prob]
+    top_disease = DISEASES[idx[:10]]
+    prediction = dict(zip(top_disease, top_prob))
+
+    result = {'result': prediction}
+    df = pd.DataFrame(result['result'].items(), columns=['Disease', 'Probability'])
+    output_file = 'prediction_results.csv'
+    output_file_path = os.path.join(output_dir, output_file)
+    df.to_csv(output_file_path, index=False)
+
+    return result, segment_result_path, cam_result_path
+
+
+# if __name__ == '__main__':
+#     image_path = r'E:\NLP\KN2024\chestX-ray-14\src\fibrosis.jpg'  # Replace with your image path
+#     result, segment_result_path, cam_result_path = process_image(image_path)
+#     print("Prediction Results:", result)
+#     print(f"Segmentation Result Saved to: {segment_result_path}")
+#     print(f"CAM Result Saved to: {cam_result_path}")

chestXray14/unet.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1768ae75ce08bc7cb1d338e26648887e5c4d79ff0b97ab89831789666ac0ed6
+size 264784168

chestXray14/unet.py

@@ -0,0 +1,142 @@
+import torch.nn as nn
+import torch.nn.functional as F
+import torch
+from layers import SaveFeature
+import pretrainedmodels
+from torchvision.models import resnet34, resnet50, resnet101, resnet152
+from pathlib import Path
+from torchvision.models.resnet import conv3x3, BasicBlock, Bottleneck
+import skimage
+from scipy import ndimage
+import numpy as np
+import torchvision.transforms as transforms
+import cv2
+from constant import IMAGENET_MEAN, IMAGENET_STD
+
+device="cuda" if torch.cuda.is_available() else "cpu"
+class UpBlock(nn.Module):
+    expansion = 1
+
+    def __init__(self, inplanes, planes, expansion=1):
+        super().__init__()
+        inplanes = inplanes * expansion
+        planes = planes * expansion
+        self.upconv = nn.ConvTranspose2d(inplanes, planes, 2, 2, 0)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.relu = nn.ReLU(inplace=True)
+        self.conv1 = conv3x3(inplanes, planes)
+        self.bn2 = nn.BatchNorm2d(planes)
+
+    def forward(self, u, x):
+        up = self.relu(self.bn1(self.upconv(u)))
+        out = torch.cat([x, up], dim=1)  # cat along channel
+        out = self.relu(self.bn2(self.conv1(out)))
+        return out
+
+class UpLayer(nn.Module):
+
+    def __init__(self, block, inplanes, planes, blocks):
+        super().__init__()
+        self.up = UpBlock(inplanes, planes, block.expansion)
+        layers = [block(planes * block.expansion, planes) for _ in range(1, blocks)]
+        self.conv = nn.Sequential(*layers)
+
+    def forward(self, u, x):
+        x = self.up(u, x)
+        x = self.conv(x)
+        return x
+
+from pathlib import Path
+
+
+
+class Unet(nn.Module):
+    tfm = transforms.Compose([
+        transforms.Resize((256, 256)),
+        transforms.ToTensor(),
+        transforms.Normalize(IMAGENET_MEAN, IMAGENET_STD)
+    ])
+
+    def __init__(self, trained=False, model_name=None):
+        super().__init__()
+        self.layers = [3, 4, 6]
+        self.block = Bottleneck
+        if trained:
+            assert model_name is not None
+            self.load_model(model_name)
+        else:
+            self.load_pretrained()
+
+    def cut_model(self, model, cut):
+        return list(model.children())[:cut]
+
+    def load_model(self, model_name):
+        resnet = resnet50(False)
+        self.backbone = nn.Sequential(*self.cut_model(resnet, 8))
+        self.init_head()
+
+        model_path = Path(__file__).parent / 'unet.h5'
+        state_dict = torch.load(model_path, map_location=torch.device(device))
+        self.load_state_dict(state_dict)
+
+    def load_pretrained(self, torch=False):
+        if torch:
+            resnet = resnet50(True)
+        else:
+            resnet = pretrainedmodels.__dict__['resnet50']()
+        self.backbone = nn.Sequential(*self.cut_model(resnet, 8))
+        self.init_head()
+
+    def init_head(self):
+        self.sfs = [SaveFeature(self.backbone[i]) for i in [2, 4, 5, 6]]
+        self.up_layer1 = UpLayer(self.block, 512, 256, self.layers[-1])
+        self.up_layer2 = UpLayer(self.block, 256, 128, self.layers[-2])
+        self.up_layer3 = UpLayer(self.block, 128, 64, self.layers[-3])
+
+        self.map = conv3x3(64 * self.block.expansion, 64)  # 64e -> 64
+        self.conv = conv3x3(128, 64)
+        self.bn_conv = nn.BatchNorm2d(64)
+        self.up_conv = nn.ConvTranspose2d(64, 1, 2, 2, 0)
+        self.bn_up = nn.BatchNorm2d(1)
+
+    def forward(self, x):
+        x = F.relu(self.backbone(x))
+        x = self.up_layer1(x, self.sfs[3].features)
+        x = self.up_layer2(x, self.sfs[2].features)
+        x = self.up_layer3(x, self.sfs[1].features)
+        x = self.map(x)
+        x = F.interpolate(x, scale_factor=2)
+        x = torch.cat([self.sfs[0].features, x], dim=1)
+        x = F.relu(self.bn_conv(self.conv(x)))
+        x = F.relu(self.bn_up(self.up_conv(x)))
+        return x
+
+    def close(self):
+        for sf in self.sfs:
+            sf.remove()
+
+    def segment(self, image):
+        """
+        image: cropped CXR PIL Image (h, w, 3)
+        """
+        kernel = np.ones((10, 10))
+        iw, ih = image.size
+
+        image_tensor = self.tfm(image).unsqueeze(0).to(next(self.parameters()).device)
+        with torch.no_grad():
+            py = torch.sigmoid(self(image_tensor))
+        py = (py[0].cpu() > 0.5).type(torch.FloatTensor)  # 1, 256, 256
+
+        mask = py[0].numpy()
+        mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
+        mask = cv2.resize(mask, (iw, ih))
+        slice_y, slice_x = ndimage.find_objects(mask, 1)[0]
+        h, w = slice_y.stop - slice_y.start, slice_x.stop - slice_x.start
+
+        nw, nh = int(w / .875), int(h / .875)
+        dw, dh = (nw - w) // 2, (nh - h) // 2
+        t = max(slice_y.start - dh, 0)
+        l = max(slice_x.start - dw, 0)
+        b = min(slice_y.stop + dh, ih)
+        r = min(slice_x.stop + dw, iw)
+        return (t, l, b, r), mask

chexnet.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d8e4e04574c0044faff5630f942ac873681be8914ba4968b6b2070362b1a340
+size 28360928

config.yaml

@@ -0,0 +1,38 @@
+cond_image_size: 512
+
+image_tokenizer_cls: tsr.models.tokenizers.image.DINOSingleImageTokenizer
+image_tokenizer:
+  pretrained_model_name_or_path: "facebook/dino-vitb16"
+
+tokenizer_cls: tsr.models.tokenizers.triplane.Triplane1DTokenizer
+tokenizer:
+  plane_size: 32
+  num_channels: 1024
+
+backbone_cls: tsr.models.transformer.transformer_1d.Transformer1D
+backbone:
+  in_channels: ${tokenizer.num_channels}
+  num_attention_heads: 16
+  attention_head_dim: 64
+  num_layers: 16
+  cross_attention_dim: 768
+
+post_processor_cls: tsr.models.network_utils.TriplaneUpsampleNetwork
+post_processor:
+  in_channels: 1024
+  out_channels: 40
+
+decoder_cls: tsr.models.network_utils.NeRFMLP
+decoder:
+  in_channels: 120 # 3 * 40
+  n_neurons: 64
+  n_hidden_layers: 9
+  activation: silu
+
+renderer_cls: tsr.models.nerf_renderer.TriplaneNeRFRenderer
+renderer:
+  radius: 0.87 # slightly larger than 0.5 * sqrt(3)
+  feature_reduction: concat
+  density_activation: exp
+  density_bias: -1.0
+  num_samples_per_ray: 128

image_to_3D/3d_model_requirements.txt

@@ -0,0 +1,14 @@
+omegaconf==2.3.0
+Pillow==10.1.0
+einops==0.7.0
+git+https://github.com/tatsy/torchmcubes.git
+transformers==4.35.0
+trimesh==4.0.5
+rembg
+huggingface-hub
+imageio[ffmpeg]
+gradio
+xatlas==0.0.9
+moderngl==5.10.0
+torch==2.0.0
+setuptools==68.2.0

image_to_3D/__init__.py

@@ -0,0 +1,12 @@
+from attention import *
+from basic_transformer_block import *
+from image import *
+from isosurface import *
+from nerf_renderer import *
+from network_utils import *
+from rotate import *
+from run import *
+from transformer_1d import *
+from triplane import *
+from ui import *
+from x3D_utils import *