files

PSSL_app.py

@@ -0,0 +1,566 @@
+import gradio as gr
+import os
+import matplotlib.pyplot as plt
+from scipy.integrate import odeint
+import torch
+from torch.utils import data
+from torch.utils.data import DataLoader, Dataset
+from torch import nn, optim
+import os
+from skimage.transform import rescale, resize
+from torch import nn, optim
+import torch.nn.functional as F
+from torch.utils.data import Subset
+from scipy.interpolate import interp1d
+
+#for pvloop simulator:
+import pandas as pd
+from scipy.integrate import odeint 
+from scipy import interpolate
+from scipy.interpolate import RegularGridInterpolator
+from matplotlib import pyplot
+import sys
+import numpy as np
+import collections
+import pandas
+import skimage.draw
+import torchvision
+import echonet
+
+#odesolver:
+from torch.storage import T
+import argparse
+import time
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+sequences_all = []
+info_data_all = []
+path = '/Users/FDean/Desktop/Physics_Informed_Transfer_Learning/EchoNet-Dynamic'
+output_path = '/Users/FDean/Desktop/Physics_Informed_Transfer_Learning'
+
+class Echo(torchvision.datasets.VisionDataset):
+    """EchoNet-Dynamic Dataset.
+    Args:
+        root (string): Root directory of dataset (defaults to `echonet.config.DATA_DIR`)
+        split (string): One of {``train'', ``val'', ``test'', ``all'', or ``external_test''}
+        target_type (string or list, optional): Type of target to use,
+            ``Filename'', ``EF'', ``EDV'', ``ESV'', ``LargeIndex'',
+            ``SmallIndex'', ``LargeFrame'', ``SmallFrame'', ``LargeTrace'',
+            or ``SmallTrace''
+            Can also be a list to output a tuple with all specified target types.
+            The targets represent:
+                ``Filename'' (string): filename of video
+                ``EF'' (float): ejection fraction
+                ``EDV'' (float): end-diastolic volume
+                ``ESV'' (float): end-systolic volume
+                ``LargeIndex'' (int): index of large (diastolic) frame in video
+                ``SmallIndex'' (int): index of small (systolic) frame in video
+                ``LargeFrame'' (np.array shape=(3, height, width)): normalized large (diastolic) frame
+                ``SmallFrame'' (np.array shape=(3, height, width)): normalized small (systolic) frame
+                ``LargeTrace'' (np.array shape=(height, width)): left ventricle large (diastolic) segmentation
+                    value of 0 indicates pixel is outside left ventricle
+                             1 indicates pixel is inside left ventricle
+                ``SmallTrace'' (np.array shape=(height, width)): left ventricle small (systolic) segmentation
+                    value of 0 indicates pixel is outside left ventricle
+                             1 indicates pixel is inside left ventricle
+            Defaults to ``EF''.
+        mean (int, float, or np.array shape=(3,), optional): means for all (if scalar) or each (if np.array) channel.
+            Used for normalizing the video. Defaults to 0 (video is not shifted).
+        std (int, float, or np.array shape=(3,), optional): standard deviation for all (if scalar) or each (if np.array) channel.
+            Used for normalizing the video. Defaults to 0 (video is not scaled).
+        length (int or None, optional): Number of frames to clip from video. If ``None'', longest possible clip is returned.
+            Defaults to 16.
+        period (int, optional): Sampling period for taking a clip from the video (i.e. every ``period''-th frame is taken)
+            Defaults to 2.
+        max_length (int or None, optional): Maximum number of frames to clip from video (main use is for shortening excessively
+            long videos when ``length'' is set to None). If ``None'', shortening is not applied to any video.
+            Defaults to 250.
+        clips (int, optional): Number of clips to sample. Main use is for test-time augmentation with random clips.
+            Defaults to 1.
+        pad (int or None, optional): Number of pixels to pad all frames on each side (used as augmentation).
+            and a window of the original size is taken. If ``None'', no padding occurs.
+            Defaults to ``None''.
+        noise (float or None, optional): Fraction of pixels to black out as simulated noise. If ``None'', no simulated noise is added.
+            Defaults to ``None''.
+        target_transform (callable, optional): A function/transform that takes in the target and transforms it.
+        external_test_location (string): Path to videos to use for external testing.
+    """
+
+    def __init__(self, root=None,
+                 split="train", target_type="EF",
+                 mean=0., std=1.,
+                 length=16, period=2,
+                 max_length=250,
+                 clips=1,
+                 pad=None,
+                 noise=None,
+                 target_transform=None,
+                 external_test_location=None):
+        if root is None:
+            root = path
+
+        super().__init__(root, target_transform=target_transform)
+
+        self.split = split.upper()
+        if not isinstance(target_type, list):
+            target_type = [target_type]
+        self.target_type = target_type
+        self.mean = mean
+        self.std = std
+        self.length = length
+        self.max_length = max_length
+        self.period = period
+        self.clips = clips
+        self.pad = pad
+        self.noise = noise
+        self.target_transform = target_transform
+        self.external_test_location = external_test_location
+
+        self.fnames, self.outcome = [], []
+
+        if self.split == "EXTERNAL_TEST":
+            self.fnames = sorted(os.listdir(self.external_test_location))
+        else:
+            # Load video-level labels
+            with open(os.path.join(self.root, "FileList.csv")) as f:
+                data = pandas.read_csv(f)
+            data["Split"].map(lambda x: x.upper())
+
+            if self.split != "ALL":
+                data = data[data["Split"] == self.split]
+
+            self.header = data.columns.tolist()
+            self.fnames = data["FileName"].tolist()
+            self.fnames = [fn + ".avi" for fn in self.fnames if os.path.splitext(fn)[1] == ""]  # Assume avi if no suffix
+            self.outcome = data.values.tolist()
+
+            # Check that files are present
+            """
+            missing = set(self.fnames) - set(os.listdir(os.path.join(self.root, "Videos")))
+            if len(missing) != 0:
+                print("{} videos could not be found in {}:".format(len(missing), os.path.join(self.root, "Videos")))
+                for f in sorted(missing):
+                    print("\t", f)
+                raise FileNotFoundError(os.path.join(self.root, "Videos", sorted(missing)[0]))
+            """
+
+            # Load traces
+            self.frames = collections.defaultdict(list)
+            self.trace = collections.defaultdict(_defaultdict_of_lists)
+
+            with open(os.path.join(self.root, "VolumeTracings.csv")) as f:
+                header = f.readline().strip().split(",")
+                assert header == ["FileName", "X1", "Y1", "X2", "Y2", "Frame"]
+
+                for line in f:
+                    filename, x1, y1, x2, y2, frame = line.strip().split(',')
+                    x1 = float(x1)
+                    y1 = float(y1)
+                    x2 = float(x2)
+                    y2 = float(y2)
+                    frame = int(frame)
+                    if frame not in self.trace[filename]:
+                        self.frames[filename].append(frame)
+                    self.trace[filename][frame].append((x1, y1, x2, y2))
+            for filename in self.frames:
+                for frame in self.frames[filename]:
+                    self.trace[filename][frame] = np.array(self.trace[filename][frame])
+
+            # A small number of videos are missing traces; remove these videos
+            keep = [len(self.frames[f]) >= 2 for f in self.fnames]
+            self.fnames = [f for (f, k) in zip(self.fnames, keep) if k]
+            self.outcome = [f for (f, k) in zip(self.outcome, keep) if k]
+
+    def __getitem__(self, index):
+        # Find filename of video
+        if self.split == "EXTERNAL_TEST":
+            video = os.path.join(self.external_test_location, self.fnames[index])
+        elif self.split == "CLINICAL_TEST":
+            video = os.path.join(self.root, "ProcessedStrainStudyA4c", self.fnames[index])
+        else:
+            video = os.path.join(self.root, "Videos", self.fnames[index])
+
+        # Load video into np.array
+        video = echonet.utils.loadvideo(video).astype(np.float32)
+
+        # Add simulated noise (black out random pixels)
+        # 0 represents black at this point (video has not been normalized yet)
+        if self.noise is not None:
+            n = video.shape[1] * video.shape[2] * video.shape[3]
+            ind = np.random.choice(n, round(self.noise * n), replace=False)
+            f = ind % video.shape[1]
+            ind //= video.shape[1]
+            i = ind % video.shape[2]
+            ind //= video.shape[2]
+            j = ind
+            video[:, f, i, j] = 0
+
+        # Apply normalization
+        if isinstance(self.mean, (float, int)):
+            video -= self.mean
+        else:
+            video -= self.mean.reshape(3, 1, 1, 1)
+
+        if isinstance(self.std, (float, int)):
+            video /= self.std
+        else:
+            video /= self.std.reshape(3, 1, 1, 1)
+
+        # Set number of frames
+        c, f, h, w = video.shape
+        if self.length is None:
+            # Take as many frames as possible
+            length = f // self.period
+        else:
+            # Take specified number of frames
+            length = self.length
+
+        if self.max_length is not None:
+            # Shorten videos to max_length
+            length = min(length, self.max_length)
+
+        if f < length * self.period:
+            # Pad video with frames filled with zeros if too short
+            # 0 represents the mean color (dark grey), since this is after normalization
+            video = np.concatenate((video, np.zeros((c, length * self.period - f, h, w), video.dtype)), axis=1)
+            c, f, h, w = video.shape  # pylint: disable=E0633
+
+        if self.clips == "all":
+            # Take all possible clips of desired length
+            start = np.arange(f - (length - 1) * self.period)
+        else:
+            # Take random clips from video
+            start = np.random.choice(f - (length - 1) * self.period, self.clips)
+
+        # Gather targets
+        target = []
+        for t in self.target_type:
+            key = self.fnames[index]
+            if t == "Filename":
+                target.append(self.fnames[index])
+            elif t == "LargeIndex":
+                # Traces are sorted by cross-sectional area
+                # Largest (diastolic) frame is last
+                target.append(int(self.frames[key][-1]))
+            elif t == "SmallIndex":
+                # Largest (diastolic) frame is first
+                target.append(int(self.frames[key][0]))
+            elif t == "LargeFrame":
+                target.append(video[:, self.frames[key][-1], :, :])
+            elif t == "SmallFrame":
+                target.append(video[:, self.frames[key][0], :, :])
+            elif t in ["LargeTrace", "SmallTrace"]:
+                if t == "LargeTrace":
+                    t = self.trace[key][self.frames[key][-1]]
+                else:
+                    t = self.trace[key][self.frames[key][0]]
+                x1, y1, x2, y2 = t[:, 0], t[:, 1], t[:, 2], t[:, 3]
+                x = np.concatenate((x1[1:], np.flip(x2[1:])))
+                y = np.concatenate((y1[1:], np.flip(y2[1:])))
+
+                r, c = skimage.draw.polygon(np.rint(y).astype(np.int), np.rint(x).astype(np.int), (video.shape[2], video.shape[3]))
+                mask = np.zeros((video.shape[2], video.shape[3]), np.float32)
+                mask[r, c] = 1
+                target.append(mask)
+            else:
+                if self.split == "CLINICAL_TEST" or self.split == "EXTERNAL_TEST":
+                    target.append(np.float32(0))
+                else:
+                    target.append(np.float32(self.outcome[index][self.header.index(t)]))
+
+        if target != []:
+            target = tuple(target) if len(target) > 1 else target[0]
+            if self.target_transform is not None:
+                target = self.target_transform(target)
+
+        # Select clips from video
+        video = tuple(video[:, s + self.period * np.arange(length), :, :] for s in start)
+        if self.clips == 1:
+            video = video[0]
+        else:
+            video = np.stack(video)
+
+        if self.pad is not None:
+            # Add padding of zeros (mean color of videos)
+            # Crop of original size is taken out
+            # (Used as augmentation)
+            c, l, h, w = video.shape
+            temp = np.zeros((c, l, h + 2 * self.pad, w + 2 * self.pad), dtype=video.dtype)
+            temp[:, :, self.pad:-self.pad, self.pad:-self.pad] = video  # pylint: disable=E1130
+            i, j = np.random.randint(0, 2 * self.pad, 2)
+            video = temp[:, :, i:(i + h), j:(j + w)]
+
+        return video, target
+
+    def __len__(self):
+        return len(self.fnames)
+
+    def extra_repr(self) -> str:
+        """Additional information to add at end of __repr__."""
+        lines = ["Target type: {target_type}", "Split: {split}"]
+        return '\n'.join(lines).format(**self.__dict__)
+
+
+def _defaultdict_of_lists():
+    """Returns a defaultdict of lists.
+    This is used to avoid issues with Windows (if this function is anonymous,
+    the Echo dataset cannot be used in a dataloader).
+    """
+
+    return collections.defaultdict(list)
+## 
+print("Done loading training data!")
+# define normalization layer to make sure output xi in an interval [ai, bi]:
+# define normalization layer to make sure output xi in an interval [ai, bi]:
+
+
+class IntervalNormalizationLayer(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        # new_output = [Tc, start_p, Emax, Emin, Rm, Ra, Vd]
+        self.a = torch.tensor([0.4, 0., 0.5, 0.02, 0.005, 0.0001, 4.], dtype=torch.float32) #HR in 20-200->Tc in [0.3, 4]
+        self.b = torch.tensor([1.7, 280., 3.5, 0.1, 0.1, 0.25, 16.], dtype=torch.float32)
+        #taken out (initial conditions): a: 20, 5, 50; b: 400, 20, 100
+    def forward(self, inputs):
+        sigmoid_output = torch.sigmoid(inputs)
+        scaled_output = sigmoid_output * (self.b - self.a) + self.a
+        return scaled_output
+
+class NEW3DCNN(nn.Module):
+    def __init__(self, num_parameters):
+        super(NEW3DCNN, self).__init__()
+        
+        self.conv1      = nn.Conv3d(3, 8, kernel_size=3, padding=1)
+        self.batchnorm1 = nn.BatchNorm3d(8)
+        self.conv2      = nn.Conv3d(8, 16, kernel_size=3, padding=1)
+        self.batchnorm2 = nn.BatchNorm3d(16)
+        self.conv3      = nn.Conv3d(16, 32, kernel_size=3, padding=1)
+        self.batchnorm3 = nn.BatchNorm3d(32)
+        self.conv4      = nn.Conv3d(32, 64, kernel_size=3, padding=1)
+        self.batchnorm4 = nn.BatchNorm3d(64)
+        self.conv5      = nn.Conv3d(64, 128, kernel_size=3, padding=1)
+        self.batchnorm5 = nn.BatchNorm3d(128)
+        self.pool       = nn.AdaptiveAvgPool3d(1)
+        self.fc1        = nn.Linear(128, 512)
+        self.fc2        = nn.Linear(512, num_parameters)
+        self.norm1      = IntervalNormalizationLayer()
+        
+    def forward(self, x):
+        x = F.relu(self.batchnorm1(self.conv1(x)))
+        x = F.max_pool3d(x, kernel_size=2, stride=2)
+        x = F.relu(self.batchnorm2(self.conv2(x)))
+        x = F.max_pool3d(x, kernel_size=2, stride=2)
+        x = F.relu(self.batchnorm3(self.conv3(x)))
+        x = F.max_pool3d(x, kernel_size=2, stride=2)
+        x = F.relu(self.batchnorm4(self.conv4(x)))
+        x = F.max_pool3d(x, kernel_size=2, stride=2)
+        x = F.relu(self.batchnorm5(self.conv5(x)))
+        x = self.pool(x)
+        x = x.view(x.size(0), -1)
+        x = F.relu(self.fc1(x))
+        x = self.fc2(x)
+        x = self.norm1(x)
+
+        return x
+
+
+# Define a neural network with one hidden layer
+class Interpolator(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.fc1 = nn.Linear(6, 250).double()
+        self.fc2 = nn.Linear(250, 2).double()
+
+    def forward(self, x):
+        x = torch.relu(self.fc1(x))
+        x = self.fc2(x)
+        return x
+
+# Initialize the neural network
+net = Interpolator()
+net.load_state_dict(torch.load('/Users/FDean/Desktop/Physics_Informed_Transfer_Learning/final_model_weights/interp6_7param_weight.pt'))
+print("Done loading interpolator!")
+
+weights_path = '/Users/FDean/Desktop/Physics_Informed_Transfer_Learning/final_model_weights/202_full_echonet_7param_Vloss_epoch_200_lr_0.001_weight_best_model.pt'
+model = NEW3DCNN(num_parameters = 7)
+model.load_state_dict(torch.load(weights_path))
+model.to(device)
+
+## PV loops
+
+#returns Plv at time t using Elastance(t) and Vlv(t)-Vd=x1
+def Plv(volume, Emax, Emin, t, Tc, Vd):
+    return Elastance(Emax,Emin,t, Tc)*(volume - Vd)
+
+#returns Elastance(t)
+def Elastance(Emax,Emin, t, Tc):
+    t = t-int(t/Tc)*Tc #can remove this if only want 1st ED (and the 1st ES before)
+    tn = t/(0.2+0.15*Tc)
+    return (Emax-Emin)*1.55*(tn/0.7)**1.9/((tn/0.7)**1.9+1)*1/((tn/1.17)**21.9+1) + Emin
+
+def solve_ODE_for_volume(Rm, Ra, Emax, Emin, Vd, Tc, start_v, t):
+    
+    # the ODE from Simaan et al 2008
+    def heart_ode(y, t, Rs, Rm, Ra, Rc, Ca, Cs, Cr, Ls, Emax, Emin, Tc):
+        x1, x2, x3, x4, x5 = y #here y is a vector of 5 values (not functions), at time t, used for getting (dy/dt)(t)
+        P_lv = Plv(x1+Vd,Emax,Emin,t,Tc,Vd)
+        dydt = [r(x2-P_lv)/Rm-r(P_lv-x4)/Ra, (x3-x2)/(Rs*Cr)-r(x2-P_lv)/(Cr*Rm), (x2-x3)/(Rs*Cs)+x5/Cs, -x5/Ca+r(P_lv-x4)/(Ca*Ra), (x4-x3-Rc*x5)/Ls]
+        return dydt
+
+    # RELU for diodes
+    def r(u):
+        return max(u, 0.)
+    
+    # Define fixed parameters
+    Rs = 1.0
+    Rc = 0.0398
+    Ca = 0.08
+    Cs = 1.33
+    Cr = 4.400
+    Ls = 0.0005
+    startp = 75.
+
+    # Initial conditions
+    start_pla = float(start_v*Elastance(Emax, Emin, 0, Tc))
+    start_pao = startp
+    start_pa = start_pao
+    start_qt = 0 #aortic flow is Q_T and is 0 at ED, also see Fig5 in simaan2008dynamical
+    y0 = [start_v, start_pla, start_pa, start_pao, start_qt]
+
+    # Solve
+    sol = odeint(heart_ode, y0, t, args = (Rs, Rm, Ra, Rc, Ca, Cs, Cr, Ls, Emax, Emin, Tc)) #t: list of values
+
+    # volume is the first state variable plus theoretical zero pressure volume
+    volumes = np.array(sol[:, 0]) + Vd
+    
+    return volumes
+
+def pvloop_simulator(Rm, Ra, Emax, Emin, Vd, Tc, start_v):
+    
+
+    # Define initial parameters
+    init_Emax = Emax # 3.0 # .5 to 3.5
+    init_Emin = Emin # 0.04 # .02 to .1
+    # init_Tc = Tc # .4 # .4 to 1.7
+    init_Vd = Vd # 10.0 # 0 to 25
+    
+    # DUMMY VOLUME
+    # def volume(t, Tc):
+        # return 50*np.sin(2 * np.pi * t*(1/Tc))+100
+    
+    # SOLVE the ODE model for the VOLUME CURVE
+    N = 100
+    t =  np.linspace(0, Tc*N, int(60000*N)) #np.linspace(1, 100, 1000000)
+    volumes = solve_ODE_for_volume(Rm, Ra, Emax, Emin, Vd, Tc, start_v, t)
+    
+    # FUNCTIONS for PRESSURE CURVE
+    vectorized_Elastance = np.vectorize(Elastance)
+    vectorized_Plv = np.vectorize(Plv)
+
+    def pressure(t, volume, Emax, Emin, Tc, Vd):
+        return vectorized_Plv(volume, Emax, Emin, t, Tc, Vd)
+
+    # calculate PRESSURE
+    pressures = pressure(t, volumes, init_Emax, init_Emin, Tc, init_Vd)
+    
+    # Create the figure and the loop that we will manipulate
+    fig, ax = plt.subplots()
+    plt.ylim((0,280))
+    plt.xlim((0,280))
+    line = ax.plot(volumes[(N-2)*60000:(N)*60000], pressures[(N-2)*60000:(N)*60000], lw=1)
+    #print(line)
+    line = line[0]
+    #print(line)
+    
+    fig.suptitle('Predicted PI-SSL LV Pressure Volume Loop', fontsize=16)
+    #plt.rcParams['fig.suptitle'] = -2.0
+    #ax.set_title(f'Mitral valve circuit resistance (Rm): {Rm} mmHg*s/ml \n Aortic valve circuit resistance (Ra): {Ra} mmHg*s/ml', fontsize=6)
+    ax.set_xlabel('LV Volume (ml)')
+    ax.set_ylabel('LV Pressure (mmHg)')
+
+    # adjust the main plot to make room for the sliders
+    fig.subplots_adjust(left=0.25, bottom=0.25)
+    
+    return plt, Rm, Ra, Emax, Emin, Vd, Tc, start_v
+
+def pvloop_simulator_plot_only(Rm, Ra, Emax, Emin, Vd, Tc, start_v):
+    plot,_,_,_,_,_,_,_ =pvloop_simulator(Rm, Ra, Emax, Emin, Vd, Tc, start_v)
+    return plot
+
+## Demo 
+
+def generate_example():
+        # get random input
+        data_path = '/Users/FDean/Desktop/Physics_Informed_Transfer_Learning/EchoNet-Dynamic'
+        image_data = Echo(root = data_path, split = 'all', target_type=['Filename','LargeIndex','SmallIndex'])
+        image_loaded_data = DataLoader(image_data, batch_size=1, shuffle=True)
+        val_data = next(iter(image_loaded_data))
+        #create_echo_clip(val_data,'test')
+        val_seq = val_data[0]
+        filename = val_data[1][0][0]
+        video = os.path.join(os.getcwd(), f"EchoNet-Dynamic/Videos/{filename}")
+        val_tensor = torch.tensor(val_seq, dtype=torch.float32) 
+        results = model(val_tensor)
+
+        plot, Rm, Ra, Emax, Emin, Vd,Tc, start_v = pvloop_simulator(Rm=round(results[0][4].item(),2), Ra=round(results[0][5].item(),2), Emax=results[0][2].item(), Emin=round(results[0][3].item(),2), Vd=round(results[0][6].item(),2), Tc=round(results[0][0].item(),2), start_v=round(results[0][1].item(),2))
+    
+        return video, plot, Rm, Ra, Emax, Emin, Vd, Tc, start_v
+
+title = "Physics-informed self-supervised learning for predicting cardiac digital twins with echocardiography"
+
+description = """
+<p style='text-align: center'> Keying Kuang, Frances Dean, Jack B. Jedlicki, David Ouyang, Anthony Philippakis, David Sontag, Ahmed Alaa  <br>
+ <a href='https://github.com/AlaaLab/CardioPINN' target='_blank'>Code</a></p>
+We develop methodology for predicting digital twins from non-invasive cardiac ultrasound images in <a href='https://arxiv.org/abs/2403.00177'>Non-Invasive Medical Digital Twins using Physics-Informed Self-Supervised Learning</a>. \n\n 
+We demonstrate the ability of our model to predict left ventricular pressure-volume loops using image data here.
+"""
+
+gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>" + title + "</h1>")
+gr.Markdown(description)
+
+with gr.Blocks() as demo:
+        
+    # text
+    gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>" + title + "</h1>")
+    gr.Markdown(description)
+        
+    with gr.Row():
+        with gr.Column(scale=1.5, min_width=100):
+            
+            generate_button = gr.Button("Load sample echocardiogram and generate result")
+            with gr.Row():
+                video = gr.PlayableVideo(format="avi")
+                plot = gr.Plot()
+            
+            with gr.Row():
+                Rm = gr.Number(label="Mitral valve circuit resistance (Rm) mmHg*s/ml:")
+                Ra = gr.Number(label="Aortic valve circuit resistance (Ra) mmHg*s/ml:")
+                Emax = gr.Number(label="Maximum elastance (Emax) mmHg/ml:")
+                Emin = gr.Number(label="Minimum elastance (Emin) mmHg/ml:")
+                Vd = gr.Number(label="Theoretical zero pressure volume (Vd) ml:")
+                Tc = gr.Number(label="Cycle duration (Tc) s:")
+                start_v = gr.Number(label="Initial volume (start_v) ml:")
+                
+            simulation_button = gr.Button("Run simulation")
+            
+            
+            
+            with gr.Row():
+                sl1 = gr.Slider(0.005, 0.1, value=Rm, label="Rm")
+                sl2 = gr.Slider(0.0001, 0.25, value=Ra, label="Ra")
+                sl3 = gr.Slider(0.5, 3.5, value=Emax, label="Emax")
+                sl4 = gr.Slider(0.02, 0.1, value= Emin, label="Emin")
+                sl5 = gr.Slider(4.0, 25.0, value=Vd, label="Vd")
+                sl6 = gr.Slider(0.4, 1.7, value=Tc, label="Tc") 
+                sl7 = gr.Slider(0.0, 280.0, value=start_v, label="start_v")
+    
+    
+    generate_button.click(fn=generate_example, outputs = [video,plot,Rm,Ra,Emax,Emin,Vd,Tc,start_v])
+    
+    
+    simulation_button.click(fn=pvloop_simulator_plot_only, inputs = [sl1,sl2,sl3,sl4,sl5,sl6,sl7], outputs = [gr.Plot()])
+    
+    
+    
+demo.launch(share=True)

dynamic/.gitignore

@@ -0,0 +1,7 @@
+.ipynb_checkpoints/
+__pycache__/
+*.swp
+echonet.cfg
+.echonet.cfg
+*.pyc
+echonet.egg-info/

dynamic/.travis.yml

@@ -0,0 +1,68 @@
+language: minimal
+
+os:
+  - linux
+
+env:
+  # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.2 (torchvision 0.2 does not have VisionDataset)
+  # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.3  (torchvision 0.3 has a cuda issue)
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.5
+    # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.2
+  # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.3
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.5
+  # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.2
+  # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.3
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.5
+  # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.2
+  # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.3
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.5
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.2
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.3
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.5
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.2
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.3
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.5
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.2
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.3
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.5
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.2
+  # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.3
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.4
+  - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.5 
+
+install:
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]];
+    then
+        MINICONDA_OS=Linux;
+        sudo apt-get update;
+    else
+        MINICONDA_OS=MacOSX;
+        brew update;
+    fi
+  - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-${MINICONDA_OS}-x86_64.sh -O miniconda.sh
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - source "$HOME/miniconda/etc/profile.d/conda.sh"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  # Useful for debugging any issues with conda
+  - conda info -a
+  - conda search pytorch || true
+
+  - conda create -q -n test-environment python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} 
+  - conda activate test-environment
+  - pip install -q torchvision==${TORCHVISION_VERSION} "pillow<7.0.0"
+  - pip install -q .
+  - pip install -q flake8 pylint
+
+script:
+  - flake8 --ignore=E501
+  - pylint --disable=C0103,C0301,R0401,R0801,R0902,R0912,R0913,R0914,R0915 --extension-pkg-whitelist=cv2,torch --generated-members=torch.* echonet/ scripts/*.py setup.py
+  - python -c "import echonet"

dynamic/LICENSE.txt

@@ -0,0 +1,3 @@
+Copyright Notice
+The authors are the proprietor of certain copyrights of and to EchoNet-Dynamic software, source code and associated material.  Code also contains source code created by certain third parties.  Redistribution and use of the Code with or without modification is not permitted without explicit written permission by the authors. 
+Copyright 2019 The authors.  All rights reserved.

dynamic/README.md

@@ -0,0 +1,97 @@
+EchoNet-Dynamic:<br/>Interpretable AI for beat-to-beat cardiac function assessment
+------------------------------------------------------------------------------
+
+EchoNet-Dynamic is a end-to-end beat-to-beat deep learning model for
+  1) semantic segmentation of the left ventricle
+  2) prediction of ejection fraction by entire video or subsampled clips, and
+  3) assessment of cardiomyopathy with reduced ejection fraction.
+
+For more details, see the accompanying paper,
+
+> [**Video-based AI for beat-to-beat assessment of cardiac function**](https://www.nature.com/articles/s41586-020-2145-8)<br/>
+  David Ouyang, Bryan He, Amirata Ghorbani, Neal Yuan, Joseph Ebinger, Curt P. Langlotz, Paul A. Heidenreich, Robert A. Harrington, David H. Liang, Euan A. Ashley, and James Y. Zou. <b>Nature</b>, March 25, 2020. https://doi.org/10.1038/s41586-020-2145-8
+
+Dataset
+-------
+We share a deidentified set of 10,030 echocardiogram images which were used for training EchoNet-Dynamic.
+Preprocessing of these images, including deidentification and conversion from DICOM format to AVI format videos, were performed with OpenCV and pydicom. Additional information is at https://echonet.github.io/dynamic/. These deidentified images are shared with a non-commerical data use agreement.
+
+Examples
+--------
+
+We show examples of our semantic segmentation for nine distinct patients below.
+Three patients have normal cardiac function, three have low ejection fractions, and three have arrhythmia.
+No human tracings for these patients were used by EchoNet-Dynamic.
+
+| Normal                                 | Low Ejection Fraction                  | Arrhythmia                             |
+| ------                                 | ---------------------                  | ----------                             |
+| ![](docs/media/0X10A28877E97DF540.gif) | ![](docs/media/0X129133A90A61A59D.gif) | ![](docs/media/0X132C1E8DBB715D1D.gif) |
+| ![](docs/media/0X1167650B8BEFF863.gif) | ![](docs/media/0X13CE2039E2D706A.gif ) | ![](docs/media/0X18BA5512BE5D6FFA.gif) |
+| ![](docs/media/0X148FFCBF4D0C398F.gif) | ![](docs/media/0X16FC9AA0AD5D8136.gif) | ![](docs/media/0X1E12EEE43FD913E5.gif) |
+
+Installation
+------------
+
+First, clone this repository and enter the directory by running:
+
+    git clone https://github.com/echonet/dynamic.git
+    cd dynamic
+
+EchoNet-Dynamic is implemented for Python 3, and depends on the following packages:
+  - NumPy
+  - PyTorch
+  - Torchvision
+  - OpenCV
+  - skimage
+  - sklearn
+  - tqdm
+
+Echonet-Dynamic and its dependencies can be installed by navigating to the cloned directory and running
+
+    pip install --user .
+
+Usage
+-----
+### Preprocessing DICOM Videos
+
+The input of EchoNet-Dynamic is an apical-4-chamber view echocardiogram video of any length. The easiest way to run our code is to use videos from our dataset, but we also provide a Jupyter Notebook, `ConvertDICOMToAVI.ipynb`, to convert DICOM files to AVI files used for input to EchoNet-Dynamic. The Notebook deidentifies the video by cropping out information outside of the ultrasound sector, resizes the input video, and saves the video in AVI format. 
+
+### Setting Path to Data
+
+By default, EchoNet-Dynamic assumes that a copy of the data is saved in a folder named `a4c-video-dir/` in this directory.
+This path can be changed by creating a configuration file named `echonet.cfg` (an example configuration file is `example.cfg`).
+
+### Running Code
+
+EchoNet-Dynamic has three main components: segmenting the left ventricle, predicting ejection fraction from subsampled clips, and assessing cardiomyopathy with beat-by-beat predictions.
+Each of these components can be run with reasonable choices of hyperparameters with the scripts below.
+We describe our full hyperparameter sweep in the next section.
+
+#### Frame-by-frame Semantic Segmentation of the Left Ventricle
+
+    echonet segmentation --save_video
+
+This creates a directory named `output/segmentation/deeplabv3_resnet50_random/`, which will contain
+  - log.csv: training and validation losses
+  - best.pt: checkpoint of weights for the model with the lowest validation loss
+  - size.csv: estimated size of left ventricle for each frame and indicator for beginning of beat
+  - videos: directory containing videos with segmentation overlay
+
+#### Prediction of Ejection Fraction from Subsampled Clips
+
+  echonet video
+
+This creates a directory named `output/video/r2plus1d_18_32_2_pretrained/`, which will contain
+  - log.csv: training and validation losses
+  - best.pt: checkpoint of weights for the model with the lowest validation loss
+  - test_predictions.csv: ejection fraction prediction for subsampled clips
+
+#### Beat-by-beat Prediction of Ejection Fraction from Full Video and Assesment of Cardiomyopathy
+
+The final beat-by-beat prediction and analysis is performed with `scripts/beat_analysis.R`.
+This script combines the results from segmentation output in `size.csv` and the clip-level ejection fraction prediction in `test_predictions.csv`. The beginning of each systolic phase is detected by using the peak detection algorithm from scipy (`scipy.signal.find_peaks`) and a video clip centered around the beat is used for beat-by-beat prediction.
+
+### Hyperparameter Sweeps
+
+The full set of hyperparameter sweeps from the paper can be run via `run_experiments.sh`.
+In particular, we choose between pretrained and random initialization for the weights, the model (selected from `r2plus1d_18`, `r3d_18`, and `mc3_18`), the length of the video (1, 4, 8, 16, 32, 64, and 96 frames), and the sampling period (1, 2, 4, 6, and 8 frames).

dynamic/echonet/__init__.py

@@ -0,0 +1,26 @@
+"""
+The echonet package contains code for loading echocardiogram videos, and
+functions for training and testing segmentation and ejection fraction
+prediction models.
+"""
+
+import click
+
+from echonet.__version__ import __version__
+from echonet.config import CONFIG as config
+import echonet.datasets as datasets
+import echonet.utils as utils
+
+
+@click.group()
+def main():
+    """Entry point for command line interface."""
+
+
+del click
+
+
+main.add_command(utils.segmentation.run)
+main.add_command(utils.video.run)
+
+__all__ = ["__version__", "config", "datasets", "main", "utils"]

dynamic/echonet/__main__.py

@@ -0,0 +1,7 @@
+"""Entry point for command line."""
+
+import echonet
+
+
+if __name__ == '__main__':
+    echonet.main()

dynamic/echonet/__version__.py

@@ -0,0 +1,3 @@
+"""Version number for Echonet package."""
+
+__version__ = "1.0.0"

dynamic/echonet/config.py

@@ -0,0 +1,24 @@
+"""Sets paths based on configuration files."""
+
+import configparser
+import os
+import types
+
+_FILENAME = None
+_PARAM = {}
+for filename in ["echonet.cfg",
+                 ".echonet.cfg",
+                 os.path.expanduser("~/echonet.cfg"),
+                 os.path.expanduser("~/.echonet.cfg"),
+                 ]:
+    if os.path.isfile(filename):
+        _FILENAME = filename
+        config = configparser.ConfigParser()
+        with open(filename, "r") as f:
+            config.read_string("[config]\n" + f.read())
+            _PARAM = config["config"]
+        break
+
+CONFIG = types.SimpleNamespace(
+    FILENAME=_FILENAME,
+    DATA_DIR=_PARAM.get("data_dir", "a4c-video-dir/"))

dynamic/echonet/datasets/__init__.py

@@ -0,0 +1,8 @@
+"""
+The echonet.datasets submodule defines a Pytorch dataset for loading
+echocardiogram videos.
+"""
+
+from .echo import Echo
+
+__all__ = ["Echo"]

dynamic/echonet/datasets/echo.py

@@ -0,0 +1,282 @@
+"""EchoNet-Dynamic Dataset."""
+
+import os
+import collections
+import pandas
+
+import numpy as np
+import skimage.draw
+import torchvision
+import echonet
+
+
+class Echo(torchvision.datasets.VisionDataset):
+    """EchoNet-Dynamic Dataset.
+
+    Args:
+        root (string): Root directory of dataset (defaults to `echonet.config.DATA_DIR`)
+        split (string): One of {``train'', ``val'', ``test'', ``all'', or ``external_test''}
+        target_type (string or list, optional): Type of target to use,
+            ``Filename'', ``EF'', ``EDV'', ``ESV'', ``LargeIndex'',
+            ``SmallIndex'', ``LargeFrame'', ``SmallFrame'', ``LargeTrace'',
+            or ``SmallTrace''
+            Can also be a list to output a tuple with all specified target types.
+            The targets represent:
+                ``Filename'' (string): filename of video
+                ``EF'' (float): ejection fraction
+                ``EDV'' (float): end-diastolic volume
+                ``ESV'' (float): end-systolic volume
+                ``LargeIndex'' (int): index of large (diastolic) frame in video
+                ``SmallIndex'' (int): index of small (systolic) frame in video
+                ``LargeFrame'' (np.array shape=(3, height, width)): normalized large (diastolic) frame
+                ``SmallFrame'' (np.array shape=(3, height, width)): normalized small (systolic) frame
+                ``LargeTrace'' (np.array shape=(height, width)): left ventricle large (diastolic) segmentation
+                    value of 0 indicates pixel is outside left ventricle
+                             1 indicates pixel is inside left ventricle
+                ``SmallTrace'' (np.array shape=(height, width)): left ventricle small (systolic) segmentation
+                    value of 0 indicates pixel is outside left ventricle
+                             1 indicates pixel is inside left ventricle
+            Defaults to ``EF''.
+        mean (int, float, or np.array shape=(3,), optional): means for all (if scalar) or each (if np.array) channel.
+            Used for normalizing the video. Defaults to 0 (video is not shifted).
+        std (int, float, or np.array shape=(3,), optional): standard deviation for all (if scalar) or each (if np.array) channel.
+            Used for normalizing the video. Defaults to 0 (video is not scaled).
+        length (int or None, optional): Number of frames to clip from video. If ``None'', longest possible clip is returned.
+            Defaults to 16.
+        period (int, optional): Sampling period for taking a clip from the video (i.e. every ``period''-th frame is taken)
+            Defaults to 2.
+        max_length (int or None, optional): Maximum number of frames to clip from video (main use is for shortening excessively
+            long videos when ``length'' is set to None). If ``None'', shortening is not applied to any video.
+            Defaults to 250.
+        clips (int, optional): Number of clips to sample. Main use is for test-time augmentation with random clips.
+            Defaults to 1.
+        pad (int or None, optional): Number of pixels to pad all frames on each side (used as augmentation).
+            and a window of the original size is taken. If ``None'', no padding occurs.
+            Defaults to ``None''.
+        noise (float or None, optional): Fraction of pixels to black out as simulated noise. If ``None'', no simulated noise is added.
+            Defaults to ``None''.
+        target_transform (callable, optional): A function/transform that takes in the target and transforms it.
+        external_test_location (string): Path to videos to use for external testing.
+    """
+
+    def __init__(self, root=None,
+                 split="train", target_type="EF",
+                 mean=0., std=1.,
+                 length=16, period=2,
+                 max_length=250,
+                 clips=1,
+                 pad=None,
+                 noise=None,
+                 target_transform=None,
+                 external_test_location=None):
+        if root is None:
+            root = echonet.config.DATA_DIR
+
+        super().__init__(root, target_transform=target_transform)
+
+        self.split = split.upper()
+        if not isinstance(target_type, list):
+            target_type = [target_type]
+        self.target_type = target_type
+        self.mean = mean
+        self.std = std
+        self.length = length
+        self.max_length = max_length
+        self.period = period
+        self.clips = clips
+        self.pad = pad
+        self.noise = noise
+        self.target_transform = target_transform
+        self.external_test_location = external_test_location
+
+        self.fnames, self.outcome = [], []
+
+        if self.split == "EXTERNAL_TEST":
+            self.fnames = sorted(os.listdir(self.external_test_location))
+        else:
+            # Load video-level labels
+            with open(os.path.join(self.root, "FileList.csv")) as f:
+                data = pandas.read_csv(f)
+            data["Split"].map(lambda x: x.upper())
+
+            if self.split != "ALL":
+                data = data[data["Split"] == self.split]
+
+            self.header = data.columns.tolist()
+            self.fnames = data["FileName"].tolist()
+            self.fnames = [fn + ".avi" for fn in self.fnames if os.path.splitext(fn)[1] == ""]  # Assume avi if no suffix
+            self.outcome = data.values.tolist()
+
+            # Check that files are present
+            missing = set(self.fnames) - set(os.listdir(os.path.join(self.root, "Videos")))
+            if len(missing) != 0:
+                print("{} videos could not be found in {}:".format(len(missing), os.path.join(self.root, "Videos")))
+                for f in sorted(missing):
+                    print("\t", f)
+                raise FileNotFoundError(os.path.join(self.root, "Videos", sorted(missing)[0]))
+
+            # Load traces
+            self.frames = collections.defaultdict(list)
+            self.trace = collections.defaultdict(_defaultdict_of_lists)
+
+            with open(os.path.join(self.root, "VolumeTracings.csv")) as f:
+                header = f.readline().strip().split(",")
+                assert header == ["FileName", "X1", "Y1", "X2", "Y2", "Frame"]
+
+                for line in f:
+                    filename, x1, y1, x2, y2, frame = line.strip().split(',')
+                    x1 = float(x1)
+                    y1 = float(y1)
+                    x2 = float(x2)
+                    y2 = float(y2)
+                    frame = int(frame)
+                    if frame not in self.trace[filename]:
+                        self.frames[filename].append(frame)
+                    self.trace[filename][frame].append((x1, y1, x2, y2))
+            for filename in self.frames:
+                for frame in self.frames[filename]:
+                    self.trace[filename][frame] = np.array(self.trace[filename][frame])
+
+            # A small number of videos are missing traces; remove these videos
+            keep = [len(self.frames[f]) >= 2 for f in self.fnames]
+            self.fnames = [f for (f, k) in zip(self.fnames, keep) if k]
+            self.outcome = [f for (f, k) in zip(self.outcome, keep) if k]
+
+    def __getitem__(self, index):
+        # Find filename of video
+        if self.split == "EXTERNAL_TEST":
+            video = os.path.join(self.external_test_location, self.fnames[index])
+        elif self.split == "CLINICAL_TEST":
+            video = os.path.join(self.root, "ProcessedStrainStudyA4c", self.fnames[index])
+        else:
+            video = os.path.join(self.root, "Videos", self.fnames[index])
+
+        # Load video into np.array
+        video = echonet.utils.loadvideo(video).astype(np.float32)
+
+        # Add simulated noise (black out random pixels)
+        # 0 represents black at this point (video has not been normalized yet)
+        if self.noise is not None:
+            n = video.shape[1] * video.shape[2] * video.shape[3]
+            ind = np.random.choice(n, round(self.noise * n), replace=False)
+            f = ind % video.shape[1]
+            ind //= video.shape[1]
+            i = ind % video.shape[2]
+            ind //= video.shape[2]
+            j = ind
+            video[:, f, i, j] = 0
+
+        # Apply normalization
+        if isinstance(self.mean, (float, int)):
+            video -= self.mean
+        else:
+            video -= self.mean.reshape(3, 1, 1, 1)
+
+        if isinstance(self.std, (float, int)):
+            video /= self.std
+        else:
+            video /= self.std.reshape(3, 1, 1, 1)
+
+        # Set number of frames
+        c, f, h, w = video.shape
+        if self.length is None:
+            # Take as many frames as possible
+            length = f // self.period
+        else:
+            # Take specified number of frames
+            length = self.length
+
+        if self.max_length is not None:
+            # Shorten videos to max_length
+            length = min(length, self.max_length)
+
+        if f < length * self.period:
+            # Pad video with frames filled with zeros if too short
+            # 0 represents the mean color (dark grey), since this is after normalization
+            video = np.concatenate((video, np.zeros((c, length * self.period - f, h, w), video.dtype)), axis=1)
+            c, f, h, w = video.shape  # pylint: disable=E0633
+
+        if self.clips == "all":
+            # Take all possible clips of desired length
+            start = np.arange(f - (length - 1) * self.period)
+        else:
+            # Take random clips from video
+            start = np.random.choice(f - (length - 1) * self.period, self.clips)
+
+        # Gather targets
+        target = []
+        for t in self.target_type:
+            key = self.fnames[index]
+            if t == "Filename":
+                target.append(self.fnames[index])
+            elif t == "LargeIndex":
+                # Traces are sorted by cross-sectional area
+                # Largest (diastolic) frame is last
+                target.append(np.int(self.frames[key][-1]))
+            elif t == "SmallIndex":
+                # Largest (diastolic) frame is first
+                target.append(np.int(self.frames[key][0]))
+            elif t == "LargeFrame":
+                target.append(video[:, self.frames[key][-1], :, :])
+            elif t == "SmallFrame":
+                target.append(video[:, self.frames[key][0], :, :])
+            elif t in ["LargeTrace", "SmallTrace"]:
+                if t == "LargeTrace":
+                    t = self.trace[key][self.frames[key][-1]]
+                else:
+                    t = self.trace[key][self.frames[key][0]]
+                x1, y1, x2, y2 = t[:, 0], t[:, 1], t[:, 2], t[:, 3]
+                x = np.concatenate((x1[1:], np.flip(x2[1:])))
+                y = np.concatenate((y1[1:], np.flip(y2[1:])))
+
+                r, c = skimage.draw.polygon(np.rint(y).astype(np.int), np.rint(x).astype(np.int), (video.shape[2], video.shape[3]))
+                mask = np.zeros((video.shape[2], video.shape[3]), np.float32)
+                mask[r, c] = 1
+                target.append(mask)
+            else:
+                if self.split == "CLINICAL_TEST" or self.split == "EXTERNAL_TEST":
+                    target.append(np.float32(0))
+                else:
+                    target.append(np.float32(self.outcome[index][self.header.index(t)]))
+
+        if target != []:
+            target = tuple(target) if len(target) > 1 else target[0]
+            if self.target_transform is not None:
+                target = self.target_transform(target)
+
+        # Select clips from video
+        video = tuple(video[:, s + self.period * np.arange(length), :, :] for s in start)
+        if self.clips == 1:
+            video = video[0]
+        else:
+            video = np.stack(video)
+
+        if self.pad is not None:
+            # Add padding of zeros (mean color of videos)
+            # Crop of original size is taken out
+            # (Used as augmentation)
+            c, l, h, w = video.shape
+            temp = np.zeros((c, l, h + 2 * self.pad, w + 2 * self.pad), dtype=video.dtype)
+            temp[:, :, self.pad:-self.pad, self.pad:-self.pad] = video  # pylint: disable=E1130
+            i, j = np.random.randint(0, 2 * self.pad, 2)
+            video = temp[:, :, i:(i + h), j:(j + w)]
+
+        return video, target
+
+    def __len__(self):
+        return len(self.fnames)
+
+    def extra_repr(self) -> str:
+        """Additional information to add at end of __repr__."""
+        lines = ["Target type: {target_type}", "Split: {split}"]
+        return '\n'.join(lines).format(**self.__dict__)
+
+
+def _defaultdict_of_lists():
+    """Returns a defaultdict of lists.
+
+    This is used to avoid issues with Windows (if this function is anonymous,
+    the Echo dataset cannot be used in a dataloader).
+    """
+
+    return collections.defaultdict(list)

dynamic/echonet/utils/__init__.py

@@ -0,0 +1,179 @@
+"""Utility functions for videos, plotting and computing performance metrics."""
+
+import os
+import typing
+
+import cv2  # pytype: disable=attribute-error
+import matplotlib
+import numpy as np
+import torch
+import tqdm
+
+from . import video
+from . import segmentation
+
+
+def loadvideo(filename: str) -> np.ndarray:
+    """Loads a video from a file.
+
+    Args:
+        filename (str): filename of video
+
+    Returns:
+        A np.ndarray with dimensions (channels=3, frames, height, width). The
+        values will be uint8's ranging from 0 to 255.
+
+    Raises:
+        FileNotFoundError: Could not find `filename`
+        ValueError: An error occurred while reading the video
+    """
+
+    if not os.path.exists(filename):
+        raise FileNotFoundError(filename)
+    capture = cv2.VideoCapture(filename)
+
+    frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+    frame_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+    frame_height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    v = np.zeros((frame_count, frame_height, frame_width, 3), np.uint8)
+
+    for count in range(frame_count):
+        ret, frame = capture.read()
+        if not ret:
+            raise ValueError("Failed to load frame #{} of {}.".format(count, filename))
+
+        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        v[count, :, :] = frame
+
+    v = v.transpose((3, 0, 1, 2))
+
+    return v
+
+
+def savevideo(filename: str, array: np.ndarray, fps: typing.Union[float, int] = 1):
+    """Saves a video to a file.
+
+    Args:
+        filename (str): filename of video
+        array (np.ndarray): video of uint8's with shape (channels=3, frames, height, width)
+        fps (float or int): frames per second
+
+    Returns:
+        None
+    """
+
+    c, _, height, width = array.shape
+
+    if c != 3:
+        raise ValueError("savevideo expects array of shape (channels=3, frames, height, width), got shape ({})".format(", ".join(map(str, array.shape))))
+    fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
+    out = cv2.VideoWriter(filename, fourcc, fps, (width, height))
+
+    for frame in array.transpose((1, 2, 3, 0)):
+        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+        out.write(frame)
+
+
+def get_mean_and_std(dataset: torch.utils.data.Dataset,
+                     samples: int = 128,
+                     batch_size: int = 8,
+                     num_workers: int = 4):
+    """Computes mean and std from samples from a Pytorch dataset.
+
+    Args:
+        dataset (torch.utils.data.Dataset): A Pytorch dataset.
+            ``dataset[i][0]'' is expected to be the i-th video in the dataset, which
+            should be a ``torch.Tensor'' of dimensions (channels=3, frames, height, width)
+        samples (int or None, optional): Number of samples to take from dataset. If ``None'', mean and
+            standard deviation are computed over all elements.
+            Defaults to 128.
+        batch_size (int, optional): how many samples per batch to load
+            Defaults to 8.
+        num_workers (int, optional): how many subprocesses to use for data
+            loading. If 0, the data will be loaded in the main process.
+            Defaults to 4.
+
+    Returns:
+       A tuple of the mean and standard deviation. Both are represented as np.array's of dimension (channels,).
+    """
+
+    if samples is not None and len(dataset) > samples:
+        indices = np.random.choice(len(dataset), samples, replace=False)
+        dataset = torch.utils.data.Subset(dataset, indices)
+    dataloader = torch.utils.data.DataLoader(
+        dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True)
+
+    n = 0  # number of elements taken (should be equal to samples by end of for loop)
+    s1 = 0.  # sum of elements along channels (ends up as np.array of dimension (channels,))
+    s2 = 0.  # sum of squares of elements along channels (ends up as np.array of dimension (channels,))
+    for (x, *_) in tqdm.tqdm(dataloader):
+        x = x.transpose(0, 1).contiguous().view(3, -1)
+        n += x.shape[1]
+        s1 += torch.sum(x, dim=1).numpy()
+        s2 += torch.sum(x ** 2, dim=1).numpy()
+    mean = s1 / n  # type: np.ndarray
+    std = np.sqrt(s2 / n - mean ** 2)  # type: np.ndarray
+
+    mean = mean.astype(np.float32)
+    std = std.astype(np.float32)
+
+    return mean, std
+
+
+def bootstrap(a, b, func, samples=10000):
+    """Computes a bootstrapped confidence intervals for ``func(a, b)''.
+
+    Args:
+        a (array_like): first argument to `func`.
+        b (array_like): second argument to `func`.
+        func (callable): Function to compute confidence intervals for.
+            ``dataset[i][0]'' is expected to be the i-th video in the dataset, which
+            should be a ``torch.Tensor'' of dimensions (channels=3, frames, height, width)
+        samples (int, optional): Number of samples to compute.
+            Defaults to 10000.
+
+    Returns:
+       A tuple of (`func(a, b)`, estimated 5-th percentile, estimated 95-th percentile).
+    """
+    a = np.array(a)
+    b = np.array(b)
+
+    bootstraps = []
+    for _ in range(samples):
+        ind = np.random.choice(len(a), len(a))
+        bootstraps.append(func(a[ind], b[ind]))
+    bootstraps = sorted(bootstraps)
+
+    return func(a, b), bootstraps[round(0.05 * len(bootstraps))], bootstraps[round(0.95 * len(bootstraps))]
+
+
+def latexify():
+    """Sets matplotlib params to appear more like LaTeX.
+
+    Based on https://nipunbatra.github.io/blog/2014/latexify.html
+    """
+    params = {'backend': 'pdf',
+              'axes.titlesize': 8,
+              'axes.labelsize': 8,
+              'font.size': 8,
+              'legend.fontsize': 8,
+              'xtick.labelsize': 8,
+              'ytick.labelsize': 8,
+              'font.family': 'DejaVu Serif',
+              'font.serif': 'Computer Modern',
+              }
+    matplotlib.rcParams.update(params)
+
+
+def dice_similarity_coefficient(inter, union):
+    """Computes the dice similarity coefficient.
+
+    Args:
+        inter (iterable): iterable of the intersections
+        union (iterable): iterable of the unions
+    """
+    return 2 * sum(inter) / (sum(union) + sum(inter))
+
+
+__all__ = ["video", "segmentation", "loadvideo", "savevideo", "get_mean_and_std", "bootstrap", "latexify", "dice_similarity_coefficient"]

dynamic/echonet/utils/segmentation.py

@@ -0,0 +1,498 @@
+"""Functions for training and running segmentation."""
+
+import math
+import os
+import time
+
+import click
+import matplotlib.pyplot as plt
+import numpy as np
+import scipy.signal
+import skimage.draw
+import torch
+import torchvision
+import tqdm
+
+import echonet
+
+
+@click.command("segmentation")
+@click.option("--data_dir", type=click.Path(exists=True, file_okay=False), default=None)
+@click.option("--output", type=click.Path(file_okay=False), default=None)
+@click.option("--model_name", type=click.Choice(
+    sorted(name for name in torchvision.models.segmentation.__dict__
+           if name.islower() and not name.startswith("__") and callable(torchvision.models.segmentation.__dict__[name]))),
+    default="deeplabv3_resnet50")
+@click.option("--pretrained/--random", default=False)
+@click.option("--weights", type=click.Path(exists=True, dir_okay=False), default=None)
+@click.option("--run_test/--skip_test", default=False)
+@click.option("--save_video/--skip_video", default=False)
+@click.option("--num_epochs", type=int, default=50)
+@click.option("--lr", type=float, default=1e-5)
+@click.option("--weight_decay", type=float, default=0)
+@click.option("--lr_step_period", type=int, default=None)
+@click.option("--num_train_patients", type=int, default=None)
+@click.option("--num_workers", type=int, default=4)
+@click.option("--batch_size", type=int, default=20)
+@click.option("--device", type=str, default=None)
+@click.option("--seed", type=int, default=0)
+def run(
+    data_dir=None,
+    output=None,
+
+    model_name="deeplabv3_resnet50",
+    pretrained=False,
+    weights=None,
+
+    run_test=False,
+    save_video=False,
+    num_epochs=50,
+    lr=1e-5,
+    weight_decay=1e-5,
+    lr_step_period=None,
+    num_train_patients=None,
+    num_workers=4,
+    batch_size=20,
+    device=None,
+    seed=0,
+):
+    """Trains/tests segmentation model.
+
+    Args:
+        data_dir (str, optional): Directory containing dataset. Defaults to
+            `echonet.config.DATA_DIR`.
+        output (str, optional): Directory to place outputs. Defaults to
+            output/segmentation/<model_name>_<pretrained/random>/.
+        model_name (str, optional): Name of segmentation model. One of ``deeplabv3_resnet50'',
+            ``deeplabv3_resnet101'', ``fcn_resnet50'', or ``fcn_resnet101''
+            (options are torchvision.models.segmentation.<model_name>)
+            Defaults to ``deeplabv3_resnet50''.
+        pretrained (bool, optional): Whether to use pretrained weights for model
+            Defaults to False.
+        weights (str, optional): Path to checkpoint containing weights to
+            initialize model. Defaults to None.
+        run_test (bool, optional): Whether or not to run on test.
+            Defaults to False.
+        save_video (bool, optional): Whether to save videos with segmentations.
+            Defaults to False.
+        num_epochs (int, optional): Number of epochs during training
+            Defaults to 50.
+        lr (float, optional): Learning rate for SGD
+            Defaults to 1e-5.
+        weight_decay (float, optional): Weight decay for SGD
+            Defaults to 0.
+        lr_step_period (int or None, optional): Period of learning rate decay
+            (learning rate is decayed by a multiplicative factor of 0.1)
+            Defaults to math.inf (never decay learning rate).
+        num_train_patients (int or None, optional): Number of training patients
+            for ablations. Defaults to all patients.
+        num_workers (int, optional): Number of subprocesses to use for data
+            loading. If 0, the data will be loaded in the main process.
+            Defaults to 4.
+        device (str or None, optional): Name of device to run on. Options from
+            https://pytorch.org/docs/stable/tensor_attributes.html#torch.torch.device
+            Defaults to ``cuda'' if available, and ``cpu'' otherwise.
+        batch_size (int, optional): Number of samples to load per batch
+            Defaults to 20.
+        seed (int, optional): Seed for random number generator. Defaults to 0.
+    """
+
+    # Seed RNGs
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+
+    # Set default output directory
+    if output is None:
+        output = os.path.join("output", "segmentation", "{}_{}".format(model_name, "pretrained" if pretrained else "random"))
+    os.makedirs(output, exist_ok=True)
+
+    # Set device for computations
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # Set up model
+    model = torchvision.models.segmentation.__dict__[model_name](pretrained=pretrained, aux_loss=False)
+
+    model.classifier[-1] = torch.nn.Conv2d(model.classifier[-1].in_channels, 1, kernel_size=model.classifier[-1].kernel_size)  # change number of outputs to 1
+    if device.type == "cuda":
+        model = torch.nn.DataParallel(model)
+    model.to(device)
+
+    if weights is not None:
+        checkpoint = torch.load(weights)
+        model.load_state_dict(checkpoint['state_dict'])
+
+    # Set up optimizer
+    optim = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay)
+    if lr_step_period is None:
+        lr_step_period = math.inf
+    scheduler = torch.optim.lr_scheduler.StepLR(optim, lr_step_period)
+
+    # Compute mean and std
+    mean, std = echonet.utils.get_mean_and_std(echonet.datasets.Echo(root=data_dir, split="train"))
+    tasks = ["LargeFrame", "SmallFrame", "LargeTrace", "SmallTrace"]
+    kwargs = {"target_type": tasks,
+              "mean": mean,
+              "std": std
+              }
+
+    # Set up datasets and dataloaders
+    dataset = {}
+    dataset["train"] = echonet.datasets.Echo(root=data_dir, split="train", **kwargs)
+    if num_train_patients is not None and len(dataset["train"]) > num_train_patients:
+        # Subsample patients (used for ablation experiment)
+        indices = np.random.choice(len(dataset["train"]), num_train_patients, replace=False)
+        dataset["train"] = torch.utils.data.Subset(dataset["train"], indices)
+    dataset["val"] = echonet.datasets.Echo(root=data_dir, split="val", **kwargs)
+
+    # Run training and testing loops
+    with open(os.path.join(output, "log.csv"), "a") as f:
+        epoch_resume = 0
+        bestLoss = float("inf")
+        try:
+            # Attempt to load checkpoint
+            checkpoint = torch.load(os.path.join(output, "checkpoint.pt"))
+            model.load_state_dict(checkpoint['state_dict'])
+            optim.load_state_dict(checkpoint['opt_dict'])
+            scheduler.load_state_dict(checkpoint['scheduler_dict'])
+            epoch_resume = checkpoint["epoch"] + 1
+            bestLoss = checkpoint["best_loss"]
+            f.write("Resuming from epoch {}\n".format(epoch_resume))
+        except FileNotFoundError:
+            f.write("Starting run from scratch\n")
+
+        for epoch in range(epoch_resume, num_epochs):
+            print("Epoch #{}".format(epoch), flush=True)
+            for phase in ['train', 'val']:
+                start_time = time.time()
+                for i in range(torch.cuda.device_count()):
+                    torch.cuda.reset_peak_memory_stats(i)
+
+                ds = dataset[phase]
+                dataloader = torch.utils.data.DataLoader(
+                    ds, batch_size=batch_size, num_workers=num_workers, shuffle=True, pin_memory=(device.type == "cuda"), drop_last=(phase == "train"))
+
+                loss, large_inter, large_union, small_inter, small_union = echonet.utils.segmentation.run_epoch(model, dataloader, phase == "train", optim, device)
+                overall_dice = 2 * (large_inter.sum() + small_inter.sum()) / (large_union.sum() + large_inter.sum() + small_union.sum() + small_inter.sum())
+                large_dice = 2 * large_inter.sum() / (large_union.sum() + large_inter.sum())
+                small_dice = 2 * small_inter.sum() / (small_union.sum() + small_inter.sum())
+                f.write("{},{},{},{},{},{},{},{},{},{},{}\n".format(epoch,
+                                                                    phase,
+                                                                    loss,
+                                                                    overall_dice,
+                                                                    large_dice,
+                                                                    small_dice,
+                                                                    time.time() - start_time,
+                                                                    large_inter.size,
+                                                                    sum(torch.cuda.max_memory_allocated() for i in range(torch.cuda.device_count())),
+                                                                    sum(torch.cuda.max_memory_reserved() for i in range(torch.cuda.device_count())),
+                                                                    batch_size))
+                f.flush()
+            scheduler.step()
+
+            # Save checkpoint
+            save = {
+                'epoch': epoch,
+                'state_dict': model.state_dict(),
+                'best_loss': bestLoss,
+                'loss': loss,
+                'opt_dict': optim.state_dict(),
+                'scheduler_dict': scheduler.state_dict(),
+            }
+            torch.save(save, os.path.join(output, "checkpoint.pt"))
+            if loss < bestLoss:
+                torch.save(save, os.path.join(output, "best.pt"))
+                bestLoss = loss
+
+        # Load best weights
+        if num_epochs != 0:
+            checkpoint = torch.load(os.path.join(output, "best.pt"))
+            model.load_state_dict(checkpoint['state_dict'])
+            f.write("Best validation loss {} from epoch {}\n".format(checkpoint["loss"], checkpoint["epoch"]))
+
+        if run_test:
+            # Run on validation and test
+            for split in ["val", "test"]:
+                dataset = echonet.datasets.Echo(root=data_dir, split=split, **kwargs)
+                dataloader = torch.utils.data.DataLoader(dataset,
+                                                         batch_size=batch_size, num_workers=num_workers, shuffle=False, pin_memory=(device.type == "cuda"))
+                loss, large_inter, large_union, small_inter, small_union = echonet.utils.segmentation.run_epoch(model, dataloader, False, None, device)
+
+                overall_dice = 2 * (large_inter + small_inter) / (large_union + large_inter + small_union + small_inter)
+                large_dice = 2 * large_inter / (large_union + large_inter)
+                small_dice = 2 * small_inter / (small_union + small_inter)
+                with open(os.path.join(output, "{}_dice.csv".format(split)), "w") as g:
+                    g.write("Filename, Overall, Large, Small\n")
+                    for (filename, overall, large, small) in zip(dataset.fnames, overall_dice, large_dice, small_dice):
+                        g.write("{},{},{},{}\n".format(filename, overall, large, small))
+
+                f.write("{} dice (overall): {:.4f} ({:.4f} - {:.4f})\n".format(split, *echonet.utils.bootstrap(np.concatenate((large_inter, small_inter)), np.concatenate((large_union, small_union)), echonet.utils.dice_similarity_coefficient)))
+                f.write("{} dice (large):   {:.4f} ({:.4f} - {:.4f})\n".format(split, *echonet.utils.bootstrap(large_inter, large_union, echonet.utils.dice_similarity_coefficient)))
+                f.write("{} dice (small):   {:.4f} ({:.4f} - {:.4f})\n".format(split, *echonet.utils.bootstrap(small_inter, small_union, echonet.utils.dice_similarity_coefficient)))
+                f.flush()
+
+    # Saving videos with segmentations
+    dataset = echonet.datasets.Echo(root=data_dir, split="test",
+                                    target_type=["Filename", "LargeIndex", "SmallIndex"],  # Need filename for saving, and human-selected frames to annotate
+                                    mean=mean, std=std,  # Normalization
+                                    length=None, max_length=None, period=1  # Take all frames
+                                    )
+    dataloader = torch.utils.data.DataLoader(dataset, batch_size=10, num_workers=num_workers, shuffle=False, pin_memory=False, collate_fn=_video_collate_fn)
+
+    # Save videos with segmentation
+    if save_video and not all(os.path.isfile(os.path.join(output, "videos", f)) for f in dataloader.dataset.fnames):
+        # Only run if missing videos
+
+        model.eval()
+
+        os.makedirs(os.path.join(output, "videos"), exist_ok=True)
+        os.makedirs(os.path.join(output, "size"), exist_ok=True)
+        echonet.utils.latexify()
+
+        with torch.no_grad():
+            with open(os.path.join(output, "size.csv"), "w") as g:
+                g.write("Filename,Frame,Size,HumanLarge,HumanSmall,ComputerSmall\n")
+                for (x, (filenames, large_index, small_index), length) in tqdm.tqdm(dataloader):
+                    # Run segmentation model on blocks of frames one-by-one
+                    # The whole concatenated video may be too long to run together
+                    y = np.concatenate([model(x[i:(i + batch_size), :, :, :].to(device))["out"].detach().cpu().numpy() for i in range(0, x.shape[0], batch_size)])
+
+                    start = 0
+                    x = x.numpy()
+                    for (i, (filename, offset)) in enumerate(zip(filenames, length)):
+                        # Extract one video and segmentation predictions
+                        video = x[start:(start + offset), ...]
+                        logit = y[start:(start + offset), 0, :, :]
+
+                        # Un-normalize video
+                        video *= std.reshape(1, 3, 1, 1)
+                        video += mean.reshape(1, 3, 1, 1)
+
+                        # Get frames, channels, height, and width
+                        f, c, h, w = video.shape  # pylint: disable=W0612
+                        assert c == 3
+
+                        # Put two copies of the video side by side
+                        video = np.concatenate((video, video), 3)
+
+                        # If a pixel is in the segmentation, saturate blue channel
+                        # Leave alone otherwise
+                        video[:, 0, :, w:] = np.maximum(255. * (logit > 0), video[:, 0, :, w:])  # pylint: disable=E1111
+
+                        # Add blank canvas under pair of videos
+                        video = np.concatenate((video, np.zeros_like(video)), 2)
+
+                        # Compute size of segmentation per frame
+                        size = (logit > 0).sum((1, 2))
+
+                        # Identify systole frames with peak detection
+                        trim_min = sorted(size)[round(len(size) ** 0.05)]
+                        trim_max = sorted(size)[round(len(size) ** 0.95)]
+                        trim_range = trim_max - trim_min
+                        systole = set(scipy.signal.find_peaks(-size, distance=20, prominence=(0.50 * trim_range))[0])
+
+                        # Write sizes and frames to file
+                        for (frame, s) in enumerate(size):
+                            g.write("{},{},{},{},{},{}\n".format(filename, frame, s, 1 if frame == large_index[i] else 0, 1 if frame == small_index[i] else 0, 1 if frame in systole else 0))
+
+                        # Plot sizes
+                        fig = plt.figure(figsize=(size.shape[0] / 50 * 1.5, 3))
+                        plt.scatter(np.arange(size.shape[0]) / 50, size, s=1)
+                        ylim = plt.ylim()
+                        for s in systole:
+                            plt.plot(np.array([s, s]) / 50, ylim, linewidth=1)
+                        plt.ylim(ylim)
+                        plt.title(os.path.splitext(filename)[0])
+                        plt.xlabel("Seconds")
+                        plt.ylabel("Size (pixels)")
+                        plt.tight_layout()
+                        plt.savefig(os.path.join(output, "size", os.path.splitext(filename)[0] + ".pdf"))
+                        plt.close(fig)
+
+                        # Normalize size to [0, 1]
+                        size -= size.min()
+                        size = size / size.max()
+                        size = 1 - size
+
+                        # Iterate the frames in this video
+                        for (f, s) in enumerate(size):
+
+                            # On all frames, mark a pixel for the size of the frame
+                            video[:, :, int(round(115 + 100 * s)), int(round(f / len(size) * 200 + 10))] = 255.
+
+                            if f in systole:
+                                # If frame is computer-selected systole, mark with a line
+                                video[:, :, 115:224, int(round(f / len(size) * 200 + 10))] = 255.
+
+                            def dash(start, stop, on=10, off=10):
+                                buf = []
+                                x = start
+                                while x < stop:
+                                    buf.extend(range(x, x + on))
+                                    x += on
+                                    x += off
+                                buf = np.array(buf)
+                                buf = buf[buf < stop]
+                                return buf
+                            d = dash(115, 224)
+
+                            if f == large_index[i]:
+                                # If frame is human-selected diastole, mark with green dashed line on all frames
+                                video[:, :, d, int(round(f / len(size) * 200 + 10))] = np.array([0, 225, 0]).reshape((1, 3, 1))
+                            if f == small_index[i]:
+                                # If frame is human-selected systole, mark with red dashed line on all frames
+                                video[:, :, d, int(round(f / len(size) * 200 + 10))] = np.array([0, 0, 225]).reshape((1, 3, 1))
+
+                            # Get pixels for a circle centered on the pixel
+                            r, c = skimage.draw.disk((int(round(115 + 100 * s)), int(round(f / len(size) * 200 + 10))), 4.1)
+
+                            # On the frame that's being shown, put a circle over the pixel
+                            video[f, :, r, c] = 255.
+
+                        # Rearrange dimensions and save
+                        video = video.transpose(1, 0, 2, 3)
+                        video = video.astype(np.uint8)
+                        echonet.utils.savevideo(os.path.join(output, "videos", filename), video, 50)
+
+                        # Move to next video
+                        start += offset
+
+
+def run_epoch(model, dataloader, train, optim, device):
+    """Run one epoch of training/evaluation for segmentation.
+
+    Args:
+        model (torch.nn.Module): Model to train/evaulate.
+        dataloder (torch.utils.data.DataLoader): Dataloader for dataset.
+        train (bool): Whether or not to train model.
+        optim (torch.optim.Optimizer): Optimizer
+        device (torch.device): Device to run on
+    """
+
+    total = 0.
+    n = 0
+
+    pos = 0
+    neg = 0
+    pos_pix = 0
+    neg_pix = 0
+
+    model.train(train)
+
+    large_inter = 0
+    large_union = 0
+    small_inter = 0
+    small_union = 0
+    large_inter_list = []
+    large_union_list = []
+    small_inter_list = []
+    small_union_list = []
+
+    with torch.set_grad_enabled(train):
+        with tqdm.tqdm(total=len(dataloader)) as pbar:
+            for (_, (large_frame, small_frame, large_trace, small_trace)) in dataloader:
+                # Count number of pixels in/out of human segmentation
+                pos += (large_trace == 1).sum().item()
+                pos += (small_trace == 1).sum().item()
+                neg += (large_trace == 0).sum().item()
+                neg += (small_trace == 0).sum().item()
+
+                # Count number of pixels in/out of computer segmentation
+                pos_pix += (large_trace == 1).sum(0).to("cpu").detach().numpy()
+                pos_pix += (small_trace == 1).sum(0).to("cpu").detach().numpy()
+                neg_pix += (large_trace == 0).sum(0).to("cpu").detach().numpy()
+                neg_pix += (small_trace == 0).sum(0).to("cpu").detach().numpy()
+
+                # Run prediction for diastolic frames and compute loss
+                large_frame = large_frame.to(device)
+                large_trace = large_trace.to(device)
+                y_large = model(large_frame)["out"]
+                loss_large = torch.nn.functional.binary_cross_entropy_with_logits(y_large[:, 0, :, :], large_trace, reduction="sum")
+                # Compute pixel intersection and union between human and computer segmentations
+                large_inter += np.logical_and(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum()
+                large_union += np.logical_or(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum()
+                large_inter_list.extend(np.logical_and(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2)))
+                large_union_list.extend(np.logical_or(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2)))
+
+                # Run prediction for systolic frames and compute loss
+                small_frame = small_frame.to(device)
+                small_trace = small_trace.to(device)
+                y_small = model(small_frame)["out"]
+                loss_small = torch.nn.functional.binary_cross_entropy_with_logits(y_small[:, 0, :, :], small_trace, reduction="sum")
+                # Compute pixel intersection and union between human and computer segmentations
+                small_inter += np.logical_and(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum()
+                small_union += np.logical_or(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum()
+                small_inter_list.extend(np.logical_and(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2)))
+                small_union_list.extend(np.logical_or(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2)))
+
+                # Take gradient step if training
+                loss = (loss_large + loss_small) / 2
+                if train:
+                    optim.zero_grad()
+                    loss.backward()
+                    optim.step()
+
+                # Accumulate losses and compute baselines
+                total += loss.item()
+                n += large_trace.size(0)
+                p = pos / (pos + neg)
+                p_pix = (pos_pix + 1) / (pos_pix + neg_pix + 2)
+
+                # Show info on process bar
+                pbar.set_postfix_str("{:.4f} ({:.4f}) / {:.4f} {:.4f}, {:.4f}, {:.4f}".format(total / n / 112 / 112, loss.item() / large_trace.size(0) / 112 / 112, -p * math.log(p) - (1 - p) * math.log(1 - p), (-p_pix * np.log(p_pix) - (1 - p_pix) * np.log(1 - p_pix)).mean(), 2 * large_inter / (large_union + large_inter), 2 * small_inter / (small_union + small_inter)))
+                pbar.update()
+
+    large_inter_list = np.array(large_inter_list)
+    large_union_list = np.array(large_union_list)
+    small_inter_list = np.array(small_inter_list)
+    small_union_list = np.array(small_union_list)
+
+    return (total / n / 112 / 112,
+            large_inter_list,
+            large_union_list,
+            small_inter_list,
+            small_union_list,
+            )
+
+
+def _video_collate_fn(x):
+    """Collate function for Pytorch dataloader to merge multiple videos.
+
+    This function should be used in a dataloader for a dataset that returns
+    a video as the first element, along with some (non-zero) tuple of
+    targets. Then, the input x is a list of tuples:
+      - x[i][0] is the i-th video in the batch
+      - x[i][1] are the targets for the i-th video
+
+    This function returns a 3-tuple:
+      - The first element is the videos concatenated along the frames
+        dimension. This is done so that videos of different lengths can be
+        processed together (tensors cannot be "jagged", so we cannot have
+        a dimension for video, and another for frames).
+      - The second element is contains the targets with no modification.
+      - The third element is a list of the lengths of the videos in frames.
+    """
+    video, target = zip(*x)  # Extract the videos and targets
+
+    # ``video'' is a tuple of length ``batch_size''
+    #   Each element has shape (channels=3, frames, height, width)
+    #   height and width are expected to be the same across videos, but
+    #   frames can be different.
+
+    # ``target'' is also a tuple of length ``batch_size''
+    # Each element is a tuple of the targets for the item.
+
+    i = list(map(lambda t: t.shape[1], video))  # Extract lengths of videos in frames
+
+    # This contatenates the videos along the the frames dimension (basically
+    # playing the videos one after another). The frames dimension is then
+    # moved to be first.
+    # Resulting shape is (total frames, channels=3, height, width)
+    video = torch.as_tensor(np.swapaxes(np.concatenate(video, 1), 0, 1))
+
+    # Swap dimensions (approximately a transpose)
+    # Before: target[i][j] is the j-th target of element i
+    # After:  target[i][j] is the i-th target of element j
+    target = zip(*target)
+
+    return video, target, i

dynamic/echonet/utils/video.py

@@ -0,0 +1,361 @@
+"""Functions for training and running EF prediction."""
+
+import math
+import os
+import time
+
+import click
+import matplotlib.pyplot as plt
+import numpy as np
+import sklearn.metrics
+import torch
+import torchvision
+import tqdm
+
+import echonet
+
+
+@click.command("video")
+@click.option("--data_dir", type=click.Path(exists=True, file_okay=False), default=None)
+@click.option("--output", type=click.Path(file_okay=False), default=None)
+@click.option("--task", type=str, default="EF")
+@click.option("--model_name", type=click.Choice(
+    sorted(name for name in torchvision.models.video.__dict__
+           if name.islower() and not name.startswith("__") and callable(torchvision.models.video.__dict__[name]))),
+    default="r2plus1d_18")
+@click.option("--pretrained/--random", default=True)
+@click.option("--weights", type=click.Path(exists=True, dir_okay=False), default=None)
+@click.option("--run_test/--skip_test", default=False)
+@click.option("--num_epochs", type=int, default=45)
+@click.option("--lr", type=float, default=1e-4)
+@click.option("--weight_decay", type=float, default=1e-4)
+@click.option("--lr_step_period", type=int, default=15)
+@click.option("--frames", type=int, default=32)
+@click.option("--period", type=int, default=2)
+@click.option("--num_train_patients", type=int, default=None)
+@click.option("--num_workers", type=int, default=4)
+@click.option("--batch_size", type=int, default=20)
+@click.option("--device", type=str, default=None)
+@click.option("--seed", type=int, default=0)
+def run(
+    data_dir=None,
+    output=None,
+    task="EF",
+
+    model_name="r2plus1d_18",
+    pretrained=True,
+    weights=None,
+
+    run_test=False,
+    num_epochs=45,
+    lr=1e-4,
+    weight_decay=1e-4,
+    lr_step_period=15,
+    frames=32,
+    period=2,
+    num_train_patients=None,
+    num_workers=4,
+    batch_size=20,
+    device=None,
+    seed=0,
+):
+    """Trains/tests EF prediction model.
+
+    \b
+    Args:
+        data_dir (str, optional): Directory containing dataset. Defaults to
+            `echonet.config.DATA_DIR`.
+        output (str, optional): Directory to place outputs. Defaults to
+            output/video/<model_name>_<pretrained/random>/.
+        task (str, optional): Name of task to predict. Options are the headers
+            of FileList.csv. Defaults to ``EF''.
+        model_name (str, optional): Name of model. One of ``mc3_18'',
+            ``r2plus1d_18'', or ``r3d_18''
+            (options are torchvision.models.video.<model_name>)
+            Defaults to ``r2plus1d_18''.
+        pretrained (bool, optional): Whether to use pretrained weights for model
+            Defaults to True.
+        weights (str, optional): Path to checkpoint containing weights to
+            initialize model. Defaults to None.
+        run_test (bool, optional): Whether or not to run on test.
+            Defaults to False.
+        num_epochs (int, optional): Number of epochs during training.
+            Defaults to 45.
+        lr (float, optional): Learning rate for SGD
+            Defaults to 1e-4.
+        weight_decay (float, optional): Weight decay for SGD
+            Defaults to 1e-4.
+        lr_step_period (int or None, optional): Period of learning rate decay
+            (learning rate is decayed by a multiplicative factor of 0.1)
+            Defaults to 15.
+        frames (int, optional): Number of frames to use in clip
+            Defaults to 32.
+        period (int, optional): Sampling period for frames
+            Defaults to 2.
+        n_train_patients (int or None, optional): Number of training patients
+            for ablations. Defaults to all patients.
+        num_workers (int, optional): Number of subprocesses to use for data
+            loading. If 0, the data will be loaded in the main process.
+            Defaults to 4.
+        device (str or None, optional): Name of device to run on. Options from
+            https://pytorch.org/docs/stable/tensor_attributes.html#torch.torch.device
+            Defaults to ``cuda'' if available, and ``cpu'' otherwise.
+        batch_size (int, optional): Number of samples to load per batch
+            Defaults to 20.
+        seed (int, optional): Seed for random number generator. Defaults to 0.
+    """
+
+    # Seed RNGs
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+
+    # Set default output directory
+    if output is None:
+        output = os.path.join("output", "video", "{}_{}_{}_{}".format(model_name, frames, period, "pretrained" if pretrained else "random"))
+    os.makedirs(output, exist_ok=True)
+
+    # Set device for computations
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # Set up model
+    model = torchvision.models.video.__dict__[model_name](pretrained=pretrained)
+
+    model.fc = torch.nn.Linear(model.fc.in_features, 1)
+    model.fc.bias.data[0] = 55.6
+    if device.type == "cuda":
+        model = torch.nn.DataParallel(model)
+    model.to(device)
+
+    if weights is not None:
+        checkpoint = torch.load(weights)
+        model.load_state_dict(checkpoint['state_dict'])
+
+    # Set up optimizer
+    optim = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay)
+    if lr_step_period is None:
+        lr_step_period = math.inf
+    scheduler = torch.optim.lr_scheduler.StepLR(optim, lr_step_period)
+
+    # Compute mean and std
+    mean, std = echonet.utils.get_mean_and_std(echonet.datasets.Echo(root=data_dir, split="train"))
+    kwargs = {"target_type": task,
+              "mean": mean,
+              "std": std,
+              "length": frames,
+              "period": period,
+              }
+
+    # Set up datasets and dataloaders
+    dataset = {}
+    dataset["train"] = echonet.datasets.Echo(root=data_dir, split="train", **kwargs, pad=12)
+    if num_train_patients is not None and len(dataset["train"]) > num_train_patients:
+        # Subsample patients (used for ablation experiment)
+        indices = np.random.choice(len(dataset["train"]), num_train_patients, replace=False)
+        dataset["train"] = torch.utils.data.Subset(dataset["train"], indices)
+    dataset["val"] = echonet.datasets.Echo(root=data_dir, split="val", **kwargs)
+
+    # Run training and testing loops
+    with open(os.path.join(output, "log.csv"), "a") as f:
+        epoch_resume = 0
+        bestLoss = float("inf")
+        try:
+            # Attempt to load checkpoint
+            checkpoint = torch.load(os.path.join(output, "checkpoint.pt"))
+            model.load_state_dict(checkpoint['state_dict'])
+            optim.load_state_dict(checkpoint['opt_dict'])
+            scheduler.load_state_dict(checkpoint['scheduler_dict'])
+            epoch_resume = checkpoint["epoch"] + 1
+            bestLoss = checkpoint["best_loss"]
+            f.write("Resuming from epoch {}\n".format(epoch_resume))
+        except FileNotFoundError:
+            f.write("Starting run from scratch\n")
+
+        for epoch in range(epoch_resume, num_epochs):
+            print("Epoch #{}".format(epoch), flush=True)
+            for phase in ['train', 'val']:
+                start_time = time.time()
+                for i in range(torch.cuda.device_count()):
+                    torch.cuda.reset_peak_memory_stats(i)
+
+                ds = dataset[phase]
+                dataloader = torch.utils.data.DataLoader(
+                    ds, batch_size=batch_size, num_workers=num_workers, shuffle=True, pin_memory=(device.type == "cuda"), drop_last=(phase == "train"))
+
+                loss, yhat, y = echonet.utils.video.run_epoch(model, dataloader, phase == "train", optim, device)
+                f.write("{},{},{},{},{},{},{},{},{}\n".format(epoch,
+                                                              phase,
+                                                              loss,
+                                                              sklearn.metrics.r2_score(y, yhat),
+                                                              time.time() - start_time,
+                                                              y.size,
+                                                              sum(torch.cuda.max_memory_allocated() for i in range(torch.cuda.device_count())),
+                                                              sum(torch.cuda.max_memory_reserved() for i in range(torch.cuda.device_count())),
+                                                              batch_size))
+                f.flush()
+            scheduler.step()
+
+            # Save checkpoint
+            save = {
+                'epoch': epoch,
+                'state_dict': model.state_dict(),
+                'period': period,
+                'frames': frames,
+                'best_loss': bestLoss,
+                'loss': loss,
+                'r2': sklearn.metrics.r2_score(y, yhat),
+                'opt_dict': optim.state_dict(),
+                'scheduler_dict': scheduler.state_dict(),
+            }
+            torch.save(save, os.path.join(output, "checkpoint.pt"))
+            if loss < bestLoss:
+                torch.save(save, os.path.join(output, "best.pt"))
+                bestLoss = loss
+
+        # Load best weights
+        if num_epochs != 0:
+            checkpoint = torch.load(os.path.join(output, "best.pt"))
+            model.load_state_dict(checkpoint['state_dict'])
+            f.write("Best validation loss {} from epoch {}\n".format(checkpoint["loss"], checkpoint["epoch"]))
+            f.flush()
+
+        if run_test:
+            for split in ["val", "test"]:
+                # Performance without test-time augmentation
+                dataloader = torch.utils.data.DataLoader(
+                    echonet.datasets.Echo(root=data_dir, split=split, **kwargs),
+                    batch_size=batch_size, num_workers=num_workers, shuffle=True, pin_memory=(device.type == "cuda"))
+                loss, yhat, y = echonet.utils.video.run_epoch(model, dataloader, False, None, device)
+                f.write("{} (one clip) R2:   {:.3f} ({:.3f} - {:.3f})\n".format(split, *echonet.utils.bootstrap(y, yhat, sklearn.metrics.r2_score)))
+                f.write("{} (one clip) MAE:  {:.2f} ({:.2f} - {:.2f})\n".format(split, *echonet.utils.bootstrap(y, yhat, sklearn.metrics.mean_absolute_error)))
+                f.write("{} (one clip) RMSE: {:.2f} ({:.2f} - {:.2f})\n".format(split, *tuple(map(math.sqrt, echonet.utils.bootstrap(y, yhat, sklearn.metrics.mean_squared_error)))))
+                f.flush()
+
+                # Performance with test-time augmentation
+                ds = echonet.datasets.Echo(root=data_dir, split=split, **kwargs, clips="all")
+                dataloader = torch.utils.data.DataLoader(
+                    ds, batch_size=1, num_workers=num_workers, shuffle=False, pin_memory=(device.type == "cuda"))
+                loss, yhat, y = echonet.utils.video.run_epoch(model, dataloader, False, None, device, save_all=True, block_size=batch_size)
+                f.write("{} (all clips) R2:   {:.3f} ({:.3f} - {:.3f})\n".format(split, *echonet.utils.bootstrap(y, np.array(list(map(lambda x: x.mean(), yhat))), sklearn.metrics.r2_score)))
+                f.write("{} (all clips) MAE:  {:.2f} ({:.2f} - {:.2f})\n".format(split, *echonet.utils.bootstrap(y, np.array(list(map(lambda x: x.mean(), yhat))), sklearn.metrics.mean_absolute_error)))
+                f.write("{} (all clips) RMSE: {:.2f} ({:.2f} - {:.2f})\n".format(split, *tuple(map(math.sqrt, echonet.utils.bootstrap(y, np.array(list(map(lambda x: x.mean(), yhat))), sklearn.metrics.mean_squared_error)))))
+                f.flush()
+
+                # Write full performance to file
+                with open(os.path.join(output, "{}_predictions.csv".format(split)), "w") as g:
+                    for (filename, pred) in zip(ds.fnames, yhat):
+                        for (i, p) in enumerate(pred):
+                            g.write("{},{},{:.4f}\n".format(filename, i, p))
+                echonet.utils.latexify()
+                yhat = np.array(list(map(lambda x: x.mean(), yhat)))
+
+                # Plot actual and predicted EF
+                fig = plt.figure(figsize=(3, 3))
+                lower = min(y.min(), yhat.min())
+                upper = max(y.max(), yhat.max())
+                plt.scatter(y, yhat, color="k", s=1, edgecolor=None, zorder=2)
+                plt.plot([0, 100], [0, 100], linewidth=1, zorder=3)
+                plt.axis([lower - 3, upper + 3, lower - 3, upper + 3])
+                plt.gca().set_aspect("equal", "box")
+                plt.xlabel("Actual EF (%)")
+                plt.ylabel("Predicted EF (%)")
+                plt.xticks([10, 20, 30, 40, 50, 60, 70, 80])
+                plt.yticks([10, 20, 30, 40, 50, 60, 70, 80])
+                plt.grid(color="gainsboro", linestyle="--", linewidth=1, zorder=1)
+                plt.tight_layout()
+                plt.savefig(os.path.join(output, "{}_scatter.pdf".format(split)))
+                plt.close(fig)
+
+                # Plot AUROC
+                fig = plt.figure(figsize=(3, 3))
+                plt.plot([0, 1], [0, 1], linewidth=1, color="k", linestyle="--")
+                for thresh in [35, 40, 45, 50]:
+                    fpr, tpr, _ = sklearn.metrics.roc_curve(y > thresh, yhat)
+                    print(thresh, sklearn.metrics.roc_auc_score(y > thresh, yhat))
+                    plt.plot(fpr, tpr)
+
+                plt.axis([-0.01, 1.01, -0.01, 1.01])
+                plt.xlabel("False Positive Rate")
+                plt.ylabel("True Positive Rate")
+                plt.tight_layout()
+                plt.savefig(os.path.join(output, "{}_roc.pdf".format(split)))
+                plt.close(fig)
+
+
+def run_epoch(model, dataloader, train, optim, device, save_all=False, block_size=None):
+    """Run one epoch of training/evaluation for segmentation.
+
+    Args:
+        model (torch.nn.Module): Model to train/evaulate.
+        dataloder (torch.utils.data.DataLoader): Dataloader for dataset.
+        train (bool): Whether or not to train model.
+        optim (torch.optim.Optimizer): Optimizer
+        device (torch.device): Device to run on
+        save_all (bool, optional): If True, return predictions for all
+            test-time augmentations separately. If False, return only
+            the mean prediction.
+            Defaults to False.
+        block_size (int or None, optional): Maximum number of augmentations
+            to run on at the same time. Use to limit the amount of memory
+            used. If None, always run on all augmentations simultaneously.
+            Default is None.
+    """
+
+    model.train(train)
+
+    total = 0  # total training loss
+    n = 0      # number of videos processed
+    s1 = 0     # sum of ground truth EF
+    s2 = 0     # Sum of ground truth EF squared
+
+    yhat = []
+    y = []
+
+    with torch.set_grad_enabled(train):
+        with tqdm.tqdm(total=len(dataloader)) as pbar:
+            for (X, outcome) in dataloader:
+
+                y.append(outcome.numpy())
+                X = X.to(device)
+                outcome = outcome.to(device)
+
+                average = (len(X.shape) == 6)
+                if average:
+                    batch, n_clips, c, f, h, w = X.shape
+                    X = X.view(-1, c, f, h, w)
+
+                s1 += outcome.sum()
+                s2 += (outcome ** 2).sum()
+
+                if block_size is None:
+                    outputs = model(X)
+                else:
+                    outputs = torch.cat([model(X[j:(j + block_size), ...]) for j in range(0, X.shape[0], block_size)])
+
+                if save_all:
+                    yhat.append(outputs.view(-1).to("cpu").detach().numpy())
+
+                if average:
+                    outputs = outputs.view(batch, n_clips, -1).mean(1)
+
+                if not save_all:
+                    yhat.append(outputs.view(-1).to("cpu").detach().numpy())
+
+                loss = torch.nn.functional.mse_loss(outputs.view(-1), outcome)
+
+                if train:
+                    optim.zero_grad()
+                    loss.backward()
+                    optim.step()
+
+                total += loss.item() * X.size(0)
+                n += X.size(0)
+
+                pbar.set_postfix_str("{:.2f} ({:.2f}) / {:.2f}".format(total / n, loss.item(), s2 / n - (s1 / n) ** 2))
+                pbar.update()
+
+    if not save_all:
+        yhat = np.concatenate(yhat)
+    y = np.concatenate(y)
+
+    return total / n, yhat, y

dynamic/example.cfg

@@ -0,0 +1 @@
+DATA_DIR = a4c-video-dir/

dynamic/requirements.txt

@@ -0,0 +1,28 @@
+certifi==2020.12.5
+cycler==0.10.0
+decorator==4.4.2
+echonet==1.0.0
+imageio==2.9.0
+joblib==1.0.1
+kiwisolver==1.3.1
+matplotlib==3.3.4
+networkx==2.5
+numpy==1.20.1
+opencv-python==4.5.1.48
+pandas==1.2.3
+Pillow==8.1.2
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+PyWavelets==1.1.1
+scikit-image==0.18.1
+scikit-learn==0.24.1
+scipy==1.6.1
+six==1.15.0
+sklearn==0.0
+threadpoolctl==2.1.0
+tifffile==2021.3.17
+torch==1.8.0
+torchvision==0.9.0
+tqdm==4.59.0
+typing-extensions==3.7.4.3

dynamic/scripts/ConvertDICOMToAVI.ipynb

@@ -0,0 +1,215 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# David Ouyang 10/2/2019\n",
+    "\n",
+    "# Notebook which iterates through a folder, including subfolders, \n",
+    "# and convert DICOM files to AVI files of a defined size (natively 112 x 112)\n",
+    "\n",
+    "import re\n",
+    "import os, os.path\n",
+    "from os.path import splitext\n",
+    "import pydicom as dicom\n",
+    "import numpy as np\n",
+    "from pydicom.uid import UID, generate_uid\n",
+    "import shutil\n",
+    "from multiprocessing import dummy as multiprocessing\n",
+    "import time\n",
+    "import subprocess\n",
+    "import datetime\n",
+    "from datetime import date\n",
+    "import sys\n",
+    "import cv2\n",
+    "#from scipy.misc import imread\n",
+    "import matplotlib.pyplot as plt\n",
+    "import sys\n",
+    "from shutil import copy\n",
+    "import math\n",
+    "\n",
+    "destinationFolder = \"Output Folder Name\"\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: pillow in c:\\programdata\\anaconda3\\lib\\site-packages (6.2.0)\n",
+      "Requirement already satisfied: scipy in c:\\programdata\\anaconda3\\lib\\site-packages (1.3.1)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Dependencies you might need to run code\n",
+    "# Commonly missing\n",
+    "\n",
+    "#!pip install pydicom\n",
+    "#!pip install opencv-python\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def mask(output):\n",
+    "    dimension = output.shape[0]\n",
+    "    \n",
+    "    # Mask pixels outside of scanning sector\n",
+    "    m1, m2 = np.meshgrid(np.arange(dimension), np.arange(dimension))\n",
+    "    \n",
+    "\n",
+    "    mask = ((m1+m2)>int(dimension/2) + int(dimension/10)) \n",
+    "    mask *=  ((m1-m2)<int(dimension/2) + int(dimension/10))\n",
+    "    mask = np.reshape(mask, (dimension, dimension)).astype(np.int8)\n",
+    "    maskedImage = cv2.bitwise_and(output, output, mask = mask)\n",
+    "    \n",
+    "    #print(maskedImage.shape)\n",
+    "    \n",
+    "    return maskedImage\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def makeVideo(fileToProcess, destinationFolder):\n",
+    "    try:\n",
+    "        fileName = fileToProcess.split('\\\\')[-1] #\\\\ if windows, / if on mac or sherlock\n",
+    "                                                 #hex(abs(hash(fileToProcess.split('/')[-1]))).upper()\n",
+    "\n",
+    "        if not os.path.isdir(os.path.join(destinationFolder,fileName)):\n",
+    "\n",
+    "            dataset = dicom.dcmread(fileToProcess, force=True)\n",
+    "            testarray = dataset.pixel_array\n",
+    "\n",
+    "            frame0 = testarray[0]\n",
+    "            mean = np.mean(frame0, axis=1)\n",
+    "            mean = np.mean(mean, axis=1)\n",
+    "            yCrop = np.where(mean<1)[0][0]\n",
+    "            testarray = testarray[:, yCrop:, :, :]\n",
+    "\n",
+    "            bias = int(np.abs(testarray.shape[2] - testarray.shape[1])/2)\n",
+    "            if bias>0:\n",
+    "                if testarray.shape[1] < testarray.shape[2]:\n",
+    "                    testarray = testarray[:, :, bias:-bias, :]\n",
+    "                else:\n",
+    "                    testarray = testarray[:, bias:-bias, :, :]\n",
+    "\n",
+    "\n",
+    "            print(testarray.shape)\n",
+    "            frames,height,width,channels = testarray.shape\n",
+    "\n",
+    "            fps = 30\n",
+    "\n",
+    "            try:\n",
+    "                fps = dataset[(0x18, 0x40)].value\n",
+    "            except:\n",
+    "                print(\"couldn't find frame rate, default to 30\")\n",
+    "\n",
+    "            fourcc = cv2.VideoWriter_fourcc('M','J','P','G')\n",
+    "            video_filename = os.path.join(destinationFolder, fileName + '.avi')\n",
+    "            out = cv2.VideoWriter(video_filename, fourcc, fps, cropSize)\n",
+    "\n",
+    "\n",
+    "            for i in range(frames):\n",
+    "\n",
+    "                outputA = testarray[i,:,:,0]\n",
+    "                smallOutput = outputA[int(height/10):(height - int(height/10)), int(height/10):(height - int(height/10))]\n",
+    "\n",
+    "                # Resize image\n",
+    "                output = cv2.resize(smallOutput, cropSize, interpolation = cv2.INTER_CUBIC)\n",
+    "\n",
+    "                finaloutput = mask(output)\n",
+    "\n",
+    "\n",
+    "                finaloutput = cv2.merge([finaloutput,finaloutput,finaloutput])\n",
+    "                out.write(finaloutput)\n",
+    "\n",
+    "            out.release()\n",
+    "\n",
+    "        else:\n",
+    "            print(fileName,\"hasAlreadyBeenProcessed\")\n",
+    "    except:\n",
+    "        print(\"something filed, not sure what, have to debug\", fileName)\n",
+    "    return 0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "AllA4cNames = \"Input Folder Name\"\n",
+    "\n",
+    "count = 0\n",
+    "    \n",
+    "cropSize = (112,112)\n",
+    "subfolders = os.listdir(AllA4cNames)\n",
+    "\n",
+    "\n",
+    "for folder in subfolders:\n",
+    "    print(folder)\n",
+    "\n",
+    "    for content in os.listdir(os.path.join(AllA4cNames, folder)):\n",
+    "        for subcontent in os.listdir(os.path.join(AllA4cNames, folder, content)):\n",
+    "            count += 1\n",
+    "            \n",
+    "\n",
+    "            VideoPath = os.path.join(AllA4cNames, folder, content, subcontent)\n",
+    "\n",
+    "            print(count, folder, content, subcontent)\n",
+    "\n",
+    "            if not os.path.exists(os.path.join(destinationFolder,subcontent + \".avi\")):\n",
+    "                makeVideo(VideoPath, destinationFolder)\n",
+    "            else:\n",
+    "                print(\"Already did this file\", VideoPath)\n",
+    "\n",
+    "\n",
+    "print(len(AllA4cFilenames))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

dynamic/scripts/InitializationNotebook.ipynb

@@ -0,0 +1,288 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# David Ouyang 12/5/2019\n",
+    "\n",
+    "# Notebook which:\n",
+    "# 1. Downloads weights\n",
+    "# 2. Initializes model and imports weights\n",
+    "# 3. Performs test time evaluation of videos (already preprocessed with ConvertDICOMToAVI.ipynb)\n",
+    "\n",
+    "import re\n",
+    "import os, os.path\n",
+    "from os.path import splitext\n",
+    "import pydicom as dicom\n",
+    "import numpy as np\n",
+    "from pydicom.uid import UID, generate_uid\n",
+    "import shutil\n",
+    "from multiprocessing import dummy as multiprocessing\n",
+    "import time\n",
+    "import subprocess\n",
+    "import datetime\n",
+    "from datetime import date\n",
+    "import sys\n",
+    "import cv2\n",
+    "import matplotlib.pyplot as plt\n",
+    "import sys\n",
+    "from shutil import copy\n",
+    "import math\n",
+    "import torch\n",
+    "import torchvision\n",
+    "\n",
+    "sys.path.append(\"..\")\n",
+    "import echonet\n",
+    "\n",
+    "import wget \n",
+    "\n",
+    "#destinationFolder = \"/Users/davidouyang/Dropbox/Echo Research/CodeBase/Output\"\n",
+    "destinationFolder = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\Output\"\n",
+    "#videosFolder = \"/Users/davidouyang/Dropbox/Echo Research/CodeBase/a4c-video-dir\"\n",
+    "videosFolder = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\a4c-video-dir\"\n",
+    "#DestinationForWeights = \"/Users/davidouyang/Dropbox/Echo Research/CodeBase/EchoNetDynamic-Weights\"\n",
+    "DestinationForWeights = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\EchoNetDynamic-Weights\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The weights are at C:\\Users\\Windows\\Dropbox\\Echo Research\\CodeBase\\EchoNetDynamic-Weights\n",
+      "Segmentation Weights already present\n",
+      "EF Weights already present\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Download model weights\n",
+    "\n",
+    "if os.path.exists(DestinationForWeights):\n",
+    "    print(\"The weights are at\", DestinationForWeights)\n",
+    "else:\n",
+    "    print(\"Creating folder at \", DestinationForWeights, \" to store weights\")\n",
+    "    os.mkdir(DestinationForWeights)\n",
+    "    \n",
+    "segmentationWeightsURL = 'https://github.com/douyang/EchoNetDynamic/releases/download/v1.0.0/deeplabv3_resnet50_random.pt'\n",
+    "ejectionFractionWeightsURL = 'https://github.com/douyang/EchoNetDynamic/releases/download/v1.0.0/r2plus1d_18_32_2_pretrained.pt'\n",
+    "\n",
+    "\n",
+    "if not os.path.exists(os.path.join(DestinationForWeights, os.path.basename(segmentationWeightsURL))):\n",
+    "    print(\"Downloading Segmentation Weights, \", segmentationWeightsURL,\" to \",os.path.join(DestinationForWeights,os.path.basename(segmentationWeightsURL)))\n",
+    "    filename = wget.download(segmentationWeightsURL, out = DestinationForWeights)\n",
+    "else:\n",
+    "    print(\"Segmentation Weights already present\")\n",
+    "    \n",
+    "if not os.path.exists(os.path.join(DestinationForWeights, os.path.basename(ejectionFractionWeightsURL))):\n",
+    "    print(\"Downloading EF Weights, \", ejectionFractionWeightsURL,\" to \",os.path.join(DestinationForWeights,os.path.basename(ejectionFractionWeightsURL)))\n",
+    "    filename = wget.download(ejectionFractionWeightsURL, out = DestinationForWeights)\n",
+    "else:\n",
+    "    print(\"EF Weights already present\")\n",
+    "        \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading weights from  C:\\Users\\Windows\\Dropbox\\Echo Research\\CodeBase\\EchoNetDynamic-Weights\\r2plus1d_18_32_2_pretrained\n",
+      "cuda is available, original weights\n",
+      "external_test ['0X1A05DFFFCAFB253B.avi', '0X1A0A263B22CCD966.avi', '0X1A2A76BDB5B98BED.avi', '0X1A2C60147AF9FDAE.avi', '0X1A2E9496910EFF5B.avi', '0X1A3D565B371DC573.avi', '0X1A3E7BF1DFB132FB.avi', '0X1A5FAE3F9D37794E.avi', '0X1A6ACFE7B286DAFC.avi', '0X1A8D85542DBE8204.avi', '23_Apical_4_chamber_view.dcm.avi', '62_Apical_4_chamber_view.dcm.avi', '64_Apical_4_chamber_view.dcm.avi']\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████████████████████████████████████████████████████████████████████████████| 10/10 [00:10<00:00,  1.00s/it]\n",
+      "100%|████████████████████████████████████████████████████████| 13/13 [00:29<00:00,  2.26s/it, 3122.29 (3440.26) / 0.00]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Initialize and Run EF model\n",
+    "\n",
+    "frames = 32\n",
+    "period = 1 #2\n",
+    "batch_size = 20\n",
+    "model = torchvision.models.video.r2plus1d_18(pretrained=False)\n",
+    "model.fc = torch.nn.Linear(model.fc.in_features, 1)\n",
+    "\n",
+    "\n",
+    "\n",
+    "print(\"loading weights from \", os.path.join(DestinationForWeights, \"r2plus1d_18_32_2_pretrained\"))\n",
+    "\n",
+    "if torch.cuda.is_available():\n",
+    "    print(\"cuda is available, original weights\")\n",
+    "    device = torch.device(\"cuda\")\n",
+    "    model = torch.nn.DataParallel(model)\n",
+    "    model.to(device)\n",
+    "    checkpoint = torch.load(os.path.join(DestinationForWeights, os.path.basename(ejectionFractionWeightsURL)))\n",
+    "    model.load_state_dict(checkpoint['state_dict'])\n",
+    "else:\n",
+    "    print(\"cuda is not available, cpu weights\")\n",
+    "    device = torch.device(\"cpu\")\n",
+    "    checkpoint = torch.load(os.path.join(DestinationForWeights, os.path.basename(ejectionFractionWeightsURL)), map_location = \"cpu\")\n",
+    "    state_dict_cpu = {k[7:]: v for (k, v) in checkpoint['state_dict'].items()}\n",
+    "    model.load_state_dict(state_dict_cpu)\n",
+    "\n",
+    "\n",
+    "# try some random weights: final_r2+1d_model_regression_EF_sgd_skip1_32frames.pth.tar\n",
+    "# scp ouyangd@arthur2:~/Echo-Tracing-Analysis/final_r2+1d_model_regression_EF_sgd_skip1_32frames.pth.tar \"C:\\Users\\Windows\\Dropbox\\Echo Research\\CodeBase\\EchoNetDynamic-Weights\"\n",
+    "#Weights = \"final_r2+1d_model_regression_EF_sgd_skip1_32frames.pth.tar\"\n",
+    "\n",
+    "\n",
+    "output = os.path.join(destinationFolder, \"cedars_ef_output.csv\")\n",
+    "\n",
+    "ds = echonet.datasets.Echo(split = \"external_test\", external_test_location = videosFolder, crops=\"all\")\n",
+    "print(ds.split, ds.fnames)\n",
+    "\n",
+    "mean, std = echonet.utils.get_mean_and_std(ds)\n",
+    "\n",
+    "kwargs = {\"target_type\": \"EF\",\n",
+    "          \"mean\": mean,\n",
+    "          \"std\": std,\n",
+    "          \"length\": frames,\n",
+    "          \"period\": period,\n",
+    "          }\n",
+    "\n",
+    "ds = echonet.datasets.Echo(split = \"external_test\", external_test_location = videosFolder, **kwargs, crops=\"all\")\n",
+    "\n",
+    "test_dataloader = torch.utils.data.DataLoader(ds, batch_size = 1, num_workers = 5, shuffle = True, pin_memory=(device.type == \"cuda\"))\n",
+    "loss, yhat, y = echonet.utils.video.run_epoch(model, test_dataloader, \"test\", None, device, save_all=True, blocks=25)\n",
+    "\n",
+    "with open(output, \"w\") as g:\n",
+    "    for (filename, pred) in zip(ds.fnames, yhat):\n",
+    "        for (i,p) in enumerate(pred):\n",
+    "            g.write(\"{},{},{:.4f}\\n\".format(filename, i, p))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize and Run Segmentation model\n",
+    "\n",
+    "torch.cuda.empty_cache()\n",
+    "\n",
+    "\n",
+    "videosFolder = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\View Classification\\\\AppearsA4c\\\\Resized2\"\n",
+    "\n",
+    "def collate_fn(x):\n",
+    "    x, f = zip(*x)\n",
+    "    i = list(map(lambda t: t.shape[1], x))\n",
+    "    x = torch.as_tensor(np.swapaxes(np.concatenate(x, 1), 0, 1))\n",
+    "    return x, f, i\n",
+    "\n",
+    "dataloader = torch.utils.data.DataLoader(echonet.datasets.Echo(split=\"external_test\", external_test_location = videosFolder, target_type=[\"Filename\"], length=None, period=1, mean=mean, std=std),\n",
+    "                                         batch_size=10, num_workers=0, shuffle=False, pin_memory=(device.type == \"cuda\"), collate_fn=collate_fn)\n",
+    "if not all([os.path.isfile(os.path.join(destinationFolder, \"labels\", os.path.splitext(f)[0] + \".npy\")) for f in dataloader.dataset.fnames]):\n",
+    "    # Save segmentations for all frames\n",
+    "    # Only run if missing files\n",
+    "\n",
+    "    pathlib.Path(os.path.join(destinationFolder, \"labels\")).mkdir(parents=True, exist_ok=True)\n",
+    "    block = 1024\n",
+    "    model.eval()\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        for (x, f, i) in tqdm.tqdm(dataloader):\n",
+    "            x = x.to(device)\n",
+    "            y = np.concatenate([model(x[i:(i + block), :, :, :])[\"out\"].detach().cpu().numpy() for i in range(0, x.shape[0], block)]).astype(np.float16)\n",
+    "            start = 0\n",
+    "            for (filename, offset) in zip(f, i):\n",
+    "                np.save(os.path.join(destinationFolder, \"labels\", os.path.splitext(filename)[0]), y[start:(start + offset), 0, :, :])\n",
+    "                start += offset\n",
+    "                \n",
+    "dataloader = torch.utils.data.DataLoader(echonet.datasets.Echo(split=\"external_test\", external_test_location = videosFolder, target_type=[\"Filename\"], length=None, period=1, segmentation=os.path.join(destinationFolder, \"labels\")),\n",
+    "                                         batch_size=1, num_workers=8, shuffle=False, pin_memory=False)\n",
+    "if not all(os.path.isfile(os.path.join(destinationFolder, \"videos\", f)) for f in dataloader.dataset.fnames):\n",
+    "    pathlib.Path(os.path.join(destinationFolder, \"videos\")).mkdir(parents=True, exist_ok=True)\n",
+    "    pathlib.Path(os.path.join(destinationFolder, \"size\")).mkdir(parents=True, exist_ok=True)\n",
+    "    echonet.utils.latexify()\n",
+    "    with open(os.path.join(destinationFolder, \"size.csv\"), \"w\") as g:\n",
+    "        g.write(\"Filename,Frame,Size,ComputerSmall\\n\")\n",
+    "        for (x, filename) in tqdm.tqdm(dataloader):\n",
+    "            x = x.numpy()\n",
+    "            for i in range(len(filename)):\n",
+    "                img = x[i, :, :, :, :].copy()\n",
+    "                logit = img[2, :, :, :].copy()\n",
+    "                img[1, :, :, :] = img[0, :, :, :]\n",
+    "                img[2, :, :, :] = img[0, :, :, :]\n",
+    "                img = np.concatenate((img, img), 3)\n",
+    "                img[0, :, :, 112:] = np.maximum(255. * (logit > 0), img[0, :, :, 112:])\n",
+    "\n",
+    "                img = np.concatenate((img, np.zeros_like(img)), 2)\n",
+    "                size = (logit > 0).sum(2).sum(1)\n",
+    "                try:\n",
+    "                    trim_min = sorted(size)[round(len(size) ** 0.05)]\n",
+    "                except:\n",
+    "                    import code; code.interact(local=dict(globals(), **locals()))\n",
+    "                trim_max = sorted(size)[round(len(size) ** 0.95)]\n",
+    "                trim_range = trim_max - trim_min\n",
+    "                peaks = set(scipy.signal.find_peaks(-size, distance=20, prominence=(0.50 * trim_range))[0])\n",
+    "                for (x, y) in enumerate(size):\n",
+    "                    g.write(\"{},{},{},{}\\n\".format(filename[0], x, y, 1 if x in peaks else 0))\n",
+    "                fig = plt.figure(figsize=(size.shape[0] / 50 * 1.5, 3))\n",
+    "                plt.scatter(np.arange(size.shape[0]) / 50, size, s=1)\n",
+    "                ylim = plt.ylim()\n",
+    "                for p in peaks:\n",
+    "                    plt.plot(np.array([p, p]) / 50, ylim, linewidth=1)\n",
+    "                plt.ylim(ylim)\n",
+    "                plt.title(os.path.splitext(filename[i])[0])\n",
+    "                plt.xlabel(\"Seconds\")\n",
+    "                plt.ylabel(\"Size (pixels)\")\n",
+    "                plt.tight_layout()\n",
+    "                plt.savefig(os.path.join(destinationFolder, \"size\", os.path.splitext(filename[i])[0] + \".pdf\"))\n",
+    "                plt.close(fig)\n",
+    "                size -= size.min()\n",
+    "                size = size / size.max()\n",
+    "                size = 1 - size\n",
+    "                for (x, y) in enumerate(size):\n",
+    "                    img[:, :, int(round(115 + 100 * y)), int(round(x / len(size) * 200 + 10))] = 255.\n",
+    "                    interval = np.array([-3, -2, -1, 0, 1, 2, 3])\n",
+    "                    for a in interval:\n",
+    "                        for b in interval:\n",
+    "                            img[:, x, a + int(round(115 + 100 * y)), b + int(round(x / len(size) * 200 + 10))] = 255.\n",
+    "                    if x in peaks:\n",
+    "                        img[:, :, 200:225, b + int(round(x / len(size) * 200 + 10))] = 255.\n",
+    "                echonet.utils.savevideo(os.path.join(destinationFolder, \"videos\", filename[i]), img.astype(np.uint8), 50)                "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

dynamic/scripts/beat_by_beat_analysis.R

@@ -0,0 +1,100 @@
+library(ggplot2)
+library(stringr)
+library(plyr)
+library(dplyr)
+library(lubridate)
+library(reshape2)
+library(scales)
+library(ggthemes)
+library(Metrics)
+
+data <- read.csv("r2plus1d_18_32_2_pretrained_test_predictions.csv", header = FALSE)
+str(data)
+
+
+dataNoAugmentation <- data[data$V2 == 0,]
+str(dataNoAugmentation)
+
+
+dataGlobalAugmentation <- data %>% group_by(V1) %>% summarize(meanPrediction = mean(V3), sdPred = sd(V3))
+str(dataGlobalAugmentation)
+
+
+sizeData <- read.csv("size.csv")
+sizeData <- sizeData[sizeData$ComputerSmall == 1,]
+str(sizeData)
+
+sizeRelevantFrames <- sizeData[c(1,2)]
+sizeRelevantFrames$Frame <- sizeRelevantFrames$Frame - 32
+sizeRelevantFrames[sizeRelevantFrames$Frame < 0,]$Frame <- 0
+
+
+beatByBeat <- merge(sizeRelevantFrames, data, by.x = c("Filename", "Frame"), by.y = c("V1", "V2"))
+beatByBeat <- beatByBeat %>% group_by(Filename) %>% summarize(meanPrediction = mean(V3), sdPred = sd(V3))
+str(beatByBeat)
+
+### For use, need to specify file directory
+fileLocation <- "/Users/davidouyang/Local Medical Data/"
+ActualNumbers <- read.csv(paste0(fileLocation, "FileList.csv", sep = ""))
+ActualNumbers <- ActualNumbers[c(1,2)]
+str(ActualNumbers)
+
+
+
+dataNoAugmentation <- merge(dataNoAugmentation, ActualNumbers, by.x = "V1", by.y = "Filename", all.x = TRUE)
+dataNoAugmentation$AbsErr <- abs(dataNoAugmentation$V3 - dataNoAugmentation$EF)
+str(dataNoAugmentation)
+
+summary(abs(dataNoAugmentation$V3 - dataNoAugmentation$EF))
+# Mean of 4.216
+
+rmse(dataNoAugmentation$V3,dataNoAugmentation$EF) 
+## 5.56
+
+modelNoAugmentation <- lm(dataNoAugmentation$EF ~ dataNoAugmentation$V3)
+summary(modelNoAugmentation)$r.squared
+# 0.79475
+
+
+beatByBeat <- merge(beatByBeat, ActualNumbers, by.x = "Filename", by.y = "Filename", all.x = TRUE)
+summary(abs(beatByBeat$meanPrediction - beatByBeat$EF))
+# Mean of 4.051697
+
+rmse(beatByBeat$meanPrediction, beatByBeat$EF) 
+# 5.325237
+
+modelBeatByBeat <- lm(beatByBeat$EF ~ beatByBeat$meanPrediction)
+summary(modelBeatByBeat)$r.squared
+# 0.8093174
+
+
+beatByBeatAnalysis <- merge(sizeRelevantFrames, data, by.x = c("Filename", "Frame"), by.y = c("V1", "V2"))
+str(beatByBeatAnalysis)
+
+
+MAEdata <- data.frame(counter = 1:500)
+MAEdata$sample <- -9999
+MAEdata$error <- -9999
+
+str(MAEdata)
+
+for (i in 1:500){
+
+
+samplingBeat <-  sample_n(beatByBeatAnalysis %>% group_by(Filename), 1 + floor((i-1)/100), replace = TRUE) %>% group_by(Filename) %>% dplyr::summarize(meanPred = mean(V3))
+samplingBeat <- merge(samplingBeat, ActualNumbers, by.x = "Filename", by.y = "Filename", all.x = TRUE)
+samplingBeat$error <- abs(samplingBeat$meanPred - samplingBeat$EF)
+
+MAEdata$sample[i] <-  1 + floor((i-1)/100)
+MAEdata$error[i] <- mean(samplingBeat$error )
+
+
+}
+
+str(MAEdata)
+
+beatBoxPlot <- ggplot(data = MAEdata) + geom_boxplot(aes(x = sample, y = error, group = sample), outlier.shape = NA
+) + theme_classic() + theme(legend.position = "none", axis.text.y = element_text( size=7)) + xlab("Number of Sampled Beats") + ylab("Mean Absolute Error") + scale_fill_brewer(palette = "Set1", direction = -1) 
+
+beatBoxPlot
+

dynamic/scripts/plot_complexity.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+"""Code to generate plots for Extended Data Fig. 4."""
+
+import os
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+
+import echonet
+
+
+def main(root=os.path.join("timing", "video"),
+         fig_root=os.path.join("figure", "complexity"),
+         FRAMES=(1, 8, 16, 32, 64, 96),
+         pretrained=True):
+    """Generate plots for Extended Data Fig. 4."""
+
+    echonet.utils.latexify()
+
+    os.makedirs(fig_root, exist_ok=True)
+    fig = plt.figure(figsize=(6.50, 2.50))
+    gs = matplotlib.gridspec.GridSpec(1, 3, width_ratios=[2.5, 2.5, 1.50])
+    ax = (plt.subplot(gs[0]), plt.subplot(gs[1]), plt.subplot(gs[2]))
+
+    # Create legend
+    for (model, color) in zip(["EchoNet-Dynamic (EF)", "R3D", "MC3"], matplotlib.colors.TABLEAU_COLORS):
+        ax[2].plot([float("nan")], [float("nan")], "-", color=color, label=model)
+    ax[2].set_title("")
+    ax[2].axis("off")
+    ax[2].legend(loc="center")
+
+    for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"], matplotlib.colors.TABLEAU_COLORS):
+        for split in ["val"]:  # ["val", "train"]:
+            print(model, split)
+            data = [load(root, model, frames, 1, pretrained, split) for frames in FRAMES]
+            time = np.array(list(map(lambda x: x[0], data)))
+            n = np.array(list(map(lambda x: x[1], data)))
+            mem_allocated = np.array(list(map(lambda x: x[2], data)))
+            # mem_cached = np.array(list(map(lambda x: x[3], data)))
+            batch_size = np.array(list(map(lambda x: x[4], data)))
+
+            # Plot Time (panel a)
+            ax[0].plot(FRAMES, time / n, "-" if pretrained else "--", marker=".", color=color, linewidth=(1 if split == "train" else None))
+            print("Time:\n" + "\n".join(map(lambda x: "{:8d}: {:f}".format(*x), zip(FRAMES, time / n))))
+
+            # Plot Memory (panel b)
+            ax[1].plot(FRAMES, mem_allocated / batch_size / 1e9, "-" if pretrained else "--", marker=".", color=color, linewidth=(1 if split == "train" else None))
+            print("Memory:\n" + "\n".join(map(lambda x: "{:8d}: {:f}".format(*x), zip(FRAMES, mem_allocated / batch_size / 1e9))))
+            print()
+
+    # Labels for panel a
+    ax[0].set_xticks(FRAMES)
+    ax[0].text(-0.05, 1.10, "(a)", transform=ax[0].transAxes)
+    ax[0].set_xlabel("Clip length (frames)")
+    ax[0].set_ylabel("Time Per Clip (seconds)")
+
+    # Labels for panel b
+    ax[1].set_xticks(FRAMES)
+    ax[1].text(-0.05, 1.10, "(b)", transform=ax[1].transAxes)
+    ax[1].set_xlabel("Clip length (frames)")
+    ax[1].set_ylabel("Memory Per Clip (GB)")
+
+    # Save figure
+    plt.tight_layout()
+    plt.savefig(os.path.join(fig_root, "complexity.pdf"))
+    plt.savefig(os.path.join(fig_root, "complexity.eps"))
+    plt.close(fig)
+
+
+def load(root, model, frames, period, pretrained, split):
+    """Loads runtime and memory usage for specified hyperparameter choice."""
+    with open(os.path.join(root, "{}_{}_{}_{}".format(model, frames, period, "pretrained" if pretrained else "random"), "log.csv"), "r") as f:
+        for line in f:
+            line = line.split(",")
+            if len(line) < 4:
+                # Skip lines that are not csv (these lines log information)
+                continue
+            if line[1] == split:
+                *_, time, n, mem_allocated, mem_cached, batch_size = line
+                time = float(time)
+                n = int(n)
+                mem_allocated = int(mem_allocated)
+                mem_cached = int(mem_cached)
+                batch_size = int(batch_size)
+                return time, n, mem_allocated, mem_cached, batch_size
+    raise ValueError("File missing information.")
+
+
+if __name__ == "__main__":
+    main()

dynamic/scripts/plot_hyperparameter_sweep.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+"""Code to generate plots for Extended Data Fig. 1."""
+
+import os
+
+import matplotlib
+import matplotlib.pyplot as plt
+
+import echonet
+
+
+def main(root=os.path.join("output", "video"),
+         fig_root=os.path.join("figure", "hyperparameter"),
+         FRAMES=(1, 8, 16, 32, 64, 96, None),
+         PERIOD=(1, 2, 4, 6, 8)
+         ):
+    """Generate plots for Extended Data Fig. 1."""
+
+    echonet.utils.latexify()
+    os.makedirs(fig_root, exist_ok=True)
+
+    # Parameters for plotting length sweep
+    MAX = FRAMES[-2]
+    START = 1    # Starting point for normal range
+    TERM0 = 104  # Ending point for normal range
+    BREAK = 112  # Location for break
+    TERM1 = 120  # Starting point for "all" section
+    ALL = 128    # Location of "all" point
+    END = 135    # Ending point for "all" section
+    RATIO = (BREAK - START) / (END - BREAK)
+
+    # Set up figure
+    fig = plt.figure(figsize=(3 + 2.5 + 1.5, 2.75))
+    outer = matplotlib.gridspec.GridSpec(1, 3, width_ratios=[3, 2.5, 1.50])
+    ax = plt.subplot(outer[2])   # Legend
+    ax2 = plt.subplot(outer[1])  # Period plot
+    gs = matplotlib.gridspec.GridSpecFromSubplotSpec(
+        1, 2, subplot_spec=outer[0], width_ratios=[RATIO, 1], wspace=0.020)  # Length plot
+
+    # Plot legend
+    for (model, color) in zip(["EchoNet-Dynamic (EF)", "R3D", "MC3"],
+                              matplotlib.colors.TABLEAU_COLORS):
+        ax.plot([float("nan")], [float("nan")], "-", color=color, label=model)
+    ax.plot([float("nan")], [float("nan")], "-", color="k", label="Pretrained")
+    ax.plot([float("nan")], [float("nan")], "--", color="k", label="Random")
+    ax.set_title("")
+    ax.axis("off")
+    ax.legend(loc="center")
+
+    # Plot length sweep (panel a)
+    ax0 = plt.subplot(gs[0])
+    ax1 = plt.subplot(gs[1], sharey=ax0)
+    print("FRAMES")
+    for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"],
+                              matplotlib.colors.TABLEAU_COLORS):
+        for pretrained in [True, False]:
+            loss = [load(root, model, frames, 1, pretrained) for frames in FRAMES]
+            print(model, pretrained)
+            print("    ".join(list(map(lambda x: "{:.1f}".format(x) if x is not None else None, loss))))
+
+            l0 = loss[-2]
+            l1 = loss[-1]
+            ax0.plot(FRAMES[:-1] + (TERM0,),
+                     loss[:-1] + [l0 + (l1 - l0) * (TERM0 - MAX) / (ALL - MAX)],
+                     "-" if pretrained else "--", color=color)
+            ax1.plot([TERM1, ALL],
+                     [l0 + (l1 - l0) * (TERM1 - MAX) / (ALL - MAX)] + [loss[-1]],
+                     "-" if pretrained else "--", color=color)
+            ax0.scatter(list(map(lambda x: x if x is not None else ALL, FRAMES)), loss, color=color, s=4)
+            ax1.scatter(list(map(lambda x: x if x is not None else ALL, FRAMES)), loss, color=color, s=4)
+
+    ax0.set_xticks(list(map(lambda x: x if x is not None else ALL, FRAMES)))
+    ax1.set_xticks(list(map(lambda x: x if x is not None else ALL, FRAMES)))
+    ax0.set_xticklabels(list(map(lambda x: x if x is not None else "All", FRAMES)))
+    ax1.set_xticklabels(list(map(lambda x: x if x is not None else "All", FRAMES)))
+
+    # https://stackoverflow.com/questions/5656798/python-matplotlib-is-there-a-way-to-make-a-discontinuous-axis/43684155
+    # zoom-in / limit the view to different portions of the data
+    ax0.set_xlim(START, BREAK)  # most of the data
+    ax1.set_xlim(BREAK, END)
+
+    # hide the spines between ax and ax2
+    ax0.spines['right'].set_visible(False)
+    ax1.spines['left'].set_visible(False)
+
+    ax1.get_yaxis().set_visible(False)
+
+    d = 0.015  # how big to make the diagonal lines in axes coordinates
+    # arguments to pass plot, just so we don't keep repeating them
+    kwargs = dict(transform=ax0.transAxes, color='k', clip_on=False, linewidth=1)
+    x0, x1, y0, y1 = ax0.axis()
+    scale = (y1 - y0) / (x1 - x0) / 2
+    ax0.plot((1 - scale * d, 1 + scale * d), (-d, +d), **kwargs)  # top-left diagonal
+    ax0.plot((1 - scale * d, 1 + scale * d), (1 - d, 1 + d), **kwargs)  # bottom-left diagonal
+
+    kwargs.update(transform=ax1.transAxes)  # switch to the bottom 1xes
+    x0, x1, y0, y1 = ax1.axis()
+    scale = (y1 - y0) / (x1 - x0) / 2
+    ax1.plot((-scale * d, scale * d), (-d, +d), **kwargs)  # top-right diagonal
+    ax1.plot((-scale * d, scale * d), (1 - d, 1 + d), **kwargs)  # bottom-right diagonal
+
+    # ax0.xaxis.label.set_transform(matplotlib.transforms.blended_transform_factory(
+    #        matplotlib.transforms.IdentityTransform(), fig.transFigure # specify x, y transform
+    #        )) # changed from default blend (IdentityTransform(), a[0].transAxes)
+    ax0.xaxis.label.set_position((0.6, 0.0))
+    ax0.text(-0.05, 1.10, "(a)", transform=ax0.transAxes)
+    ax0.set_xlabel("Clip length (frames)")
+    ax0.set_ylabel("Validation Loss")
+
+    # Plot period sweep (panel b)
+    print("PERIOD")
+    for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"], matplotlib.colors.TABLEAU_COLORS):
+        for pretrained in [True, False]:
+            loss = [load(root, model, 64 // period, period, pretrained) for period in PERIOD]
+            print(model, pretrained)
+            print("    ".join(list(map(lambda x: "{:.1f}".format(x) if x is not None else None, loss))))
+
+            ax2.plot(PERIOD, loss, "-" if pretrained else "--", marker=".", color=color)
+    ax2.set_xticks(PERIOD)
+    ax2.text(-0.05, 1.10, "(b)", transform=ax2.transAxes)
+    ax2.set_xlabel("Sampling Period (frames)")
+    ax2.set_ylabel("Validation Loss")
+
+    # Save figure
+    plt.tight_layout()
+    plt.savefig(os.path.join(fig_root, "hyperparameter.pdf"))
+    plt.savefig(os.path.join(fig_root, "hyperparameter.eps"))
+    plt.savefig(os.path.join(fig_root, "hyperparameter.png"))
+    plt.close(fig)
+
+
+def load(root, model, frames, period, pretrained):
+    """Loads best validation loss for specified hyperparameter choice."""
+    pretrained = ("pretrained" if pretrained else "random")
+    f = os.path.join(
+        root,
+        "{}_{}_{}_{}".format(model, frames, period, pretrained),
+        "log.csv")
+    with open(f, "r") as f:
+        for line in f:
+            if "Best validation loss " in line:
+                return float(line.split()[3])
+
+    raise ValueError("File missing information.")
+
+
+if __name__ == "__main__":
+    main()

dynamic/scripts/plot_loss.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+"""Code to generate plots for Extended Data Fig. 3."""
+
+import argparse
+import os
+import matplotlib
+import matplotlib.pyplot as plt
+
+import echonet
+
+
+def main():
+    """Generate plots for Extended Data Fig. 3."""
+
+    # Select paths and hyperparameter to plot
+    parser = argparse.ArgumentParser()
+    parser.add_argument("dir", nargs="?", default="output")
+    parser.add_argument("fig", nargs="?", default=os.path.join("figure", "loss"))
+    parser.add_argument("--frames", type=int, default=32)
+    parser.add_argument("--period", type=int, default=2)
+    args = parser.parse_args()
+
+    # Set up figure
+    echonet.utils.latexify()
+    os.makedirs(args.fig, exist_ok=True)
+    fig = plt.figure(figsize=(7, 5))
+    gs = matplotlib.gridspec.GridSpec(ncols=3, nrows=2, figure=fig, width_ratios=[2.75, 2.75, 1.50])
+
+    # Plot EF loss curve
+    ax0 = fig.add_subplot(gs[0, 0])
+    ax1 = fig.add_subplot(gs[0, 1], sharey=ax0)
+    for pretrained in [True]:
+        for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"], matplotlib.colors.TABLEAU_COLORS):
+            loss = load(os.path.join(args.dir, "video", "{}_{}_{}_{}".format(model, args.frames, args.period, "pretrained" if pretrained else "random"), "log.csv"))
+            ax0.plot(range(1, 1 + len(loss["train"])), loss["train"], "-" if pretrained else "--", color=color)
+            ax1.plot(range(1, 1 + len(loss["val"])), loss["val"], "-" if pretrained else "--", color=color)
+
+    plt.axis([0, max(len(loss["train"]), len(loss["val"])), 0, max(max(loss["train"]), max(loss["val"]))])
+    ax0.text(-0.25, 1.00, "(a)", transform=ax0.transAxes)
+    ax1.text(-0.25, 1.00, "(b)", transform=ax1.transAxes)
+    ax0.set_xlabel("Epochs")
+    ax1.set_xlabel("Epochs")
+    ax0.set_xticks([0, 15, 30, 45])
+    ax1.set_xticks([0, 15, 30, 45])
+    ax0.set_ylabel("Training MSE Loss")
+    ax1.set_ylabel("Validation MSE Loss")
+
+    # Plot segmentation loss curve
+    ax0 = fig.add_subplot(gs[1, 0])
+    ax1 = fig.add_subplot(gs[1, 1], sharey=ax0)
+    pretrained = False
+    for (model, color) in zip(["deeplabv3_resnet50"], list(matplotlib.colors.TABLEAU_COLORS)[3:]):
+        loss = load(os.path.join(args.dir, "segmentation", "{}_{}".format(model, "pretrained" if pretrained else "random"), "log.csv"))
+        ax0.plot(range(1, 1 + len(loss["train"])), loss["train"], "--", color=color)
+        ax1.plot(range(1, 1 + len(loss["val"])), loss["val"], "--", color=color)
+
+    ax0.text(-0.25, 1.00, "(c)", transform=ax0.transAxes)
+    ax1.text(-0.25, 1.00, "(d)", transform=ax1.transAxes)
+    ax0.set_ylim([0, 0.13])
+    ax0.set_xlabel("Epochs")
+    ax1.set_xlabel("Epochs")
+    ax0.set_xticks([0, 25, 50])
+    ax1.set_xticks([0, 25, 50])
+    ax0.set_ylabel("Training Cross Entropy Loss")
+    ax1.set_ylabel("Validation Cross Entropy Loss")
+
+    # Legend
+    ax = fig.add_subplot(gs[:, 2])
+    for (model, color) in zip(["EchoNet-Dynamic (EF)", "R3D", "MC3", "EchoNet-Dynamic (Seg)"], matplotlib.colors.TABLEAU_COLORS):
+        ax.plot([float("nan")], [float("nan")], "-", color=color, label=model)
+    ax.set_title("")
+    ax.axis("off")
+    ax.legend(loc="center")
+
+    plt.tight_layout()
+    plt.savefig(os.path.join(args.fig, "loss.pdf"))
+    plt.savefig(os.path.join(args.fig, "loss.eps"))
+    plt.savefig(os.path.join(args.fig, "loss.png"))
+    plt.close(fig)
+
+
+def load(filename):
+    """Loads losses from specified file."""
+
+    losses = {"train": [], "val": []}
+    with open(filename, "r") as f:
+        for line in f:
+            line = line.split(",")
+            if len(line) < 4:
+                continue
+            epoch, split, loss, *_ = line
+            epoch = int(epoch)
+            loss = float(loss)
+            assert(split in ["train", "val"])
+            if epoch == len(losses[split]):
+                losses[split].append(loss)
+            elif epoch == len(losses[split]) - 1:
+                losses[split][-1] = loss
+            else:
+                raise ValueError("File has uninterpretable formatting.")
+    return losses
+
+
+if __name__ == "__main__":
+    main()

dynamic/scripts/plot_simulated_noise.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+
+"""Code to generate plots for Extended Data Fig. 6."""
+
+import os
+import pickle
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import PIL
+import sklearn
+import torch
+import torchvision
+
+import echonet
+
+
+def main(fig_root=os.path.join("figure", "noise"),
+         video_output=os.path.join("output", "video", "r2plus1d_18_32_2_pretrained"),
+         seg_output=os.path.join("output", "segmentation", "deeplabv3_resnet50_random"),
+         NOISE=(0, 0.1, 0.2, 0.3, 0.4, 0.5)):
+    """Generate plots for Extended Data Fig. 6."""
+
+    device = torch.device("cuda")
+
+    filename = os.path.join(fig_root, "data.pkl")  # Cache of results
+    try:
+        # Attempt to load cache
+        with open(filename, "rb") as f:
+            Y, YHAT, INTER, UNION = pickle.load(f)
+    except FileNotFoundError:
+        # Generate results if no cache available
+        os.makedirs(fig_root, exist_ok=True)
+
+        # Load trained video model
+        model_v = torchvision.models.video.r2plus1d_18()
+        model_v.fc = torch.nn.Linear(model_v.fc.in_features, 1)
+        if device.type == "cuda":
+            model_v = torch.nn.DataParallel(model_v)
+        model_v.to(device)
+
+        checkpoint = torch.load(os.path.join(video_output, "checkpoint.pt"))
+        model_v.load_state_dict(checkpoint['state_dict'])
+
+        # Load trained segmentation model
+        model_s = torchvision.models.segmentation.deeplabv3_resnet50(aux_loss=False)
+        model_s.classifier[-1] = torch.nn.Conv2d(model_s.classifier[-1].in_channels, 1, kernel_size=model_s.classifier[-1].kernel_size)
+        if device.type == "cuda":
+            model_s = torch.nn.DataParallel(model_s)
+        model_s.to(device)
+
+        checkpoint = torch.load(os.path.join(seg_output, "checkpoint.pt"))
+        model_s.load_state_dict(checkpoint['state_dict'])
+
+        # Run simulation
+        dice = []
+        mse = []
+        r2 = []
+        Y = []
+        YHAT = []
+        INTER = []
+        UNION = []
+        for noise in NOISE:
+            Y.append([])
+            YHAT.append([])
+            INTER.append([])
+            UNION.append([])
+
+            dataset = echonet.datasets.Echo(split="test", noise=noise)
+            PIL.Image.fromarray(dataset[0][0][:, 0, :, :].astype(np.uint8).transpose(1, 2, 0)).save(os.path.join(fig_root, "noise_{}.tif".format(round(100 * noise))))
+
+            mean, std = echonet.utils.get_mean_and_std(echonet.datasets.Echo(split="train"))
+
+            tasks = ["LargeFrame", "SmallFrame", "LargeTrace", "SmallTrace"]
+            kwargs = {
+                "target_type": tasks,
+                "mean": mean,
+                "std": std,
+                "noise": noise
+            }
+            dataset = echonet.datasets.Echo(split="test", **kwargs)
+
+            dataloader = torch.utils.data.DataLoader(dataset,
+                                                     batch_size=16, num_workers=5, shuffle=True, pin_memory=(device.type == "cuda"))
+
+            loss, large_inter, large_union, small_inter, small_union = echonet.utils.segmentation.run_epoch(model_s, dataloader, "test", None, device)
+            inter = np.concatenate((large_inter, small_inter)).sum()
+            union = np.concatenate((large_union, small_union)).sum()
+            dice.append(2 * inter / (union + inter))
+
+            INTER[-1].extend(large_inter.tolist() + small_inter.tolist())
+            UNION[-1].extend(large_union.tolist() + small_union.tolist())
+
+            kwargs = {"target_type": "EF",
+                      "mean": mean,
+                      "std": std,
+                      "length": 32,
+                      "period": 2,
+                      "noise": noise
+                      }
+
+            dataset = echonet.datasets.Echo(split="test", **kwargs)
+
+            dataloader = torch.utils.data.DataLoader(dataset,
+                                                     batch_size=16, num_workers=5, shuffle=True, pin_memory=(device.type == "cuda"))
+            loss, yhat, y = echonet.utils.video.run_epoch(model_v, dataloader, "test", None, device)
+            mse.append(loss)
+            r2.append(sklearn.metrics.r2_score(y, yhat))
+            Y[-1].extend(y.tolist())
+            YHAT[-1].extend(yhat.tolist())
+
+        # Save results in cache
+        with open(filename, "wb") as f:
+            pickle.dump((Y, YHAT, INTER, UNION), f)
+
+    # Set up plot
+    echonet.utils.latexify()
+
+    NOISE = list(map(lambda x: round(100 * x), NOISE))
+    fig = plt.figure(figsize=(6.50, 4.75))
+    gs = matplotlib.gridspec.GridSpec(3, 1, height_ratios=[2.0, 2.0, 0.75])
+    ax = (plt.subplot(gs[0]), plt.subplot(gs[1]), plt.subplot(gs[2]))
+
+    # Plot EF prediction results (R^2)
+    r2 = [sklearn.metrics.r2_score(y, yhat) for (y, yhat) in zip(Y, YHAT)]
+    ax[0].plot(NOISE, r2, color="k", linewidth=1, marker=".")
+    ax[0].set_xticks([])
+    ax[0].set_ylabel("R$^2$")
+    l, h = min(r2), max(r2)
+    l, h = l - 0.1 * (h - l), h + 0.1 * (h - l)
+    ax[0].axis([min(NOISE) - 5, max(NOISE) + 5, 0, 1])
+
+    # Plot segmentation results (DSC)
+    dice = [echonet.utils.dice_similarity_coefficient(inter, union) for (inter, union) in zip(INTER, UNION)]
+    ax[1].plot(NOISE, dice, color="k", linewidth=1, marker=".")
+    ax[1].set_xlabel("Pixels Removed (%)")
+    ax[1].set_ylabel("DSC")
+    l, h = min(dice), max(dice)
+    l, h = l - 0.1 * (h - l), h + 0.1 * (h - l)
+    ax[1].axis([min(NOISE) - 5, max(NOISE) + 5, 0, 1])
+
+    # Add example images below
+    for noise in NOISE:
+        image = matplotlib.image.imread(os.path.join(fig_root, "noise_{}.tif".format(noise)))
+        imagebox = matplotlib.offsetbox.OffsetImage(image, zoom=0.4)
+        ab = matplotlib.offsetbox.AnnotationBbox(imagebox, (noise, 0.0), frameon=False)
+        ax[2].add_artist(ab)
+        ax[2].axis("off")
+    ax[2].axis([min(NOISE) - 5, max(NOISE) + 5, -1, 1])
+
+    fig.tight_layout()
+    plt.savefig(os.path.join(fig_root, "noise.pdf"), dpi=1200)
+    plt.savefig(os.path.join(fig_root, "noise.eps"), dpi=300)
+    plt.savefig(os.path.join(fig_root, "noise.png"), dpi=600)
+    plt.close(fig)
+
+
+if __name__ == "__main__":
+    main()

dynamic/scripts/run_experiments.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+for pretrained in True False
+do
+    for model in r2plus1d_18 r3d_18 mc3_18
+    do
+        for frames in 96 64 32 16 8 4 1
+        do
+            batch=$((256 / frames))
+            batch=$(( batch > 16 ? 16 : batch ))
+
+            cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=${frames}, period=1, pretrained=${pretrained}, batch_size=${batch})"
+            python3 -c "${cmd}"
+        done
+        for period in 2 4 6 8
+        do
+            batch=$((256 / 64 * period))
+            batch=$(( batch > 16 ? 16 : batch ))
+
+            cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=(64 // ${period}), period=${period}, pretrained=${pretrained}, batch_size=${batch})"
+            python3 -c "${cmd}"
+        done
+    done
+done
+
+period=2
+pretrained=True
+for model in r2plus1d_18 r3d_18 mc3_18
+do
+    cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=(64 // ${period}), period=${period}, pretrained=${pretrained}, run_test=True)"
+    python3 -c "${cmd}"
+done
+
+python3 -c "import echonet; echonet.utils.segmentation.run(modelname=\"deeplabv3_resnet50\",  save_segmentation=True, pretrained=False)"
+
+pretrained=True
+model=r2plus1d_18
+period=2
+batch=$((256 / 64 * period))
+batch=$(( batch > 16 ? 16 : batch ))
+for patients in 16 32 64 128 256 512 1024 2048 4096 7460
+do
+    cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=(64 // ${period}), period=${period}, pretrained=${pretrained}, batch_size=${batch}, num_epochs=min(50 * (8192 // ${patients}), 200), output=\"output/training_size/video/${patients}\", n_train_patients=${patients})"
+    python3 -c "${cmd}"
+    cmd="import echonet; echonet.utils.segmentation.run(modelname=\"deeplabv3_resnet50\", pretrained=False, num_epochs=min(50 * (8192 // ${patients}), 200), output=\"output/training_size/segmentation/${patients}\", n_train_patients=${patients})"
+    python3 -c "${cmd}"
+
+done
+

dynamic/setup.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Metadata for package to allow installation with pip."""
+
+import os
+
+import setuptools
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+# Use same version from code
+# See 3 from
+# https://packaging.python.org/guides/single-sourcing-package-version/
+version = {}
+with open(os.path.join("echonet", "__version__.py")) as f:
+    exec(f.read(), version)  # pylint: disable=W0122
+
+setuptools.setup(
+    name="echonet",
+    description="Video-based AI for beat-to-beat cardiac function assessment.",
+    version=version["__version__"],
+    url="https://echonet.github.io/dynamic",
+    packages=setuptools.find_packages(),
+    install_requires=[
+        "click",
+        "numpy",
+        "pandas",
+        "torch",
+        "torchvision",
+        "opencv-python",
+        "scikit-image",
+        "tqdm",
+        "sklearn"
+    ],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+    ],
+    entry_points={
+        "console_scripts": [
+            "echonet=echonet:main",
+        ],
+    }
+
+)

echonet/__init__.py

@@ -0,0 +1,26 @@
+"""
+The echonet package contains code for loading echocardiogram videos, and
+functions for training and testing segmentation and ejection fraction
+prediction models.
+"""
+
+import click
+
+from echonet.__version__ import __version__
+from echonet.config import CONFIG as config
+import echonet.datasets as datasets
+import echonet.utils as utils
+
+
+@click.group()
+def main():
+    """Entry point for command line interface."""
+
+
+del click
+
+
+main.add_command(utils.segmentation.run)
+main.add_command(utils.video.run)
+
+__all__ = ["__version__", "config", "datasets", "main", "utils"]

echonet/__main__.py

@@ -0,0 +1,7 @@
+"""Entry point for command line."""
+
+import echonet
+
+
+if __name__ == '__main__':
+    echonet.main()

echonet/__version__.py

@@ -0,0 +1,3 @@
+"""Version number for Echonet package."""
+
+__version__ = "1.0.0"

echonet/config.py

@@ -0,0 +1,24 @@
+"""Sets paths based on configuration files."""
+
+import configparser
+import os
+import types
+
+_FILENAME = None
+_PARAM = {}
+for filename in ["echonet.cfg",
+                 ".echonet.cfg",
+                 os.path.expanduser("~/echonet.cfg"),
+                 os.path.expanduser("~/.echonet.cfg"),
+                 ]:
+    if os.path.isfile(filename):
+        _FILENAME = filename
+        config = configparser.ConfigParser()
+        with open(filename, "r") as f:
+            config.read_string("[config]\n" + f.read())
+            _PARAM = config["config"]
+        break
+
+CONFIG = types.SimpleNamespace(
+    FILENAME=_FILENAME,
+    DATA_DIR=_PARAM.get("data_dir", "a4c-video-dir/"))

echonet/datasets/__init__.py

@@ -0,0 +1,8 @@
+"""
+The echonet.datasets submodule defines a Pytorch dataset for loading
+echocardiogram videos.
+"""
+
+from .echo import Echo
+
+__all__ = ["Echo"]

echonet/datasets/echo.py

@@ -0,0 +1,282 @@
+"""EchoNet-Dynamic Dataset."""
+
+import os
+import collections
+import pandas
+
+import numpy as np
+import skimage.draw
+import torchvision
+import echonet
+
+
+class Echo(torchvision.datasets.VisionDataset):
+    """EchoNet-Dynamic Dataset.
+
+    Args:
+        root (string): Root directory of dataset (defaults to `echonet.config.DATA_DIR`)
+        split (string): One of {``train'', ``val'', ``test'', ``all'', or ``external_test''}
+        target_type (string or list, optional): Type of target to use,
+            ``Filename'', ``EF'', ``EDV'', ``ESV'', ``LargeIndex'',
+            ``SmallIndex'', ``LargeFrame'', ``SmallFrame'', ``LargeTrace'',
+            or ``SmallTrace''
+            Can also be a list to output a tuple with all specified target types.
+            The targets represent:
+                ``Filename'' (string): filename of video
+                ``EF'' (float): ejection fraction
+                ``EDV'' (float): end-diastolic volume
+                ``ESV'' (float): end-systolic volume
+                ``LargeIndex'' (int): index of large (diastolic) frame in video
+                ``SmallIndex'' (int): index of small (systolic) frame in video
+                ``LargeFrame'' (np.array shape=(3, height, width)): normalized large (diastolic) frame
+                ``SmallFrame'' (np.array shape=(3, height, width)): normalized small (systolic) frame
+                ``LargeTrace'' (np.array shape=(height, width)): left ventricle large (diastolic) segmentation
+                    value of 0 indicates pixel is outside left ventricle
+                             1 indicates pixel is inside left ventricle
+                ``SmallTrace'' (np.array shape=(height, width)): left ventricle small (systolic) segmentation
+                    value of 0 indicates pixel is outside left ventricle
+                             1 indicates pixel is inside left ventricle
+            Defaults to ``EF''.
+        mean (int, float, or np.array shape=(3,), optional): means for all (if scalar) or each (if np.array) channel.
+            Used for normalizing the video. Defaults to 0 (video is not shifted).
+        std (int, float, or np.array shape=(3,), optional): standard deviation for all (if scalar) or each (if np.array) channel.
+            Used for normalizing the video. Defaults to 0 (video is not scaled).
+        length (int or None, optional): Number of frames to clip from video. If ``None'', longest possible clip is returned.
+            Defaults to 16.
+        period (int, optional): Sampling period for taking a clip from the video (i.e. every ``period''-th frame is taken)
+            Defaults to 2.
+        max_length (int or None, optional): Maximum number of frames to clip from video (main use is for shortening excessively
+            long videos when ``length'' is set to None). If ``None'', shortening is not applied to any video.
+            Defaults to 250.
+        clips (int, optional): Number of clips to sample. Main use is for test-time augmentation with random clips.
+            Defaults to 1.
+        pad (int or None, optional): Number of pixels to pad all frames on each side (used as augmentation).
+            and a window of the original size is taken. If ``None'', no padding occurs.
+            Defaults to ``None''.
+        noise (float or None, optional): Fraction of pixels to black out as simulated noise. If ``None'', no simulated noise is added.
+            Defaults to ``None''.
+        target_transform (callable, optional): A function/transform that takes in the target and transforms it.
+        external_test_location (string): Path to videos to use for external testing.
+    """
+
+    def __init__(self, root=None,
+                 split="train", target_type="EF",
+                 mean=0., std=1.,
+                 length=16, period=2,
+                 max_length=250,
+                 clips=1,
+                 pad=None,
+                 noise=None,
+                 target_transform=None,
+                 external_test_location=None):
+        if root is None:
+            root = echonet.config.DATA_DIR
+
+        super().__init__(root, target_transform=target_transform)
+
+        self.split = split.upper()
+        if not isinstance(target_type, list):
+            target_type = [target_type]
+        self.target_type = target_type
+        self.mean = mean
+        self.std = std
+        self.length = length
+        self.max_length = max_length
+        self.period = period
+        self.clips = clips
+        self.pad = pad
+        self.noise = noise
+        self.target_transform = target_transform
+        self.external_test_location = external_test_location
+
+        self.fnames, self.outcome = [], []
+
+        if self.split == "EXTERNAL_TEST":
+            self.fnames = sorted(os.listdir(self.external_test_location))
+        else:
+            # Load video-level labels
+            with open(os.path.join(self.root, "FileList.csv")) as f:
+                data = pandas.read_csv(f)
+            data["Split"].map(lambda x: x.upper())
+
+            if self.split != "ALL":
+                data = data[data["Split"] == self.split]
+
+            self.header = data.columns.tolist()
+            self.fnames = data["FileName"].tolist()
+            self.fnames = [fn + ".avi" for fn in self.fnames if os.path.splitext(fn)[1] == ""]  # Assume avi if no suffix
+            self.outcome = data.values.tolist()
+
+            # Check that files are present
+            missing = set(self.fnames) - set(os.listdir(os.path.join(self.root, "Videos")))
+            if len(missing) != 0:
+                print("{} videos could not be found in {}:".format(len(missing), os.path.join(self.root, "Videos")))
+                for f in sorted(missing):
+                    print("\t", f)
+                raise FileNotFoundError(os.path.join(self.root, "Videos", sorted(missing)[0]))
+
+            # Load traces
+            self.frames = collections.defaultdict(list)
+            self.trace = collections.defaultdict(_defaultdict_of_lists)
+
+            with open(os.path.join(self.root, "VolumeTracings.csv")) as f:
+                header = f.readline().strip().split(",")
+                assert header == ["FileName", "X1", "Y1", "X2", "Y2", "Frame"]
+
+                for line in f:
+                    filename, x1, y1, x2, y2, frame = line.strip().split(',')
+                    x1 = float(x1)
+                    y1 = float(y1)
+                    x2 = float(x2)
+                    y2 = float(y2)
+                    frame = int(frame)
+                    if frame not in self.trace[filename]:
+                        self.frames[filename].append(frame)
+                    self.trace[filename][frame].append((x1, y1, x2, y2))
+            for filename in self.frames:
+                for frame in self.frames[filename]:
+                    self.trace[filename][frame] = np.array(self.trace[filename][frame])
+
+            # A small number of videos are missing traces; remove these videos
+            keep = [len(self.frames[f]) >= 2 for f in self.fnames]
+            self.fnames = [f for (f, k) in zip(self.fnames, keep) if k]
+            self.outcome = [f for (f, k) in zip(self.outcome, keep) if k]
+
+    def __getitem__(self, index):
+        # Find filename of video
+        if self.split == "EXTERNAL_TEST":
+            video = os.path.join(self.external_test_location, self.fnames[index])
+        elif self.split == "CLINICAL_TEST":
+            video = os.path.join(self.root, "ProcessedStrainStudyA4c", self.fnames[index])
+        else:
+            video = os.path.join(self.root, "Videos", self.fnames[index])
+
+        # Load video into np.array
+        video = echonet.utils.loadvideo(video).astype(np.float32)
+
+        # Add simulated noise (black out random pixels)
+        # 0 represents black at this point (video has not been normalized yet)
+        if self.noise is not None:
+            n = video.shape[1] * video.shape[2] * video.shape[3]
+            ind = np.random.choice(n, round(self.noise * n), replace=False)
+            f = ind % video.shape[1]
+            ind //= video.shape[1]
+            i = ind % video.shape[2]
+            ind //= video.shape[2]
+            j = ind
+            video[:, f, i, j] = 0
+
+        # Apply normalization
+        if isinstance(self.mean, (float, int)):
+            video -= self.mean
+        else:
+            video -= self.mean.reshape(3, 1, 1, 1)
+
+        if isinstance(self.std, (float, int)):
+            video /= self.std
+        else:
+            video /= self.std.reshape(3, 1, 1, 1)
+
+        # Set number of frames
+        c, f, h, w = video.shape
+        if self.length is None:
+            # Take as many frames as possible
+            length = f // self.period
+        else:
+            # Take specified number of frames
+            length = self.length
+
+        if self.max_length is not None:
+            # Shorten videos to max_length
+            length = min(length, self.max_length)
+
+        if f < length * self.period:
+            # Pad video with frames filled with zeros if too short
+            # 0 represents the mean color (dark grey), since this is after normalization
+            video = np.concatenate((video, np.zeros((c, length * self.period - f, h, w), video.dtype)), axis=1)
+            c, f, h, w = video.shape  # pylint: disable=E0633
+
+        if self.clips == "all":
+            # Take all possible clips of desired length
+            start = np.arange(f - (length - 1) * self.period)
+        else:
+            # Take random clips from video
+            start = np.random.choice(f - (length - 1) * self.period, self.clips)
+
+        # Gather targets
+        target = []
+        for t in self.target_type:
+            key = self.fnames[index]
+            if t == "Filename":
+                target.append(self.fnames[index])
+            elif t == "LargeIndex":
+                # Traces are sorted by cross-sectional area
+                # Largest (diastolic) frame is last
+                target.append(np.int(self.frames[key][-1]))
+            elif t == "SmallIndex":
+                # Largest (diastolic) frame is first
+                target.append(np.int(self.frames[key][0]))
+            elif t == "LargeFrame":
+                target.append(video[:, self.frames[key][-1], :, :])
+            elif t == "SmallFrame":
+                target.append(video[:, self.frames[key][0], :, :])
+            elif t in ["LargeTrace", "SmallTrace"]:
+                if t == "LargeTrace":
+                    t = self.trace[key][self.frames[key][-1]]
+                else:
+                    t = self.trace[key][self.frames[key][0]]
+                x1, y1, x2, y2 = t[:, 0], t[:, 1], t[:, 2], t[:, 3]
+                x = np.concatenate((x1[1:], np.flip(x2[1:])))
+                y = np.concatenate((y1[1:], np.flip(y2[1:])))
+
+                r, c = skimage.draw.polygon(np.rint(y).astype(np.int), np.rint(x).astype(np.int), (video.shape[2], video.shape[3]))
+                mask = np.zeros((video.shape[2], video.shape[3]), np.float32)
+                mask[r, c] = 1
+                target.append(mask)
+            else:
+                if self.split == "CLINICAL_TEST" or self.split == "EXTERNAL_TEST":
+                    target.append(np.float32(0))
+                else:
+                    target.append(np.float32(self.outcome[index][self.header.index(t)]))
+
+        if target != []:
+            target = tuple(target) if len(target) > 1 else target[0]
+            if self.target_transform is not None:
+                target = self.target_transform(target)
+
+        # Select clips from video
+        video = tuple(video[:, s + self.period * np.arange(length), :, :] for s in start)
+        if self.clips == 1:
+            video = video[0]
+        else:
+            video = np.stack(video)
+
+        if self.pad is not None:
+            # Add padding of zeros (mean color of videos)
+            # Crop of original size is taken out
+            # (Used as augmentation)
+            c, l, h, w = video.shape
+            temp = np.zeros((c, l, h + 2 * self.pad, w + 2 * self.pad), dtype=video.dtype)
+            temp[:, :, self.pad:-self.pad, self.pad:-self.pad] = video  # pylint: disable=E1130
+            i, j = np.random.randint(0, 2 * self.pad, 2)
+            video = temp[:, :, i:(i + h), j:(j + w)]
+
+        return video, target
+
+    def __len__(self):
+        return len(self.fnames)
+
+    def extra_repr(self) -> str:
+        """Additional information to add at end of __repr__."""
+        lines = ["Target type: {target_type}", "Split: {split}"]
+        return '\n'.join(lines).format(**self.__dict__)
+
+
+def _defaultdict_of_lists():
+    """Returns a defaultdict of lists.
+
+    This is used to avoid issues with Windows (if this function is anonymous,
+    the Echo dataset cannot be used in a dataloader).
+    """
+
+    return collections.defaultdict(list)

echonet/utils/__init__.py

@@ -0,0 +1,179 @@
+"""Utility functions for videos, plotting and computing performance metrics."""
+
+import os
+import typing
+
+import cv2  # pytype: disable=attribute-error
+import matplotlib
+import numpy as np
+import torch
+import tqdm
+
+from . import video
+from . import segmentation
+
+
+def loadvideo(filename: str) -> np.ndarray:
+    """Loads a video from a file.
+
+    Args:
+        filename (str): filename of video
+
+    Returns:
+        A np.ndarray with dimensions (channels=3, frames, height, width). The
+        values will be uint8's ranging from 0 to 255.
+
+    Raises:
+        FileNotFoundError: Could not find `filename`
+        ValueError: An error occurred while reading the video
+    """
+
+    if not os.path.exists(filename):
+        raise FileNotFoundError(filename)
+    capture = cv2.VideoCapture(filename)
+
+    frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+    frame_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+    frame_height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    v = np.zeros((frame_count, frame_height, frame_width, 3), np.uint8)
+
+    for count in range(frame_count):
+        ret, frame = capture.read()
+        if not ret:
+            raise ValueError("Failed to load frame #{} of {}.".format(count, filename))
+
+        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        v[count, :, :] = frame
+
+    v = v.transpose((3, 0, 1, 2))
+
+    return v
+
+
+def savevideo(filename: str, array: np.ndarray, fps: typing.Union[float, int] = 1):
+    """Saves a video to a file.
+
+    Args:
+        filename (str): filename of video
+        array (np.ndarray): video of uint8's with shape (channels=3, frames, height, width)
+        fps (float or int): frames per second
+
+    Returns:
+        None
+    """
+
+    c, _, height, width = array.shape
+
+    if c != 3:
+        raise ValueError("savevideo expects array of shape (channels=3, frames, height, width), got shape ({})".format(", ".join(map(str, array.shape))))
+    fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
+    out = cv2.VideoWriter(filename, fourcc, fps, (width, height))
+
+    for frame in array.transpose((1, 2, 3, 0)):
+        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+        out.write(frame)
+
+
+def get_mean_and_std(dataset: torch.utils.data.Dataset,
+                     samples: int = 128,
+                     batch_size: int = 8,
+                     num_workers: int = 4):
+    """Computes mean and std from samples from a Pytorch dataset.
+
+    Args:
+        dataset (torch.utils.data.Dataset): A Pytorch dataset.
+            ``dataset[i][0]'' is expected to be the i-th video in the dataset, which
+            should be a ``torch.Tensor'' of dimensions (channels=3, frames, height, width)
+        samples (int or None, optional): Number of samples to take from dataset. If ``None'', mean and
+            standard deviation are computed over all elements.
+            Defaults to 128.
+        batch_size (int, optional): how many samples per batch to load
+            Defaults to 8.
+        num_workers (int, optional): how many subprocesses to use for data
+            loading. If 0, the data will be loaded in the main process.
+            Defaults to 4.
+
+    Returns:
+       A tuple of the mean and standard deviation. Both are represented as np.array's of dimension (channels,).
+    """
+
+    if samples is not None and len(dataset) > samples:
+        indices = np.random.choice(len(dataset), samples, replace=False)
+        dataset = torch.utils.data.Subset(dataset, indices)
+    dataloader = torch.utils.data.DataLoader(
+        dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True)
+
+    n = 0  # number of elements taken (should be equal to samples by end of for loop)
+    s1 = 0.  # sum of elements along channels (ends up as np.array of dimension (channels,))
+    s2 = 0.  # sum of squares of elements along channels (ends up as np.array of dimension (channels,))
+    for (x, *_) in tqdm.tqdm(dataloader):
+        x = x.transpose(0, 1).contiguous().view(3, -1)
+        n += x.shape[1]
+        s1 += torch.sum(x, dim=1).numpy()
+        s2 += torch.sum(x ** 2, dim=1).numpy()
+    mean = s1 / n  # type: np.ndarray
+    std = np.sqrt(s2 / n - mean ** 2)  # type: np.ndarray
+
+    mean = mean.astype(np.float32)
+    std = std.astype(np.float32)
+
+    return mean, std
+
+
+def bootstrap(a, b, func, samples=10000):
+    """Computes a bootstrapped confidence intervals for ``func(a, b)''.
+
+    Args:
+        a (array_like): first argument to `func`.
+        b (array_like): second argument to `func`.
+        func (callable): Function to compute confidence intervals for.
+            ``dataset[i][0]'' is expected to be the i-th video in the dataset, which
+            should be a ``torch.Tensor'' of dimensions (channels=3, frames, height, width)
+        samples (int, optional): Number of samples to compute.
+            Defaults to 10000.
+
+    Returns:
+       A tuple of (`func(a, b)`, estimated 5-th percentile, estimated 95-th percentile).
+    """
+    a = np.array(a)
+    b = np.array(b)
+
+    bootstraps = []
+    for _ in range(samples):
+        ind = np.random.choice(len(a), len(a))
+        bootstraps.append(func(a[ind], b[ind]))
+    bootstraps = sorted(bootstraps)
+
+    return func(a, b), bootstraps[round(0.05 * len(bootstraps))], bootstraps[round(0.95 * len(bootstraps))]
+
+
+def latexify():
+    """Sets matplotlib params to appear more like LaTeX.
+
+    Based on https://nipunbatra.github.io/blog/2014/latexify.html
+    """
+    params = {'backend': 'pdf',
+              'axes.titlesize': 8,
+              'axes.labelsize': 8,
+              'font.size': 8,
+              'legend.fontsize': 8,
+              'xtick.labelsize': 8,
+              'ytick.labelsize': 8,
+              'font.family': 'DejaVu Serif',
+              'font.serif': 'Computer Modern',
+              }
+    matplotlib.rcParams.update(params)
+
+
+def dice_similarity_coefficient(inter, union):
+    """Computes the dice similarity coefficient.
+
+    Args:
+        inter (iterable): iterable of the intersections
+        union (iterable): iterable of the unions
+    """
+    return 2 * sum(inter) / (sum(union) + sum(inter))
+
+
+__all__ = ["video", "segmentation", "loadvideo", "savevideo", "get_mean_and_std", "bootstrap", "latexify", "dice_similarity_coefficient"]