Upload 9 files

models/FullyConectedModels/model.py

@@ -0,0 +1,120 @@
+from tensorflow.data import Dataset
+import tensorflow.keras as keras
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.layers import (
+    Conv2D,
+    Input,
+    MaxPooling2D,
+    Dense,
+    Dropout,
+    MaxPool1D,
+    Flatten,
+    AveragePooling1D,
+    BatchNormalization,
+)
+from tensorflow.keras import Model
+import numpy as np
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+def basemodel(weight_decay):
+    # 2 hidden layers
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        kernel_regularizer=l2(weight_decay),
+        activation="relu",
+    )(model_input)
+    model = Conv2D(
+        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model
+
+
+def model_2(weight_decay):
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        kernel_regularizer=l2(weight_decay),
+        activation="relu",
+    )(model_input)
+    model = Conv2D(
+        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        128, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model
+
+
+def model_3(weight_decay):
+    # 4 hidden layers
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        kernel_regularizer=l2(weight_decay),
+        activation="relu",
+    )(model_input)
+    model = Conv2D(
+        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        128, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        256, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model

models/FullyConectedModels/parseval.py

@@ -0,0 +1,83 @@
+from tensorflow.data import Dataset
+import tensorflow.keras as keras
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.layers import (
+    Conv2D,
+    Input,
+    MaxPooling2D,
+    Dense,
+    Dropout,
+    MaxPool1D,
+    Flatten,
+    AveragePooling1D,
+    BatchNormalization,
+)
+from tensorflow.keras import Model
+import numpy as np
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+from constraint import tight_frame
+
+warnings.filterwarnings("ignore")
+
+
+def model_parseval(weight_decay):
+
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        activation="relu",
+        input_shape=(32, 32, 1),
+        kernel_regularizer=l2(weight_decay),
+        kernel_constraint=tight_frame(0.001),
+        kernel_initializer="Orthogonal",
+    )(model_input)
+    model = Conv2D(
+        64,
+        kernel_size=(3, 3),
+        activation="relu",
+        kernel_regularizer=l2(weight_decay),
+        kernel_initializer="Orthogonal",
+        kernel_constraint=tight_frame(0.001),
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        128,
+        kernel_size=(3, 3),
+        activation="relu",
+        kernel_initializer="Orthogonal",
+        kernel_regularizer=l2(weight_decay),
+        kernel_constraint=tight_frame(0.001),
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        256,
+        kernel_size=(3, 3),
+        activation="relu",
+        kernel_initializer="Orthogonal",
+        kernel_regularizer=l2(weight_decay),
+        kernel_constraint=tight_frame(0.001),
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, activation="softmax", kernel_regularizer=l2(weight_decay))(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model

models/Parseval_Networks/README.md

@@ -0,0 +1,3 @@
+## ParsevalNetworks
+* Orthogonality Constraint
+* Convexity Constraint

models/Parseval_Networks/constraint.py

@@ -0,0 +1,81 @@
+from tensorflow.python.keras.constraints import Constraint
+from tensorflow.python.ops import math_ops, array_ops
+
+
+class TightFrame(Constraint):
+    """
+    Parseval (tight) frame contstraint, as introduced in https://arxiv.org/abs/1704.08847
+
+    Constraints the weight matrix to be a tight frame, so that the Lipschitz
+    constant of the layer is <= 1. This increases the robustness of the network
+    to adversarial noise.
+
+    Warning: This constraint simply performs the update step on the weight matrix
+    (or the unfolded weight matrix for convolutional layers). Thus, it does not
+    handle the necessary scalings for convolutional layers.
+
+    Args:
+        scale (float):    Retraction parameter (length of retraction step).
+        num_passes (int): Number of retraction steps.
+
+    Returns:
+        Weight matrix after applying regularizer.
+    """
+
+    def __init__(self, scale, num_passes=1):
+        """[summary]
+
+        Args:
+            scale ([type]): [description]
+            num_passes (int, optional): [description]. Defaults to 1.
+
+        Raises:
+            ValueError: [description]
+        """
+        self.scale = scale
+
+        if num_passes < 1:
+            raise ValueError(
+                "Number of passes cannot be non-positive! (got {})".format(num_passes)
+            )
+        self.num_passes = num_passes
+
+    def __call__(self, w):
+        """[summary]
+
+        Args:
+            w ([type]): weight of conv or linear layers
+
+        Returns:
+            [type]: returns new weights
+        """
+        transpose_channels = len(w.shape) == 4
+
+        # Move channels_num to the front in order to make the dimensions correct for matmul
+        if transpose_channels:
+            w_reordered = array_ops.reshape(w, (-1, w.shape[3]))
+
+        else:
+            w_reordered = w
+
+        last = w_reordered
+        for i in range(self.num_passes):
+            temp1 = math_ops.matmul(last, last, transpose_a=True)
+            temp2 = (1 + self.scale) * w_reordered - self.scale * math_ops.matmul(
+                w_reordered, temp1
+            )
+
+            last = temp2
+
+        # Move channels_num to the back again
+        if transpose_channels:
+            return array_ops.reshape(last, w.shape)
+        else:
+            return last
+
+    def get_config(self):
+        return {"scale": self.scale, "num_passes": self.num_passes}
+
+
+# Alias
+tight_frame = TightFrame

models/Parseval_Networks/convexity_constraint.py

@@ -0,0 +1,53 @@
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.framework import dtypes
+import numpy as _np
+
+
+def convex_add(input_layer, layer_3, initial_convex_par=0.5, trainable=False):
+    """
+    Do a convex combination of input_layer and layer_3. That is, return the output of
+
+        lamda* input_layer + (1 - lamda) * layer_3
+
+
+    Args:
+            input_layer (tf.Tensor):	Input to take convex combinatio of
+            layer_3 (tf.Tensor):		Input to take convex combinatio of
+            initial_convex_par (float):	Initial value for convex parameter. Must be
+                                                                    in [0, 1].
+            trainable (bool):			Whether convex parameter should be trainable
+                                                                    or not.
+
+    Returns:
+            tf.Tensor: Result of convex combination
+    """
+    # Will implement this as sigmoid(p)*input_layer + (1-sigmoid(p))*layer_3 to ensure
+    # convex parameter to be in the unit interval without constraints during
+    # optimization
+
+    # Find value for p, also check for legal initial_convex_par
+    if initial_convex_par < 0:
+        raise ValueError("Convex parameter must be >=0")
+
+    elif initial_convex_par == 0:
+        # sigmoid(-16) is approximately a 32bit roundoff error, practically 0
+        initial_p_value = -16
+
+    elif initial_convex_par < 1:
+        # Compute inverse of sigmoid to find initial p value
+        initial_p_value = -_np.log(1 / initial_convex_par - 1)
+
+    elif initial_convex_par == 1:
+        # Same argument as for 0
+        initial_p_value = 16
+
+    else:
+        raise ValueError("Convex parameter must be <=1")
+
+    p = variables.Variable(
+        initial_value=initial_p_value, dtype=dtypes.float32, trainable=trainable
+    )
+
+    lam = math_ops.sigmoid(p)
+    return input_layer * lam + (1 - lam) * layer_3

models/Parseval_Networks/parsevalnet.py

@@ -0,0 +1,328 @@
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+from constraint import tight_frame
+from convexity_constraint import convex_add
+
+warnings.filterwarnings("ignore")
+
+
+class ParsevalNetwork(Model):
+    def __init__(
+        self,
+        input_dim,
+        weight_decay,
+        momentum,
+        nb_classes=4,
+        N=2,
+        k=1,
+        dropout=0.0,
+        verbose=1,
+    ):
+        """[Assign the initial parameters of the wide residual network]
+
+        Args:
+            weight_decay ([float]): [description]
+            input_dim ([tuple]): [input dimension]
+            nb_classes (int, optional): [output class]. Defaults to 4.
+            N (int, optional): [the number of blocks]. Defaults to 2.
+            k (int, optional): [network width]. Defaults to 1.
+            dropout (float, optional): [dropout value to prevent overfitting]. Defaults to 0.0.
+            verbose (int, optional): [description]. Defaults to 1.
+
+        Returns:
+            [Model]: [parsevalnetwork]
+        """
+        self.weight_decay = weight_decay
+        self.input_dim = input_dim
+        self.nb_classes = nb_classes
+        self.N = N
+        self.k = k
+        self.dropout = dropout
+        self.verbose = verbose
+
+    def initial_conv(self, input):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            16,
+            (3, 3),
+            padding="same",
+            kernel_initializer="orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(input)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        return x
+
+    def expand_conv(self, init, base, k, strides=(1, 1)):
+        """[summary]
+
+        Args:
+            init ([type]): [description]
+            base ([type]): [description]
+            k ([type]): [description]
+            strides (tuple, optional): [description]. Defaults to (1, 1).
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            strides=strides,
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(init)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        skip = Convolution2D(
+            base * k,
+            (1, 1),
+            padding="same",
+            strides=strides,
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(init)
+
+        m = Add()([x, skip])
+
+        return m
+
+    def conv1_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
+        return m
+
+    def conv2_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
+        return m
+
+    def conv3_block(self, input, k=1, dropout=0.0):
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_constraint=tight_frame(0.001),
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_constraint=tight_frame(0.001),
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
+        return m
+
+    def create_wide_residual_network(self):
+        """create a wide residual network model
+
+
+        Returns:
+            [Model]: [wide residual network]
+        """
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        ip = Input(shape=self.input_dim)
+
+        x = self.initial_conv(ip)
+        nb_conv = 4
+
+        x = self.expand_conv(x, 16, self.k)
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv1_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 32, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv2_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 64, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv3_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = AveragePooling2D((8, 8))(x)
+        x = Flatten()(x)
+
+        x = Dense(
+            self.nb_classes,
+            kernel_regularizer=l2(self.weight_decay),
+            activation="softmax",
+        )(x)
+
+        model = Model(ip, x)
+
+        if self.verbose:
+            print("Parseval  Network-%d-%d created." % (nb_conv, self.k))
+        return model

models/README.md

@@ -0,0 +1,15 @@
+## Models
+
+````
+├── Parseval_network
+│   ├── __init__.py
+│   └── Parseval_resnet.py
+├── Parseval_Networks_OC
+│   ├── constraint.py
+│   ├── parsnet_oc.py
+│   └── README.md
+├── README.md
+├── _utility.py
+└── wideresnet
+    └── wresnet.py
+````

models/_utility.py

@@ -0,0 +1,109 @@
+from tensorflow.keras.callbacks import LearningRateScheduler
+
+# Define configuration parameters
+import math
+import cleverhans
+from cleverhans.tf2.attacks.fast_gradient_method import fast_gradient_method
+import tensorflow as tf
+
+import numpy as np
+
+
+def step_decay(epoch):
+    """[summary]
+
+    Args:
+        epoch (int): epoch number
+
+    Returns:
+        lrate(float): new learning rate
+    """
+    initial_lrate = 0.1
+    factor = 0.1
+    if epoch < 10:
+        lrate = initial_lrate
+    elif epoch < 20:
+        lrate = initial_lrate * math.pow(factor, 1)
+    elif epoch < 30:
+        lrate = initial_lrate * math.pow(factor, 2)
+    elif epoch < 40:
+        lrate = initial_lrate * math.pow(factor, 3)
+    else:
+        lrate = initial_lrate * math.pow(factor, 4)
+    return lrate
+
+
+def step_decay_conv(epoch):
+    """step decay for learning rate in convolutional networks
+
+    Args:
+        epoch (int): epoch number
+
+    Returns:
+        lrate(float): new learning rate
+    """
+    initial_lrate = 0.01
+    factor = 0.1
+    if epoch < 10:
+        lrate = initial_lrate
+    elif epoch < 20:
+        lrate = initial_lrate * math.pow(factor, 1)
+    elif epoch < 30:
+        lrate = initial_lrate * math.pow(factor, 2)
+    elif epoch < 40:
+        lrate = initial_lrate * math.pow(factor, 3)
+    else:
+        lrate = initial_lrate * math.pow(factor, 4)
+    return lrate
+
+
+def print_test(model, X_adv, X_test, y_test, epsilon):
+    """
+    returns the test results and show the SNR and evaluation results
+    """
+    loss, acc = model.evaluate(X_adv, y_test)
+    print("epsilon: {} and test evaluation : {}, {}".format(epsilon, loss, acc))
+    SNR = 20 * np.log10(np.linalg.norm(X_test) / np.linalg.norm(X_test - X_adv))
+    print("SNR: {}".format(SNR))
+    return loss, acc
+
+
+def get_adversarial_examples(pretrained_model, X_true, y_true, epsilon):
+    """
+    The attack requires the model to ouput the logits
+    returns the adversarial example/s of a given image/s for epsilon value using
+    fast gradient sign method
+    """
+    logits_model = tf.keras.Model(
+        pretrained_model.input, pretrained_model.layers[-1].output
+    )
+    X_adv = []
+
+    for i in range(len(X_true)):
+
+        random_index = i
+
+        original_image = X_true[random_index]
+        original_image = tf.convert_to_tensor(
+            original_image.reshape((1, 32, 32))
+        )  # The .reshape just gives it the proper form to input into the model, a batch of 1 a.k.a a tensor
+        original_label = y_true[random_index]
+        original_label = np.reshape(np.argmax(original_label), (1,)).astype("int64")
+
+        adv_example_targeted_label = fast_gradient_method(
+            logits_model,
+            original_image,
+            epsilon,
+            np.inf,
+            y=original_label,
+            targeted=False,
+        )
+        X_adv.append(np.array(adv_example_targeted_label).reshape(32, 32, 1))
+
+    X_adv = np.array(X_adv)
+
+    return X_adv
+
+
+lrate_conv = LearningRateScheduler(step_decay_conv)
+lrate = LearningRateScheduler(step_decay)

models/wideresnet/wresnet.py

@@ -0,0 +1,329 @@
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class WideResidualNetwork(object):
+    def __init__(
+        self,
+        input_dim,
+        weight_decay,
+        momentum,
+        nb_classes=100,
+        N=2,
+        k=1,
+        dropout=0.0,
+        verbose=1,
+    ):
+        """[Assign the initial parameters of the wide residual network]
+
+        Args:
+            weight_decay ([float]): [description]
+            input_dim ([tuple]): [input dimension]
+            nb_classes (int, optional): [output class]. Defaults to 100.
+            N (int, optional): [the number of blocks]. Defaults to 2.
+            k (int, optional): [network width]. Defaults to 1.
+            dropout (float, optional): [dropout value to prevent overfitting]. Defaults to 0.0.
+            verbose (int, optional): [description]. Defaults to 1.
+
+        Returns:
+            [Model]: [wideresnet]
+        """
+        self.weight_decay = weight_decay
+        self.input_dim = input_dim
+        self.nb_classes = nb_classes
+        self.N = N
+        self.k = k
+        self.dropout = dropout
+        self.verbose = verbose
+
+    def initial_conv(self, input):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            16,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(input)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        return x
+
+    def expand_conv(self, init, base, k, strides=(1, 1)):
+        """[summary]
+
+        Args:
+            init ([type]): [description]
+            base ([type]): [description]
+            k ([type]): [description]
+            strides (tuple, optional): [description]. Defaults to (1, 1).
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            strides=strides,
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(init)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        skip = Convolution2D(
+            base * k,
+            (1, 1),
+            padding="same",
+            strides=strides,
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(init)
+
+        m = Add()([x, skip])
+
+        return m
+
+    def conv1_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = Add()([init, x])
+        return m
+
+    def conv2_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+        print("conv2:channel:  {}".format(channel_axis))
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = Add()([init, x])
+        return m
+
+    def conv3_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = Add()([init, x])
+        return m
+
+    def create_wide_residual_network(self):
+        """create a wide residual network model
+
+
+        Returns:
+            [Model]: [wide residual network]
+        """
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        ip = Input(shape=self.input_dim)
+
+        x = self.initial_conv(ip)
+        nb_conv = 4
+
+        x = self.expand_conv(x, 16, self.k)
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv1_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 32, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv2_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 64, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv3_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = AveragePooling2D((8, 8))(x)
+        x = Flatten()(x)
+
+        x = Dense(
+            self.nb_classes,
+            kernel_regularizer=l2(self.weight_decay),
+            activation="softmax",
+        )(x)
+
+        model = Model(ip, x)
+
+        if self.verbose:
+            print("Wide Residual Network-%d-%d created." % (nb_conv, self.k))
+        return model