File size: 9,866 Bytes
90555c2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 |
#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
plt.rcParams.update({"font.size": 14})
## add your path
prefix = +"logs/"
def plot_figure(x_acc, x_size, y_err, title, epsilon, exp):
plt.figure(figsize=(10, 8))
plt.errorbar(x_001, y_001_acc, yerr=yerr, fmt="o", label="ACC")
plt.title("{0} = {0}".format(title, epsilon))
plt.xlabel("Percent of Adversarial Examples")
plt.ylabel("Accuracy")
plt.savefig(prefix + exp + "/performance/" + str(epsilon) + ".png")
Experiment = ["AEModels", "RandomNoisemodels"]
for exp in Experiment:
columns = ["Model_ID", "percent", "epsilon", "accuracy", "attack"]
table = pd.DataFrame(columns=columns)
table
for file in os.listdir(prefix + exp):
format = file.split("_")[3].split(".")[-1]
if format == "h5":
epsilon = file.split("_")[1]
percent = file.split("_")[2]
ModelID = file.split("_")[3].split(".")[0]
resnet = WideResidualNetwork(
init, 0.0001, 0.9, nb_classes=4, N=2, k=1, dropout=0.0
)
resnet_model = resnet.create_wide_residual_network()
resnet_model.compile(
loss="categorical_crossentropy", optimizer=sgd, metrics=["acc"]
)
resnet_model.load_weights(prefix + exp + "/" + file)
acc = resnet_model.evaluate(X_test, y_test)
row = {
"Model_ID": ModelID,
"percent": percent,
"epsilon": epsilon,
"accuracy": acc[1],
"attack": 0,
}
table = table.append(row, ignore_index=True)
tab_25 = table[table["percent"] == "0.25"]
tab_5 = table[table["percent"] == "0.5"]
tab_75 = table[table["percent"] == "0.75"]
tab_1 = table[table["percent"] == "1.0"]
tab_25_001 = tab_25[tab_25["epsilon"] == "0.001"]
tab_25_001["mean_acc"], tab_25_001["mean_att"] = (
sum(tab_25_001["accuracy"]) / 10,
sum(tab_25_001["attack"]) / 10,
)
tab_25_01 = tab_25[tab_25["epsilon"] == "0.01"]
tab_25_01["mean_acc"], tab_25_01["mean_att"] = (
sum(tab_25_01["accuracy"]) / 10,
sum(tab_25_01["attack"]) / 10,
)
tab_25_03 = tab_25[tab_25["epsilon"] == "0.03"]
tab_25_03["mean_acc"], tab_25_03["mean_att"] = (
sum(tab_25_03["accuracy"]) / 10,
sum(tab_25_03["attack"]) / 10,
)
tab_25_005 = tab_25[tab_25["epsilon"] == "0.005"]
tab_25_005["mean_acc"], tab_25_005["mean_att"] = (
sum(tab_25_005["accuracy"]) / 10,
sum(tab_25_005["attack"]) / 10,
)
tab_25_003 = tab_25[tab_25["epsilon"] == "0.003"]
tab_25_003["mean_acc"], tab_25_003["mean_att"] = (
sum(tab_25_003["accuracy"]) / 10,
sum(tab_25_003["attack"]) / 10,
)
tab_5_001 = tab_5[tab_5["epsilon"] == "0.001"]
tab_5_001["mean_acc"], tab_5_001["mean_att"] = (
sum(tab_5_001["accuracy"]) / 10,
sum(tab_5_001["attack"]) / 10,
)
tab_5_01 = tab_5[tab_5["epsilon"] == "0.01"]
tab_5_01["mean_acc"], tab_5_01["mean_att"] = (
sum(tab_5_01["accuracy"]) / 10,
sum(tab_5_01["attack"]) / 10,
)
tab_5_03 = tab_5[tab_5["epsilon"] == "0.03"]
tab_5_03["mean_acc"], tab_5_03["mean_att"] = (
sum(tab_5_03["accuracy"]) / 10,
sum(tab_5_03["attack"]) / 10,
)
tab_5_005 = tab_5[tab_5["epsilon"] == "0.005"]
tab_5_005["mean_acc"], tab_5_005["mean_att"] = (
sum(tab_5_005["accuracy"]) / 10,
sum(tab_5_005["attack"]) / 10,
)
tab_5_003 = tab_5[tab_5["epsilon"] == "0.003"]
tab_5_003["mean_acc"], tab_5_003["mean_att"] = (
sum(tab_5_003["accuracy"]) / 10,
sum(tab_5_003["attack"]) / 10,
)
tab_1_001 = tab_1[tab_1["epsilon"] == "0.001"]
tab_1_001["mean_acc"], tab_1_001["mean_att"] = (
sum(tab_1_001["accuracy"]) / 10,
sum(tab_1_001["attack"]) / 10,
)
tab_1_01 = tab_1[tab_1["epsilon"] == "0.01"]
tab_1_01["mean_acc"], tab_1_01["mean_att"] = (
sum(tab_1_01["accuracy"]) / 10,
sum(tab_1_01["attack"]) / 10,
)
tab_1_03 = tab_1[tab_1["epsilon"] == "0.03"]
tab_1_03["mean_acc"], tab_1_03["mean_att"] = (
sum(tab_1_03["accuracy"]) / 10,
sum(tab_1_03["attack"]) / 10,
)
tab_1_005 = tab_1[tab_1["epsilon"] == "0.005"]
tab_1_005["mean_acc"], tab_1_005["mean_att"] = (
sum(tab_1_005["accuracy"]) / 10,
sum(tab_1_005["attack"]) / 10,
)
tab_1_003 = tab_1[tab_1["epsilon"] == "0.003"]
tab_1_003["mean_acc"], tab_1_003["mean_att"] = (
sum(tab_1_003["accuracy"]) / 10,
sum(tab_1_003["attack"]) / 10,
)
tab_75_001 = tab_75[tab_75["epsilon"] == "0.001"]
tab_75_001["mean_acc"], tab_75_001["mean_att"] = (
sum(tab_75_001["accuracy"]) / 10,
sum(tab_75_001["attack"]) / 10,
)
tab_75_01 = tab_75[tab_75["epsilon"] == "0.01"]
tab_75_01["mean_acc"], tab_75_01["mean_att"] = (
sum(tab_75_01["accuracy"]) / 10,
sum(tab_75_01["attack"]) / 10,
)
tab_75_03 = tab_75[tab_75["epsilon"] == "0.03"]
tab_75_03["mean_acc"], tab_75_03["mean_att"] = (
sum(tab_75_03["accuracy"]) / 10,
sum(tab_75_03["attack"]) / 10,
)
tab_75_005 = tab_75[tab_75["epsilon"] == "0.005"]
tab_75_005["mean_acc"], tab_75_005["mean_att"] = (
sum(tab_75_005["accuracy"]) / 10,
sum(tab_75_005["attack"]) / 10,
)
tab_75_003 = tab_75[tab_75["epsilon"] == "0.003"]
tab_75_003["mean_acc"], tab_75_003["mean_att"] = (
sum(tab_75_003["accuracy"]) / 10,
sum(tab_75_003["attack"]) / 10,
)
table_mean = table_mean.append(tab_25_001.head(1), ignore_index=True)
table_mean = table_mean.append(tab_25_01.head(1), ignore_index=True)
table_mean = table_mean.append(tab_25_03.head(1), ignore_index=True)
table_mean = table_mean.append(tab_25_005.head(1), ignore_index=True)
table_mean = table_mean.append(tab_25_003.head(1), ignore_index=True)
###
table_mean = table_mean.append(tab_5_001.head(1), ignore_index=True)
table_mean = table_mean.append(tab_5_01.head(1), ignore_index=True)
table_mean = table_mean.append(tab_5_03.head(1), ignore_index=True)
table_mean = table_mean.append(tab_5_005.head(1), ignore_index=True)
table_mean = table_mean.append(tab_5_003.head(1), ignore_index=True)
###
table_mean = table_mean.append(tab_75_001.head(1), ignore_index=True)
table_mean = table_mean.append(tab_75_01.head(1), ignore_index=True)
table_mean = table_mean.append(tab_75_03.head(1), ignore_index=True)
table_mean = table_mean.append(tab_75_005.head(1), ignore_index=True)
table_mean = table_mean.append(tab_75_003.head(1), ignore_index=True)
###
table_mean = table_mean.append(tab_1_001.head(1), ignore_index=True)
table_mean = table_mean.append(tab_1_01.head(1), ignore_index=True)
table_mean = table_mean.append(tab_1_03.head(1), ignore_index=True)
table_mean = table_mean.append(tab_1_005.head(1), ignore_index=True)
table_mean = table_mean.append(tab_1_003.head(1), ignore_index=True)
print(
table_mean[column].sort_values(by=["epsilon", "percent"]).to_latex(index=False)
)
y_001 = table_mean[table_mean["epsilon"] == "0.001"]["mean_att"]
y_001_acc = table_mean[table_mean["epsilon"] == "0.001"]["mean_acc"]
x_001 = table_mean[table_mean["epsilon"] == "0.001"]["percent"]
y_003 = table_mean[table_mean["epsilon"] == "0.003"]["mean_att"]
y_003_acc = table_mean[table_mean["epsilon"] == "0.003"]["mean_acc"]
x_003 = table_mean[table_mean["epsilon"] == "0.003"]["percent"]
y_005 = table_mean[table_mean["epsilon"] == "0.005"]["mean_att"]
y_005_acc = table_mean[table_mean["epsilon"] == "0.005"]["mean_acc"]
x_005 = table_mean[table_mean["epsilon"] == "0.005"]["percent"]
y_01 = table_mean[table_mean["epsilon"] == "0.01"]["mean_att"]
y_01_acc = table_mean[table_mean["epsilon"] == "0.01"]["mean_acc"]
x_01 = table_mean[table_mean["epsilon"] == "0.01"]["percent"]
y_03 = table_mean[table_mean["epsilon"] == "0.03"]["mean_att"]
y_03_acc = table_mean[table_mean["epsilon"] == "0.03"]["mean_acc"]
x_03 = table_mean[table_mean["epsilon"] == "0.03"]["percent"]
l_25 = []
l_25.append(np.std(tab_25_001["accuracy"], axis=0))
l_25.append(np.std(tab_25_01["accuracy"], axis=0))
l_25.append(np.std(tab_25_03["accuracy"], axis=0))
l_25.append(np.std(tab_25_005["accuracy"], axis=0))
l_25.append(np.std(tab_25_003["accuracy"], axis=0))
# ###
l_5 = []
l_5.append(np.std(tab_5_001["accuracy"], axis=0))
l_5.append(np.std(tab_5_01["accuracy"], axis=0))
l_5.append(np.std(tab_5_03["accuracy"], axis=0))
l_5.append(np.std(tab_5_005["accuracy"], axis=0))
l_5.append(np.std(tab_5_003["accuracy"], axis=0))
# ###
l_75 = []
l_75.append(np.std(tab_75_001["accuracy"], axis=0))
l_75.append(np.std(tab_75_01["accuracy"], axis=0))
l_75.append(np.std(tab_75_03["accuracy"], axis=0))
l_75.append(np.std(tab_75_005["accuracy"], axis=0))
l_75.append(np.std(tab_75_003["accuracy"], axis=0))
# ###
l_1 = []
l_1.append(np.std(tab_1_001["accuracy"], axis=0))
l_1.append(np.std(tab_1_01["accuracy"], axis=0))
l_1.append(np.std(tab_1_03["accuracy"], axis=0))
l_1.append(np.std(tab_1_005["accuracy"], axis=0))
l_1.append(np.std(tab_1_003["accuracy"], axis=0))
x_acc = [x_001_acc, x_01_acc, x_03_acc, x_005_acc, x_003_acc]
x_sizes = [x_001, x_01, x_03, x_005, x_003]
epsilons = [0.001, 0.01, 0.03, 0.005, 0.003]
# example variable error bar values
for i in range(5):
yerr = [0.01, l_25[i], l_5[i], l_75[i], l_1[i]]
plot_figure(x_acc[i], x_sizes[i], yerr, epsilons[i], exp)
|