|
import cv2 |
|
import os |
|
from tqdm import tqdm |
|
import torch |
|
import numpy as np |
|
from extract import extract_method |
|
|
|
use_cuda = torch.cuda.is_available() |
|
device = torch.device("cuda" if use_cuda else "cpu") |
|
|
|
methods = [ |
|
"d2", |
|
"lfnet", |
|
"superpoint", |
|
"r2d2", |
|
"aslfeat", |
|
"disk", |
|
"alike-n", |
|
"alike-l", |
|
"alike-n-ms", |
|
"alike-l-ms", |
|
] |
|
names = [ |
|
"D2-Net(MS)", |
|
"LF-Net(MS)", |
|
"SuperPoint", |
|
"R2D2(MS)", |
|
"ASLFeat(MS)", |
|
"DISK", |
|
"ALike-N", |
|
"ALike-L", |
|
"ALike-N(MS)", |
|
"ALike-L(MS)", |
|
] |
|
|
|
top_k = None |
|
n_i = 52 |
|
n_v = 56 |
|
cache_dir = "hseq/cache" |
|
dataset_path = "hseq/hpatches-sequences-release" |
|
|
|
|
|
def generate_read_function(method, extension="ppm"): |
|
def read_function(seq_name, im_idx): |
|
aux = np.load( |
|
os.path.join( |
|
dataset_path, seq_name, "%d.%s.%s" % (im_idx, extension, method) |
|
) |
|
) |
|
if top_k is None: |
|
return aux["keypoints"], aux["descriptors"] |
|
else: |
|
assert "scores" in aux |
|
ids = np.argsort(aux["scores"])[-top_k:] |
|
return aux["keypoints"][ids, :], aux["descriptors"][ids, :] |
|
|
|
return read_function |
|
|
|
|
|
def mnn_matcher(descriptors_a, descriptors_b): |
|
device = descriptors_a.device |
|
sim = descriptors_a @ descriptors_b.t() |
|
nn12 = torch.max(sim, dim=1)[1] |
|
nn21 = torch.max(sim, dim=0)[1] |
|
ids1 = torch.arange(0, sim.shape[0], device=device) |
|
mask = ids1 == nn21[nn12] |
|
matches = torch.stack([ids1[mask], nn12[mask]]) |
|
return matches.t().data.cpu().numpy() |
|
|
|
|
|
def homo_trans(coord, H): |
|
kpt_num = coord.shape[0] |
|
homo_coord = np.concatenate((coord, np.ones((kpt_num, 1))), axis=-1) |
|
proj_coord = np.matmul(H, homo_coord.T).T |
|
proj_coord = proj_coord / proj_coord[:, 2][..., None] |
|
proj_coord = proj_coord[:, 0:2] |
|
return proj_coord |
|
|
|
|
|
def benchmark_features(read_feats): |
|
lim = [1, 5] |
|
rng = np.arange(lim[0], lim[1] + 1) |
|
|
|
seq_names = sorted(os.listdir(dataset_path)) |
|
|
|
n_feats = [] |
|
n_matches = [] |
|
seq_type = [] |
|
i_err = {thr: 0 for thr in rng} |
|
v_err = {thr: 0 for thr in rng} |
|
|
|
i_err_homo = {thr: 0 for thr in rng} |
|
v_err_homo = {thr: 0 for thr in rng} |
|
|
|
for seq_idx, seq_name in tqdm(enumerate(seq_names), total=len(seq_names)): |
|
keypoints_a, descriptors_a = read_feats(seq_name, 1) |
|
n_feats.append(keypoints_a.shape[0]) |
|
|
|
|
|
ref_img = cv2.imread(os.path.join(dataset_path, seq_name, "1.ppm")) |
|
ref_img_shape = ref_img.shape |
|
|
|
for im_idx in range(2, 7): |
|
keypoints_b, descriptors_b = read_feats(seq_name, im_idx) |
|
n_feats.append(keypoints_b.shape[0]) |
|
|
|
matches = mnn_matcher( |
|
torch.from_numpy(descriptors_a).to(device=device), |
|
torch.from_numpy(descriptors_b).to(device=device), |
|
) |
|
|
|
homography = np.loadtxt( |
|
os.path.join(dataset_path, seq_name, "H_1_" + str(im_idx)) |
|
) |
|
|
|
pos_a = keypoints_a[matches[:, 0], :2] |
|
pos_a_h = np.concatenate([pos_a, np.ones([matches.shape[0], 1])], axis=1) |
|
pos_b_proj_h = np.transpose(np.dot(homography, np.transpose(pos_a_h))) |
|
pos_b_proj = pos_b_proj_h[:, :2] / pos_b_proj_h[:, 2:] |
|
|
|
pos_b = keypoints_b[matches[:, 1], :2] |
|
|
|
dist = np.sqrt(np.sum((pos_b - pos_b_proj) ** 2, axis=1)) |
|
|
|
n_matches.append(matches.shape[0]) |
|
seq_type.append(seq_name[0]) |
|
|
|
if dist.shape[0] == 0: |
|
dist = np.array([float("inf")]) |
|
|
|
for thr in rng: |
|
if seq_name[0] == "i": |
|
i_err[thr] += np.mean(dist <= thr) |
|
else: |
|
v_err[thr] += np.mean(dist <= thr) |
|
|
|
|
|
gt_homo = homography |
|
pred_homo, _ = cv2.findHomography( |
|
keypoints_a[matches[:, 0], :2], |
|
keypoints_b[matches[:, 1], :2], |
|
cv2.RANSAC, |
|
) |
|
if pred_homo is None: |
|
homo_dist = np.array([float("inf")]) |
|
else: |
|
corners = np.array( |
|
[ |
|
[0, 0], |
|
[ref_img_shape[1] - 1, 0], |
|
[0, ref_img_shape[0] - 1], |
|
[ref_img_shape[1] - 1, ref_img_shape[0] - 1], |
|
] |
|
) |
|
real_warped_corners = homo_trans(corners, gt_homo) |
|
warped_corners = homo_trans(corners, pred_homo) |
|
homo_dist = np.mean( |
|
np.linalg.norm(real_warped_corners - warped_corners, axis=1) |
|
) |
|
|
|
for thr in rng: |
|
if seq_name[0] == "i": |
|
i_err_homo[thr] += np.mean(homo_dist <= thr) |
|
else: |
|
v_err_homo[thr] += np.mean(homo_dist <= thr) |
|
|
|
seq_type = np.array(seq_type) |
|
n_feats = np.array(n_feats) |
|
n_matches = np.array(n_matches) |
|
|
|
return i_err, v_err, i_err_homo, v_err_homo, [seq_type, n_feats, n_matches] |
|
|
|
|
|
if __name__ == "__main__": |
|
errors = {} |
|
for method in methods: |
|
output_file = os.path.join(cache_dir, method + ".npy") |
|
read_function = generate_read_function(method) |
|
if os.path.exists(output_file): |
|
errors[method] = np.load(output_file, allow_pickle=True) |
|
else: |
|
extract_method(method) |
|
errors[method] = benchmark_features(read_function) |
|
np.save(output_file, errors[method]) |
|
|
|
for name, method in zip(names, methods): |
|
i_err, v_err, i_err_hom, v_err_hom, _ = errors[method] |
|
|
|
print(f"====={name}=====") |
|
print(f"MMA@1 MMA@2 MMA@3 MHA@1 MHA@2 MHA@3: ", end="") |
|
for thr in range(1, 4): |
|
err = (i_err[thr] + v_err[thr]) / ((n_i + n_v) * 5) |
|
print(f"{err * 100:.2f}%", end=" ") |
|
for thr in range(1, 4): |
|
err_hom = (i_err_hom[thr] + v_err_hom[thr]) / ((n_i + n_v) * 5) |
|
print(f"{err_hom * 100:.2f}%", end=" ") |
|
print("") |
|
|
