| ''' | |
| COTR two view reconstruction with known extrinsic/intrinsic demo | |
| ''' | |
| import argparse | |
| import os | |
| import time | |
| import numpy as np | |
| import torch | |
| import imageio | |
| import open3d as o3d | |
| from COTR.utils import utils, debug_utils | |
| from COTR.models import build_model | |
| from COTR.options.options import * | |
| from COTR.options.options_utils import * | |
| from COTR.inference.sparse_engine import SparseEngine, FasterSparseEngine | |
| from COTR.projector import pcd_projector | |
| utils.fix_randomness(0) | |
| torch.set_grad_enabled(False) | |
| def triangulate_rays_to_pcd(center_a, dir_a, center_b, dir_b): | |
| A = center_a | |
| a = dir_a / np.linalg.norm(dir_a, axis=1, keepdims=True) | |
| B = center_b | |
| b = dir_b / np.linalg.norm(dir_b, axis=1, keepdims=True) | |
| c = B - A | |
| D = A + a * ((-np.sum(a * b, axis=1) * np.sum(b * c, axis=1) + np.sum(a * c, axis=1) * np.sum(b * b, axis=1)) / (np.sum(a * a, axis=1) * np.sum(b * b, axis=1) - np.sum(a * b, axis=1) * np.sum(a * b, axis=1)))[..., None] | |
| return D | |
| def main(opt): | |
| model = build_model(opt) | |
| model = model.cuda() | |
| weights = torch.load(opt.load_weights_path, map_location='cpu')['model_state_dict'] | |
| utils.safe_load_weights(model, weights) | |
| model = model.eval() | |
| img_a = imageio.imread('./sample_data/imgs/img_0.jpg', pilmode='RGB') | |
| img_b = imageio.imread('./sample_data/imgs/img_1.jpg', pilmode='RGB') | |
| if opt.faster_infer: | |
| engine = FasterSparseEngine(model, 32, mode='tile') | |
| else: | |
| engine = SparseEngine(model, 32, mode='tile') | |
| t0 = time.time() | |
| corrs = engine.cotr_corr_multiscale_with_cycle_consistency(img_a, img_b, np.linspace(0.5, 0.0625, 4), 1, max_corrs=opt.max_corrs, queries_a=None) | |
| t1 = time.time() | |
| print(f'spent {t1-t0} seconds for {opt.max_corrs} correspondences.') | |
| camera_a = np.load('./sample_data/camera_0.npy', allow_pickle=True).item() | |
| camera_b = np.load('./sample_data/camera_1.npy', allow_pickle=True).item() | |
| center_a = camera_a['cam_center'] | |
| center_b = camera_b['cam_center'] | |
| rays_a = pcd_projector.PointCloudProjector.pcd_2d_to_pcd_3d_np(corrs[:, :2], np.ones([corrs.shape[0], 1]) * 2, camera_a['intrinsic'], motion=camera_a['c2w']) | |
| rays_b = pcd_projector.PointCloudProjector.pcd_2d_to_pcd_3d_np(corrs[:, 2:], np.ones([corrs.shape[0], 1]) * 2, camera_b['intrinsic'], motion=camera_b['c2w']) | |
| dir_a = rays_a - center_a | |
| dir_b = rays_b - center_b | |
| center_a = np.array([center_a] * corrs.shape[0]) | |
| center_b = np.array([center_b] * corrs.shape[0]) | |
| points = triangulate_rays_to_pcd(center_a, dir_a, center_b, dir_b) | |
| colors = (img_a[tuple(np.floor(corrs[:, :2]).astype(int)[:, ::-1].T)] / 255 + img_b[tuple(np.floor(corrs[:, 2:]).astype(int)[:, ::-1].T)] / 255) / 2 | |
| colors = np.array(colors) | |
| pcd = o3d.geometry.PointCloud() | |
| pcd.points = o3d.utility.Vector3dVector(points) | |
| pcd.colors = o3d.utility.Vector3dVector(colors) | |
| o3d.visualization.draw_geometries([pcd]) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| set_COTR_arguments(parser) | |
| parser.add_argument('--out_dir', type=str, default=general_config['out'], help='out directory') | |
| parser.add_argument('--load_weights', type=str, default=None, help='load a pretrained set of weights, you need to provide the model id') | |
| parser.add_argument('--max_corrs', type=int, default=2048, help='number of correspondences') | |
| parser.add_argument('--faster_infer', type=str2bool, default=False, help='use fatser inference') | |
| opt = parser.parse_args() | |
| opt.command = ' '.join(sys.argv) | |
| layer_2_channels = {'layer1': 256, | |
| 'layer2': 512, | |
| 'layer3': 1024, | |
| 'layer4': 2048, } | |
| opt.dim_feedforward = layer_2_channels[opt.layer] | |
| if opt.load_weights: | |
| opt.load_weights_path = os.path.join(opt.out_dir, opt.load_weights, 'checkpoint.pth.tar') | |
| print_opt(opt) | |
| main(opt) | |