| import torch | |
| import torch.nn as nn | |
| import time | |
| eps=1e-8 | |
| def sinkhorn(M,r,c,iteration): | |
| p = torch.softmax(M, dim=-1) | |
| u = torch.ones_like(r) | |
| v = torch.ones_like(c) | |
| for _ in range(iteration): | |
| u = r / ((p * v.unsqueeze(-2)).sum(-1) + eps) | |
| v = c / ((p * u.unsqueeze(-1)).sum(-2) + eps) | |
| p = p * u.unsqueeze(-1) * v.unsqueeze(-2) | |
| return p | |
| def sink_algorithm(M,dustbin,iteration): | |
| M = torch.cat([M, dustbin.expand([M.shape[0], M.shape[1], 1])], dim=-1) | |
| M = torch.cat([M, dustbin.expand([M.shape[0], 1, M.shape[2]])], dim=-2) | |
| r = torch.ones([M.shape[0], M.shape[1] - 1],device='cuda') | |
| r = torch.cat([r, torch.ones([M.shape[0], 1],device='cuda') * M.shape[1]], dim=-1) | |
| c = torch.ones([M.shape[0], M.shape[2] - 1],device='cuda') | |
| c = torch.cat([c, torch.ones([M.shape[0], 1],device='cuda') * M.shape[2]], dim=-1) | |
| p=sinkhorn(M,r,c,iteration) | |
| return p | |
| class attention_block(nn.Module): | |
| def __init__(self,channels,head,type): | |
| assert type=='self' or type=='cross','invalid attention type' | |
| nn.Module.__init__(self) | |
| self.head=head | |
| self.type=type | |
| self.head_dim=channels//head | |
| self.query_filter=nn.Conv1d(channels, channels, kernel_size=1) | |
| self.key_filter=nn.Conv1d(channels,channels,kernel_size=1) | |
| self.value_filter=nn.Conv1d(channels,channels,kernel_size=1) | |
| self.attention_filter=nn.Sequential(nn.Conv1d(2*channels,2*channels, kernel_size=1),nn.SyncBatchNorm(2*channels), nn.ReLU(), | |
| nn.Conv1d(2*channels, channels, kernel_size=1)) | |
| self.mh_filter=nn.Conv1d(channels, channels, kernel_size=1) | |
| def forward(self,fea1,fea2): | |
| batch_size,n,m=fea1.shape[0],fea1.shape[2],fea2.shape[2] | |
| query1, key1, value1 = self.query_filter(fea1).view(batch_size,self.head_dim,self.head,-1), self.key_filter(fea1).view(batch_size,self.head_dim,self.head,-1), \ | |
| self.value_filter(fea1).view(batch_size,self.head_dim,self.head,-1) | |
| query2, key2, value2 = self.query_filter(fea2).view(batch_size,self.head_dim,self.head,-1), self.key_filter(fea2).view(batch_size,self.head_dim,self.head,-1), \ | |
| self.value_filter(fea2).view(batch_size,self.head_dim,self.head,-1) | |
| if(self.type=='self'): | |
| score1,score2=torch.softmax(torch.einsum('bdhn,bdhm->bhnm',query1,key1)/self.head_dim**0.5,dim=-1),\ | |
| torch.softmax(torch.einsum('bdhn,bdhm->bhnm',query2,key2)/self.head_dim**0.5,dim=-1) | |
| add_value1, add_value2 = torch.einsum('bhnm,bdhm->bdhn', score1, value1), torch.einsum('bhnm,bdhm->bdhn',score2, value2) | |
| else: | |
| score1,score2 = torch.softmax(torch.einsum('bdhn,bdhm->bhnm', query1, key2) / self.head_dim ** 0.5,dim=-1), \ | |
| torch.softmax(torch.einsum('bdhn,bdhm->bhnm', query2, key1) / self.head_dim ** 0.5, dim=-1) | |
| add_value1, add_value2 =torch.einsum('bhnm,bdhm->bdhn',score1,value2),torch.einsum('bhnm,bdhm->bdhn',score2,value1) | |
| add_value1,add_value2=self.mh_filter(add_value1.contiguous().view(batch_size,self.head*self.head_dim,n)),self.mh_filter(add_value2.contiguous().view(batch_size,self.head*self.head_dim,m)) | |
| fea11, fea22 = torch.cat([fea1, add_value1], dim=1), torch.cat([fea2, add_value2], dim=1) | |
| fea1, fea2 = fea1+self.attention_filter(fea11), fea2+self.attention_filter(fea22) | |
| return fea1,fea2 | |
| class matcher(nn.Module): | |
| def __init__(self, config): | |
| nn.Module.__init__(self) | |
| self.use_score_encoding=config.use_score_encoding | |
| self.layer_num=config.layer_num | |
| self.sink_iter=config.sink_iter | |
| self.position_encoder = nn.Sequential(nn.Conv1d(3, 32, kernel_size=1) if config.use_score_encoding else nn.Conv1d(2, 32, kernel_size=1), | |
| nn.SyncBatchNorm(32), nn.ReLU(), | |
| nn.Conv1d(32, 64, kernel_size=1), nn.SyncBatchNorm(64),nn.ReLU(), | |
| nn.Conv1d(64, 128, kernel_size=1), nn.SyncBatchNorm(128), nn.ReLU(), | |
| nn.Conv1d(128, 256, kernel_size=1), nn.SyncBatchNorm(256), nn.ReLU(), | |
| nn.Conv1d(256, config.net_channels, kernel_size=1)) | |
| self.dustbin=nn.Parameter(torch.tensor(1,dtype=torch.float32,device='cuda')) | |
| self.self_attention_block=nn.Sequential(*[attention_block(config.net_channels,config.head,'self') for _ in range(config.layer_num)]) | |
| self.cross_attention_block=nn.Sequential(*[attention_block(config.net_channels,config.head,'cross') for _ in range(config.layer_num)]) | |
| self.final_project=nn.Conv1d(config.net_channels, config.net_channels, kernel_size=1) | |
| def forward(self,data,test_mode=True): | |
| desc1, desc2 = data['desc1'], data['desc2'] | |
| desc1, desc2 = torch.nn.functional.normalize(desc1,dim=-1), torch.nn.functional.normalize(desc2,dim=-1) | |
| desc1,desc2=desc1.transpose(1,2),desc2.transpose(1,2) | |
| if test_mode: | |
| encode_x1,encode_x2=data['x1'],data['x2'] | |
| else: | |
| encode_x1,encode_x2=data['aug_x1'], data['aug_x2'] | |
| if not self.use_score_encoding: | |
| encode_x1,encode_x2=encode_x1[:,:,:2],encode_x2[:,:,:2] | |
| encode_x1,encode_x2=encode_x1.transpose(1,2),encode_x2.transpose(1,2) | |
| x1_pos_embedding, x2_pos_embedding = self.position_encoder(encode_x1), self.position_encoder(encode_x2) | |
| aug_desc1, aug_desc2 = x1_pos_embedding + desc1, x2_pos_embedding+desc2 | |
| for i in range(self.layer_num): | |
| aug_desc1,aug_desc2=self.self_attention_block[i](aug_desc1,aug_desc2) | |
| aug_desc1,aug_desc2=self.cross_attention_block[i](aug_desc1,aug_desc2) | |
| aug_desc1,aug_desc2=self.final_project(aug_desc1),self.final_project(aug_desc2) | |
| desc_mat = torch.matmul(aug_desc1.transpose(1, 2), aug_desc2) | |
| p = sink_algorithm(desc_mat, self.dustbin,self.sink_iter[0]) | |
| return {'p':p} | |