from loguru import logger import copy import torch import torch.nn as nn import torch.nn.functional as F from .linear_attention import LinearAttention, FullAttention class LoFTREncoderLayer(nn.Module): def __init__(self, d_model, nhead, attention='linear'): super(LoFTREncoderLayer, self).__init__() self.dim = d_model // nhead self.nhead = nhead # multi-head attention self.q_proj = nn.Linear(d_model, d_model, bias=False) self.k_proj = nn.Linear(d_model, d_model, bias=False) self.v_proj = nn.Linear(d_model, d_model, bias=False) self.attention = LinearAttention() if attention == 'linear' else FullAttention() self.merge = nn.Linear(d_model, d_model, bias=False) # feed-forward network self.mlp = nn.Sequential( nn.Linear(d_model*2, d_model*2, bias=False), nn.GELU(), nn.Linear(d_model*2, d_model, bias=False), ) # norm and dropout self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) def forward(self, x, source, x_mask=None, source_mask=None): """ Args: x (torch.Tensor): [N, L, C] source (torch.Tensor): [N, S, C] x_mask (torch.Tensor): [N, L] (optional) source_mask (torch.Tensor): [N, S] (optional) """ bs = x.shape[0] query, key, value = x, source, source # multi-head attention query = self.q_proj(query).view(bs, -1, self.nhead, self.dim) # [N, L, (H, D)] key = self.k_proj(key).view(bs, -1, self.nhead, self.dim) # [N, S, (H, D)] value = self.v_proj(value).view(bs, -1, self.nhead, self.dim) message = self.attention(query, key, value, q_mask=x_mask, kv_mask=source_mask) # [N, L, (H, D)] message = self.merge(message.view(bs, -1, self.nhead*self.dim)) # [N, L, C] message = self.norm1(message) # feed-forward network message = self.mlp(torch.cat([x, message], dim=2)) message = self.norm2(message) return x + message class TopicFormer(nn.Module): """A Local Feature Transformer (LoFTR) module.""" def __init__(self, config): super(TopicFormer, self).__init__() self.config = config self.d_model = config['d_model'] self.nhead = config['nhead'] self.layer_names = config['layer_names'] encoder_layer = LoFTREncoderLayer(config['d_model'], config['nhead'], config['attention']) self.layers = nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(len(self.layer_names))]) self.topic_transformers = nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(2*config['n_topic_transformers'])]) if config['n_samples'] > 0 else None #nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(2)]) self.n_iter_topic_transformer = config['n_topic_transformers'] self.seed_tokens = nn.Parameter(torch.randn(config['n_topics'], config['d_model'])) self.register_parameter('seed_tokens', self.seed_tokens) self.n_samples = config['n_samples'] self._reset_parameters() def _reset_parameters(self): for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def sample_topic(self, prob_topics, topics, L): """ Args: topics (torch.Tensor): [N, L+S, K] """ prob_topics0, prob_topics1 = prob_topics[:, :L], prob_topics[:, L:] topics0, topics1 = topics[:, :L], topics[:, L:] theta0 = F.normalize(prob_topics0.sum(dim=1), p=1, dim=-1) # [N, K] theta1 = F.normalize(prob_topics1.sum(dim=1), p=1, dim=-1) theta = F.normalize(theta0 * theta1, p=1, dim=-1) if self.n_samples == 0: return None if self.training: sampled_inds = torch.multinomial(theta, self.n_samples) sampled_values = torch.gather(theta, dim=-1, index=sampled_inds) else: sampled_values, sampled_inds = torch.topk(theta, self.n_samples, dim=-1) sampled_topics0 = torch.gather(topics0, dim=-1, index=sampled_inds.unsqueeze(1).repeat(1, topics0.shape[1], 1)) sampled_topics1 = torch.gather(topics1, dim=-1, index=sampled_inds.unsqueeze(1).repeat(1, topics1.shape[1], 1)) return sampled_topics0, sampled_topics1 def reduce_feat(self, feat, topick, N, C): len_topic = topick.sum(dim=-1).int() max_len = len_topic.max().item() selected_ids = topick.bool() resized_feat = torch.zeros((N, max_len, C), dtype=torch.float32, device=feat.device) new_mask = torch.zeros_like(resized_feat[..., 0]).bool() for i in range(N): new_mask[i, :len_topic[i]] = True resized_feat[new_mask, :] = feat[selected_ids, :] return resized_feat, new_mask, selected_ids def forward(self, feat0, feat1, mask0=None, mask1=None): """ Args: feat0 (torch.Tensor): [N, L, C] feat1 (torch.Tensor): [N, S, C] mask0 (torch.Tensor): [N, L] (optional) mask1 (torch.Tensor): [N, S] (optional) """ assert self.d_model == feat0.shape[2], "the feature number of src and transformer must be equal" N, L, S, C, K = feat0.shape[0], feat0.shape[1], feat1.shape[1], feat0.shape[2], self.config['n_topics'] seeds = self.seed_tokens.unsqueeze(0).repeat(N, 1, 1) feat = torch.cat((feat0, feat1), dim=1) if mask0 is not None: mask = torch.cat((mask0, mask1), dim=-1) else: mask = None for layer, name in zip(self.layers, self.layer_names): if name == 'seed': # seeds = layer(seeds, feat0, None, mask0) # seeds = layer(seeds, feat1, None, mask1) seeds = layer(seeds, feat, None, mask) elif name == 'feat': feat0 = layer(feat0, seeds, mask0, None) feat1 = layer(feat1, seeds, mask1, None) dmatrix = torch.einsum("nmd,nkd->nmk", feat, seeds) prob_topics = F.softmax(dmatrix, dim=-1) feat_topics = torch.zeros_like(dmatrix).scatter_(-1, torch.argmax(dmatrix, dim=-1, keepdim=True), 1.0) if mask is not None: feat_topics = feat_topics * mask.unsqueeze(-1) prob_topics = prob_topics * mask.unsqueeze(-1) if (feat_topics.detach().sum(dim=1).sum(dim=0) > 100).sum() <= 3: logger.warning("topic distribution is highly sparse!") sampled_topics = self.sample_topic(prob_topics.detach(), feat_topics, L) if sampled_topics is not None: updated_feat0, updated_feat1 = torch.zeros_like(feat0), torch.zeros_like(feat1) s_topics0, s_topics1 = sampled_topics for k in range(s_topics0.shape[-1]): topick0, topick1 = s_topics0[..., k], s_topics1[..., k] # [N, L+S] if (topick0.sum() > 0) and (topick1.sum() > 0): new_feat0, new_mask0, selected_ids0 = self.reduce_feat(feat0, topick0, N, C) new_feat1, new_mask1, selected_ids1 = self.reduce_feat(feat1, topick1, N, C) for idt in range(self.n_iter_topic_transformer): new_feat0 = self.topic_transformers[idt*2](new_feat0, new_feat0, new_mask0, new_mask0) new_feat1 = self.topic_transformers[idt*2](new_feat1, new_feat1, new_mask1, new_mask1) new_feat0 = self.topic_transformers[idt*2+1](new_feat0, new_feat1, new_mask0, new_mask1) new_feat1 = self.topic_transformers[idt*2+1](new_feat1, new_feat0, new_mask1, new_mask0) updated_feat0[selected_ids0, :] = new_feat0[new_mask0, :] updated_feat1[selected_ids1, :] = new_feat1[new_mask1, :] feat0 = (1 - s_topics0.sum(dim=-1, keepdim=True)) * feat0 + updated_feat0 feat1 = (1 - s_topics1.sum(dim=-1, keepdim=True)) * feat1 + updated_feat1 conf_matrix = torch.einsum("nlc,nsc->nls", feat0, feat1) / C**.5 #(C * temperature) if self.training: topic_matrix = torch.einsum("nlk,nsk->nls", prob_topics[:, :L], prob_topics[:, L:]) outlier_mask = torch.einsum("nlk,nsk->nls", feat_topics[:, :L], feat_topics[:, L:]) else: topic_matrix = {"img0": feat_topics[:, :L], "img1": feat_topics[:, L:]} outlier_mask = torch.ones_like(conf_matrix) if mask0 is not None: outlier_mask = (outlier_mask * mask0[..., None] * mask1[:, None]) #.bool() conf_matrix.masked_fill_(~outlier_mask.bool(), -1e9) conf_matrix = F.softmax(conf_matrix, 1) * F.softmax(conf_matrix, 2) # * topic_matrix return feat0, feat1, conf_matrix, topic_matrix class LocalFeatureTransformer(nn.Module): """A Local Feature Transformer (LoFTR) module.""" def __init__(self, config): super(LocalFeatureTransformer, self).__init__() self.config = config self.d_model = config['d_model'] self.nhead = config['nhead'] self.layer_names = config['layer_names'] encoder_layer = LoFTREncoderLayer(config['d_model'], config['nhead'], config['attention']) self.layers = nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(2)]) #len(self.layer_names))]) self._reset_parameters() def _reset_parameters(self): for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def forward(self, feat0, feat1, mask0=None, mask1=None): """ Args: feat0 (torch.Tensor): [N, L, C] feat1 (torch.Tensor): [N, S, C] mask0 (torch.Tensor): [N, L] (optional) mask1 (torch.Tensor): [N, S] (optional) """ assert self.d_model == feat0.shape[2], "the feature number of src and transformer must be equal" feat0 = self.layers[0](feat0, feat1, mask0, mask1) feat1 = self.layers[1](feat1, feat0, mask1, mask0) return feat0, feat1