third_party/DarkFeat/nets/reliability_loss.py

import torch
import torch.nn as nn
import numpy as np


class APLoss(nn.Module):
    """differentiable AP loss, through quantization.

    Input: (N, M)   values in [min, max]
    label: (N, M)   values in {0, 1}

    Returns: list of query AP (for each n in {1..N})
             Note: typically, you want to minimize 1 - mean(AP)
    """

    def __init__(self, nq=25, min=0, max=1, euc=False):
        nn.Module.__init__(self)
        assert isinstance(nq, int) and 2 <= nq <= 100
        self.nq = nq
        self.min = min
        self.max = max
        self.euc = euc
        gap = max - min
        assert gap > 0

        # init quantizer = non-learnable (fixed) convolution
        self.quantizer = q = nn.Conv1d(1, 2 * nq, kernel_size=1, bias=True)
        a = (nq - 1) / gap
        # 1st half = lines passing to (min+x,1) and (min+x+1/a,0) with x = {nq-1..0}*gap/(nq-1)
        q.weight.data[:nq] = -a
        q.bias.data[:nq] = torch.from_numpy(
            a * min + np.arange(nq, 0, -1)
        )  # b = 1 + a*(min+x)
        # 2nd half = lines passing to (min+x,1) and (min+x-1/a,0) with x = {nq-1..0}*gap/(nq-1)
        q.weight.data[nq:] = a
        q.bias.data[nq:] = torch.from_numpy(
            np.arange(2 - nq, 2, 1) - a * min
        )  # b = 1 - a*(min+x)
        # first and last one are special: just horizontal straight line
        q.weight.data[0] = q.weight.data[-1] = 0
        q.bias.data[0] = q.bias.data[-1] = 1

    def compute_AP(self, x, label):
        N, M = x.shape
        # print(x.shape, label.shape)
        if self.euc:  # euclidean distance in same range than similarities
            x = 1 - torch.sqrt(2.001 - 2 * x)

        # quantize all predictions
        q = self.quantizer(x.unsqueeze(1))
        q = torch.min(q[:, : self.nq], q[:, self.nq :]).clamp(
            min=0
        )  # N x Q x M [1600, 20, 1681]

        nbs = q.sum(dim=-1)  # number of samples  N x Q = c
        rec = (q * label.view(N, 1, M).float()).sum(
            dim=-1
        )  # nb of correct samples = c+ N x Q
        prec = rec.cumsum(dim=-1) / (1e-16 + nbs.cumsum(dim=-1))  # precision
        rec /= rec.sum(dim=-1).unsqueeze(1)  # norm in [0,1]

        ap = (prec * rec).sum(dim=-1)  # per-image AP
        return ap

    def forward(self, x, label):
        assert x.shape == label.shape  # N x M
        return self.compute_AP(x, label)


class PixelAPLoss(nn.Module):
    """Computes the pixel-wise AP loss:
    Given two images and ground-truth optical flow, computes the AP per pixel.

    feat1:  (B, C, H, W)   pixel-wise features extracted from img1
    feat2:  (B, C, H, W)   pixel-wise features extracted from img2
    aflow:  (B, 2, H, W)   absolute flow: aflow[...,y1,x1] = x2,y2
    """

    def __init__(self, sampler, nq=20):
        nn.Module.__init__(self)
        self.aploss = APLoss(nq, min=0, max=1, euc=False)
        self.name = "pixAP"
        self.sampler = sampler

    def loss_from_ap(self, ap, rel):
        return 1 - ap

    def forward(self, feat0, feat1, conf0, conf1, pos0, pos1, B, H, W, N=1200):
        # subsample things
        scores, gt, msk, qconf = self.sampler(
            feat0, feat1, conf0, conf1, pos0, pos1, B, H, W, N=1200
        )

        # compute pixel-wise AP
        n = qconf.numel()
        if n == 0:
            return 0
        scores, gt = scores.view(n, -1), gt.view(n, -1)
        ap = self.aploss(scores, gt).view(msk.shape)

        pixel_loss = self.loss_from_ap(ap, qconf)

        loss = pixel_loss[msk].mean()
        return loss


class ReliabilityLoss(PixelAPLoss):
    """same than PixelAPLoss, but also train a pixel-wise confidence
    that this pixel is going to have a good AP.
    """

    def __init__(self, sampler, base=0.5, **kw):
        PixelAPLoss.__init__(self, sampler, **kw)
        assert 0 <= base < 1
        self.base = base

    def loss_from_ap(self, ap, rel):
        return 1 - ap * rel - (1 - rel) * self.base