init

Base-RCNN-FPN.yaml

@@ -0,0 +1,50 @@
+MODEL:
+    META_ARCHITECTURE: "GeneralizedRCNN"
+
+    BACKBONE:
+        NAME: "build_resnet_fpn_backbone"
+
+    RESNETS:
+        OUT_FEATURES: ["res2", "res3", "res4", "res5"]
+
+    FPN:
+        IN_FEATURES: ["res2", "res3", "res4", "res5"]
+
+    ANCHOR_GENERATOR:
+        # One size for each in feature map
+        SIZES: [[32], [64], [128], [256], [512]]
+        # Three aspect ratios (same for all in feature maps)
+        ASPECT_RATIOS: [[0.5, 1.0, 2.0]]
+
+    RPN:
+        IN_FEATURES: ["p2", "p3", "p4", "p5", "p6"]
+        PRE_NMS_TOPK_TRAIN: 2000  # Per FPN level
+        PRE_NMS_TOPK_TEST: 1000  # Per FPN level
+        # Detectron1 uses 2000 proposals per-batch,
+        # (See "modeling/rpn/rpn_outputs.py" for details of this legacy issue)
+        # which is approximately 1000 proposals per-image since the default
+        # batch size for FPN is 2.
+        POST_NMS_TOPK_TRAIN: 1000
+        POST_NMS_TOPK_TEST: 1000
+
+    ROI_HEADS:
+        NAME: "StandardROIHeads"
+        IN_FEATURES: ["p2", "p3", "p4", "p5"]
+
+    ROI_BOX_HEAD:
+        NAME: "FastRCNNConvFCHead"
+        NUM_FC: 2
+        POOLER_RESOLUTION: 7
+
+    ROI_MASK_HEAD:
+        NAME: "MaskRCNNConvUpsampleHead"
+        NUM_CONV: 4
+        POOLER_RESOLUTION: 14
+
+INPUT:
+    MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800)
+
+SOLVER:
+    CHECKPOINT_PERIOD: 210000
+
+VERSION: 2

abstractions/Box.py

@@ -0,0 +1,89 @@
+from copy import deepcopy
+
+class Box:
+    def __init__(self):
+        self.dimensions = None
+        self.ivals = []
+        self.element_indexes = [] # record this box is built for what samples
+        self.low_bound_indexes = dict() # record which samples visit the low bound for each dimension
+        self.high_bound_indexes = dict() # record which samples visit the low bound for each dimension
+
+    def build(self, dimensions, points):
+        # a point is a tuple (index, n-dim numpy)
+        # index = point[0]
+        # value = point[1]
+        piter = iter(points)
+        self.dimensions = dimensions
+        self.ivals = []
+        self.element_indexes = []
+        self.low_bound_indexes = dict()
+        self.high_bound_indexes = dict()
+
+        try:
+            point = next(piter)
+        except StopIteration:
+            return
+        else:
+            self.element_indexes.append(point[0]) # update index list
+            i = 0
+            for coord in point[1]:
+                if(i >= self.dimensions):
+                    break
+                self.ivals.append([coord, coord])
+                self.low_bound_indexes["n"+str(i+1)] = [point[0]] # update low bound visiting index list
+                self.high_bound_indexes["n"+str(i+1)] = [point[0]] # update upper bound visiting index list
+                i += 1
+            if(len(self.ivals) != self.dimensions):
+                raise "IllegalArgument"
+
+        while True:
+            try:
+                point = next(piter)
+            except StopIteration:
+                break
+            else:
+                self.element_indexes.append(point[0]) # update index list
+                i = 0
+                for coord in point[1]:
+                    if(i >= self.dimensions):
+                        break
+                    ival = self.ivals[i]
+                    if(coord < ival[0]):
+                        ival[0] = coord
+                        self.low_bound_indexes["n"+str(i+1)] = [point[0]] # update the bound and its index
+                    elif(coord == ival[0]):
+                        low_index_list = self.low_bound_indexes["n"+str(i+1)]
+                        low_index_list.append(point[0])
+
+                    if(coord > ival[1]):
+                        ival[1] = coord
+                        self.high_bound_indexes["n"+str(i+1)] = [point[0]] # update the bound and its index
+                    elif(coord == ival[1]):
+                        high_index_list = self.high_bound_indexes["n"+str(i+1)]
+                        high_index_list.append(point[0])
+                    i += 1
+
+    def query(self, point):
+        i = 0
+        for coord in point:
+            if(i >= self.dimensions):
+                break
+            ival = self.ivals[i]
+            if(coord < ival[0] or coord > ival[1]):
+                return False
+            i += 1
+        return True
+
+    def __str__(self):
+        return self.ivals.__str__()
+
+
+def boxes_query(point, boxes):
+    for box in boxes:
+        if len(box.ivals):
+            if box.query(point):
+                return True
+    return False
+
+
+                

abstractions/__init__.py

@@ -0,0 +1 @@
+from .Box import *

app.py

@@ -0,0 +1,226 @@
+# %%
+try:
+    import detectron2
+except:
+    import os 
+    os.system('pip install git+https://github.com/facebookresearch/detectron2.git')
+import torch
+from detectron2.utils.logger import setup_logger
+setup_logger()
+
+from detectron2.config import get_cfg
+import detectron2.data.transforms as T
+from detectron2.checkpoint import DetectionCheckpointer
+from detectron2.modeling import build_model
+from detectron2.utils.visualizer import Visualizer
+from detectron2.data.detection_utils import read_image
+from detectron2.data import MetadataCatalog
+
+import numpy as np
+import cv2
+from PIL import Image
+import random
+import gradio as gr
+import pandas as pd
+import matplotlib.pyplot as plt
+import io
+from pickle import load
+
+torch.manual_seed(0)
+np.random.seed(0)
+random.seed(0)
+
+from base_cam import EigenCAM
+from pytorch_grad_cam.utils.model_targets import FasterRCNNBoxScoreTarget
+
+class Detectron2Monitor():
+    def __init__(self, label_list, label_dict, config_file, model_file):
+        self.label_list = label_list
+        self.cfg = self._setup_cfg(config_file, model_file)
+        self.model = build_model(self.cfg)
+        self.model.eval()
+        checkpointer = DetectionCheckpointer(self.model)
+        checkpointer.load(self.cfg.MODEL.WEIGHTS)
+        self.monitors_dict = self._load_monitor()
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.class_dict = label_dict
+        
+    def _setup_cfg(self, config_file, model_file):  
+        cfg = get_cfg()
+        cfg.merge_from_file(config_file)
+        cfg.MODEL.WEIGHTS = model_file
+        cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5
+        if torch.cuda.is_available():
+            cfg.MODEL.DEVICE = "cuda"
+        else:
+            cfg.MODEL.DEVICE = "cpu"
+        cfg.freeze()
+        return cfg
+    
+    def _get_input_dict(self, original_image):
+        height, width = original_image.shape[:2]
+        transform_gen = T.ResizeShortestEdge(
+        [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST
+        )
+        image = transform_gen.get_transform(original_image).apply_image(original_image)
+        image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))
+        inputs = {"image": image, "height": height, "width": width}
+        return inputs
+    
+    def _postprocess_cam(self, raw_cam, img_width, img_height):
+        cam_orig = np.sum(raw_cam, axis=0)  # [H,W]
+        cam_orig = np.maximum(cam_orig, 0)  # ReLU
+        cam_orig -= np.min(cam_orig)
+        cam_orig /= np.max(cam_orig)
+        cam = cv2.resize(cam_orig, (img_width, img_height))
+        return cam
+
+    def _inference(self, inputs):
+        with torch.no_grad():
+            images = self.model.preprocess_image(inputs)  
+            features = self.model.backbone(images.tensor)  
+            proposals, _ = self.model.proposal_generator(images, features, None)  # RPN
+
+            features_ = [features[f] for f in self.model.roi_heads.box_in_features]
+            box_features = self.model.roi_heads.box_pooler(features_, [x.proposal_boxes for x in proposals])
+            box_features = self.model.roi_heads.box_head(box_features)  # features of all 1k candidates
+            predictions = self.model.roi_heads.box_predictor(box_features)
+            pred_instances, pred_inds = self.model.roi_heads.box_predictor.inference(predictions, proposals)
+            pred_instances = self.model.roi_heads.forward_with_given_boxes(features, pred_instances)
+
+            # output boxes, masks, scores, etc
+            pred_instances = self.model._postprocess(pred_instances, inputs, images.image_sizes)  # scale box to orig size
+            # features of the proposed boxes
+            feats = box_features[pred_inds]
+        return pred_instances, feats 
+     
+    def _load_monitor(self):
+        with open("monitors_dict.pkl", 'rb') as f:
+            monitors_dict = load(f)
+        return monitors_dict
+        # monitors_dict = {}
+        # for class_name in self.label_list:
+        #     if class_name == "train" or class_name == "OOD":
+        #         continue
+        #     monitor_path = f"monitors/{class_name}/monitor_for_clustering_parameter" + "_tau_" + str(tau) + ".pkl"
+        #     with open(monitor_path, 'rb') as f:
+        #         monitor = load(f)
+        #     monitors_dict[class_name] = monitor
+        # return monitors_dict
+    
+    def _fasterrcnn_reshape_transform(self, x):
+        target_size = x['p6'].size()[-2 : ]
+        activations = []
+        for key, value in x.items():
+            activations.append(torch.nn.functional.interpolate(torch.abs(value), target_size, mode='bilinear'))
+        activations = torch.cat(activations, axis=1)
+        return activations
+
+    def get_output(self, img):
+        image = read_image(img, format="BGR")
+        input_image_dict = [self._get_input_dict(image)]
+        
+        pred_instances, feats = self._inference(input_image_dict)
+        feats = feats.cpu().detach().numpy()
+        detections = pred_instances[0]["instances"].to("cpu")
+        cls_idxs = detections.pred_classes.detach().numpy()
+        # get labels from class indices
+        labels = [self.class_dict[i] for i in cls_idxs]
+        # count values in labels, and return a dictionary
+        labels_count_dict = dict((i, labels.count(i)) for i in labels)
+        v = Visualizer(image[..., ::-1], MetadataCatalog.get("bdd_dataset"), scale=1)
+        v = v.draw_instance_predictions(detections)
+        img_detection = v.get_image()
+        df = pd.DataFrame(list(labels_count_dict.items()), columns=['Object', 'Count'])
+        verdicts = []
+        for label, feat in zip(labels, feats):
+            verdict = self.monitors_dict[label].make_verdicts(feat[np.newaxis,:])[0] if label in self.monitors_dict else True
+            verdicts.append(verdict)
+        detections_ood = detections[[i for i, x in enumerate(verdicts) if not x]]
+        detections_ood.pred_classes = torch.tensor([10]*len(detections_ood.pred_classes))
+        v = Visualizer(image[..., ::-1], MetadataCatalog.get("bdd_dataset"), scale=1)
+        v = v.draw_instance_predictions(detections_ood)
+        img_ood = v.get_image()
+        pred_bboxes = detections.pred_boxes.tensor.numpy().astype(np.int32)
+        target_layers = [self.model.backbone]
+        targets = [FasterRCNNBoxScoreTarget(labels=labels, bounding_boxes=pred_bboxes)]
+        cam = EigenCAM(self.model,
+                        target_layers, 
+                        use_cuda=False,
+                        reshape_transform=self._fasterrcnn_reshape_transform)
+        grayscale_cam = cam(input_image_dict, targets)
+        cam = self._postprocess_cam(grayscale_cam, input_image_dict[0]["width"], input_image_dict[0]["height"])
+        # plt.rcParams["figure.figsize"] = (30,10)
+        # plt.imshow(img_detection[..., ::-1], interpolation='none')
+        # plt.imshow(cam, cmap='jet', alpha=0.5)
+        # plt.axis("off")
+        # img_buff = io.BytesIO()
+        # plt.savefig(img_buff, format='png', bbox_inches='tight', pad_inches=0)
+        # img_cam = Image.open(img_buff)
+        img_cam = image
+        image_dict = {}
+        image_dict["image"] = image
+        image_dict["cam"] = img_cam
+        image_dict["detection"] = img_detection
+        image_dict["verdict"] = img_ood
+        return image_dict, df
+
+config_file = "vanilla.yaml"
+model_file = "model_final_vos_resnet_bdd.pth"
+label_dict = {
+            0: 'pedestrian',
+            1: 'rider',
+            2: 'car',
+            3: 'truck',
+            4: 'bus',
+            5: 'train',
+            6: 'motor',
+            7: 'bike',
+            8: 'traffic light',
+            9: 'traffic sign',
+            10: 'OOD'
+        }
+label_list = list(label_dict.values())
+MetadataCatalog.get("bdd_dataset").set(thing_classes=label_list)
+extractor = Detectron2Monitor(config_file=config_file, label_list=label_list, label_dict=label_dict, model_file=model_file)
+
+# %%
+def inference_gd(file):
+    image_dict, df = extractor.get_output(file)
+    return image_dict["detection"], df, image_dict["verdict"], image_dict["cam"]
+
+
+examples = ["examples/0.jpg", "examples/1.jpg", "examples/2.jpg", "examples/3.jpg"]
+with gr.Blocks(theme="gradio/monochrome") as demo:
+    gr.Markdown("# Runtime Monitoring Object Detection")
+    gr.Markdown(
+        """This interactive demo is based on the box abstraction-based monitors for Faster R-CNN model. The model is trained using [Detectron2](https://github.com/facebookresearch/detectron2) library on the in-distribution dataset [Berkeley DeepDrive-100k](https://www.bdd100k.com/), which contains objects within autonomous driving domain. The monitors are constructed by abstraction of extracted feature from the training data. The demo showcases the monitors' capacity to reject problematic detections due to out-of-distribution(OOD) objects. 
+
+To utilize the demo, upload an image or select one from the provided examples, and click on "Infer" to view the following results: 
+
+- **Detection**: outputs of Object Detector
+- **Detection summary**: a summary of the detection outputs
+- **Verdict**: verdicts from Monitors
+- **Explainable AI**: visual explanation generated by [grad-cam](https://github.com/jacobgil/pytorch-grad-cam) library which is based on Class Activation Mapping(CAM) method.
+
+In case the output image seems too small, simply right-click on the image, and choose “Open image in new tab” to visualize it in full size.
+    """
+    )
+    with gr.Row():
+        with gr.Column():
+            image = gr.inputs.Image(type="filepath", label="Input")
+            button = gr.Button("Infer")
+            eamples_block = gr.Examples(examples, image)
+        with gr.Column():
+            with gr.Tab("Detection"):
+                detection = gr.Image(label="Output")
+            with gr.Tab("Verdict"):
+                verdict = gr.Image(label="Output")
+            with gr.Tab("Explainable AI"):
+                cam = gr.Image(label="Output")
+            df = gr.Dataframe(label="Detection summary")
+    button.click(fn=inference_gd, inputs=image, outputs=[detection, df, verdict, cam])
+
+demo.launch()
+
+

base_cam.py

@@ -0,0 +1,223 @@
+import numpy as np
+import torch
+import ttach as tta
+from typing import Callable, List, Tuple
+from pytorch_grad_cam.activations_and_gradients import ActivationsAndGradients
+from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
+from pytorch_grad_cam.utils.image import scale_cam_image
+from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
+from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
+
+# https://arxiv.org/abs/2008.00299
+
+class BaseCAM:
+    def __init__(self,
+                 model: torch.nn.Module,
+                 target_layers: List[torch.nn.Module],
+                 use_cuda: bool = False,
+                 reshape_transform: Callable = None,
+                 compute_input_gradient: bool = False,
+                 uses_gradients: bool = True) -> None:
+        self.model = model.eval()
+        self.target_layers = target_layers
+        self.cuda = use_cuda
+        if self.cuda:
+            self.model = model.cuda()
+        self.reshape_transform = reshape_transform
+        self.compute_input_gradient = compute_input_gradient
+        self.uses_gradients = uses_gradients
+        self.activations_and_grads = ActivationsAndGradients(
+            self.model, target_layers, reshape_transform)
+
+    """ Get a vector of weights for every channel in the target layer.
+        Methods that return weights channels,
+        will typically need to only implement this function. """
+
+    def get_cam_weights(self,
+                        input_tensor: torch.Tensor,
+                        target_layers: List[torch.nn.Module],
+                        targets: List[torch.nn.Module],
+                        activations: torch.Tensor,
+                        grads: torch.Tensor) -> np.ndarray:
+        raise Exception("Not Implemented")
+
+    def get_cam_image(self,
+                      input_tensor: torch.Tensor,
+                      target_layer: torch.nn.Module,
+                      targets: List[torch.nn.Module],
+                      activations: torch.Tensor,
+                      grads: torch.Tensor,
+                      eigen_smooth: bool = False) -> np.ndarray:
+
+        weights = self.get_cam_weights(input_tensor,
+                                       target_layer,
+                                       targets,
+                                       activations,
+                                       grads)
+        weighted_activations = weights[:, :, None, None] * activations
+        if eigen_smooth:
+            cam = get_2d_projection(weighted_activations)
+        else:
+            cam = weighted_activations.sum(axis=1)
+        return cam
+
+    def forward(self,
+                input_tensor: torch.Tensor,
+                targets: List[torch.nn.Module],
+                eigen_smooth: bool = False) -> np.ndarray:
+        if self.cuda:
+            input_tensor = input_tensor.cuda()
+
+        if self.compute_input_gradient:
+            input_tensor = torch.autograd.Variable(input_tensor,
+                                                   requires_grad=True)
+
+        outputs = self.activations_and_grads(input_tensor)
+        if targets is None:
+            target_categories = np.argmax(outputs.cpu().data.numpy(), axis=-1)
+            targets = [ClassifierOutputTarget(
+                category) for category in target_categories]
+
+        if self.uses_gradients:
+            self.model.zero_grad()
+            loss = sum([target(output)
+                       for target, output in zip(targets, outputs)])
+            loss.backward(retain_graph=True)
+
+        # In most of the saliency attribution papers, the saliency is
+        # computed with a single target layer.
+        # Commonly it is the last convolutional layer.
+        # Here we support passing a list with multiple target layers.
+        # It will compute the saliency image for every image,
+        # and then aggregate them (with a default mean aggregation).
+        # This gives you more flexibility in case you just want to
+        # use all conv layers for example, all Batchnorm layers,
+        # or something else.
+        cam_per_layer = self.compute_cam_per_layer(input_tensor,
+                                                   targets,
+                                                   eigen_smooth)
+        return self.aggregate_multi_layers(cam_per_layer)
+
+    def get_target_width_height(self,
+                                input_tensor: torch.Tensor) -> Tuple[int, int]:
+        width, height = input_tensor.size(-1), input_tensor.size(-2)
+        return width, height
+
+    def compute_cam_per_layer(
+            self,
+            input_tensor: torch.Tensor,
+            targets: List[torch.nn.Module],
+            eigen_smooth: bool) -> np.ndarray:
+        activations_list = [a.cpu().data.numpy()
+                            for a in self.activations_and_grads.activations]
+        grads_list = [g.cpu().data.numpy()
+                      for g in self.activations_and_grads.gradients]
+        target_size = self.get_target_width_height(input_tensor[0]["image"])
+        
+
+        cam_per_target_layer = []
+        # Loop over the saliency image from every layer
+        for i in range(len(self.target_layers)):
+            target_layer = self.target_layers[i]
+            layer_activations = None
+            layer_grads = None
+            if i < len(activations_list):
+                layer_activations = activations_list[i]
+            if i < len(grads_list):
+                layer_grads = grads_list[i]
+
+            cam = self.get_cam_image(input_tensor,
+                                     target_layer,
+                                     targets,
+                                     layer_activations,
+                                     layer_grads,
+                                     eigen_smooth)
+            cam = np.maximum(cam, 0)
+            scaled = scale_cam_image(cam, target_size)
+            cam_per_target_layer.append(scaled[:, None, :])
+
+        return cam_per_target_layer
+
+    def aggregate_multi_layers(
+            self,
+            cam_per_target_layer: np.ndarray) -> np.ndarray:
+        cam_per_target_layer = np.concatenate(cam_per_target_layer, axis=1)
+        cam_per_target_layer = np.maximum(cam_per_target_layer, 0)
+        result = np.mean(cam_per_target_layer, axis=1)
+        return scale_cam_image(result)
+
+    def forward_augmentation_smoothing(self,
+                                       input_tensor: torch.Tensor,
+                                       targets: List[torch.nn.Module],
+                                       eigen_smooth: bool = False) -> np.ndarray:
+        transforms = tta.Compose(
+            [
+                tta.HorizontalFlip(),
+                tta.Multiply(factors=[0.9, 1, 1.1]),
+            ]
+        )
+        cams = []
+        for transform in transforms:
+            augmented_tensor = transform.augment_image(input_tensor)
+            cam = self.forward(augmented_tensor,
+                               targets,
+                               eigen_smooth)
+
+            # The ttach library expects a tensor of size BxCxHxW
+            cam = cam[:, None, :, :]
+            cam = torch.from_numpy(cam)
+            cam = transform.deaugment_mask(cam)
+
+            # Back to numpy float32, HxW
+            cam = cam.numpy()
+            cam = cam[:, 0, :, :]
+            cams.append(cam)
+
+        cam = np.mean(np.float32(cams), axis=0)
+        return cam
+
+    def __call__(self,
+                 input_tensor: torch.Tensor,
+                 targets: List[torch.nn.Module] = None,
+                 aug_smooth: bool = False,
+                 eigen_smooth: bool = False) -> np.ndarray:
+
+        # Smooth the CAM result with test time augmentation
+        if aug_smooth is True:
+            return self.forward_augmentation_smoothing(
+                input_tensor, targets, eigen_smooth)
+
+        return self.forward(input_tensor,
+                            targets, eigen_smooth)
+
+    def __del__(self):
+        self.activations_and_grads.release()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self.activations_and_grads.release()
+        if isinstance(exc_value, IndexError):
+            # Handle IndexError here...
+            print(
+                f"An exception occurred in CAM with block: {exc_type}. Message: {exc_value}")
+            return True
+
+class EigenCAM(BaseCAM):
+    def __init__(self, model, target_layers, use_cuda=False,
+                 reshape_transform=None):
+        super(EigenCAM, self).__init__(model,
+                                       target_layers,
+                                       use_cuda,
+                                       reshape_transform,
+                                       uses_gradients=False)
+
+    def get_cam_image(self,
+                      input_tensor,
+                      target_layer,
+                      target_category,
+                      activations,
+                      grads,
+                      eigen_smooth):
+        return get_2d_projection(activations)

examples/0.jpg:Zone.Identifier

@@ -0,0 +1,4 @@
+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=https://www.file.io/
+HostUrl=https://file.io/oAt6o2nEO4Dh

examples/1.jpg:Zone.Identifier

@@ -0,0 +1,3 @@
+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=C:\Users\adm-hew\Downloads\AVP\03_Scenario_onlycamsForUGA.zip

examples/2.jpg:Zone.Identifier

@@ -0,0 +1,3 @@
+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=C:\Users\adm-hew\Downloads\AVP\03_Scenario_onlycamsForUGA.zip

examples/3.jpg:Zone.Identifier

@@ -0,0 +1,3 @@
+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=C:\Users\adm-hew\Downloads\AVP\03_Scenario_onlycamsForUGA.zip

examples/4.jpg:Zone.Identifier

@@ -0,0 +1,3 @@
+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=C:\Users\adm-hew\Downloads\AVP\03_Scenario_onlycamsForUGA.zip

runtime_monitors/Monitor.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from abstractions import *
+import pickle
+import numpy as np
+
+class Monitor(object):
+
+    def __init__(self, good_ref=None):
+        # self.abs_type = abs_type
+        self.good_ref = good_ref
+
+
+    def set_reference(self, good_ref):
+        self.good_ref = good_ref
+    
+    # def get_identity(self):
+    #     print("Monitor for network:" + self.netName + "class: " + str(self.classification) + "at layer " + str(self.location))
+
+
+    def make_verdicts(self, features):
+        if len(self.good_ref):
+            verdicts = ref_query(features, self.good_ref)
+        else:
+            raise RuntimeError("No reference exists!")
+        return verdicts
+
+def ref_query(features, reference):
+    query_results = [boxes_query(x, reference) for x in features]
+    return query_results
+
+
+# def query_infusion(in_good_ref, in_bad_ref):
+#     if len(in_good_ref) == len(in_bad_ref): #0: acceptance (true, false), 1: rejection (false, true or false), 2: uncertainty (true, true)
+#         verdicts = np.zeros(len(in_good_ref), dtype=int)
+#         for i in range(len(in_good_ref)):
+#             if not in_good_ref[i]:
+#                 verdicts[i] = 1
+#             elif in_bad_ref[i]:
+#                 verdicts[i] = 2
+#         return verdicts
+#     else:
+#         print("Error: IllegalArgument")       

runtime_monitors/__init__.py

@@ -0,0 +1,2 @@
+from .Monitor import *
+

vanilla.yaml

@@ -0,0 +1,28 @@
+_BASE_: "Base-RCNN-FPN.yaml"
+MODEL:
+  META_ARCHITECTURE: "GeneralizedRCNN"
+  WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl"
+#  WEIGHTS: "./data/VOC-Detection/faster-rcnn/faster_rcnn_R_50_FPN_all_logistic/random_seed_0/model_final.pth"
+
+#  PROPOSAL_GENERATOR:
+#    NAME: "RPNLogistic"
+  MASK_ON: False
+  RESNETS:
+    DEPTH: 50
+  ROI_HEADS:
+    NAME: "StandardROIHeads"
+    NUM_CLASSES: 10
+INPUT:
+  MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800)
+  MIN_SIZE_TEST: 800
+DATASETS:
+  TRAIN: ('bdd_custom_train',)
+  TEST: ('bdd_custom_val',)
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000  # 17.4 epochs
+  WARMUP_ITERS: 100
+DATALOADER:
+    NUM_WORKERS: 8  # Depends on the available memory