Duplicate from taesiri/DeticChatGPT

Co-authored-by: taesiri <taesiri@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,38 @@
+---
+title: Detic+ChatGPT
+emoji: 👀
+colorFrom: blue
+colorTo: red
+sdk: gradio
+app_file: app.py
+pinned: false
+duplicated_from: taesiri/DeticChatGPT
+---
+
+# Configuration
+
+`title`: _string_
+Display title for the Space
+
+`emoji`: _string_
+Space emoji (emoji-only character allowed)
+
+`colorFrom`: _string_
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`colorTo`: _string_
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`sdk`: _string_
+Can be either `gradio`, `streamlit`, or `static`
+
+`sdk_version` : _string_
+Only applicable for `streamlit` SDK.  
+See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
+
+`app_file`: _string_
+Path to your main application file (which contains either `gradio` or `streamlit` Python code, or `static` html code).
+Path is relative to the root of the repository.
+
+`pinned`: _boolean_
+Whether the Space stays on top of your list.

app.py

@@ -0,0 +1,172 @@
+import os
+from pyChatGPT import ChatGPT
+
+os.system("pip install -U gradio")
+
+import sys
+import gradio as gr
+
+os.system(
+    "pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html"
+)
+
+# clone and install Detic
+os.system(
+    "git clone https://github.com/facebookresearch/Detic.git --recurse-submodules"
+)
+os.chdir("Detic")
+
+# Install detectron2
+import torch
+
+# Some basic setup:
+# Setup detectron2 logger
+import detectron2
+from detectron2.utils.logger import setup_logger
+
+setup_logger()
+
+# import some common libraries
+import sys
+import numpy as np
+import os, json, cv2, random
+
+# import some common detectron2 utilities
+from detectron2 import model_zoo
+from detectron2.engine import DefaultPredictor
+from detectron2.config import get_cfg
+from detectron2.utils.visualizer import Visualizer
+from detectron2.data import MetadataCatalog, DatasetCatalog
+
+# Detic libraries
+sys.path.insert(0, "third_party/CenterNet2/projects/CenterNet2/")
+sys.path.insert(0, "third_party/CenterNet2/")
+from centernet.config import add_centernet_config
+from detic.config import add_detic_config
+from detic.modeling.utils import reset_cls_test
+
+from PIL import Image
+
+# Build the detector and download our pretrained weights
+cfg = get_cfg()
+add_centernet_config(cfg)
+add_detic_config(cfg)
+cfg.MODEL.DEVICE = "cpu"
+cfg.merge_from_file("configs/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml")
+cfg.MODEL.WEIGHTS = "https://dl.fbaipublicfiles.com/detic/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.pth"
+cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model
+cfg.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = "rand"
+cfg.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = (
+    True  # For better visualization purpose. Set to False for all classes.
+)
+predictor = DefaultPredictor(cfg)
+
+# Setup the model's vocabulary using build-in datasets
+
+BUILDIN_CLASSIFIER = {
+    "lvis": "datasets/metadata/lvis_v1_clip_a+cname.npy",
+    "objects365": "datasets/metadata/o365_clip_a+cnamefix.npy",
+    "openimages": "datasets/metadata/oid_clip_a+cname.npy",
+    "coco": "datasets/metadata/coco_clip_a+cname.npy",
+}
+
+BUILDIN_METADATA_PATH = {
+    "lvis": "lvis_v1_val",
+    "objects365": "objects365_v2_val",
+    "openimages": "oid_val_expanded",
+    "coco": "coco_2017_val",
+}
+
+session_token = os.environ.get("SessionToken")
+
+
+def get_response_from_chatbot(text):
+    try:
+        api = ChatGPT(session_token)
+        resp = api.send_message(text)
+        api.refresh_auth()
+        api.reset_conversation()
+        response = resp["message"]
+    except:
+        response = "Sorry, I'm busy. Try again later."
+    return response
+
+
+def inference(img, vocabulary):
+    metadata = MetadataCatalog.get(BUILDIN_METADATA_PATH[vocabulary])
+    classifier = BUILDIN_CLASSIFIER[vocabulary]
+    num_classes = len(metadata.thing_classes)
+    reset_cls_test(predictor.model, classifier, num_classes)
+
+    im = cv2.imread(img)
+
+    outputs = predictor(im)
+    v = Visualizer(im[:, :, ::-1], metadata)
+    out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
+
+    detected_objects = []
+    object_list_str = []
+
+    box_locations = outputs["instances"].pred_boxes
+    box_loc_screen = box_locations.tensor.cpu().numpy()
+
+    for i, box_coord in enumerate(box_loc_screen):
+        x0, y0, x1, y1 = box_coord
+        width = x1 - x0
+        height = y1 - y0
+        predicted_label = metadata.thing_classes[outputs["instances"].pred_classes[i]]
+        detected_objects.append(
+            {
+                "prediction": predicted_label,
+                "x": int(x0),
+                "y": int(y0),
+                "w": int(width),
+                "h": int(height),
+            }
+        )
+        object_list_str.append(
+            f"{predicted_label} - X:({int(x0)} Y: {int(y0)} Width {int(width)} Height: {int(height)})"
+        )
+
+    chat_gpt_response = get_response_from_chatbot(
+        f"You are an intelligent image captioner. I will hand you the objects and their position, and you should give me a detailed description for the photo. In this photo we have the following objects\n{object_list_str}"
+    )
+
+    return (
+        Image.fromarray(np.uint8(out.get_image())).convert("RGB"),
+        chat_gpt_response,
+    )
+
+
+# create a gradio block for image classification
+with gr.Blocks() as demo:
+    gr.Markdown("# Detic+ChatGPT")
+    gr.Markdown(
+        "Use Detic to detect objects in an image and then use ChatGPT to describe the image."
+    )
+
+    gr.HTML(
+        "<p>You can duplicating this space and use your own session token: <a style='display:inline-block' href='https://huggingface.co/spaces/yizhangliu/chatGPT?duplicate=true'><img src='https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14' alt='Duplicate Space'></a></p>"
+    )
+    gr.HTML(
+        "<p> Instruction on how to get session token can be seen in video <a style='display:inline-block' href='https://www.youtube.com/watch?v=TdNSj_qgdFk'><font style='color:blue;weight:bold;'>here</font></a>. Add your session token by going to settings and add under secrets. </p>"
+    )
+
+    with gr.Column():
+        with gr.Row():
+            inp = gr.Image(label="Input Image", type="filepath")
+            vocab = gr.Dropdown(
+                ["lvis", "objects365", "openimages", "coco"],
+                label="Vocabulary",
+                value="lvis",
+            )
+
+        btn_detic = gr.Button("Run Detic+ChatGPT")
+    with gr.Row():
+        outviz = gr.Image(label="Visualization", type="pil")
+        output_desc = gr.Textbox(label="ChatGPT Description", lines=5)
+        # outputjson = gr.JSON(label="Detected Objects")
+
+    btn_detic.click(fn=inference, inputs=[inp, vocab], outputs=[outviz, output_desc])
+
+demo.launch()

configs/Base-C2_L_R5021k_640b64_4x.yaml

@@ -0,0 +1,82 @@
+MODEL:
+  META_ARCHITECTURE: "CustomRCNN"
+  MASK_ON: True
+  PROPOSAL_GENERATOR:
+    NAME: "CenterNet"
+  WEIGHTS: "models/resnet50_miil_21k.pkl"
+  BACKBONE:
+    NAME: build_p67_timm_fpn_backbone
+  TIMM:
+    BASE_NAME: resnet50_in21k
+  FPN:
+    IN_FEATURES: ["layer3", "layer4", "layer5"]
+  PIXEL_MEAN: [123.675, 116.280, 103.530]
+  PIXEL_STD: [58.395, 57.12, 57.375]
+  ROI_HEADS:
+    NAME: DeticCascadeROIHeads
+    IN_FEATURES: ["p3", "p4", "p5"]
+    IOU_THRESHOLDS: [0.6]
+    NUM_CLASSES: 1203
+    SCORE_THRESH_TEST: 0.02
+    NMS_THRESH_TEST: 0.5
+  ROI_BOX_CASCADE_HEAD:
+    IOUS: [0.6, 0.7, 0.8]
+  ROI_BOX_HEAD:
+    NAME: "FastRCNNConvFCHead"
+    NUM_FC: 2
+    POOLER_RESOLUTION: 7
+    CLS_AGNOSTIC_BBOX_REG: True
+    MULT_PROPOSAL_SCORE: True
+
+    USE_SIGMOID_CE: True
+    USE_FED_LOSS: True
+  ROI_MASK_HEAD:
+    NAME: "MaskRCNNConvUpsampleHead"
+    NUM_CONV: 4
+    POOLER_RESOLUTION: 14
+    CLS_AGNOSTIC_MASK: True
+  CENTERNET:
+    NUM_CLASSES: 1203
+    REG_WEIGHT: 1.
+    NOT_NORM_REG: True
+    ONLY_PROPOSAL: True
+    WITH_AGN_HM: True
+    INFERENCE_TH: 0.0001
+    PRE_NMS_TOPK_TRAIN: 4000
+    POST_NMS_TOPK_TRAIN: 2000
+    PRE_NMS_TOPK_TEST: 1000
+    POST_NMS_TOPK_TEST: 256
+    NMS_TH_TRAIN: 0.9
+    NMS_TH_TEST: 0.9
+    POS_WEIGHT: 0.5
+    NEG_WEIGHT: 0.5
+    IGNORE_HIGH_FP: 0.85
+DATASETS:
+  TRAIN: ("lvis_v1_train",)
+  TEST: ("lvis_v1_val",)
+DATALOADER:
+  SAMPLER_TRAIN: "RepeatFactorTrainingSampler"
+  REPEAT_THRESHOLD: 0.001
+  NUM_WORKERS: 8
+TEST:
+  DETECTIONS_PER_IMAGE: 300
+SOLVER:
+  LR_SCHEDULER_NAME: "WarmupCosineLR"
+  CHECKPOINT_PERIOD: 1000000000
+  WARMUP_ITERS: 10000
+  WARMUP_FACTOR: 0.0001
+  USE_CUSTOM_SOLVER: True
+  OPTIMIZER: "ADAMW"
+  MAX_ITER: 90000
+  IMS_PER_BATCH: 64
+  BASE_LR: 0.0002
+  CLIP_GRADIENTS:
+    ENABLED: True
+INPUT:
+  FORMAT: RGB
+  CUSTOM_AUG: EfficientDetResizeCrop
+  TRAIN_SIZE: 640
+OUTPUT_DIR: "./output/Detic/auto"
+EVAL_PROPOSAL_AR: False
+VERSION: 2
+FP16: True

configs/Base-DeformDETR_L_R50_4x.yaml

@@ -0,0 +1,59 @@
+MODEL:
+  META_ARCHITECTURE: "DeformableDetr"
+  WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+  PIXEL_MEAN: [123.675, 116.280, 103.530]
+  PIXEL_STD: [58.395, 57.120, 57.375]
+  MASK_ON: False
+  RESNETS:
+    DEPTH: 50
+    STRIDE_IN_1X1: False
+    OUT_FEATURES: ["res3", "res4", "res5"]
+  DETR:
+    CLS_WEIGHT: 2.0
+    GIOU_WEIGHT: 2.0
+    L1_WEIGHT: 5.0
+    NUM_OBJECT_QUERIES: 300
+    DIM_FEEDFORWARD: 1024
+    WITH_BOX_REFINE: True
+    TWO_STAGE: True
+    NUM_CLASSES: 1203
+    USE_FED_LOSS: True
+DATASETS:
+  TRAIN: ("lvis_v1_train",)
+  TEST: ("lvis_v1_val",)
+SOLVER:
+  CHECKPOINT_PERIOD: 10000000
+  USE_CUSTOM_SOLVER: True
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0002
+  STEPS: (150000,)
+  MAX_ITER: 180000
+  WARMUP_FACTOR: 1.0
+  WARMUP_ITERS: 10
+  WEIGHT_DECAY: 0.0001
+  OPTIMIZER: "ADAMW"
+  BACKBONE_MULTIPLIER: 0.1
+  CLIP_GRADIENTS:
+    ENABLED: True
+    CLIP_TYPE: "full_model"
+    CLIP_VALUE: 0.01
+    NORM_TYPE: 2.0
+  CUSTOM_MULTIPLIER: 0.1
+  CUSTOM_MULTIPLIER_NAME: ['reference_points', 'sampling_offsets']
+INPUT:
+  FORMAT: "RGB"
+  MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800)
+  CROP:
+    ENABLED: True
+    TYPE: "absolute_range"
+    SIZE: (384, 600)
+  CUSTOM_AUG: "DETR"
+TEST:
+  DETECTIONS_PER_IMAGE: 300
+DATALOADER:
+  FILTER_EMPTY_ANNOTATIONS: False
+  NUM_WORKERS: 4
+  SAMPLER_TRAIN: "RepeatFactorTrainingSampler"
+  REPEAT_THRESHOLD: 0.001
+OUTPUT_DIR: "output/Detic/auto"
+VERSION: 2

configs/Base_OVCOCO_C4_1x.yaml

@@ -0,0 +1,31 @@
+MODEL:
+  META_ARCHITECTURE: "CustomRCNN"
+  RPN:
+    PRE_NMS_TOPK_TEST: 6000
+    POST_NMS_TOPK_TEST: 1000
+  ROI_HEADS:
+    NAME: "CustomRes5ROIHeads"
+  WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl"
+  RESNETS:
+    DEPTH: 50
+  ROI_BOX_HEAD:
+    CLS_AGNOSTIC_BBOX_REG: True
+    USE_SIGMOID_CE: True
+    USE_ZEROSHOT_CLS: True
+    ZEROSHOT_WEIGHT_PATH: 'datasets/metadata/coco_clip_a+cname.npy'
+    IGNORE_ZERO_CATS: True
+    CAT_FREQ_PATH: 'datasets/coco/zero-shot/instances_train2017_seen_2_oriorder_cat_info.json'
+DATASETS:
+  TRAIN: ("coco_zeroshot_train_oriorder",)
+  TEST: ("coco_generalized_zeroshot_val",)
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+  CHECKPOINT_PERIOD: 1000000000
+INPUT:
+  MIN_SIZE_TRAIN: (800,)
+VERSION: 2
+OUTPUT_DIR: output/Detic-COCO/auto
+FP16: True

configs/BoxSup-C2_LCOCO_CLIP_SwinB_896b32_4x.yaml

@@ -0,0 +1,19 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+  WEIGHTS: "models/swin_base_patch4_window7_224_22k.pkl"
+  BACKBONE:
+    NAME: build_swintransformer_fpn_backbone
+  SWIN:
+    SIZE: B-22k
+  FPN:
+    IN_FEATURES: ["swin1", "swin2", "swin3"]
+SOLVER:
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0001
+INPUT:
+  TRAIN_SIZE: 896
+DATASETS:
+  TRAIN: ("lvis_v1_train+coco",)

configs/BoxSup-C2_L_CLIP_R5021k_640b64_4x.yaml

@@ -0,0 +1,4 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True

configs/BoxSup-C2_L_CLIP_SwinB_896b32_4x.yaml

@@ -0,0 +1,17 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+  WEIGHTS: "models/swin_base_patch4_window7_224_22k.pkl"
+  BACKBONE:
+    NAME: build_swintransformer_fpn_backbone
+  SWIN:
+    SIZE: B-22k
+  FPN:
+    IN_FEATURES: ["swin1", "swin2", "swin3"]
+SOLVER:
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0001
+INPUT:
+  TRAIN_SIZE: 896

configs/BoxSup-C2_Lbase_CLIP_R5021k_640b64_4x.yaml

@@ -0,0 +1,6 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+DATASETS:
+  TRAIN: ("lvis_v1_train_norare",)

configs/BoxSup-C2_Lbase_CLIP_SwinB_896b32_4x.yaml

@@ -0,0 +1,19 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+  WEIGHTS: "models/swin_base_patch4_window7_224_22k.pkl"
+  BACKBONE:
+    NAME: build_swintransformer_fpn_backbone
+  SWIN:
+    SIZE: B-22k
+  FPN:
+    IN_FEATURES: ["swin1", "swin2", "swin3"]
+SOLVER:
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0001
+INPUT:
+  TRAIN_SIZE: 896
+DATASETS:
+  TRAIN: ("lvis_v1_train_norare",)

configs/BoxSup-DeformDETR_L_R50_2x.yaml

@@ -0,0 +1,3 @@
+_BASE_: "Base-DeformDETR_L_R50_4x.yaml"
+SOLVER:
+  IMS_PER_BATCH: 16

configs/BoxSup-DeformDETR_L_R50_4x.yaml

@@ -0,0 +1 @@
+_BASE_: "Base-DeformDETR_L_R50_4x.yaml"

configs/BoxSup_OVCOCO_CLIP_R50_1x.yaml

@@ -0,0 +1 @@
+_BASE_: "Base_OVCOCO_C4_1x.yaml"

configs/Detic_DeformDETR_LI_R50_4x_ft4x.yaml

@@ -0,0 +1,22 @@
+_BASE_: "Base-DeformDETR_L_R50_4x.yaml"
+MODEL:
+  WEIGHTS: "models/BoxSup-DeformDETR_L_R50_4x.pth"
+INPUT:
+  CUSTOM_AUG: ResizeShortestEdge
+  MIN_SIZE_TRAIN_SAMPLING: range
+  MIN_SIZE_TRAIN: [480, 800]
+DATASETS:
+  TRAIN: ("lvis_v1_train","imagenet_lvis_v1")
+  TEST: ("lvis_v1_val",)
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [4, 16]
+  USE_RFS: [True, False]
+  DATASET_MIN_SIZES: [[480, 800], [240, 400]]
+  DATASET_MAX_SIZES: [1333, 667]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+WITH_IMAGE_LABELS: True

configs/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml

@@ -0,0 +1,43 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  WEIGHTS: "models/BoxSup-C2_LCOCO_CLIP_SwinB_896b32_4x.pth"
+  DYNAMIC_CLASSIFIER: True
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+    ZEROSHOT_WEIGHT_PATH: 'datasets/metadata/lvis-21k_clip_a+cname.npy'
+    USE_FED_LOSS: False # Federated loss is enabled when DYNAMIC_CLASSIFIER is on
+  ROI_HEADS:
+    NUM_CLASSES: 22047
+  BACKBONE:
+    NAME: build_swintransformer_fpn_backbone
+  SWIN:
+    SIZE: B-22k
+  FPN:
+    IN_FEATURES: ["swin1", "swin2", "swin3"]
+  RESET_CLS_TESTS: True
+  TEST_CLASSIFIERS: ("datasets/metadata/oid_clip_a+cname.npy","datasets/metadata/o365_clip_a+cnamefix.npy")
+  TEST_NUM_CLASSES: [500, 365]
+SOLVER:
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0001
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train+coco","imagenet_lvis-22k")
+  TEST: ('oid_val_expanded', 'objects365_v2_val')
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 16]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [4, 16]
+  DATASET_INPUT_SIZE: [896, 448]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 4
+  USE_TAR_DATASET: True
+WITH_IMAGE_LABELS: True

configs/Detic_LI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml

@@ -0,0 +1,43 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  WEIGHTS: "models/BoxSup-C2_L_CLIP_SwinB_896b32_4x.pth"
+  DYNAMIC_CLASSIFIER: True
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+    ZEROSHOT_WEIGHT_PATH: 'datasets/metadata/lvis-21k_clip_a+cname.npy'
+    USE_FED_LOSS: False # Federated loss is enabled when DYNAMIC_CLASSIFIER is on
+  ROI_HEADS:
+    NUM_CLASSES: 22047
+  BACKBONE:
+    NAME: build_swintransformer_fpn_backbone
+  SWIN:
+    SIZE: B-22k
+  FPN:
+    IN_FEATURES: ["swin1", "swin2", "swin3"]
+  RESET_CLS_TESTS: True
+  TEST_CLASSIFIERS: ("datasets/metadata/oid_clip_a+cname.npy","datasets/metadata/o365_clip_a+cnamefix.npy")
+  TEST_NUM_CLASSES: [500, 365]
+SOLVER:
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0001
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train","imagenet_lvis-22k")
+  TEST: ('oid_val_expanded', 'objects365_v2_val')
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 16]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [4, 16]
+  DATASET_INPUT_SIZE: [896, 448]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 4
+  USE_TAR_DATASET: True
+WITH_IMAGE_LABELS: True

configs/Detic_LI_CLIP_R5021k_640b64_4x_ft4x_max-size.yaml

@@ -0,0 +1,27 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+  WEIGHTS: "models/BoxSup-C2_L_CLIP_R5021k_640b64_4x.pth"
+SOLVER:
+  MAX_ITER: 90000
+  IMS_PER_BATCH: 64
+  BASE_LR: 0.0002
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train","imagenet_lvis_v1")
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [8, 32]
+  DATASET_INPUT_SIZE: [640, 320]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_LI_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml

@@ -0,0 +1,33 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+  BACKBONE:
+    NAME: build_swintransformer_fpn_backbone
+  SWIN:
+    SIZE: B-22k
+  FPN:
+    IN_FEATURES: ["swin1", "swin2", "swin3"]
+  WEIGHTS: "models/BoxSup-C2_L_CLIP_SwinB_896b32_4x.pth"
+SOLVER:
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0001
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train","imagenet_lvis_v1")
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [4, 16]
+  DATASET_INPUT_SIZE: [896, 448]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_LbaseCCcapimg_CLIP_R5021k_640b64_4x_ft4x_max-size.yaml

@@ -0,0 +1,30 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  WITH_CAPTION: True
+  SYNC_CAPTION_BATCH: True
+  ROI_BOX_HEAD:
+    ADD_IMAGE_BOX: True # caption loss is added to the image-box
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+  WEIGHTS: "models/BoxSup-C2_Lbase_CLIP_R5021k_640b64_4x.pth"
+SOLVER:
+  MAX_ITER: 90000
+  IMS_PER_BATCH: 64
+  BASE_LR: 0.0002
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train_norare","cc3m_v1_train_tags")
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [8, 32]
+  DATASET_INPUT_SIZE: [640, 320]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'captiontag']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_LbaseCCimg_CLIP_R5021k_640b64_4x_ft4x_max-size.yaml

@@ -0,0 +1,27 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+  WEIGHTS: "models/BoxSup-C2_Lbase_CLIP_R5021k_640b64_4x.pth"
+SOLVER:
+  MAX_ITER: 90000
+  IMS_PER_BATCH: 64
+  BASE_LR: 0.0002
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train_norare","cc3m_v1_train_tags")
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [8, 32]
+  DATASET_INPUT_SIZE: [640, 320]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_LbaseI_CLIP_R5021k_640b64_4x_ft4x_max-size.yaml

@@ -0,0 +1,27 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+  WEIGHTS: "models/BoxSup-C2_Lbase_CLIP_R5021k_640b64_4x.pth"
+SOLVER:
+  MAX_ITER: 90000
+  IMS_PER_BATCH: 64
+  BASE_LR: 0.0002
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train_norare","imagenet_lvis_v1")
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [8, 32]
+  DATASET_INPUT_SIZE: [640, 320]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_LbaseI_CLIP_R5021k_640b64_4x_ft4x_predicted.yaml

@@ -0,0 +1,27 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_score'
+  WEIGHTS: "models/BoxSup-C2_Lbase_CLIP_R5021k_640b64_4x.pth"
+SOLVER:
+  MAX_ITER: 90000
+  IMS_PER_BATCH: 64
+  BASE_LR: 0.0002
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train_norare","imagenet_lvis_v1")
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [8, 32]
+  DATASET_INPUT_SIZE: [640, 320]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_LbaseI_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml

@@ -0,0 +1,33 @@
+_BASE_: "Base-C2_L_R5021k_640b64_4x.yaml"
+MODEL:
+  ROI_BOX_HEAD:
+    USE_ZEROSHOT_CLS: True
+    IMAGE_LABEL_LOSS: 'max_size'
+  BACKBONE:
+    NAME: build_swintransformer_fpn_backbone
+  SWIN:
+    SIZE: B-22k
+  FPN:
+    IN_FEATURES: ["swin1", "swin2", "swin3"]
+  WEIGHTS: "models/BoxSup-C2_Lbase_CLIP_SwinB_896b32_4x.pth"
+SOLVER:
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 32
+  BASE_LR: 0.0001
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+DATASETS:
+  TRAIN: ("lvis_v1_train_norare","imagenet_lvis_v1")
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [4, 16]
+  DATASET_INPUT_SIZE: [896, 448]
+  USE_RFS: [True, False]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.5, 1.5]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_OVCOCO_CLIP_R50_1x_caption.yaml

@@ -0,0 +1,33 @@
+_BASE_: "Base_OVCOCO_C4_1x.yaml"
+MODEL:
+  WEIGHTS: "models/BoxSup_OVCOCO_CLIP_R50_1x.pth"
+  WITH_CAPTION: True
+  SYNC_CAPTION_BATCH: True
+  ROI_BOX_HEAD:
+    WS_NUM_PROPS: 1
+    ADD_IMAGE_BOX: True
+    NEG_CAP_WEIGHT: 1.0
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+DATASETS:
+  TRAIN: ("coco_zeroshot_train_oriorder", "coco_caption_train_tags")
+INPUT:
+  CUSTOM_AUG: ResizeShortestEdge
+  MIN_SIZE_TRAIN_SAMPLING: range
+  MIN_SIZE_TRAIN: (800, 800)
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [2, 8]
+  USE_RFS: [False, False]
+  DATASET_MIN_SIZES: [[800, 800], [400, 400]]
+  DATASET_MAX_SIZES: [1333, 667]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'caption']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_OVCOCO_CLIP_R50_1x_max-size.yaml

@@ -0,0 +1,30 @@
+_BASE_: "Base_OVCOCO_C4_1x.yaml"
+MODEL:
+  WEIGHTS: "models/BoxSup_OVCOCO_CLIP_R50_1x.pth"
+  ROI_BOX_HEAD:
+    WS_NUM_PROPS: 32
+    IMAGE_LABEL_LOSS: 'max_size'
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+DATASETS:
+  TRAIN: ("coco_zeroshot_train_oriorder", "coco_caption_train_tags")
+INPUT:
+  CUSTOM_AUG: ResizeShortestEdge
+  MIN_SIZE_TRAIN_SAMPLING: range
+  MIN_SIZE_TRAIN: (800, 800)
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [2, 8]
+  USE_RFS: [False, False]
+  DATASET_MIN_SIZES: [[800, 800], [400, 400]]
+  DATASET_MAX_SIZES: [1333, 667]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'image']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

configs/Detic_OVCOCO_CLIP_R50_1x_max-size_caption.yaml

@@ -0,0 +1,35 @@
+_BASE_: "Base_OVCOCO_C4_1x.yaml"
+MODEL:
+  WEIGHTS: "models/BoxSup_OVCOCO_CLIP_R50_1x.pth"
+  WITH_CAPTION: True
+  SYNC_CAPTION_BATCH: True
+  ROI_BOX_HEAD:
+    WS_NUM_PROPS: 32
+    ADD_IMAGE_BOX: True # caption loss is added to the image-box
+    IMAGE_LABEL_LOSS: 'max_size'
+
+    NEG_CAP_WEIGHT: 1.0
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+DATASETS:
+  TRAIN: ("coco_zeroshot_train_oriorder", "coco_caption_train_tags")
+INPUT:
+  CUSTOM_AUG: ResizeShortestEdge
+  MIN_SIZE_TRAIN_SAMPLING: range
+  MIN_SIZE_TRAIN: (800, 800)
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1, 4]
+  USE_DIFF_BS_SIZE: True
+  DATASET_BS: [2, 8]
+  USE_RFS: [False, False]
+  DATASET_MIN_SIZES: [[800, 800], [400, 400]]
+  DATASET_MAX_SIZES: [1333, 667]
+  FILTER_EMPTY_ANNOTATIONS: False
+  MULTI_DATASET_GROUPING: True
+  DATASET_ANN: ['box', 'captiontag']
+  NUM_WORKERS: 8
+WITH_IMAGE_LABELS: True

datasets/README.md

@@ -0,0 +1,207 @@
+# Prepare datasets for Detic
+
+The basic training of our model uses [LVIS](https://www.lvisdataset.org/) (which uses [COCO](https://cocodataset.org/) images) and [ImageNet-21K](https://www.image-net.org/download.php). 
+Some models are trained on [Conceptual Caption (CC3M)](https://ai.google.com/research/ConceptualCaptions/).
+Optionally, we use [Objects365](https://www.objects365.org/) and [OpenImages (Challenge 2019 version)](https://storage.googleapis.com/openimages/web/challenge2019.html) for cross-dataset evaluation. 
+Before starting processing, please download the (selected) datasets from the official websites and place or sim-link them under `$Detic_ROOT/datasets/`. 
+
+```
+$Detic_ROOT/datasets/
+    metadata/
+    lvis/
+    coco/
+    imagenet/
+    cc3m/
+    objects365/
+    oid/
+```
+`metadata/` is our preprocessed meta-data (included in the repo). See the below [section](#Metadata) for details.
+Please follow the following instruction to pre-process individual datasets.
+
+### COCO and LVIS
+
+First, download COCO and LVIS data place them in the following way:
+
+```
+lvis/
+    lvis_v1_train.json
+    lvis_v1_val.json
+coco/
+    train2017/
+    val2017/
+    annotations/
+        captions_train2017.json
+        instances_train2017.json 
+        instances_val2017.json
+```
+
+Next, prepare the open-vocabulary LVIS training set using 
+
+```
+python tools/remove_lvis_rare.py --ann datasets/lvis/lvis_v1_train.json
+```
+
+This will generate `datasets/lvis/lvis_v1_train_norare.json`.
+
+### ImageNet-21K
+
+The ImageNet-21K folder should look like:
+```
+imagenet/
+    ImageNet-21K/
+        n01593028.tar
+        n01593282.tar
+        ...
+```
+
+We first unzip the overlapping classes of LVIS (we will directly work with the .tar file for the rest classes) and convert them into LVIS annotation format.
+
+~~~
+mkdir imagenet/annotations
+python tools/unzip_imagenet_lvis.py --dst_path datasets/imagenet/ImageNet-LVIS
+python tools/create_imagenetlvis_json.py --imagenet_path datasets/imagenet/ImageNet-LVIS --out_path datasets/imagenet/annotations/imagenet_lvis_image_info.json
+~~~
+This creates `datasets/imagenet/annotations/imagenet_lvis_image_info.json`.
+
+[Optional] To train with all the 21K classes, run
+
+~~~
+python tools/get_imagenet_21k_full_tar_json.py
+python tools/create_lvis_21k.py
+~~~
+This creates `datasets/imagenet/annotations/imagenet-21k_image_info_lvis-21k.json` and `datasets/lvis/lvis_v1_train_lvis-21k.json` (combined LVIS and ImageNet-21K classes in `categories`).
+
+[Optional] To train on combined LVIS and COCO, run
+
+~~~
+python tools/merge_lvis_coco.py
+~~~
+This creates `datasets/lvis/lvis_v1_train+coco_mask.json`
+
+### Conceptual Caption
+
+
+Download the dataset from [this](https://ai.google.com/research/ConceptualCaptions/download) page and place them as:
+```
+cc3m/
+    GCC-training.tsv
+```
+
+Run the following command to download the images and convert the annotations to LVIS format (Note: download images takes long).
+
+~~~
+python tools/download_cc.py --ann datasets/cc3m/GCC-training.tsv --save_image_path datasets/cc3m/training/ --out_path datasets/cc3m/train_image_info.json
+python tools/get_cc_tags.py
+~~~
+
+This creates `datasets/cc3m/train_image_info_tags.json`.
+
+### Objects365
+Download Objects365 (v2) from the website. We only need the validation set in this project:
+```
+objects365/
+    annotations/
+        zhiyuan_objv2_val.json
+    val/
+        images/
+            v1/
+                patch0/
+                ...
+                patch15/
+            v2/
+                patch16/
+                ...
+                patch49/
+
+```
+
+The original annotation has typos in the class names, we first fix them for our following use of language embeddings.
+
+```
+python tools/fix_o365_names.py --ann datasets/objects365/annotations/zhiyuan_objv2_val.json
+```
+This creates `datasets/objects365/zhiyuan_objv2_val_fixname.json`.
+
+To train on Objects365, download the training images and use the command above.  We note some images in the training annotation do not exist.
+We use the following command to filter the missing images.
+~~~
+python tools/fix_0365_path.py
+~~~
+This creates `datasets/objects365/zhiyuan_objv2_train_fixname_fixmiss.json`.
+
+### OpenImages
+
+We followed the instructions in [UniDet](https://github.com/xingyizhou/UniDet/blob/master/projects/UniDet/unidet_docs/DATASETS.md#openimages) to convert the metadata for OpenImages.
+
+The converted folder should look like
+
+```
+oid/
+    annotations/
+        oid_challenge_2019_train_bbox.json
+        oid_challenge_2019_val_expanded.json
+    images/
+        0/
+        1/
+        2/
+        ...
+```
+
+### Open-vocabulary COCO
+
+We first follow [OVR-CNN](https://github.com/alirezazareian/ovr-cnn/blob/master/ipynb/003.ipynb) to create the open-vocabulary COCO split. The converted files should be like 
+
+```
+coco/
+    zero-shot/
+        instances_train2017_seen_2.json
+        instances_val2017_all_2.json
+```
+
+We further pre-process the annotation format for easier evaluation:
+
+```
+python tools/get_coco_zeroshot_oriorder.py --data_path datasets/coco/zero-shot/instances_train2017_seen_2.json
+python tools/get_coco_zeroshot_oriorder.py --data_path datasets/coco/zero-shot/instances_val2017_all_2.json
+```
+
+Next, we preprocess the COCO caption data:
+
+```
+python tools/get_cc_tags.py --cc_ann datasets/coco/annotations/captions_train2017.json --out_path datasets/coco/captions_train2017_tags_allcaps.json --allcaps --convert_caption
+```
+This creates `datasets/coco/captions_train2017_tags_allcaps.json`.
+
+### Metadata
+
+```
+metadata/
+    lvis_v1_train_cat_info.json
+    coco_clip_a+cname.npy
+    lvis_v1_clip_a+cname.npy
+    o365_clip_a+cnamefix.npy
+    oid_clip_a+cname.npy
+    imagenet_lvis_wnid.txt
+    Objects365_names_fix.csv
+```
+
+`lvis_v1_train_cat_info.json` is used by the Federated loss.
+This is created by 
+~~~
+python tools/get_lvis_cat_info.py --ann datasets/lvis/lvis_v1_train.json
+~~~
+
+`*_clip_a+cname.npy` is the pre-computed CLIP embeddings for each datasets.
+They are created by (taking LVIS as an example)
+~~~
+python tools/dump_clip_features.py --ann datasets/lvis/lvis_v1_val.json --out_path metadata/lvis_v1_clip_a+cname.npy
+~~~
+Note we do not include the 21K class embeddings due to the large file size.
+To create it, run
+~~~
+python tools/dump_clip_features.py --ann datasets/lvis/lvis_v1_val_lvis-21k.json --out_path datasets/metadata/lvis-21k_clip_a+cname.npy
+~~~
+
+`imagenet_lvis_wnid.txt` is the list of matched classes between ImageNet-21K and LVIS.
+
+`Objects365_names_fix.csv` is our manual fix of the Objects365 names.

datasets/metadata/Objects365_names_fix.csv

@@ -0,0 +1,365 @@
+1,Person,Person
+2,Sneakers,Sneakers
+3,Chair,Chair
+4,Other Shoes,Other Shoes
+5,Hat,Hat
+6,Car,Car
+7,Lamp,Lamp
+8,Glasses,Glasses
+9,Bottle,Bottle
+10,Desk,Desk
+11,Cup,Cup
+12,Street Lights,Street Lights
+13,Cabinet/shelf,Cabinet/shelf
+14,Handbag/Satchel,Handbag/Satchel
+15,Bracelet,Bracelet
+16,Plate,Plate
+17,Picture/Frame,Picture/Frame
+18,Helmet,Helmet
+19,Book,Book
+20,Gloves,Gloves
+21,Storage box,Storage box
+22,Boat,Boat
+23,Leather Shoes,Leather Shoes
+24,Flower,Flower
+25,Bench,Bench
+26,Potted Plant,Potted Plant
+27,Bowl/Basin,Bowl/Basin
+28,Flag,Flag
+29,Pillow,Pillow
+30,Boots,Boots
+31,Vase,Vase
+32,Microphone,Microphone
+33,Necklace,Necklace
+34,Ring,Ring
+35,SUV,SUV
+36,Wine Glass,Wine Glass
+37,Belt,Belt
+38,Moniter/TV,Monitor/TV
+39,Backpack,Backpack
+40,Umbrella,Umbrella
+41,Traffic Light,Traffic Light
+42,Speaker,Speaker
+43,Watch,Watch
+44,Tie,Tie
+45,Trash bin Can,Trash bin Can
+46,Slippers,Slippers
+47,Bicycle,Bicycle
+48,Stool,Stool
+49,Barrel/bucket,Barrel/bucket
+50,Van,Van
+51,Couch,Couch
+52,Sandals,Sandals
+53,Bakset,Basket
+54,Drum,Drum
+55,Pen/Pencil,Pen/Pencil
+56,Bus,Bus
+57,Wild Bird,Wild Bird
+58,High Heels,High Heels
+59,Motorcycle,Motorcycle
+60,Guitar,Guitar
+61,Carpet,Carpet
+62,Cell Phone,Cell Phone
+63,Bread,Bread
+64,Camera,Camera
+65,Canned,Canned
+66,Truck,Truck
+67,Traffic cone,Traffic cone
+68,Cymbal,Cymbal
+69,Lifesaver,Lifesaver
+70,Towel,Towel
+71,Stuffed Toy,Stuffed Toy
+72,Candle,Candle
+73,Sailboat,Sailboat
+74,Laptop,Laptop
+75,Awning,Awning
+76,Bed,Bed
+77,Faucet,Faucet
+78,Tent,Tent
+79,Horse,Horse
+80,Mirror,Mirror
+81,Power outlet,Power outlet
+82,Sink,Sink
+83,Apple,Apple
+84,Air Conditioner,Air Conditioner
+85,Knife,Knife
+86,Hockey Stick,Hockey Stick
+87,Paddle,Paddle
+88,Pickup Truck,Pickup Truck
+89,Fork,Fork
+90,Traffic Sign,Traffic Sign
+91,Ballon,Ballon
+92,Tripod,Tripod
+93,Dog,Dog
+94,Spoon,Spoon
+95,Clock,Clock
+96,Pot,Pot
+97,Cow,Cow
+98,Cake,Cake
+99,Dinning Table,Dining Table
+100,Sheep,Sheep
+101,Hanger,Hanger
+102,Blackboard/Whiteboard,Blackboard/Whiteboard
+103,Napkin,Napkin
+104,Other Fish,Other Fish
+105,Orange/Tangerine,Orange/Tangerine
+106,Toiletry,Toiletry
+107,Keyboard,Keyboard
+108,Tomato,Tomato
+109,Lantern,Lantern
+110,Machinery Vehicle,Machinery Vehicle
+111,Fan,Fan
+112,Green Vegetables,Green Vegetables
+113,Banana,Banana
+114,Baseball Glove,Baseball Glove
+115,Airplane,Airplane
+116,Mouse,Mouse
+117,Train,Train
+118,Pumpkin,Pumpkin
+119,Soccer,Soccer
+120,Skiboard,Skiboard
+121,Luggage,Luggage
+122,Nightstand,Nightstand
+123,Tea pot,Teapot
+124,Telephone,Telephone
+125,Trolley,Trolley
+126,Head Phone,Head Phone
+127,Sports Car,Sports Car
+128,Stop Sign,Stop Sign
+129,Dessert,Dessert
+130,Scooter,Scooter
+131,Stroller,Stroller
+132,Crane,Crane
+133,Remote,Remote
+134,Refrigerator,Refrigerator
+135,Oven,Oven
+136,Lemon,Lemon
+137,Duck,Duck
+138,Baseball Bat,Baseball Bat
+139,Surveillance Camera,Surveillance Camera
+140,Cat,Cat
+141,Jug,Jug
+142,Broccoli,Broccoli
+143,Piano,Piano
+144,Pizza,Pizza
+145,Elephant,Elephant
+146,Skateboard,Skateboard
+147,Surfboard,Surfboard
+148,Gun,Gun
+149,Skating and Skiing shoes,Skating and Skiing shoes
+150,Gas stove,Gas stove
+151,Donut,Donut
+152,Bow Tie,Bow Tie
+153,Carrot,Carrot
+154,Toilet,Toilet
+155,Kite,Kite
+156,Strawberry,Strawberry
+157,Other Balls,Other Balls
+158,Shovel,Shovel
+159,Pepper,Pepper
+160,Computer Box,Computer Box
+161,Toilet Paper,Toilet Paper
+162,Cleaning Products,Cleaning Products
+163,Chopsticks,Chopsticks
+164,Microwave,Microwave
+165,Pigeon,Pigeon
+166,Baseball,Baseball
+167,Cutting/chopping Board,Cutting/chopping Board
+168,Coffee Table,Coffee Table
+169,Side Table,Side Table
+170,Scissors,Scissors
+171,Marker,Marker
+172,Pie,Pie
+173,Ladder,Ladder
+174,Snowboard,Snowboard
+175,Cookies,Cookies
+176,Radiator,Radiator
+177,Fire Hydrant,Fire Hydrant
+178,Basketball,Basketball
+179,Zebra,Zebra
+180,Grape,Grape
+181,Giraffe,Giraffe
+182,Potato,Potato
+183,Sausage,Sausage
+184,Tricycle,Tricycle
+185,Violin,Violin
+186,Egg,Egg
+187,Fire Extinguisher,Fire Extinguisher
+188,Candy,Candy
+189,Fire Truck,Fire Truck
+190,Billards,Billards
+191,Converter,Converter
+192,Bathtub,Bathtub
+193,Wheelchair,Wheelchair
+194,Golf Club,Golf Club
+195,Briefcase,Briefcase
+196,Cucumber,Cucumber
+197,Cigar/Cigarette,Cigar/Cigarette
+198,Paint Brush,Paint Brush
+199,Pear,Pear
+200,Heavy Truck,Heavy Truck
+201,Hamburger,Hamburger
+202,Extractor,Extractor
+203,Extention Cord,Extension Cord
+204,Tong,Tong
+205,Tennis Racket,Tennis Racket
+206,Folder,Folder
+207,American Football,American Football
+208,earphone,earphone
+209,Mask,Mask
+210,Kettle,Kettle
+211,Tennis,Tennis
+212,Ship,Ship
+213,Swing,Swing
+214,Coffee Machine,Coffee Machine
+215,Slide,Slide
+216,Carriage,Carriage
+217,Onion,Onion
+218,Green beans,Green beans
+219,Projector,Projector
+220,Frisbee,Frisbee
+221,Washing Machine/Drying Machine,Washing Machine/Drying Machine
+222,Chicken,Chicken
+223,Printer,Printer
+224,Watermelon,Watermelon
+225,Saxophone,Saxophone
+226,Tissue,Tissue
+227,Toothbrush,Toothbrush
+228,Ice cream,Ice cream
+229,Hotair ballon,Hot air balloon
+230,Cello,Cello
+231,French Fries,French Fries
+232,Scale,Scale
+233,Trophy,Trophy
+234,Cabbage,Cabbage
+235,Hot dog,Hot dog
+236,Blender,Blender
+237,Peach,Peach
+238,Rice,Rice
+239,Wallet/Purse,Wallet/Purse
+240,Volleyball,Volleyball
+241,Deer,Deer
+242,Goose,Goose
+243,Tape,Tape
+244,Tablet,Tablet
+245,Cosmetics,Cosmetics
+246,Trumpet,Trumpet
+247,Pineapple,Pineapple
+248,Golf Ball,Golf Ball
+249,Ambulance,Ambulance
+250,Parking meter,Parking meter
+251,Mango,Mango
+252,Key,Key
+253,Hurdle,Hurdle
+254,Fishing Rod,Fishing Rod
+255,Medal,Medal
+256,Flute,Flute
+257,Brush,Brush
+258,Penguin,Penguin
+259,Megaphone,Megaphone
+260,Corn,Corn
+261,Lettuce,Lettuce
+262,Garlic,Garlic
+263,Swan,Swan
+264,Helicopter,Helicopter
+265,Green Onion,Green Onion
+266,Sandwich,Sandwich
+267,Nuts,Nuts
+268,Speed Limit Sign,Speed Limit Sign
+269,Induction Cooker,Induction Cooker
+270,Broom,Broom
+271,Trombone,Trombone
+272,Plum,Plum
+273,Rickshaw,Rickshaw
+274,Goldfish,Goldfish
+275,Kiwi fruit,Kiwi fruit
+276,Router/modem,Router/modem
+277,Poker Card,Poker Card
+278,Toaster,Toaster
+279,Shrimp,Shrimp
+280,Sushi,Sushi
+281,Cheese,Cheese
+282,Notepaper,Notepaper
+283,Cherry,Cherry
+284,Pliers,Pliers
+285,CD,CD
+286,Pasta,Pasta
+287,Hammer,Hammer
+288,Cue,Cue
+289,Avocado,Avocado
+290,Hamimelon,Hami melon
+291,Flask,Flask
+292,Mushroon,Mushroom
+293,Screwdriver,Screwdriver
+294,Soap,Soap
+295,Recorder,Recorder
+296,Bear,Bear
+297,Eggplant,Eggplant
+298,Board Eraser,Board Eraser
+299,Coconut,Coconut
+300,Tape Measur/ Ruler,Tape Measure/ Ruler
+301,Pig,Pig
+302,Showerhead,Showerhead
+303,Globe,Globe
+304,Chips,Chips
+305,Steak,Steak
+306,Crosswalk Sign,Crosswalk Sign
+307,Stapler,Stapler
+308,Campel,Camel
+309,Formula 1,Formula 1
+310,Pomegranate,Pomegranate
+311,Dishwasher,Dishwasher
+312,Crab,Crab
+313,Hoverboard,Hoverboard
+314,Meat ball,Meatball
+315,Rice Cooker,Rice Cooker
+316,Tuba,Tuba
+317,Calculator,Calculator
+318,Papaya,Papaya
+319,Antelope,Antelope
+320,Parrot,Parrot
+321,Seal,Seal
+322,Buttefly,Butterfly
+323,Dumbbell,Dumbbell
+324,Donkey,Donkey
+325,Lion,Lion
+326,Urinal,Urinal
+327,Dolphin,Dolphin
+328,Electric Drill,Electric Drill
+329,Hair Dryer,Hair Dryer
+330,Egg tart,Egg tart
+331,Jellyfish,Jellyfish
+332,Treadmill,Treadmill
+333,Lighter,Lighter
+334,Grapefruit,Grapefruit
+335,Game board,Game board
+336,Mop,Mop
+337,Radish,Radish
+338,Baozi,Baozi
+339,Target,Target
+340,French,French
+341,Spring Rolls,Spring Rolls
+342,Monkey,Monkey
+343,Rabbit,Rabbit
+344,Pencil Case,Pencil Case
+345,Yak,Yak
+346,Red Cabbage,Red Cabbage
+347,Binoculars,Binoculars
+348,Asparagus,Asparagus
+349,Barbell,Barbell
+350,Scallop,Scallop
+351,Noddles,Noddles
+352,Comb,Comb
+353,Dumpling,Dumpling
+354,Oyster,Oyster
+355,Table Teniis paddle,Table Tennis paddle
+356,Cosmetics Brush/Eyeliner Pencil,Cosmetics Brush/Eyeliner Pencil
+357,Chainsaw,Chainsaw
+358,Eraser,Eraser
+359,Lobster,Lobster
+360,Durian,Durian
+361,Okra,Okra
+362,Lipstick,Lipstick
+363,Cosmetics Mirror,Cosmetics Mirror
+364,Curling,Curling
+365,Table Tennis,Table Tennis

demo.py

@@ -0,0 +1,206 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import argparse
+import glob
+import multiprocessing as mp
+import numpy as np
+import os
+import tempfile
+import time
+import warnings
+import cv2
+import tqdm
+import sys
+
+from detectron2.config import get_cfg
+from detectron2.data.detection_utils import read_image
+from detectron2.utils.logger import setup_logger
+
+sys.path.insert(0, 'third_party/CenterNet2/projects/CenterNet2/')
+sys.path.insert(0, 'third_party/CenterNet2/')
+
+from centernet.config import add_centernet_config
+from detic.config import add_detic_config
+
+from detic.predictor import VisualizationDemo
+
+
+# constants
+WINDOW_NAME = "Detic"
+
+def setup_cfg(args):
+    cfg = get_cfg()
+    add_centernet_config(cfg)
+    add_detic_config(cfg)
+    cfg.merge_from_file(args.config_file)
+    cfg.merge_from_list(args.opts)
+    # Set score_threshold for builtin models
+    cfg.MODEL.RETINANET.SCORE_THRESH_TEST = args.confidence_threshold
+    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = args.confidence_threshold
+    cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = args.confidence_threshold
+    cfg.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = 'rand' # load later
+    if not args.pred_all_class:
+        cfg.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = True
+    cfg.freeze()
+    return cfg
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(description="Detectron2 demo for builtin configs")
+    parser.add_argument(
+        "--config-file",
+        default="configs/quick_schedules/mask_rcnn_R_50_FPN_inference_acc_test.yaml",
+        metavar="FILE",
+        help="path to config file",
+    )
+    parser.add_argument("--webcam", action="store_true", help="Take inputs from webcam.")
+    parser.add_argument("--video-input", help="Path to video file.")
+    parser.add_argument(
+        "--input",
+        nargs="+",
+        help="A list of space separated input images; "
+        "or a single glob pattern such as 'directory/*.jpg'",
+    )
+    parser.add_argument(
+        "--output",
+        help="A file or directory to save output visualizations. "
+        "If not given, will show output in an OpenCV window.",
+    )
+    parser.add_argument(
+        "--vocabulary",
+        default="lvis",
+        choices=['lvis', 'openimages', 'objects365', 'coco', 'custom'],
+        help="",
+    )
+    parser.add_argument(
+        "--custom_vocabulary",
+        default="",
+        help="",
+    )
+    parser.add_argument("--pred_all_class", action='store_true')
+    parser.add_argument(
+        "--confidence-threshold",
+        type=float,
+        default=0.5,
+        help="Minimum score for instance predictions to be shown",
+    )
+    parser.add_argument(
+        "--opts",
+        help="Modify config options using the command-line 'KEY VALUE' pairs",
+        default=[],
+        nargs=argparse.REMAINDER,
+    )
+    return parser
+
+
+def test_opencv_video_format(codec, file_ext):
+    with tempfile.TemporaryDirectory(prefix="video_format_test") as dir:
+        filename = os.path.join(dir, "test_file" + file_ext)
+        writer = cv2.VideoWriter(
+            filename=filename,
+            fourcc=cv2.VideoWriter_fourcc(*codec),
+            fps=float(30),
+            frameSize=(10, 10),
+            isColor=True,
+        )
+        [writer.write(np.zeros((10, 10, 3), np.uint8)) for _ in range(30)]
+        writer.release()
+        if os.path.isfile(filename):
+            return True
+        return False
+
+
+if __name__ == "__main__":
+    mp.set_start_method("spawn", force=True)
+    args = get_parser().parse_args()
+    setup_logger(name="fvcore")
+    logger = setup_logger()
+    logger.info("Arguments: " + str(args))
+
+    cfg = setup_cfg(args)
+
+    demo = VisualizationDemo(cfg, args)
+
+    if args.input:
+        if len(args.input) == 1:
+            args.input = glob.glob(os.path.expanduser(args.input[0]))
+            assert args.input, "The input path(s) was not found"
+        for path in tqdm.tqdm(args.input, disable=not args.output):
+            img = read_image(path, format="BGR")
+            start_time = time.time()
+            predictions, visualized_output = demo.run_on_image(img)
+            logger.info(
+                "{}: {} in {:.2f}s".format(
+                    path,
+                    "detected {} instances".format(len(predictions["instances"]))
+                    if "instances" in predictions
+                    else "finished",
+                    time.time() - start_time,
+                )
+            )
+
+            if args.output:
+                if os.path.isdir(args.output):
+                    assert os.path.isdir(args.output), args.output
+                    out_filename = os.path.join(args.output, os.path.basename(path))
+                else:
+                    assert len(args.input) == 1, "Please specify a directory with args.output"
+                    out_filename = args.output
+                visualized_output.save(out_filename)
+            else:
+                cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
+                cv2.imshow(WINDOW_NAME, visualized_output.get_image()[:, :, ::-1])
+                if cv2.waitKey(0) == 27:
+                    break  # esc to quit
+    elif args.webcam:
+        assert args.input is None, "Cannot have both --input and --webcam!"
+        assert args.output is None, "output not yet supported with --webcam!"
+        cam = cv2.VideoCapture(0)
+        for vis in tqdm.tqdm(demo.run_on_video(cam)):
+            cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
+            cv2.imshow(WINDOW_NAME, vis)
+            if cv2.waitKey(1) == 27:
+                break  # esc to quit
+        cam.release()
+        cv2.destroyAllWindows()
+    elif args.video_input:
+        video = cv2.VideoCapture(args.video_input)
+        width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        frames_per_second = video.get(cv2.CAP_PROP_FPS)
+        num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+        basename = os.path.basename(args.video_input)
+        codec, file_ext = (
+            ("x264", ".mkv") if test_opencv_video_format("x264", ".mkv") else ("mp4v", ".mp4")
+        )
+        if codec == ".mp4v":
+            warnings.warn("x264 codec not available, switching to mp4v")
+        if args.output:
+            if os.path.isdir(args.output):
+                output_fname = os.path.join(args.output, basename)
+                output_fname = os.path.splitext(output_fname)[0] + file_ext
+            else:
+                output_fname = args.output
+            assert not os.path.isfile(output_fname), output_fname
+            output_file = cv2.VideoWriter(
+                filename=output_fname,
+                # some installation of opencv may not support x264 (due to its license),
+                # you can try other format (e.g. MPEG)
+                fourcc=cv2.VideoWriter_fourcc(*codec),
+                fps=float(frames_per_second),
+                frameSize=(width, height),
+                isColor=True,
+            )
+        assert os.path.isfile(args.video_input)
+        for vis_frame in tqdm.tqdm(demo.run_on_video(video), total=num_frames):
+            if args.output:
+                output_file.write(vis_frame)
+            else:
+                cv2.namedWindow(basename, cv2.WINDOW_NORMAL)
+                cv2.imshow(basename, vis_frame)
+                if cv2.waitKey(1) == 27:
+                    break  # esc to quit
+        video.release()
+        if args.output:
+            output_file.release()
+        else:
+            cv2.destroyAllWindows()

detic/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+from .modeling.meta_arch import custom_rcnn
+from .modeling.roi_heads import detic_roi_heads
+from .modeling.roi_heads import res5_roi_heads
+from .modeling.backbone import swintransformer
+from .modeling.backbone import timm
+
+
+from .data.datasets import lvis_v1
+from .data.datasets import imagenet
+from .data.datasets import cc
+from .data.datasets import objects365
+from .data.datasets import oid
+from .data.datasets import coco_zeroshot
+
+try:
+    from .modeling.meta_arch import d2_deformable_detr
+except:
+    pass

detic/config.py

@@ -0,0 +1,131 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+from detectron2.config import CfgNode as CN
+
+def add_detic_config(cfg):
+    _C = cfg
+
+    _C.WITH_IMAGE_LABELS = False # Turn on co-training with classification data
+
+    # Open-vocabulary classifier
+    _C.MODEL.ROI_BOX_HEAD.USE_ZEROSHOT_CLS = False # Use fixed classifier for open-vocabulary detection
+    _C.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = 'datasets/metadata/lvis_v1_clip_a+cname.npy'
+    _C.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_DIM = 512
+    _C.MODEL.ROI_BOX_HEAD.NORM_WEIGHT = True
+    _C.MODEL.ROI_BOX_HEAD.NORM_TEMP = 50.0
+    _C.MODEL.ROI_BOX_HEAD.IGNORE_ZERO_CATS = False
+    _C.MODEL.ROI_BOX_HEAD.USE_BIAS = 0.0 # >= 0: not use
+    
+    _C.MODEL.ROI_BOX_HEAD.MULT_PROPOSAL_SCORE = False # CenterNet2
+    _C.MODEL.ROI_BOX_HEAD.USE_SIGMOID_CE = False
+    _C.MODEL.ROI_BOX_HEAD.PRIOR_PROB = 0.01
+    _C.MODEL.ROI_BOX_HEAD.USE_FED_LOSS = False # Federated Loss
+    _C.MODEL.ROI_BOX_HEAD.CAT_FREQ_PATH = \
+        'datasets/metadata/lvis_v1_train_cat_info.json'
+    _C.MODEL.ROI_BOX_HEAD.FED_LOSS_NUM_CAT = 50
+    _C.MODEL.ROI_BOX_HEAD.FED_LOSS_FREQ_WEIGHT = 0.5
+
+    # Classification data configs
+    _C.MODEL.ROI_BOX_HEAD.IMAGE_LABEL_LOSS = 'max_size' # max, softmax, sum
+    _C.MODEL.ROI_BOX_HEAD.IMAGE_LOSS_WEIGHT = 0.1
+    _C.MODEL.ROI_BOX_HEAD.IMAGE_BOX_SIZE = 1.0
+    _C.MODEL.ROI_BOX_HEAD.ADD_IMAGE_BOX = False # Used for image-box loss and caption loss
+    _C.MODEL.ROI_BOX_HEAD.WS_NUM_PROPS = 128 # num proposals for image-labeled data
+    _C.MODEL.ROI_BOX_HEAD.WITH_SOFTMAX_PROP = False # Used for WSDDN
+    _C.MODEL.ROI_BOX_HEAD.CAPTION_WEIGHT = 1.0 # Caption loss weight
+    _C.MODEL.ROI_BOX_HEAD.NEG_CAP_WEIGHT = 0.125 # Caption loss hyper-parameter
+    _C.MODEL.ROI_BOX_HEAD.ADD_FEATURE_TO_PROP = False # Used for WSDDN
+    _C.MODEL.ROI_BOX_HEAD.SOFTMAX_WEAK_LOSS = False # Used when USE_SIGMOID_CE is False
+
+    _C.MODEL.ROI_HEADS.MASK_WEIGHT = 1.0
+    _C.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = False # For demo only
+
+    # Caption losses
+    _C.MODEL.CAP_BATCH_RATIO = 4 # Ratio between detection data and caption data
+    _C.MODEL.WITH_CAPTION = False
+    _C.MODEL.SYNC_CAPTION_BATCH = False # synchronize across GPUs to enlarge # "classes"
+
+    # dynamic class sampling when training with 21K classes
+    _C.MODEL.DYNAMIC_CLASSIFIER = False
+    _C.MODEL.NUM_SAMPLE_CATS = 50
+
+    # Different classifiers in testing, used in cross-dataset evaluation
+    _C.MODEL.RESET_CLS_TESTS = False
+    _C.MODEL.TEST_CLASSIFIERS = []
+    _C.MODEL.TEST_NUM_CLASSES = []
+
+    # Backbones
+    _C.MODEL.SWIN = CN()
+    _C.MODEL.SWIN.SIZE = 'T' # 'T', 'S', 'B'
+    _C.MODEL.SWIN.USE_CHECKPOINT = False
+    _C.MODEL.SWIN.OUT_FEATURES = (1, 2, 3) # FPN stride 8 - 32
+
+    _C.MODEL.TIMM = CN()
+    _C.MODEL.TIMM.BASE_NAME = 'resnet50'
+    _C.MODEL.TIMM.OUT_LEVELS = (3, 4, 5)
+    _C.MODEL.TIMM.NORM = 'FrozenBN'
+    _C.MODEL.TIMM.FREEZE_AT = 0
+    _C.MODEL.DATASET_LOSS_WEIGHT = []
+    
+    # Multi-dataset dataloader
+    _C.DATALOADER.DATASET_RATIO = [1, 1] # sample ratio
+    _C.DATALOADER.USE_RFS = [False, False]
+    _C.DATALOADER.MULTI_DATASET_GROUPING = False # Always true when multi-dataset is enabled
+    _C.DATALOADER.DATASET_ANN = ['box', 'box'] # Annotation type of each dataset
+    _C.DATALOADER.USE_DIFF_BS_SIZE = False # Use different batchsize for each dataset
+    _C.DATALOADER.DATASET_BS = [8, 32] # Used when USE_DIFF_BS_SIZE is on
+    _C.DATALOADER.DATASET_INPUT_SIZE = [896, 384] # Used when USE_DIFF_BS_SIZE is on
+    _C.DATALOADER.DATASET_INPUT_SCALE = [(0.1, 2.0), (0.5, 1.5)] # Used when USE_DIFF_BS_SIZE is on 
+    _C.DATALOADER.DATASET_MIN_SIZES = [(640, 800), (320, 400)] # Used when USE_DIFF_BS_SIZE is on
+    _C.DATALOADER.DATASET_MAX_SIZES = [1333, 667] # Used when USE_DIFF_BS_SIZE is on
+    _C.DATALOADER.USE_TAR_DATASET = False # for ImageNet-21K, directly reading from unziped files
+    _C.DATALOADER.TARFILE_PATH = 'datasets/imagenet/metadata-22k/tar_files.npy'
+    _C.DATALOADER.TAR_INDEX_DIR = 'datasets/imagenet/metadata-22k/tarindex_npy'
+    
+    _C.SOLVER.USE_CUSTOM_SOLVER = False
+    _C.SOLVER.OPTIMIZER = 'SGD'
+    _C.SOLVER.BACKBONE_MULTIPLIER = 1.0 # Used in DETR
+    _C.SOLVER.CUSTOM_MULTIPLIER = 1.0 # Used in DETR
+    _C.SOLVER.CUSTOM_MULTIPLIER_NAME = [] # Used in DETR
+
+    # Deformable DETR
+    _C.MODEL.DETR = CN()
+    _C.MODEL.DETR.NUM_CLASSES = 80
+    _C.MODEL.DETR.FROZEN_WEIGHTS = '' # For Segmentation
+    _C.MODEL.DETR.GIOU_WEIGHT = 2.0
+    _C.MODEL.DETR.L1_WEIGHT = 5.0
+    _C.MODEL.DETR.DEEP_SUPERVISION = True
+    _C.MODEL.DETR.NO_OBJECT_WEIGHT = 0.1
+    _C.MODEL.DETR.CLS_WEIGHT = 2.0
+    _C.MODEL.DETR.NUM_FEATURE_LEVELS = 4
+    _C.MODEL.DETR.TWO_STAGE = False
+    _C.MODEL.DETR.WITH_BOX_REFINE = False
+    _C.MODEL.DETR.FOCAL_ALPHA = 0.25
+    _C.MODEL.DETR.NHEADS = 8
+    _C.MODEL.DETR.DROPOUT = 0.1
+    _C.MODEL.DETR.DIM_FEEDFORWARD = 2048
+    _C.MODEL.DETR.ENC_LAYERS = 6
+    _C.MODEL.DETR.DEC_LAYERS = 6
+    _C.MODEL.DETR.PRE_NORM = False
+    _C.MODEL.DETR.HIDDEN_DIM = 256
+    _C.MODEL.DETR.NUM_OBJECT_QUERIES = 100
+
+    _C.MODEL.DETR.USE_FED_LOSS = False
+    _C.MODEL.DETR.WEAK_WEIGHT = 0.1
+
+    _C.INPUT.CUSTOM_AUG = ''
+    _C.INPUT.TRAIN_SIZE = 640
+    _C.INPUT.TEST_SIZE = 640
+    _C.INPUT.SCALE_RANGE = (0.1, 2.)
+    # 'default' for fixed short/ long edge, 'square' for max size=INPUT.SIZE
+    _C.INPUT.TEST_INPUT_TYPE = 'default' 
+
+    _C.FIND_UNUSED_PARAM = True
+    _C.EVAL_PRED_AR = False
+    _C.EVAL_PROPOSAL_AR = False
+    _C.EVAL_CAT_SPEC_AR = False
+    _C.IS_DEBUG = False
+    _C.QUICK_DEBUG = False
+    _C.FP16 = False
+    _C.EVAL_AP_FIX = False
+    _C.GEN_PSEDO_LABELS = False
+    _C.SAVE_DEBUG_PATH = 'output/save_debug/'

detic/custom_solver.py

@@ -0,0 +1,78 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from enum import Enum
+import itertools
+from typing import Any, Callable, Dict, Iterable, List, Set, Type, Union
+import torch
+
+from detectron2.config import CfgNode
+
+from detectron2.solver.build import maybe_add_gradient_clipping
+
+def match_name_keywords(n, name_keywords):
+    out = False
+    for b in name_keywords:
+        if b in n:
+            out = True
+            break
+    return out
+
+def build_custom_optimizer(cfg: CfgNode, model: torch.nn.Module) -> torch.optim.Optimizer:
+    """
+    Build an optimizer from config.
+    """
+    params: List[Dict[str, Any]] = []
+    memo: Set[torch.nn.parameter.Parameter] = set()
+    custom_multiplier_name = cfg.SOLVER.CUSTOM_MULTIPLIER_NAME
+    optimizer_type = cfg.SOLVER.OPTIMIZER
+    for key, value in model.named_parameters(recurse=True):
+        if not value.requires_grad:
+            continue
+        # Avoid duplicating parameters
+        if value in memo:
+            continue
+        memo.add(value)
+        lr = cfg.SOLVER.BASE_LR
+        weight_decay = cfg.SOLVER.WEIGHT_DECAY
+        if "backbone" in key:
+            lr = lr * cfg.SOLVER.BACKBONE_MULTIPLIER
+        if match_name_keywords(key, custom_multiplier_name):
+            lr = lr * cfg.SOLVER.CUSTOM_MULTIPLIER
+            print('Costum LR', key, lr)
+        param = {"params": [value], "lr": lr}
+        if optimizer_type != 'ADAMW':
+            param['weight_decay'] = weight_decay
+        params += [param]
+
+    def maybe_add_full_model_gradient_clipping(optim):  # optim: the optimizer class
+        # detectron2 doesn't have full model gradient clipping now
+        clip_norm_val = cfg.SOLVER.CLIP_GRADIENTS.CLIP_VALUE
+        enable = (
+            cfg.SOLVER.CLIP_GRADIENTS.ENABLED
+            and cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model"
+            and clip_norm_val > 0.0
+        )
+
+        class FullModelGradientClippingOptimizer(optim):
+            def step(self, closure=None):
+                all_params = itertools.chain(*[x["params"] for x in self.param_groups])
+                torch.nn.utils.clip_grad_norm_(all_params, clip_norm_val)
+                super().step(closure=closure)
+
+        return FullModelGradientClippingOptimizer if enable else optim
+
+    
+    if optimizer_type == 'SGD':
+        optimizer = maybe_add_full_model_gradient_clipping(torch.optim.SGD)(
+            params, cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM, 
+            nesterov=cfg.SOLVER.NESTEROV
+        )
+    elif optimizer_type == 'ADAMW':
+        optimizer = maybe_add_full_model_gradient_clipping(torch.optim.AdamW)(
+            params, cfg.SOLVER.BASE_LR, 
+            weight_decay=cfg.SOLVER.WEIGHT_DECAY
+        )
+    else:
+        raise NotImplementedError(f"no optimizer type {optimizer_type}")
+    if not cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model":
+        optimizer = maybe_add_gradient_clipping(cfg, optimizer)
+    return optimizer

detic/data/custom_build_augmentation.py

@@ -0,0 +1,51 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import logging
+import numpy as np
+import pycocotools.mask as mask_util
+import torch
+from fvcore.common.file_io import PathManager
+from PIL import Image
+
+
+from detectron2.data import transforms as T
+from .transforms.custom_augmentation_impl import EfficientDetResizeCrop
+
+def build_custom_augmentation(cfg, is_train, scale=None, size=None, \
+    min_size=None, max_size=None):
+    """
+    Create a list of default :class:`Augmentation` from config.
+    Now it includes resizing and flipping.
+
+    Returns:
+        list[Augmentation]
+    """
+    if cfg.INPUT.CUSTOM_AUG == 'ResizeShortestEdge':
+        if is_train:
+            min_size = cfg.INPUT.MIN_SIZE_TRAIN if min_size is None else min_size
+            max_size = cfg.INPUT.MAX_SIZE_TRAIN if max_size is None else max_size
+            sample_style = cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING
+        else:
+            min_size = cfg.INPUT.MIN_SIZE_TEST
+            max_size = cfg.INPUT.MAX_SIZE_TEST
+            sample_style = "choice"
+        augmentation = [T.ResizeShortestEdge(min_size, max_size, sample_style)]
+    elif cfg.INPUT.CUSTOM_AUG == 'EfficientDetResizeCrop':
+        if is_train:
+            scale = cfg.INPUT.SCALE_RANGE if scale is None else scale
+            size = cfg.INPUT.TRAIN_SIZE if size is None else size
+        else:
+            scale = (1, 1)
+            size = cfg.INPUT.TEST_SIZE
+        augmentation = [EfficientDetResizeCrop(size, scale)]
+    else:
+        assert 0, cfg.INPUT.CUSTOM_AUG
+
+    if is_train:
+        augmentation.append(T.RandomFlip())
+    return augmentation
+
+
+build_custom_transform_gen = build_custom_augmentation
+"""
+Alias for backward-compatibility.
+"""

detic/data/custom_dataset_dataloader.py

@@ -0,0 +1,331 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# Part of the code is from https://github.com/xingyizhou/UniDet/blob/master/projects/UniDet/unidet/data/multi_dataset_dataloader.py (Apache-2.0 License)
+import copy
+import logging
+import numpy as np
+import operator
+import torch
+import torch.utils.data
+import json
+from detectron2.utils.comm import get_world_size
+from detectron2.utils.logger import _log_api_usage, log_first_n
+
+from detectron2.config import configurable
+from detectron2.data import samplers
+from torch.utils.data.sampler import BatchSampler, Sampler
+from detectron2.data.common import DatasetFromList, MapDataset
+from detectron2.data.dataset_mapper import DatasetMapper
+from detectron2.data.build import get_detection_dataset_dicts, build_batch_data_loader
+from detectron2.data.samplers import TrainingSampler, RepeatFactorTrainingSampler
+from detectron2.data.build import worker_init_reset_seed, print_instances_class_histogram
+from detectron2.data.build import filter_images_with_only_crowd_annotations
+from detectron2.data.build import filter_images_with_few_keypoints
+from detectron2.data.build import check_metadata_consistency
+from detectron2.data.catalog import MetadataCatalog, DatasetCatalog
+from detectron2.utils import comm
+import itertools
+import math
+from collections import defaultdict
+from typing import Optional
+
+
+def _custom_train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None):
+    sampler_name = cfg.DATALOADER.SAMPLER_TRAIN
+    if 'MultiDataset' in sampler_name:
+        dataset_dicts = get_detection_dataset_dicts_with_source(
+            cfg.DATASETS.TRAIN,
+            filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,
+            min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE
+            if cfg.MODEL.KEYPOINT_ON else 0,
+            proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None,
+        )
+    else:
+        dataset_dicts = get_detection_dataset_dicts(
+            cfg.DATASETS.TRAIN,
+            filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,
+            min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE
+            if cfg.MODEL.KEYPOINT_ON else 0,
+            proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None,
+        )
+
+    if mapper is None:
+        mapper = DatasetMapper(cfg, True)
+
+    if sampler is not None:
+        pass
+    elif sampler_name == "TrainingSampler":
+        sampler = TrainingSampler(len(dataset))
+    elif sampler_name == "MultiDatasetSampler":
+        sampler = MultiDatasetSampler(
+            dataset_dicts,
+            dataset_ratio = cfg.DATALOADER.DATASET_RATIO,
+            use_rfs = cfg.DATALOADER.USE_RFS,
+            dataset_ann = cfg.DATALOADER.DATASET_ANN,
+            repeat_threshold = cfg.DATALOADER.REPEAT_THRESHOLD,
+        )
+    elif sampler_name == "RepeatFactorTrainingSampler":
+        repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency(
+            dataset_dicts, cfg.DATALOADER.REPEAT_THRESHOLD
+        )
+        sampler = RepeatFactorTrainingSampler(repeat_factors)
+    else:
+        raise ValueError("Unknown training sampler: {}".format(sampler_name))
+
+    return {
+        "dataset": dataset_dicts,
+        "sampler": sampler,
+        "mapper": mapper,
+        "total_batch_size": cfg.SOLVER.IMS_PER_BATCH,
+        "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING,
+        "num_workers": cfg.DATALOADER.NUM_WORKERS,
+        'multi_dataset_grouping': cfg.DATALOADER.MULTI_DATASET_GROUPING,
+        'use_diff_bs_size': cfg.DATALOADER.USE_DIFF_BS_SIZE,
+        'dataset_bs': cfg.DATALOADER.DATASET_BS,
+        'num_datasets': len(cfg.DATASETS.TRAIN)
+    }
+
+
+@configurable(from_config=_custom_train_loader_from_config)
+def build_custom_train_loader(
+        dataset, *, mapper, sampler, 
+        total_batch_size=16,
+        aspect_ratio_grouping=True, 
+        num_workers=0,
+        num_datasets=1,
+        multi_dataset_grouping=False,
+        use_diff_bs_size=False,
+        dataset_bs=[]
+    ):
+    """
+    Modified from detectron2.data.build.build_custom_train_loader, but supports
+    different samplers
+    """
+    if isinstance(dataset, list):
+        dataset = DatasetFromList(dataset, copy=False)
+    if mapper is not None:
+        dataset = MapDataset(dataset, mapper)
+    if sampler is None:
+        sampler = TrainingSampler(len(dataset))
+    assert isinstance(sampler, torch.utils.data.sampler.Sampler)
+    if multi_dataset_grouping:
+        return build_multi_dataset_batch_data_loader(
+            use_diff_bs_size,
+            dataset_bs,
+            dataset,
+            sampler,
+            total_batch_size,
+            num_datasets=num_datasets,
+            num_workers=num_workers,
+        )
+    else:
+        return build_batch_data_loader(
+            dataset,
+            sampler,
+            total_batch_size,
+            aspect_ratio_grouping=aspect_ratio_grouping,
+            num_workers=num_workers,
+        )
+
+
+def build_multi_dataset_batch_data_loader(
+    use_diff_bs_size, dataset_bs,
+    dataset, sampler, total_batch_size, num_datasets, num_workers=0
+):
+    """
+    """
+    world_size = get_world_size()
+    assert (
+        total_batch_size > 0 and total_batch_size % world_size == 0
+    ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format(
+        total_batch_size, world_size
+    )
+
+    batch_size = total_batch_size // world_size
+    data_loader = torch.utils.data.DataLoader(
+        dataset,
+        sampler=sampler,
+        num_workers=num_workers,
+        batch_sampler=None,
+        collate_fn=operator.itemgetter(0),  # don't batch, but yield individual elements
+        worker_init_fn=worker_init_reset_seed,
+    )  # yield individual mapped dict
+    if use_diff_bs_size:
+        return DIFFMDAspectRatioGroupedDataset(
+            data_loader, dataset_bs, num_datasets)
+    else:
+        return MDAspectRatioGroupedDataset(
+            data_loader, batch_size, num_datasets)
+
+
+def get_detection_dataset_dicts_with_source(
+    dataset_names, filter_empty=True, min_keypoints=0, proposal_files=None
+):
+    assert len(dataset_names)
+    dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in dataset_names]
+    for dataset_name, dicts in zip(dataset_names, dataset_dicts):
+        assert len(dicts), "Dataset '{}' is empty!".format(dataset_name)
+    
+    for source_id, (dataset_name, dicts) in \
+        enumerate(zip(dataset_names, dataset_dicts)):
+        assert len(dicts), "Dataset '{}' is empty!".format(dataset_name)
+        for d in dicts:
+            d['dataset_source'] = source_id
+
+        if "annotations" in dicts[0]:
+            try:
+                class_names = MetadataCatalog.get(dataset_name).thing_classes
+                check_metadata_consistency("thing_classes", dataset_name)
+                print_instances_class_histogram(dicts, class_names)
+            except AttributeError:  # class names are not available for this dataset
+                pass
+
+    assert proposal_files is None
+
+    dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))
+
+    has_instances = "annotations" in dataset_dicts[0]
+    if filter_empty and has_instances:
+        dataset_dicts = filter_images_with_only_crowd_annotations(dataset_dicts)
+    if min_keypoints > 0 and has_instances:
+        dataset_dicts = filter_images_with_few_keypoints(dataset_dicts, min_keypoints)
+
+    return dataset_dicts
+
+
+class MultiDatasetSampler(Sampler):
+    def __init__(
+        self, 
+        dataset_dicts, 
+        dataset_ratio,
+        use_rfs,
+        dataset_ann,
+        repeat_threshold=0.001,
+        seed: Optional[int] = None,
+        ):
+        """
+        """
+        sizes = [0 for _ in range(len(dataset_ratio))]
+        for d in dataset_dicts:
+            sizes[d['dataset_source']] += 1
+        print('dataset sizes', sizes)
+        self.sizes = sizes
+        assert len(dataset_ratio) == len(sizes), \
+            'length of dataset ratio {} should be equal to number if dataset {}'.format(
+                len(dataset_ratio), len(sizes)
+            )
+        if seed is None:
+            seed = comm.shared_random_seed()
+        self._seed = int(seed)
+        self._rank = comm.get_rank()
+        self._world_size = comm.get_world_size()
+        
+        self.dataset_ids =  torch.tensor(
+            [d['dataset_source'] for d in dataset_dicts], dtype=torch.long)
+
+        dataset_weight = [torch.ones(s) * max(sizes) / s * r / sum(dataset_ratio) \
+            for i, (r, s) in enumerate(zip(dataset_ratio, sizes))]
+        dataset_weight = torch.cat(dataset_weight)
+        
+        rfs_factors = []
+        st = 0
+        for i, s in enumerate(sizes):
+            if use_rfs[i]:
+                if dataset_ann[i] == 'box':
+                    rfs_func = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency
+                else:
+                    rfs_func = repeat_factors_from_tag_frequency
+                rfs_factor = rfs_func(
+                    dataset_dicts[st: st + s],
+                    repeat_thresh=repeat_threshold)
+                rfs_factor = rfs_factor * (s / rfs_factor.sum())
+            else:
+                rfs_factor = torch.ones(s)
+            rfs_factors.append(rfs_factor)
+            st = st + s
+        rfs_factors = torch.cat(rfs_factors)
+
+        self.weights = dataset_weight * rfs_factors
+        self.sample_epoch_size = len(self.weights)
+
+    def __iter__(self):
+        start = self._rank
+        yield from itertools.islice(
+            self._infinite_indices(), start, None, self._world_size)
+
+
+    def _infinite_indices(self):
+        g = torch.Generator()
+        g.manual_seed(self._seed)
+        while True:
+            ids = torch.multinomial(
+                self.weights, self.sample_epoch_size, generator=g, 
+                replacement=True)
+            nums = [(self.dataset_ids[ids] == i).sum().int().item() \
+                for i in range(len(self.sizes))]
+            yield from ids
+
+
+class MDAspectRatioGroupedDataset(torch.utils.data.IterableDataset):
+    def __init__(self, dataset, batch_size, num_datasets):
+        """
+        """
+        self.dataset = dataset
+        self.batch_size = batch_size
+        self._buckets = [[] for _ in range(2 * num_datasets)]
+
+    def __iter__(self):
+        for d in self.dataset:
+            w, h = d["width"], d["height"]
+            aspect_ratio_bucket_id = 0 if w > h else 1
+            bucket_id = d['dataset_source'] * 2 + aspect_ratio_bucket_id
+            bucket = self._buckets[bucket_id]
+            bucket.append(d)
+            if len(bucket) == self.batch_size:
+                yield bucket[:]
+                del bucket[:]
+
+
+class DIFFMDAspectRatioGroupedDataset(torch.utils.data.IterableDataset):
+    def __init__(self, dataset, batch_sizes, num_datasets):
+        """
+        """
+        self.dataset = dataset
+        self.batch_sizes = batch_sizes
+        self._buckets = [[] for _ in range(2 * num_datasets)]
+
+    def __iter__(self):
+        for d in self.dataset:
+            w, h = d["width"], d["height"]
+            aspect_ratio_bucket_id = 0 if w > h else 1
+            bucket_id = d['dataset_source'] * 2 + aspect_ratio_bucket_id
+            bucket = self._buckets[bucket_id]
+            bucket.append(d)
+            if len(bucket) == self.batch_sizes[d['dataset_source']]:
+                yield bucket[:]
+                del bucket[:]
+
+
+def repeat_factors_from_tag_frequency(dataset_dicts, repeat_thresh):
+    """
+    """
+    category_freq = defaultdict(int)
+    for dataset_dict in dataset_dicts:
+        cat_ids = dataset_dict['pos_category_ids']
+        for cat_id in cat_ids:
+            category_freq[cat_id] += 1
+    num_images = len(dataset_dicts)
+    for k, v in category_freq.items():
+        category_freq[k] = v / num_images
+
+    category_rep = {
+        cat_id: max(1.0, math.sqrt(repeat_thresh / cat_freq))
+        for cat_id, cat_freq in category_freq.items()
+    }
+
+    rep_factors = []
+    for dataset_dict in dataset_dicts:
+        cat_ids = dataset_dict['pos_category_ids']
+        rep_factor = max({category_rep[cat_id] for cat_id in cat_ids}, default=1.0)
+        rep_factors.append(rep_factor)
+
+    return torch.tensor(rep_factors, dtype=torch.float32)

detic/data/custom_dataset_mapper.py

@@ -0,0 +1,280 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import copy
+import logging
+import numpy as np
+from typing import List, Optional, Union
+import torch
+import pycocotools.mask as mask_util
+
+from detectron2.config import configurable
+
+from detectron2.data import detection_utils as utils
+from detectron2.data.detection_utils import transform_keypoint_annotations
+from detectron2.data import transforms as T
+from detectron2.data.dataset_mapper import DatasetMapper
+from detectron2.structures import Boxes, BoxMode, Instances
+from detectron2.structures import Keypoints, PolygonMasks, BitMasks
+from fvcore.transforms.transform import TransformList
+from .custom_build_augmentation import build_custom_augmentation
+from .tar_dataset import DiskTarDataset
+
+__all__ = ["CustomDatasetMapper"]
+
+class CustomDatasetMapper(DatasetMapper):
+    @configurable
+    def __init__(self, is_train: bool, 
+        with_ann_type=False,
+        dataset_ann=[],
+        use_diff_bs_size=False,
+        dataset_augs=[],
+        is_debug=False,
+        use_tar_dataset=False,
+        tarfile_path='',
+        tar_index_dir='',
+        **kwargs):
+        """
+        add image labels
+        """
+        self.with_ann_type = with_ann_type
+        self.dataset_ann = dataset_ann
+        self.use_diff_bs_size = use_diff_bs_size
+        if self.use_diff_bs_size and is_train:
+            self.dataset_augs = [T.AugmentationList(x) for x in dataset_augs]
+        self.is_debug = is_debug
+        self.use_tar_dataset = use_tar_dataset
+        if self.use_tar_dataset:
+            print('Using tar dataset')
+            self.tar_dataset = DiskTarDataset(tarfile_path, tar_index_dir)
+        super().__init__(is_train, **kwargs)
+ 
+
+    @classmethod
+    def from_config(cls, cfg, is_train: bool = True):
+        ret = super().from_config(cfg, is_train)
+        ret.update({
+            'with_ann_type': cfg.WITH_IMAGE_LABELS,
+            'dataset_ann': cfg.DATALOADER.DATASET_ANN,
+            'use_diff_bs_size': cfg.DATALOADER.USE_DIFF_BS_SIZE,
+            'is_debug': cfg.IS_DEBUG,
+            'use_tar_dataset': cfg.DATALOADER.USE_TAR_DATASET,
+            'tarfile_path': cfg.DATALOADER.TARFILE_PATH,
+            'tar_index_dir': cfg.DATALOADER.TAR_INDEX_DIR,
+        })
+        if ret['use_diff_bs_size'] and is_train:
+            if cfg.INPUT.CUSTOM_AUG == 'EfficientDetResizeCrop':
+                dataset_scales = cfg.DATALOADER.DATASET_INPUT_SCALE
+                dataset_sizes = cfg.DATALOADER.DATASET_INPUT_SIZE
+                ret['dataset_augs'] = [
+                    build_custom_augmentation(cfg, True, scale, size) \
+                        for scale, size in zip(dataset_scales, dataset_sizes)]
+            else:
+                assert cfg.INPUT.CUSTOM_AUG == 'ResizeShortestEdge'
+                min_sizes = cfg.DATALOADER.DATASET_MIN_SIZES
+                max_sizes = cfg.DATALOADER.DATASET_MAX_SIZES
+                ret['dataset_augs'] = [
+                    build_custom_augmentation(
+                        cfg, True, min_size=mi, max_size=ma) \
+                        for mi, ma in zip(min_sizes, max_sizes)]
+        else:
+            ret['dataset_augs'] = []
+
+        return ret
+
+    def __call__(self, dataset_dict):
+        """
+        include image labels
+        """
+        dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below
+        # USER: Write your own image loading if it's not from a file
+        if 'file_name' in dataset_dict:
+            ori_image = utils.read_image(
+                dataset_dict["file_name"], format=self.image_format)
+        else:
+            ori_image, _, _ = self.tar_dataset[dataset_dict["tar_index"]]
+            ori_image = utils._apply_exif_orientation(ori_image)
+            ori_image = utils.convert_PIL_to_numpy(ori_image, self.image_format)
+        utils.check_image_size(dataset_dict, ori_image)
+
+        # USER: Remove if you don't do semantic/panoptic segmentation.
+        if "sem_seg_file_name" in dataset_dict:
+            sem_seg_gt = utils.read_image(
+                dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2)
+        else:
+            sem_seg_gt = None
+
+        if self.is_debug:
+            dataset_dict['dataset_source'] = 0
+
+        not_full_labeled = 'dataset_source' in dataset_dict and \
+            self.with_ann_type and \
+                self.dataset_ann[dataset_dict['dataset_source']] != 'box'
+
+        aug_input = T.AugInput(copy.deepcopy(ori_image), sem_seg=sem_seg_gt)
+        if self.use_diff_bs_size and self.is_train:
+            transforms = \
+                self.dataset_augs[dataset_dict['dataset_source']](aug_input)
+        else:
+            transforms = self.augmentations(aug_input)
+        image, sem_seg_gt = aug_input.image, aug_input.sem_seg
+
+        image_shape = image.shape[:2]  # h, w
+        dataset_dict["image"] = torch.as_tensor(
+            np.ascontiguousarray(image.transpose(2, 0, 1)))
+        
+        if sem_seg_gt is not None:
+            dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long"))
+
+        # USER: Remove if you don't use pre-computed proposals.
+        # Most users would not need this feature.
+        if self.proposal_topk is not None:
+            utils.transform_proposals(
+                dataset_dict, image_shape, transforms, 
+                proposal_topk=self.proposal_topk
+            )
+
+        if not self.is_train:
+            # USER: Modify this if you want to keep them for some reason.
+            dataset_dict.pop("annotations", None)
+            dataset_dict.pop("sem_seg_file_name", None)
+            return dataset_dict
+
+        if "annotations" in dataset_dict:
+            # USER: Modify this if you want to keep them for some reason.
+            for anno in dataset_dict["annotations"]:
+                if not self.use_instance_mask:
+                    anno.pop("segmentation", None)
+                if not self.use_keypoint:
+                    anno.pop("keypoints", None)
+
+            # USER: Implement additional transformations if you have other types of data
+            all_annos = [
+                (utils.transform_instance_annotations(
+                    obj, transforms, image_shape, 
+                    keypoint_hflip_indices=self.keypoint_hflip_indices,
+                ),  obj.get("iscrowd", 0))
+                for obj in dataset_dict.pop("annotations")
+            ]
+            annos = [ann[0] for ann in all_annos if ann[1] == 0]
+            instances = utils.annotations_to_instances(
+                annos, image_shape, mask_format=self.instance_mask_format
+            )
+            
+            del all_annos
+            if self.recompute_boxes:
+                instances.gt_boxes = instances.gt_masks.get_bounding_boxes()
+            dataset_dict["instances"] = utils.filter_empty_instances(instances)
+        if self.with_ann_type:
+            dataset_dict["pos_category_ids"] = dataset_dict.get(
+                'pos_category_ids', [])
+            dataset_dict["ann_type"] = \
+                self.dataset_ann[dataset_dict['dataset_source']]
+        if self.is_debug and (('pos_category_ids' not in dataset_dict) or \
+            (dataset_dict['pos_category_ids'] == [])):
+            dataset_dict['pos_category_ids'] = [x for x in sorted(set(
+                dataset_dict['instances'].gt_classes.tolist()
+            ))]
+        return dataset_dict
+
+# DETR augmentation
+def build_transform_gen(cfg, is_train):
+    """
+    """
+    if is_train:
+        min_size = cfg.INPUT.MIN_SIZE_TRAIN
+        max_size = cfg.INPUT.MAX_SIZE_TRAIN
+        sample_style = cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING
+    else:
+        min_size = cfg.INPUT.MIN_SIZE_TEST
+        max_size = cfg.INPUT.MAX_SIZE_TEST
+        sample_style = "choice"
+    if sample_style == "range":
+        assert len(min_size) == 2, "more than 2 ({}) min_size(s) are provided for ranges".format(len(min_size))
+
+    logger = logging.getLogger(__name__)
+    tfm_gens = []
+    if is_train:
+        tfm_gens.append(T.RandomFlip())
+    tfm_gens.append(T.ResizeShortestEdge(min_size, max_size, sample_style))
+    if is_train:
+        logger.info("TransformGens used in training: " + str(tfm_gens))
+    return tfm_gens
+
+
+class DetrDatasetMapper:
+    """
+    A callable which takes a dataset dict in Detectron2 Dataset format,
+    and map it into a format used by DETR.
+    The callable currently does the following:
+    1. Read the image from "file_name"
+    2. Applies geometric transforms to the image and annotation
+    3. Find and applies suitable cropping to the image and annotation
+    4. Prepare image and annotation to Tensors
+    """
+
+    def __init__(self, cfg, is_train=True):
+        if cfg.INPUT.CROP.ENABLED and is_train:
+            self.crop_gen = [
+                T.ResizeShortestEdge([400, 500, 600], sample_style="choice"),
+                T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE),
+            ]
+        else:
+            self.crop_gen = None
+
+        self.mask_on = cfg.MODEL.MASK_ON
+        self.tfm_gens = build_transform_gen(cfg, is_train)
+        logging.getLogger(__name__).info(
+            "Full TransformGens used in training: {}, crop: {}".format(str(self.tfm_gens), str(self.crop_gen))
+        )
+
+        self.img_format = cfg.INPUT.FORMAT
+        self.is_train = is_train
+
+    def __call__(self, dataset_dict):
+        """
+        Args:
+            dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format.
+        Returns:
+            dict: a format that builtin models in detectron2 accept
+        """
+        dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below
+        image = utils.read_image(dataset_dict["file_name"], format=self.img_format)
+        utils.check_image_size(dataset_dict, image)
+
+        if self.crop_gen is None:
+            image, transforms = T.apply_transform_gens(self.tfm_gens, image)
+        else:
+            if np.random.rand() > 0.5:
+                image, transforms = T.apply_transform_gens(self.tfm_gens, image)
+            else:
+                image, transforms = T.apply_transform_gens(
+                    self.tfm_gens[:-1] + self.crop_gen + self.tfm_gens[-1:], image
+                )
+
+        image_shape = image.shape[:2]  # h, w
+
+        # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory,
+        # but not efficient on large generic data structures due to the use of pickle & mp.Queue.
+        # Therefore it's important to use torch.Tensor.
+        dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1)))
+
+        if not self.is_train:
+            # USER: Modify this if you want to keep them for some reason.
+            dataset_dict.pop("annotations", None)
+            return dataset_dict
+
+        if "annotations" in dataset_dict:
+            # USER: Modify this if you want to keep them for some reason.
+            for anno in dataset_dict["annotations"]:
+                if not self.mask_on:
+                    anno.pop("segmentation", None)
+                anno.pop("keypoints", None)
+
+            # USER: Implement additional transformations if you have other types of data
+            annos = [
+                utils.transform_instance_annotations(obj, transforms, image_shape)
+                for obj in dataset_dict.pop("annotations")
+                if obj.get("iscrowd", 0) == 0
+            ]
+            instances = utils.annotations_to_instances(annos, image_shape)
+            dataset_dict["instances"] = utils.filter_empty_instances(instances)
+        return dataset_dict

detic/data/datasets/cc.py

@@ -0,0 +1,23 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import logging
+import os
+
+from detectron2.data.datasets.builtin_meta import _get_builtin_metadata
+from detectron2.data.datasets.lvis import get_lvis_instances_meta
+from .lvis_v1 import custom_register_lvis_instances
+
+_CUSTOM_SPLITS = {
+    "cc3m_v1_val": ("cc3m/validation/", "cc3m/val_image_info.json"),
+    "cc3m_v1_train": ("cc3m/training/", "cc3m/train_image_info.json"),
+    "cc3m_v1_train_tags": ("cc3m/training/", "cc3m/train_image_info_tags.json"),
+
+}
+
+for key, (image_root, json_file) in _CUSTOM_SPLITS.items():
+    custom_register_lvis_instances(
+        key,
+        get_lvis_instances_meta('lvis_v1'),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )
+

detic/data/datasets/coco_zeroshot.py

@@ -0,0 +1,121 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import os
+
+from detectron2.data.datasets.register_coco import register_coco_instances
+from detectron2.data.datasets.builtin_meta import _get_builtin_metadata
+from .lvis_v1 import custom_register_lvis_instances
+
+categories_seen = [
+    {'id': 1, 'name': 'person'},
+    {'id': 2, 'name': 'bicycle'},
+    {'id': 3, 'name': 'car'},
+    {'id': 4, 'name': 'motorcycle'},
+    {'id': 7, 'name': 'train'},
+    {'id': 8, 'name': 'truck'},
+    {'id': 9, 'name': 'boat'},
+    {'id': 15, 'name': 'bench'},
+    {'id': 16, 'name': 'bird'},
+    {'id': 19, 'name': 'horse'},
+    {'id': 20, 'name': 'sheep'},
+    {'id': 23, 'name': 'bear'},
+    {'id': 24, 'name': 'zebra'},
+    {'id': 25, 'name': 'giraffe'},
+    {'id': 27, 'name': 'backpack'},
+    {'id': 31, 'name': 'handbag'},
+    {'id': 33, 'name': 'suitcase'},
+    {'id': 34, 'name': 'frisbee'},
+    {'id': 35, 'name': 'skis'},
+    {'id': 38, 'name': 'kite'},
+    {'id': 42, 'name': 'surfboard'},
+    {'id': 44, 'name': 'bottle'},
+    {'id': 48, 'name': 'fork'},
+    {'id': 50, 'name': 'spoon'},
+    {'id': 51, 'name': 'bowl'},
+    {'id': 52, 'name': 'banana'},
+    {'id': 53, 'name': 'apple'},
+    {'id': 54, 'name': 'sandwich'},
+    {'id': 55, 'name': 'orange'},
+    {'id': 56, 'name': 'broccoli'},
+    {'id': 57, 'name': 'carrot'},
+    {'id': 59, 'name': 'pizza'},
+    {'id': 60, 'name': 'donut'},
+    {'id': 62, 'name': 'chair'},
+    {'id': 65, 'name': 'bed'},
+    {'id': 70, 'name': 'toilet'},
+    {'id': 72, 'name': 'tv'},
+    {'id': 73, 'name': 'laptop'},
+    {'id': 74, 'name': 'mouse'},
+    {'id': 75, 'name': 'remote'},
+    {'id': 78, 'name': 'microwave'},
+    {'id': 79, 'name': 'oven'},
+    {'id': 80, 'name': 'toaster'},
+    {'id': 82, 'name': 'refrigerator'},
+    {'id': 84, 'name': 'book'},
+    {'id': 85, 'name': 'clock'},
+    {'id': 86, 'name': 'vase'},
+    {'id': 90, 'name': 'toothbrush'},
+]
+
+categories_unseen = [
+    {'id': 5, 'name': 'airplane'},
+    {'id': 6, 'name': 'bus'},
+    {'id': 17, 'name': 'cat'},
+    {'id': 18, 'name': 'dog'},
+    {'id': 21, 'name': 'cow'},
+    {'id': 22, 'name': 'elephant'},
+    {'id': 28, 'name': 'umbrella'},
+    {'id': 32, 'name': 'tie'},
+    {'id': 36, 'name': 'snowboard'},
+    {'id': 41, 'name': 'skateboard'},
+    {'id': 47, 'name': 'cup'},
+    {'id': 49, 'name': 'knife'},
+    {'id': 61, 'name': 'cake'},
+    {'id': 63, 'name': 'couch'},
+    {'id': 76, 'name': 'keyboard'},
+    {'id': 81, 'name': 'sink'},
+    {'id': 87, 'name': 'scissors'},
+]
+
+def _get_metadata(cat):
+    if cat == 'all':
+        return _get_builtin_metadata('coco')
+    elif cat == 'seen':
+        id_to_name = {x['id']: x['name'] for x in categories_seen}
+    else:
+        assert cat == 'unseen'
+        id_to_name = {x['id']: x['name'] for x in categories_unseen}
+
+    thing_dataset_id_to_contiguous_id = {
+        x: i for i, x in enumerate(sorted(id_to_name))}
+    thing_classes = [id_to_name[k] for k in sorted(id_to_name)]
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+_PREDEFINED_SPLITS_COCO = {
+    "coco_zeroshot_train": ("coco/train2017", "coco/zero-shot/instances_train2017_seen_2.json", 'seen'),
+    "coco_zeroshot_val": ("coco/val2017", "coco/zero-shot/instances_val2017_unseen_2.json", 'unseen'),
+    "coco_not_zeroshot_val": ("coco/val2017", "coco/zero-shot/instances_val2017_seen_2.json", 'seen'),
+    "coco_generalized_zeroshot_val": ("coco/val2017", "coco/zero-shot/instances_val2017_all_2_oriorder.json", 'all'),
+    "coco_zeroshot_train_oriorder": ("coco/train2017", "coco/zero-shot/instances_train2017_seen_2_oriorder.json", 'all'),
+}
+
+for key, (image_root, json_file, cat) in _PREDEFINED_SPLITS_COCO.items():
+    register_coco_instances(
+        key,
+        _get_metadata(cat),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )
+
+_CUSTOM_SPLITS_COCO = {
+    "cc3m_coco_train_tags": ("cc3m/training/", "cc3m/coco_train_image_info_tags.json"),
+    "coco_caption_train_tags": ("coco/train2017/", "coco/annotations/captions_train2017_tags_allcaps.json"),}
+
+for key, (image_root, json_file) in _CUSTOM_SPLITS_COCO.items():
+    custom_register_lvis_instances(
+        key,
+        _get_builtin_metadata('coco'),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

detic/data/datasets/imagenet.py

@@ -0,0 +1,41 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import logging
+import os
+
+from detectron2.data import DatasetCatalog, MetadataCatalog
+from detectron2.data.datasets.lvis import get_lvis_instances_meta
+from .lvis_v1 import custom_load_lvis_json, get_lvis_22k_meta
+def custom_register_imagenet_instances(name, metadata, json_file, image_root):
+    """
+    """
+    DatasetCatalog.register(name, lambda: custom_load_lvis_json(
+        json_file, image_root, name))
+    MetadataCatalog.get(name).set(
+        json_file=json_file, image_root=image_root, 
+        evaluator_type="imagenet", **metadata
+    )
+
+_CUSTOM_SPLITS_IMAGENET = {
+    "imagenet_lvis_v1": ("imagenet/ImageNet-LVIS/", "imagenet/annotations/imagenet_lvis_image_info.json"),
+}
+
+for key, (image_root, json_file) in _CUSTOM_SPLITS_IMAGENET.items():
+    custom_register_imagenet_instances(
+        key,
+        get_lvis_instances_meta('lvis_v1'),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )
+
+
+_CUSTOM_SPLITS_IMAGENET_22K = {
+    "imagenet_lvis-22k": ("imagenet/ImageNet-LVIS/", "imagenet/annotations/imagenet-22k_image_info_lvis-22k.json"),
+}
+
+for key, (image_root, json_file) in _CUSTOM_SPLITS_IMAGENET_22K.items():
+    custom_register_imagenet_instances(
+        key,
+        get_lvis_22k_meta(),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

detic/data/datasets/lvis_v1.py

@@ -0,0 +1,155 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import logging
+import os
+
+from fvcore.common.timer import Timer
+from detectron2.structures import BoxMode
+from fvcore.common.file_io import PathManager
+from detectron2.data import DatasetCatalog, MetadataCatalog
+from detectron2.data.datasets.lvis import get_lvis_instances_meta
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["custom_load_lvis_json", "custom_register_lvis_instances"]
+
+
+def custom_register_lvis_instances(name, metadata, json_file, image_root):
+    """
+    """
+    DatasetCatalog.register(name, lambda: custom_load_lvis_json(
+        json_file, image_root, name))
+    MetadataCatalog.get(name).set(
+        json_file=json_file, image_root=image_root, 
+        evaluator_type="lvis", **metadata
+    )
+
+
+def custom_load_lvis_json(json_file, image_root, dataset_name=None):
+    '''
+    Modifications:
+      use `file_name`
+      convert neg_category_ids
+      add pos_category_ids
+    '''
+    from lvis import LVIS
+
+    json_file = PathManager.get_local_path(json_file)
+
+    timer = Timer()
+    lvis_api = LVIS(json_file)
+    if timer.seconds() > 1:
+        logger.info("Loading {} takes {:.2f} seconds.".format(
+            json_file, timer.seconds()))
+
+    catid2contid = {x['id']: i for i, x in enumerate(
+        sorted(lvis_api.dataset['categories'], key=lambda x: x['id']))}
+    if len(lvis_api.dataset['categories']) == 1203:
+        for x in lvis_api.dataset['categories']:
+            assert catid2contid[x['id']] == x['id'] - 1
+    img_ids = sorted(lvis_api.imgs.keys())
+    imgs = lvis_api.load_imgs(img_ids)
+    anns = [lvis_api.img_ann_map[img_id] for img_id in img_ids]
+
+    ann_ids = [ann["id"] for anns_per_image in anns for ann in anns_per_image]
+    assert len(set(ann_ids)) == len(ann_ids), \
+        "Annotation ids in '{}' are not unique".format(json_file)
+
+    imgs_anns = list(zip(imgs, anns))
+    logger.info("Loaded {} images in the LVIS v1 format from {}".format(
+        len(imgs_anns), json_file))
+
+    dataset_dicts = []
+
+    for (img_dict, anno_dict_list) in imgs_anns:
+        record = {}
+        if "file_name" in img_dict:
+            file_name = img_dict["file_name"]
+            if img_dict["file_name"].startswith("COCO"):
+                file_name = file_name[-16:]
+            record["file_name"] = os.path.join(image_root, file_name)
+        elif 'coco_url' in img_dict:
+            # e.g., http://images.cocodataset.org/train2017/000000391895.jpg
+            file_name = img_dict["coco_url"][30:]
+            record["file_name"] = os.path.join(image_root, file_name)
+        elif 'tar_index' in img_dict:
+            record['tar_index'] = img_dict['tar_index']
+        
+        record["height"] = img_dict["height"]
+        record["width"] = img_dict["width"]
+        record["not_exhaustive_category_ids"] = img_dict.get(
+            "not_exhaustive_category_ids", [])
+        record["neg_category_ids"] = img_dict.get("neg_category_ids", [])
+        # NOTE: modified by Xingyi: convert to 0-based
+        record["neg_category_ids"] = [
+            catid2contid[x] for x in record["neg_category_ids"]]
+        if 'pos_category_ids' in img_dict:
+            record['pos_category_ids'] = [
+                catid2contid[x] for x in img_dict.get("pos_category_ids", [])]
+        if 'captions' in img_dict:
+            record['captions'] = img_dict['captions']
+        if 'caption_features' in img_dict:
+            record['caption_features'] = img_dict['caption_features']
+        image_id = record["image_id"] = img_dict["id"]
+
+        objs = []
+        for anno in anno_dict_list:
+            assert anno["image_id"] == image_id
+            if anno.get('iscrowd', 0) > 0:
+                continue
+            obj = {"bbox": anno["bbox"], "bbox_mode": BoxMode.XYWH_ABS}
+            obj["category_id"] = catid2contid[anno['category_id']] 
+            if 'segmentation' in anno:
+                segm = anno["segmentation"]
+                valid_segm = [poly for poly in segm \
+                    if len(poly) % 2 == 0 and len(poly) >= 6]
+                # assert len(segm) == len(
+                #     valid_segm
+                # ), "Annotation contains an invalid polygon with < 3 points"
+                if not len(segm) == len(valid_segm):
+                    print('Annotation contains an invalid polygon with < 3 points')
+                assert len(segm) > 0
+                obj["segmentation"] = segm
+            objs.append(obj)
+        record["annotations"] = objs
+        dataset_dicts.append(record)
+
+    return dataset_dicts
+
+_CUSTOM_SPLITS_LVIS = {
+    "lvis_v1_train+coco": ("coco/", "lvis/lvis_v1_train+coco_mask.json"),
+    "lvis_v1_train_norare": ("coco/", "lvis/lvis_v1_train_norare.json"),
+}
+
+
+for key, (image_root, json_file) in _CUSTOM_SPLITS_LVIS.items():
+    custom_register_lvis_instances(
+        key,
+        get_lvis_instances_meta(key),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )
+
+
+def get_lvis_22k_meta():
+    from .lvis_22k_categories import CATEGORIES
+    cat_ids = [k["id"] for k in CATEGORIES]
+    assert min(cat_ids) == 1 and max(cat_ids) == len(
+        cat_ids
+    ), "Category ids are not in [1, #categories], as expected"
+    # Ensure that the category list is sorted by id
+    lvis_categories = sorted(CATEGORIES, key=lambda x: x["id"])
+    thing_classes = [k["name"] for k in lvis_categories]
+    meta = {"thing_classes": thing_classes}
+    return meta
+
+_CUSTOM_SPLITS_LVIS_22K = {
+    "lvis_v1_train_22k": ("coco/", "lvis/lvis_v1_train_lvis-22k.json"),
+}
+
+for key, (image_root, json_file) in _CUSTOM_SPLITS_LVIS_22K.items():
+    custom_register_lvis_instances(
+        key,
+        get_lvis_22k_meta(),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

detic/data/datasets/objects365.py

@@ -0,0 +1,770 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+from detectron2.data.datasets.register_coco import register_coco_instances
+import os
+
+# categories_v2 = [
+#     {'id': 1, 'name': 'Person'},
+#     {'id': 2, 'name': 'Sneakers'},
+#     {'id': 3, 'name': 'Chair'},
+#     {'id': 4, 'name': 'Other Shoes'},
+#     {'id': 5, 'name': 'Hat'},
+#     {'id': 6, 'name': 'Car'},
+#     {'id': 7, 'name': 'Lamp'},
+#     {'id': 8, 'name': 'Glasses'},
+#     {'id': 9, 'name': 'Bottle'},
+#     {'id': 10, 'name': 'Desk'},
+#     {'id': 11, 'name': 'Cup'},
+#     {'id': 12, 'name': 'Street Lights'},
+#     {'id': 13, 'name': 'Cabinet/shelf'},
+#     {'id': 14, 'name': 'Handbag/Satchel'},
+#     {'id': 15, 'name': 'Bracelet'},
+#     {'id': 16, 'name': 'Plate'},
+#     {'id': 17, 'name': 'Picture/Frame'},
+#     {'id': 18, 'name': 'Helmet'},
+#     {'id': 19, 'name': 'Book'},
+#     {'id': 20, 'name': 'Gloves'},
+#     {'id': 21, 'name': 'Storage box'},
+#     {'id': 22, 'name': 'Boat'},
+#     {'id': 23, 'name': 'Leather Shoes'},
+#     {'id': 24, 'name': 'Flower'},
+#     {'id': 25, 'name': 'Bench'},
+#     {'id': 26, 'name': 'Potted Plant'},
+#     {'id': 27, 'name': 'Bowl/Basin'},
+#     {'id': 28, 'name': 'Flag'},
+#     {'id': 29, 'name': 'Pillow'},
+#     {'id': 30, 'name': 'Boots'},
+#     {'id': 31, 'name': 'Vase'},
+#     {'id': 32, 'name': 'Microphone'},
+#     {'id': 33, 'name': 'Necklace'},
+#     {'id': 34, 'name': 'Ring'},
+#     {'id': 35, 'name': 'SUV'},
+#     {'id': 36, 'name': 'Wine Glass'},
+#     {'id': 37, 'name': 'Belt'},
+#     {'id': 38, 'name': 'Moniter/TV'},
+#     {'id': 39, 'name': 'Backpack'},
+#     {'id': 40, 'name': 'Umbrella'},
+#     {'id': 41, 'name': 'Traffic Light'},
+#     {'id': 42, 'name': 'Speaker'},
+#     {'id': 43, 'name': 'Watch'},
+#     {'id': 44, 'name': 'Tie'},
+#     {'id': 45, 'name': 'Trash bin Can'},
+#     {'id': 46, 'name': 'Slippers'},
+#     {'id': 47, 'name': 'Bicycle'},
+#     {'id': 48, 'name': 'Stool'},
+#     {'id': 49, 'name': 'Barrel/bucket'},
+#     {'id': 50, 'name': 'Van'},
+#     {'id': 51, 'name': 'Couch'},
+#     {'id': 52, 'name': 'Sandals'},
+#     {'id': 53, 'name': 'Bakset'},
+#     {'id': 54, 'name': 'Drum'},
+#     {'id': 55, 'name': 'Pen/Pencil'},
+#     {'id': 56, 'name': 'Bus'},
+#     {'id': 57, 'name': 'Wild Bird'},
+#     {'id': 58, 'name': 'High Heels'},
+#     {'id': 59, 'name': 'Motorcycle'},
+#     {'id': 60, 'name': 'Guitar'},
+#     {'id': 61, 'name': 'Carpet'},
+#     {'id': 62, 'name': 'Cell Phone'},
+#     {'id': 63, 'name': 'Bread'},
+#     {'id': 64, 'name': 'Camera'},
+#     {'id': 65, 'name': 'Canned'},
+#     {'id': 66, 'name': 'Truck'},
+#     {'id': 67, 'name': 'Traffic cone'},
+#     {'id': 68, 'name': 'Cymbal'},
+#     {'id': 69, 'name': 'Lifesaver'},
+#     {'id': 70, 'name': 'Towel'},
+#     {'id': 71, 'name': 'Stuffed Toy'},
+#     {'id': 72, 'name': 'Candle'},
+#     {'id': 73, 'name': 'Sailboat'},
+#     {'id': 74, 'name': 'Laptop'},
+#     {'id': 75, 'name': 'Awning'},
+#     {'id': 76, 'name': 'Bed'},
+#     {'id': 77, 'name': 'Faucet'},
+#     {'id': 78, 'name': 'Tent'},
+#     {'id': 79, 'name': 'Horse'},
+#     {'id': 80, 'name': 'Mirror'},
+#     {'id': 81, 'name': 'Power outlet'},
+#     {'id': 82, 'name': 'Sink'},
+#     {'id': 83, 'name': 'Apple'},
+#     {'id': 84, 'name': 'Air Conditioner'},
+#     {'id': 85, 'name': 'Knife'},
+#     {'id': 86, 'name': 'Hockey Stick'},
+#     {'id': 87, 'name': 'Paddle'},
+#     {'id': 88, 'name': 'Pickup Truck'},
+#     {'id': 89, 'name': 'Fork'},
+#     {'id': 90, 'name': 'Traffic Sign'},
+#     {'id': 91, 'name': 'Ballon'},
+#     {'id': 92, 'name': 'Tripod'},
+#     {'id': 93, 'name': 'Dog'},
+#     {'id': 94, 'name': 'Spoon'},
+#     {'id': 95, 'name': 'Clock'},
+#     {'id': 96, 'name': 'Pot'},
+#     {'id': 97, 'name': 'Cow'},
+#     {'id': 98, 'name': 'Cake'},
+#     {'id': 99, 'name': 'Dinning Table'},
+#     {'id': 100, 'name': 'Sheep'},
+#     {'id': 101, 'name': 'Hanger'},
+#     {'id': 102, 'name': 'Blackboard/Whiteboard'},
+#     {'id': 103, 'name': 'Napkin'},
+#     {'id': 104, 'name': 'Other Fish'},
+#     {'id': 105, 'name': 'Orange/Tangerine'},
+#     {'id': 106, 'name': 'Toiletry'},
+#     {'id': 107, 'name': 'Keyboard'},
+#     {'id': 108, 'name': 'Tomato'},
+#     {'id': 109, 'name': 'Lantern'},
+#     {'id': 110, 'name': 'Machinery Vehicle'},
+#     {'id': 111, 'name': 'Fan'},
+#     {'id': 112, 'name': 'Green Vegetables'},
+#     {'id': 113, 'name': 'Banana'},
+#     {'id': 114, 'name': 'Baseball Glove'},
+#     {'id': 115, 'name': 'Airplane'},
+#     {'id': 116, 'name': 'Mouse'},
+#     {'id': 117, 'name': 'Train'},
+#     {'id': 118, 'name': 'Pumpkin'},
+#     {'id': 119, 'name': 'Soccer'},
+#     {'id': 120, 'name': 'Skiboard'},
+#     {'id': 121, 'name': 'Luggage'},
+#     {'id': 122, 'name': 'Nightstand'},
+#     {'id': 123, 'name': 'Tea pot'},
+#     {'id': 124, 'name': 'Telephone'},
+#     {'id': 125, 'name': 'Trolley'},
+#     {'id': 126, 'name': 'Head Phone'},
+#     {'id': 127, 'name': 'Sports Car'},
+#     {'id': 128, 'name': 'Stop Sign'},
+#     {'id': 129, 'name': 'Dessert'},
+#     {'id': 130, 'name': 'Scooter'},
+#     {'id': 131, 'name': 'Stroller'},
+#     {'id': 132, 'name': 'Crane'},
+#     {'id': 133, 'name': 'Remote'},
+#     {'id': 134, 'name': 'Refrigerator'},
+#     {'id': 135, 'name': 'Oven'},
+#     {'id': 136, 'name': 'Lemon'},
+#     {'id': 137, 'name': 'Duck'},
+#     {'id': 138, 'name': 'Baseball Bat'},
+#     {'id': 139, 'name': 'Surveillance Camera'},
+#     {'id': 140, 'name': 'Cat'},
+#     {'id': 141, 'name': 'Jug'},
+#     {'id': 142, 'name': 'Broccoli'},
+#     {'id': 143, 'name': 'Piano'},
+#     {'id': 144, 'name': 'Pizza'},
+#     {'id': 145, 'name': 'Elephant'},
+#     {'id': 146, 'name': 'Skateboard'},
+#     {'id': 147, 'name': 'Surfboard'},
+#     {'id': 148, 'name': 'Gun'},
+#     {'id': 149, 'name': 'Skating and Skiing shoes'},
+#     {'id': 150, 'name': 'Gas stove'},
+#     {'id': 151, 'name': 'Donut'},
+#     {'id': 152, 'name': 'Bow Tie'},
+#     {'id': 153, 'name': 'Carrot'},
+#     {'id': 154, 'name': 'Toilet'},
+#     {'id': 155, 'name': 'Kite'},
+#     {'id': 156, 'name': 'Strawberry'},
+#     {'id': 157, 'name': 'Other Balls'},
+#     {'id': 158, 'name': 'Shovel'},
+#     {'id': 159, 'name': 'Pepper'},
+#     {'id': 160, 'name': 'Computer Box'},
+#     {'id': 161, 'name': 'Toilet Paper'},
+#     {'id': 162, 'name': 'Cleaning Products'},
+#     {'id': 163, 'name': 'Chopsticks'},
+#     {'id': 164, 'name': 'Microwave'},
+#     {'id': 165, 'name': 'Pigeon'},
+#     {'id': 166, 'name': 'Baseball'},
+#     {'id': 167, 'name': 'Cutting/chopping Board'},
+#     {'id': 168, 'name': 'Coffee Table'},
+#     {'id': 169, 'name': 'Side Table'},
+#     {'id': 170, 'name': 'Scissors'},
+#     {'id': 171, 'name': 'Marker'},
+#     {'id': 172, 'name': 'Pie'},
+#     {'id': 173, 'name': 'Ladder'},
+#     {'id': 174, 'name': 'Snowboard'},
+#     {'id': 175, 'name': 'Cookies'},
+#     {'id': 176, 'name': 'Radiator'},
+#     {'id': 177, 'name': 'Fire Hydrant'},
+#     {'id': 178, 'name': 'Basketball'},
+#     {'id': 179, 'name': 'Zebra'},
+#     {'id': 180, 'name': 'Grape'},
+#     {'id': 181, 'name': 'Giraffe'},
+#     {'id': 182, 'name': 'Potato'},
+#     {'id': 183, 'name': 'Sausage'},
+#     {'id': 184, 'name': 'Tricycle'},
+#     {'id': 185, 'name': 'Violin'},
+#     {'id': 186, 'name': 'Egg'},
+#     {'id': 187, 'name': 'Fire Extinguisher'},
+#     {'id': 188, 'name': 'Candy'},
+#     {'id': 189, 'name': 'Fire Truck'},
+#     {'id': 190, 'name': 'Billards'},
+#     {'id': 191, 'name': 'Converter'},
+#     {'id': 192, 'name': 'Bathtub'},
+#     {'id': 193, 'name': 'Wheelchair'},
+#     {'id': 194, 'name': 'Golf Club'},
+#     {'id': 195, 'name': 'Briefcase'},
+#     {'id': 196, 'name': 'Cucumber'},
+#     {'id': 197, 'name': 'Cigar/Cigarette '},
+#     {'id': 198, 'name': 'Paint Brush'},
+#     {'id': 199, 'name': 'Pear'},
+#     {'id': 200, 'name': 'Heavy Truck'},
+#     {'id': 201, 'name': 'Hamburger'},
+#     {'id': 202, 'name': 'Extractor'},
+#     {'id': 203, 'name': 'Extention Cord'},
+#     {'id': 204, 'name': 'Tong'},
+#     {'id': 205, 'name': 'Tennis Racket'},
+#     {'id': 206, 'name': 'Folder'},
+#     {'id': 207, 'name': 'American Football'},
+#     {'id': 208, 'name': 'earphone'},
+#     {'id': 209, 'name': 'Mask'},
+#     {'id': 210, 'name': 'Kettle'},
+#     {'id': 211, 'name': 'Tennis'},
+#     {'id': 212, 'name': 'Ship'},
+#     {'id': 213, 'name': 'Swing'},
+#     {'id': 214, 'name': 'Coffee Machine'},
+#     {'id': 215, 'name': 'Slide'},
+#     {'id': 216, 'name': 'Carriage'},
+#     {'id': 217, 'name': 'Onion'},
+#     {'id': 218, 'name': 'Green beans'},
+#     {'id': 219, 'name': 'Projector'},
+#     {'id': 220, 'name': 'Frisbee'},
+#     {'id': 221, 'name': 'Washing Machine/Drying Machine'},
+#     {'id': 222, 'name': 'Chicken'},
+#     {'id': 223, 'name': 'Printer'},
+#     {'id': 224, 'name': 'Watermelon'},
+#     {'id': 225, 'name': 'Saxophone'},
+#     {'id': 226, 'name': 'Tissue'},
+#     {'id': 227, 'name': 'Toothbrush'},
+#     {'id': 228, 'name': 'Ice cream'},
+#     {'id': 229, 'name': 'Hotair ballon'},
+#     {'id': 230, 'name': 'Cello'},
+#     {'id': 231, 'name': 'French Fries'},
+#     {'id': 232, 'name': 'Scale'},
+#     {'id': 233, 'name': 'Trophy'},
+#     {'id': 234, 'name': 'Cabbage'},
+#     {'id': 235, 'name': 'Hot dog'},
+#     {'id': 236, 'name': 'Blender'},
+#     {'id': 237, 'name': 'Peach'},
+#     {'id': 238, 'name': 'Rice'},
+#     {'id': 239, 'name': 'Wallet/Purse'},
+#     {'id': 240, 'name': 'Volleyball'},
+#     {'id': 241, 'name': 'Deer'},
+#     {'id': 242, 'name': 'Goose'},
+#     {'id': 243, 'name': 'Tape'},
+#     {'id': 244, 'name': 'Tablet'},
+#     {'id': 245, 'name': 'Cosmetics'},
+#     {'id': 246, 'name': 'Trumpet'},
+#     {'id': 247, 'name': 'Pineapple'},
+#     {'id': 248, 'name': 'Golf Ball'},
+#     {'id': 249, 'name': 'Ambulance'},
+#     {'id': 250, 'name': 'Parking meter'},
+#     {'id': 251, 'name': 'Mango'},
+#     {'id': 252, 'name': 'Key'},
+#     {'id': 253, 'name': 'Hurdle'},
+#     {'id': 254, 'name': 'Fishing Rod'},
+#     {'id': 255, 'name': 'Medal'},
+#     {'id': 256, 'name': 'Flute'},
+#     {'id': 257, 'name': 'Brush'},
+#     {'id': 258, 'name': 'Penguin'},
+#     {'id': 259, 'name': 'Megaphone'},
+#     {'id': 260, 'name': 'Corn'},
+#     {'id': 261, 'name': 'Lettuce'},
+#     {'id': 262, 'name': 'Garlic'},
+#     {'id': 263, 'name': 'Swan'},
+#     {'id': 264, 'name': 'Helicopter'},
+#     {'id': 265, 'name': 'Green Onion'},
+#     {'id': 266, 'name': 'Sandwich'},
+#     {'id': 267, 'name': 'Nuts'},
+#     {'id': 268, 'name': 'Speed Limit Sign'},
+#     {'id': 269, 'name': 'Induction Cooker'},
+#     {'id': 270, 'name': 'Broom'},
+#     {'id': 271, 'name': 'Trombone'},
+#     {'id': 272, 'name': 'Plum'},
+#     {'id': 273, 'name': 'Rickshaw'},
+#     {'id': 274, 'name': 'Goldfish'},
+#     {'id': 275, 'name': 'Kiwi fruit'},
+#     {'id': 276, 'name': 'Router/modem'},
+#     {'id': 277, 'name': 'Poker Card'},
+#     {'id': 278, 'name': 'Toaster'},
+#     {'id': 279, 'name': 'Shrimp'},
+#     {'id': 280, 'name': 'Sushi'},
+#     {'id': 281, 'name': 'Cheese'},
+#     {'id': 282, 'name': 'Notepaper'},
+#     {'id': 283, 'name': 'Cherry'},
+#     {'id': 284, 'name': 'Pliers'},
+#     {'id': 285, 'name': 'CD'},
+#     {'id': 286, 'name': 'Pasta'},
+#     {'id': 287, 'name': 'Hammer'},
+#     {'id': 288, 'name': 'Cue'},
+#     {'id': 289, 'name': 'Avocado'},
+#     {'id': 290, 'name': 'Hamimelon'},
+#     {'id': 291, 'name': 'Flask'},
+#     {'id': 292, 'name': 'Mushroon'},
+#     {'id': 293, 'name': 'Screwdriver'},
+#     {'id': 294, 'name': 'Soap'},
+#     {'id': 295, 'name': 'Recorder'},
+#     {'id': 296, 'name': 'Bear'},
+#     {'id': 297, 'name': 'Eggplant'},
+#     {'id': 298, 'name': 'Board Eraser'},
+#     {'id': 299, 'name': 'Coconut'},
+#     {'id': 300, 'name': 'Tape Measur/ Ruler'},
+#     {'id': 301, 'name': 'Pig'},
+#     {'id': 302, 'name': 'Showerhead'},
+#     {'id': 303, 'name': 'Globe'},
+#     {'id': 304, 'name': 'Chips'},
+#     {'id': 305, 'name': 'Steak'},
+#     {'id': 306, 'name': 'Crosswalk Sign'},
+#     {'id': 307, 'name': 'Stapler'},
+#     {'id': 308, 'name': 'Campel'},
+#     {'id': 309, 'name': 'Formula 1 '},
+#     {'id': 310, 'name': 'Pomegranate'},
+#     {'id': 311, 'name': 'Dishwasher'},
+#     {'id': 312, 'name': 'Crab'},
+#     {'id': 313, 'name': 'Hoverboard'},
+#     {'id': 314, 'name': 'Meat ball'},
+#     {'id': 315, 'name': 'Rice Cooker'},
+#     {'id': 316, 'name': 'Tuba'},
+#     {'id': 317, 'name': 'Calculator'},
+#     {'id': 318, 'name': 'Papaya'},
+#     {'id': 319, 'name': 'Antelope'},
+#     {'id': 320, 'name': 'Parrot'},
+#     {'id': 321, 'name': 'Seal'},
+#     {'id': 322, 'name': 'Buttefly'},
+#     {'id': 323, 'name': 'Dumbbell'},
+#     {'id': 324, 'name': 'Donkey'},
+#     {'id': 325, 'name': 'Lion'},
+#     {'id': 326, 'name': 'Urinal'},
+#     {'id': 327, 'name': 'Dolphin'},
+#     {'id': 328, 'name': 'Electric Drill'},
+#     {'id': 329, 'name': 'Hair Dryer'},
+#     {'id': 330, 'name': 'Egg tart'},
+#     {'id': 331, 'name': 'Jellyfish'},
+#     {'id': 332, 'name': 'Treadmill'},
+#     {'id': 333, 'name': 'Lighter'},
+#     {'id': 334, 'name': 'Grapefruit'},
+#     {'id': 335, 'name': 'Game board'},
+#     {'id': 336, 'name': 'Mop'},
+#     {'id': 337, 'name': 'Radish'},
+#     {'id': 338, 'name': 'Baozi'},
+#     {'id': 339, 'name': 'Target'},
+#     {'id': 340, 'name': 'French'},
+#     {'id': 341, 'name': 'Spring Rolls'},
+#     {'id': 342, 'name': 'Monkey'},
+#     {'id': 343, 'name': 'Rabbit'},
+#     {'id': 344, 'name': 'Pencil Case'},
+#     {'id': 345, 'name': 'Yak'},
+#     {'id': 346, 'name': 'Red Cabbage'},
+#     {'id': 347, 'name': 'Binoculars'},
+#     {'id': 348, 'name': 'Asparagus'},
+#     {'id': 349, 'name': 'Barbell'},
+#     {'id': 350, 'name': 'Scallop'},
+#     {'id': 351, 'name': 'Noddles'},
+#     {'id': 352, 'name': 'Comb'},
+#     {'id': 353, 'name': 'Dumpling'},
+#     {'id': 354, 'name': 'Oyster'},
+#     {'id': 355, 'name': 'Table Teniis paddle'},
+#     {'id': 356, 'name': 'Cosmetics Brush/Eyeliner Pencil'},
+#     {'id': 357, 'name': 'Chainsaw'},
+#     {'id': 358, 'name': 'Eraser'},
+#     {'id': 359, 'name': 'Lobster'},
+#     {'id': 360, 'name': 'Durian'},
+#     {'id': 361, 'name': 'Okra'},
+#     {'id': 362, 'name': 'Lipstick'},
+#     {'id': 363, 'name': 'Cosmetics Mirror'},
+#     {'id': 364, 'name': 'Curling'},
+#     {'id': 365, 'name': 'Table Tennis '},
+# ]
+
+'''
+The official Objects365 category names contains typos.
+Below is a manual fix.
+'''
+categories_v2_fix = [
+  {'id': 1, 'name': 'Person'},
+  {'id': 2, 'name': 'Sneakers'},
+  {'id': 3, 'name': 'Chair'},
+  {'id': 4, 'name': 'Other Shoes'},
+  {'id': 5, 'name': 'Hat'},
+  {'id': 6, 'name': 'Car'},
+  {'id': 7, 'name': 'Lamp'},
+  {'id': 8, 'name': 'Glasses'},
+  {'id': 9, 'name': 'Bottle'},
+  {'id': 10, 'name': 'Desk'},
+  {'id': 11, 'name': 'Cup'},
+  {'id': 12, 'name': 'Street Lights'},
+  {'id': 13, 'name': 'Cabinet/shelf'},
+  {'id': 14, 'name': 'Handbag/Satchel'},
+  {'id': 15, 'name': 'Bracelet'},
+  {'id': 16, 'name': 'Plate'},
+  {'id': 17, 'name': 'Picture/Frame'},
+  {'id': 18, 'name': 'Helmet'},
+  {'id': 19, 'name': 'Book'},
+  {'id': 20, 'name': 'Gloves'},
+  {'id': 21, 'name': 'Storage box'},
+  {'id': 22, 'name': 'Boat'},
+  {'id': 23, 'name': 'Leather Shoes'},
+  {'id': 24, 'name': 'Flower'},
+  {'id': 25, 'name': 'Bench'},
+  {'id': 26, 'name': 'Potted Plant'},
+  {'id': 27, 'name': 'Bowl/Basin'},
+  {'id': 28, 'name': 'Flag'},
+  {'id': 29, 'name': 'Pillow'},
+  {'id': 30, 'name': 'Boots'},
+  {'id': 31, 'name': 'Vase'},
+  {'id': 32, 'name': 'Microphone'},
+  {'id': 33, 'name': 'Necklace'},
+  {'id': 34, 'name': 'Ring'},
+  {'id': 35, 'name': 'SUV'},
+  {'id': 36, 'name': 'Wine Glass'},
+  {'id': 37, 'name': 'Belt'},
+  {'id': 38, 'name': 'Monitor/TV'},
+  {'id': 39, 'name': 'Backpack'},
+  {'id': 40, 'name': 'Umbrella'},
+  {'id': 41, 'name': 'Traffic Light'},
+  {'id': 42, 'name': 'Speaker'},
+  {'id': 43, 'name': 'Watch'},
+  {'id': 44, 'name': 'Tie'},
+  {'id': 45, 'name': 'Trash bin Can'},
+  {'id': 46, 'name': 'Slippers'},
+  {'id': 47, 'name': 'Bicycle'},
+  {'id': 48, 'name': 'Stool'},
+  {'id': 49, 'name': 'Barrel/bucket'},
+  {'id': 50, 'name': 'Van'},
+  {'id': 51, 'name': 'Couch'},
+  {'id': 52, 'name': 'Sandals'},
+  {'id': 53, 'name': 'Basket'},
+  {'id': 54, 'name': 'Drum'},
+  {'id': 55, 'name': 'Pen/Pencil'},
+  {'id': 56, 'name': 'Bus'},
+  {'id': 57, 'name': 'Wild Bird'},
+  {'id': 58, 'name': 'High Heels'},
+  {'id': 59, 'name': 'Motorcycle'},
+  {'id': 60, 'name': 'Guitar'},
+  {'id': 61, 'name': 'Carpet'},
+  {'id': 62, 'name': 'Cell Phone'},
+  {'id': 63, 'name': 'Bread'},
+  {'id': 64, 'name': 'Camera'},
+  {'id': 65, 'name': 'Canned'},
+  {'id': 66, 'name': 'Truck'},
+  {'id': 67, 'name': 'Traffic cone'},
+  {'id': 68, 'name': 'Cymbal'},
+  {'id': 69, 'name': 'Lifesaver'},
+  {'id': 70, 'name': 'Towel'},
+  {'id': 71, 'name': 'Stuffed Toy'},
+  {'id': 72, 'name': 'Candle'},
+  {'id': 73, 'name': 'Sailboat'},
+  {'id': 74, 'name': 'Laptop'},
+  {'id': 75, 'name': 'Awning'},
+  {'id': 76, 'name': 'Bed'},
+  {'id': 77, 'name': 'Faucet'},
+  {'id': 78, 'name': 'Tent'},
+  {'id': 79, 'name': 'Horse'},
+  {'id': 80, 'name': 'Mirror'},
+  {'id': 81, 'name': 'Power outlet'},
+  {'id': 82, 'name': 'Sink'},
+  {'id': 83, 'name': 'Apple'},
+  {'id': 84, 'name': 'Air Conditioner'},
+  {'id': 85, 'name': 'Knife'},
+  {'id': 86, 'name': 'Hockey Stick'},
+  {'id': 87, 'name': 'Paddle'},
+  {'id': 88, 'name': 'Pickup Truck'},
+  {'id': 89, 'name': 'Fork'},
+  {'id': 90, 'name': 'Traffic Sign'},
+  {'id': 91, 'name': 'Ballon'},
+  {'id': 92, 'name': 'Tripod'},
+  {'id': 93, 'name': 'Dog'},
+  {'id': 94, 'name': 'Spoon'},
+  {'id': 95, 'name': 'Clock'},
+  {'id': 96, 'name': 'Pot'},
+  {'id': 97, 'name': 'Cow'},
+  {'id': 98, 'name': 'Cake'},
+  {'id': 99, 'name': 'Dining Table'},
+  {'id': 100, 'name': 'Sheep'},
+  {'id': 101, 'name': 'Hanger'},
+  {'id': 102, 'name': 'Blackboard/Whiteboard'},
+  {'id': 103, 'name': 'Napkin'},
+  {'id': 104, 'name': 'Other Fish'},
+  {'id': 105, 'name': 'Orange/Tangerine'},
+  {'id': 106, 'name': 'Toiletry'},
+  {'id': 107, 'name': 'Keyboard'},
+  {'id': 108, 'name': 'Tomato'},
+  {'id': 109, 'name': 'Lantern'},
+  {'id': 110, 'name': 'Machinery Vehicle'},
+  {'id': 111, 'name': 'Fan'},
+  {'id': 112, 'name': 'Green Vegetables'},
+  {'id': 113, 'name': 'Banana'},
+  {'id': 114, 'name': 'Baseball Glove'},
+  {'id': 115, 'name': 'Airplane'},
+  {'id': 116, 'name': 'Mouse'},
+  {'id': 117, 'name': 'Train'},
+  {'id': 118, 'name': 'Pumpkin'},
+  {'id': 119, 'name': 'Soccer'},
+  {'id': 120, 'name': 'Skiboard'},
+  {'id': 121, 'name': 'Luggage'},
+  {'id': 122, 'name': 'Nightstand'},
+  {'id': 123, 'name': 'Teapot'},
+  {'id': 124, 'name': 'Telephone'},
+  {'id': 125, 'name': 'Trolley'},
+  {'id': 126, 'name': 'Head Phone'},
+  {'id': 127, 'name': 'Sports Car'},
+  {'id': 128, 'name': 'Stop Sign'},
+  {'id': 129, 'name': 'Dessert'},
+  {'id': 130, 'name': 'Scooter'},
+  {'id': 131, 'name': 'Stroller'},
+  {'id': 132, 'name': 'Crane'},
+  {'id': 133, 'name': 'Remote'},
+  {'id': 134, 'name': 'Refrigerator'},
+  {'id': 135, 'name': 'Oven'},
+  {'id': 136, 'name': 'Lemon'},
+  {'id': 137, 'name': 'Duck'},
+  {'id': 138, 'name': 'Baseball Bat'},
+  {'id': 139, 'name': 'Surveillance Camera'},
+  {'id': 140, 'name': 'Cat'},
+  {'id': 141, 'name': 'Jug'},
+  {'id': 142, 'name': 'Broccoli'},
+  {'id': 143, 'name': 'Piano'},
+  {'id': 144, 'name': 'Pizza'},
+  {'id': 145, 'name': 'Elephant'},
+  {'id': 146, 'name': 'Skateboard'},
+  {'id': 147, 'name': 'Surfboard'},
+  {'id': 148, 'name': 'Gun'},
+  {'id': 149, 'name': 'Skating and Skiing shoes'},
+  {'id': 150, 'name': 'Gas stove'},
+  {'id': 151, 'name': 'Donut'},
+  {'id': 152, 'name': 'Bow Tie'},
+  {'id': 153, 'name': 'Carrot'},
+  {'id': 154, 'name': 'Toilet'},
+  {'id': 155, 'name': 'Kite'},
+  {'id': 156, 'name': 'Strawberry'},
+  {'id': 157, 'name': 'Other Balls'},
+  {'id': 158, 'name': 'Shovel'},
+  {'id': 159, 'name': 'Pepper'},
+  {'id': 160, 'name': 'Computer Box'},
+  {'id': 161, 'name': 'Toilet Paper'},
+  {'id': 162, 'name': 'Cleaning Products'},
+  {'id': 163, 'name': 'Chopsticks'},
+  {'id': 164, 'name': 'Microwave'},
+  {'id': 165, 'name': 'Pigeon'},
+  {'id': 166, 'name': 'Baseball'},
+  {'id': 167, 'name': 'Cutting/chopping Board'},
+  {'id': 168, 'name': 'Coffee Table'},
+  {'id': 169, 'name': 'Side Table'},
+  {'id': 170, 'name': 'Scissors'},
+  {'id': 171, 'name': 'Marker'},
+  {'id': 172, 'name': 'Pie'},
+  {'id': 173, 'name': 'Ladder'},
+  {'id': 174, 'name': 'Snowboard'},
+  {'id': 175, 'name': 'Cookies'},
+  {'id': 176, 'name': 'Radiator'},
+  {'id': 177, 'name': 'Fire Hydrant'},
+  {'id': 178, 'name': 'Basketball'},
+  {'id': 179, 'name': 'Zebra'},
+  {'id': 180, 'name': 'Grape'},
+  {'id': 181, 'name': 'Giraffe'},
+  {'id': 182, 'name': 'Potato'},
+  {'id': 183, 'name': 'Sausage'},
+  {'id': 184, 'name': 'Tricycle'},
+  {'id': 185, 'name': 'Violin'},
+  {'id': 186, 'name': 'Egg'},
+  {'id': 187, 'name': 'Fire Extinguisher'},
+  {'id': 188, 'name': 'Candy'},
+  {'id': 189, 'name': 'Fire Truck'},
+  {'id': 190, 'name': 'Billards'},
+  {'id': 191, 'name': 'Converter'},
+  {'id': 192, 'name': 'Bathtub'},
+  {'id': 193, 'name': 'Wheelchair'},
+  {'id': 194, 'name': 'Golf Club'},
+  {'id': 195, 'name': 'Briefcase'},
+  {'id': 196, 'name': 'Cucumber'},
+  {'id': 197, 'name': 'Cigar/Cigarette '},
+  {'id': 198, 'name': 'Paint Brush'},
+  {'id': 199, 'name': 'Pear'},
+  {'id': 200, 'name': 'Heavy Truck'},
+  {'id': 201, 'name': 'Hamburger'},
+  {'id': 202, 'name': 'Extractor'},
+  {'id': 203, 'name': 'Extension Cord'},
+  {'id': 204, 'name': 'Tong'},
+  {'id': 205, 'name': 'Tennis Racket'},
+  {'id': 206, 'name': 'Folder'},
+  {'id': 207, 'name': 'American Football'},
+  {'id': 208, 'name': 'earphone'},
+  {'id': 209, 'name': 'Mask'},
+  {'id': 210, 'name': 'Kettle'},
+  {'id': 211, 'name': 'Tennis'},
+  {'id': 212, 'name': 'Ship'},
+  {'id': 213, 'name': 'Swing'},
+  {'id': 214, 'name': 'Coffee Machine'},
+  {'id': 215, 'name': 'Slide'},
+  {'id': 216, 'name': 'Carriage'},
+  {'id': 217, 'name': 'Onion'},
+  {'id': 218, 'name': 'Green beans'},
+  {'id': 219, 'name': 'Projector'},
+  {'id': 220, 'name': 'Frisbee'},
+  {'id': 221, 'name': 'Washing Machine/Drying Machine'},
+  {'id': 222, 'name': 'Chicken'},
+  {'id': 223, 'name': 'Printer'},
+  {'id': 224, 'name': 'Watermelon'},
+  {'id': 225, 'name': 'Saxophone'},
+  {'id': 226, 'name': 'Tissue'},
+  {'id': 227, 'name': 'Toothbrush'},
+  {'id': 228, 'name': 'Ice cream'},
+  {'id': 229, 'name': 'Hot air balloon'},
+  {'id': 230, 'name': 'Cello'},
+  {'id': 231, 'name': 'French Fries'},
+  {'id': 232, 'name': 'Scale'},
+  {'id': 233, 'name': 'Trophy'},
+  {'id': 234, 'name': 'Cabbage'},
+  {'id': 235, 'name': 'Hot dog'},
+  {'id': 236, 'name': 'Blender'},
+  {'id': 237, 'name': 'Peach'},
+  {'id': 238, 'name': 'Rice'},
+  {'id': 239, 'name': 'Wallet/Purse'},
+  {'id': 240, 'name': 'Volleyball'},
+  {'id': 241, 'name': 'Deer'},
+  {'id': 242, 'name': 'Goose'},
+  {'id': 243, 'name': 'Tape'},
+  {'id': 244, 'name': 'Tablet'},
+  {'id': 245, 'name': 'Cosmetics'},
+  {'id': 246, 'name': 'Trumpet'},
+  {'id': 247, 'name': 'Pineapple'},
+  {'id': 248, 'name': 'Golf Ball'},
+  {'id': 249, 'name': 'Ambulance'},
+  {'id': 250, 'name': 'Parking meter'},
+  {'id': 251, 'name': 'Mango'},
+  {'id': 252, 'name': 'Key'},
+  {'id': 253, 'name': 'Hurdle'},
+  {'id': 254, 'name': 'Fishing Rod'},
+  {'id': 255, 'name': 'Medal'},
+  {'id': 256, 'name': 'Flute'},
+  {'id': 257, 'name': 'Brush'},
+  {'id': 258, 'name': 'Penguin'},
+  {'id': 259, 'name': 'Megaphone'},
+  {'id': 260, 'name': 'Corn'},
+  {'id': 261, 'name': 'Lettuce'},
+  {'id': 262, 'name': 'Garlic'},
+  {'id': 263, 'name': 'Swan'},
+  {'id': 264, 'name': 'Helicopter'},
+  {'id': 265, 'name': 'Green Onion'},
+  {'id': 266, 'name': 'Sandwich'},
+  {'id': 267, 'name': 'Nuts'},
+  {'id': 268, 'name': 'Speed Limit Sign'},
+  {'id': 269, 'name': 'Induction Cooker'},
+  {'id': 270, 'name': 'Broom'},
+  {'id': 271, 'name': 'Trombone'},
+  {'id': 272, 'name': 'Plum'},
+  {'id': 273, 'name': 'Rickshaw'},
+  {'id': 274, 'name': 'Goldfish'},
+  {'id': 275, 'name': 'Kiwi fruit'},
+  {'id': 276, 'name': 'Router/modem'},
+  {'id': 277, 'name': 'Poker Card'},
+  {'id': 278, 'name': 'Toaster'},
+  {'id': 279, 'name': 'Shrimp'},
+  {'id': 280, 'name': 'Sushi'},
+  {'id': 281, 'name': 'Cheese'},
+  {'id': 282, 'name': 'Notepaper'},
+  {'id': 283, 'name': 'Cherry'},
+  {'id': 284, 'name': 'Pliers'},
+  {'id': 285, 'name': 'CD'},
+  {'id': 286, 'name': 'Pasta'},
+  {'id': 287, 'name': 'Hammer'},
+  {'id': 288, 'name': 'Cue'},
+  {'id': 289, 'name': 'Avocado'},
+  {'id': 290, 'name': 'Hami melon'},
+  {'id': 291, 'name': 'Flask'},
+  {'id': 292, 'name': 'Mushroom'},
+  {'id': 293, 'name': 'Screwdriver'},
+  {'id': 294, 'name': 'Soap'},
+  {'id': 295, 'name': 'Recorder'},
+  {'id': 296, 'name': 'Bear'},
+  {'id': 297, 'name': 'Eggplant'},
+  {'id': 298, 'name': 'Board Eraser'},
+  {'id': 299, 'name': 'Coconut'},
+  {'id': 300, 'name': 'Tape Measure/ Ruler'},
+  {'id': 301, 'name': 'Pig'},
+  {'id': 302, 'name': 'Showerhead'},
+  {'id': 303, 'name': 'Globe'},
+  {'id': 304, 'name': 'Chips'},
+  {'id': 305, 'name': 'Steak'},
+  {'id': 306, 'name': 'Crosswalk Sign'},
+  {'id': 307, 'name': 'Stapler'},
+  {'id': 308, 'name': 'Camel'},
+  {'id': 309, 'name': 'Formula 1 '},
+  {'id': 310, 'name': 'Pomegranate'},
+  {'id': 311, 'name': 'Dishwasher'},
+  {'id': 312, 'name': 'Crab'},
+  {'id': 313, 'name': 'Hoverboard'},
+  {'id': 314, 'name': 'Meatball'},
+  {'id': 315, 'name': 'Rice Cooker'},
+  {'id': 316, 'name': 'Tuba'},
+  {'id': 317, 'name': 'Calculator'},
+  {'id': 318, 'name': 'Papaya'},
+  {'id': 319, 'name': 'Antelope'},
+  {'id': 320, 'name': 'Parrot'},
+  {'id': 321, 'name': 'Seal'},
+  {'id': 322, 'name': 'Butterfly'},
+  {'id': 323, 'name': 'Dumbbell'},
+  {'id': 324, 'name': 'Donkey'},
+  {'id': 325, 'name': 'Lion'},
+  {'id': 326, 'name': 'Urinal'},
+  {'id': 327, 'name': 'Dolphin'},
+  {'id': 328, 'name': 'Electric Drill'},
+  {'id': 329, 'name': 'Hair Dryer'},
+  {'id': 330, 'name': 'Egg tart'},
+  {'id': 331, 'name': 'Jellyfish'},
+  {'id': 332, 'name': 'Treadmill'},
+  {'id': 333, 'name': 'Lighter'},
+  {'id': 334, 'name': 'Grapefruit'},
+  {'id': 335, 'name': 'Game board'},
+  {'id': 336, 'name': 'Mop'},
+  {'id': 337, 'name': 'Radish'},
+  {'id': 338, 'name': 'Baozi'},
+  {'id': 339, 'name': 'Target'},
+  {'id': 340, 'name': 'French'},
+  {'id': 341, 'name': 'Spring Rolls'},
+  {'id': 342, 'name': 'Monkey'},
+  {'id': 343, 'name': 'Rabbit'},
+  {'id': 344, 'name': 'Pencil Case'},
+  {'id': 345, 'name': 'Yak'},
+  {'id': 346, 'name': 'Red Cabbage'},
+  {'id': 347, 'name': 'Binoculars'},
+  {'id': 348, 'name': 'Asparagus'},
+  {'id': 349, 'name': 'Barbell'},
+  {'id': 350, 'name': 'Scallop'},
+  {'id': 351, 'name': 'Noddles'},
+  {'id': 352, 'name': 'Comb'},
+  {'id': 353, 'name': 'Dumpling'},
+  {'id': 354, 'name': 'Oyster'},
+  {'id': 355, 'name': 'Table Tennis paddle'},
+  {'id': 356, 'name': 'Cosmetics Brush/Eyeliner Pencil'},
+  {'id': 357, 'name': 'Chainsaw'},
+  {'id': 358, 'name': 'Eraser'},
+  {'id': 359, 'name': 'Lobster'},
+  {'id': 360, 'name': 'Durian'},
+  {'id': 361, 'name': 'Okra'},
+  {'id': 362, 'name': 'Lipstick'},
+  {'id': 363, 'name': 'Cosmetics Mirror'},
+  {'id': 364, 'name': 'Curling'},
+  {'id': 365, 'name': 'Table Tennis '},
+]
+
+
+def _get_builtin_metadata():
+    id_to_name = {x['id']: x['name'] for x in categories_v2_fix}
+    thing_dataset_id_to_contiguous_id = {
+        x['id']: i for i, x in enumerate(
+            sorted(categories_v2_fix, key=lambda x: x['id']))}
+    thing_classes = [id_to_name[k] for k in sorted(id_to_name)]
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+
+_PREDEFINED_SPLITS_OBJECTS365 = {
+    "objects365_v2_train": ("objects365/train", "objects365/annotations/zhiyuan_objv2_train_fixname_fixmiss.json"),
+    # 80,000 images, 1,240,587 annotations
+    "objects365_v2_val": ("objects365/val", "objects365/annotations/zhiyuan_objv2_val_fixname.json"),
+    "objects365_v2_val_rare": ("objects365/val", "objects365/annotations/zhiyuan_objv2_val_fixname_rare.json"),
+}
+
+for key, (image_root, json_file) in _PREDEFINED_SPLITS_OBJECTS365.items():
+    register_coco_instances(
+        key,
+        _get_builtin_metadata(),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

detic/data/datasets/oid.py

@@ -0,0 +1,535 @@
+# Part of the code is from https://github.com/xingyizhou/UniDet/blob/master/projects/UniDet/unidet/data/datasets/oid.py
+# Copyright (c) Facebook, Inc. and its affiliates.
+from .register_oid import register_oid_instances
+import os
+
+categories = [
+    {'id': 1, 'name': 'Infant bed', 'freebase_id': '/m/061hd_'},
+    {'id': 2, 'name': 'Rose', 'freebase_id': '/m/06m11'},
+    {'id': 3, 'name': 'Flag', 'freebase_id': '/m/03120'},
+    {'id': 4, 'name': 'Flashlight', 'freebase_id': '/m/01kb5b'},
+    {'id': 5, 'name': 'Sea turtle', 'freebase_id': '/m/0120dh'},
+    {'id': 6, 'name': 'Camera', 'freebase_id': '/m/0dv5r'},
+    {'id': 7, 'name': 'Animal', 'freebase_id': '/m/0jbk'},
+    {'id': 8, 'name': 'Glove', 'freebase_id': '/m/0174n1'},
+    {'id': 9, 'name': 'Crocodile', 'freebase_id': '/m/09f_2'},
+    {'id': 10, 'name': 'Cattle', 'freebase_id': '/m/01xq0k1'},
+    {'id': 11, 'name': 'House', 'freebase_id': '/m/03jm5'},
+    {'id': 12, 'name': 'Guacamole', 'freebase_id': '/m/02g30s'},
+    {'id': 13, 'name': 'Penguin', 'freebase_id': '/m/05z6w'},
+    {'id': 14, 'name': 'Vehicle registration plate', 'freebase_id': '/m/01jfm_'},
+    {'id': 15, 'name': 'Bench', 'freebase_id': '/m/076lb9'},
+    {'id': 16, 'name': 'Ladybug', 'freebase_id': '/m/0gj37'},
+    {'id': 17, 'name': 'Human nose', 'freebase_id': '/m/0k0pj'},
+    {'id': 18, 'name': 'Watermelon', 'freebase_id': '/m/0kpqd'},
+    {'id': 19, 'name': 'Flute', 'freebase_id': '/m/0l14j_'},
+    {'id': 20, 'name': 'Butterfly', 'freebase_id': '/m/0cyf8'},
+    {'id': 21, 'name': 'Washing machine', 'freebase_id': '/m/0174k2'},
+    {'id': 22, 'name': 'Raccoon', 'freebase_id': '/m/0dq75'},
+    {'id': 23, 'name': 'Segway', 'freebase_id': '/m/076bq'},
+    {'id': 24, 'name': 'Taco', 'freebase_id': '/m/07crc'},
+    {'id': 25, 'name': 'Jellyfish', 'freebase_id': '/m/0d8zb'},
+    {'id': 26, 'name': 'Cake', 'freebase_id': '/m/0fszt'},
+    {'id': 27, 'name': 'Pen', 'freebase_id': '/m/0k1tl'},
+    {'id': 28, 'name': 'Cannon', 'freebase_id': '/m/020kz'},
+    {'id': 29, 'name': 'Bread', 'freebase_id': '/m/09728'},
+    {'id': 30, 'name': 'Tree', 'freebase_id': '/m/07j7r'},
+    {'id': 31, 'name': 'Shellfish', 'freebase_id': '/m/0fbdv'},
+    {'id': 32, 'name': 'Bed', 'freebase_id': '/m/03ssj5'},
+    {'id': 33, 'name': 'Hamster', 'freebase_id': '/m/03qrc'},
+    {'id': 34, 'name': 'Hat', 'freebase_id': '/m/02dl1y'},
+    {'id': 35, 'name': 'Toaster', 'freebase_id': '/m/01k6s3'},
+    {'id': 36, 'name': 'Sombrero', 'freebase_id': '/m/02jfl0'},
+    {'id': 37, 'name': 'Tiara', 'freebase_id': '/m/01krhy'},
+    {'id': 38, 'name': 'Bowl', 'freebase_id': '/m/04kkgm'},
+    {'id': 39, 'name': 'Dragonfly', 'freebase_id': '/m/0ft9s'},
+    {'id': 40, 'name': 'Moths and butterflies', 'freebase_id': '/m/0d_2m'},
+    {'id': 41, 'name': 'Antelope', 'freebase_id': '/m/0czz2'},
+    {'id': 42, 'name': 'Vegetable', 'freebase_id': '/m/0f4s2w'},
+    {'id': 43, 'name': 'Torch', 'freebase_id': '/m/07dd4'},
+    {'id': 44, 'name': 'Building', 'freebase_id': '/m/0cgh4'},
+    {'id': 45, 'name': 'Power plugs and sockets', 'freebase_id': '/m/03bbps'},
+    {'id': 46, 'name': 'Blender', 'freebase_id': '/m/02pjr4'},
+    {'id': 47, 'name': 'Billiard table', 'freebase_id': '/m/04p0qw'},
+    {'id': 48, 'name': 'Cutting board', 'freebase_id': '/m/02pdsw'},
+    {'id': 49, 'name': 'Bronze sculpture', 'freebase_id': '/m/01yx86'},
+    {'id': 50, 'name': 'Turtle', 'freebase_id': '/m/09dzg'},
+    {'id': 51, 'name': 'Broccoli', 'freebase_id': '/m/0hkxq'},
+    {'id': 52, 'name': 'Tiger', 'freebase_id': '/m/07dm6'},
+    {'id': 53, 'name': 'Mirror', 'freebase_id': '/m/054_l'},
+    {'id': 54, 'name': 'Bear', 'freebase_id': '/m/01dws'},
+    {'id': 55, 'name': 'Zucchini', 'freebase_id': '/m/027pcv'},
+    {'id': 56, 'name': 'Dress', 'freebase_id': '/m/01d40f'},
+    {'id': 57, 'name': 'Volleyball', 'freebase_id': '/m/02rgn06'},
+    {'id': 58, 'name': 'Guitar', 'freebase_id': '/m/0342h'},
+    {'id': 59, 'name': 'Reptile', 'freebase_id': '/m/06bt6'},
+    {'id': 60, 'name': 'Golf cart', 'freebase_id': '/m/0323sq'},
+    {'id': 61, 'name': 'Tart', 'freebase_id': '/m/02zvsm'},
+    {'id': 62, 'name': 'Fedora', 'freebase_id': '/m/02fq_6'},
+    {'id': 63, 'name': 'Carnivore', 'freebase_id': '/m/01lrl'},
+    {'id': 64, 'name': 'Car', 'freebase_id': '/m/0k4j'},
+    {'id': 65, 'name': 'Lighthouse', 'freebase_id': '/m/04h7h'},
+    {'id': 66, 'name': 'Coffeemaker', 'freebase_id': '/m/07xyvk'},
+    {'id': 67, 'name': 'Food processor', 'freebase_id': '/m/03y6mg'},
+    {'id': 68, 'name': 'Truck', 'freebase_id': '/m/07r04'},
+    {'id': 69, 'name': 'Bookcase', 'freebase_id': '/m/03__z0'},
+    {'id': 70, 'name': 'Surfboard', 'freebase_id': '/m/019w40'},
+    {'id': 71, 'name': 'Footwear', 'freebase_id': '/m/09j5n'},
+    {'id': 72, 'name': 'Bench', 'freebase_id': '/m/0cvnqh'},
+    {'id': 73, 'name': 'Necklace', 'freebase_id': '/m/01llwg'},
+    {'id': 74, 'name': 'Flower', 'freebase_id': '/m/0c9ph5'},
+    {'id': 75, 'name': 'Radish', 'freebase_id': '/m/015x5n'},
+    {'id': 76, 'name': 'Marine mammal', 'freebase_id': '/m/0gd2v'},
+    {'id': 77, 'name': 'Frying pan', 'freebase_id': '/m/04v6l4'},
+    {'id': 78, 'name': 'Tap', 'freebase_id': '/m/02jz0l'},
+    {'id': 79, 'name': 'Peach', 'freebase_id': '/m/0dj6p'},
+    {'id': 80, 'name': 'Knife', 'freebase_id': '/m/04ctx'},
+    {'id': 81, 'name': 'Handbag', 'freebase_id': '/m/080hkjn'},
+    {'id': 82, 'name': 'Laptop', 'freebase_id': '/m/01c648'},
+    {'id': 83, 'name': 'Tent', 'freebase_id': '/m/01j61q'},
+    {'id': 84, 'name': 'Ambulance', 'freebase_id': '/m/012n7d'},
+    {'id': 85, 'name': 'Christmas tree', 'freebase_id': '/m/025nd'},
+    {'id': 86, 'name': 'Eagle', 'freebase_id': '/m/09csl'},
+    {'id': 87, 'name': 'Limousine', 'freebase_id': '/m/01lcw4'},
+    {'id': 88, 'name': 'Kitchen & dining room table', 'freebase_id': '/m/0h8n5zk'},
+    {'id': 89, 'name': 'Polar bear', 'freebase_id': '/m/0633h'},
+    {'id': 90, 'name': 'Tower', 'freebase_id': '/m/01fdzj'},
+    {'id': 91, 'name': 'Football', 'freebase_id': '/m/01226z'},
+    {'id': 92, 'name': 'Willow', 'freebase_id': '/m/0mw_6'},
+    {'id': 93, 'name': 'Human head', 'freebase_id': '/m/04hgtk'},
+    {'id': 94, 'name': 'Stop sign', 'freebase_id': '/m/02pv19'},
+    {'id': 95, 'name': 'Banana', 'freebase_id': '/m/09qck'},
+    {'id': 96, 'name': 'Mixer', 'freebase_id': '/m/063rgb'},
+    {'id': 97, 'name': 'Binoculars', 'freebase_id': '/m/0lt4_'},
+    {'id': 98, 'name': 'Dessert', 'freebase_id': '/m/0270h'},
+    {'id': 99, 'name': 'Bee', 'freebase_id': '/m/01h3n'},
+    {'id': 100, 'name': 'Chair', 'freebase_id': '/m/01mzpv'},
+    {'id': 101, 'name': 'Wood-burning stove', 'freebase_id': '/m/04169hn'},
+    {'id': 102, 'name': 'Flowerpot', 'freebase_id': '/m/0fm3zh'},
+    {'id': 103, 'name': 'Beaker', 'freebase_id': '/m/0d20w4'},
+    {'id': 104, 'name': 'Oyster', 'freebase_id': '/m/0_cp5'},
+    {'id': 105, 'name': 'Woodpecker', 'freebase_id': '/m/01dy8n'},
+    {'id': 106, 'name': 'Harp', 'freebase_id': '/m/03m5k'},
+    {'id': 107, 'name': 'Bathtub', 'freebase_id': '/m/03dnzn'},
+    {'id': 108, 'name': 'Wall clock', 'freebase_id': '/m/0h8mzrc'},
+    {'id': 109, 'name': 'Sports uniform', 'freebase_id': '/m/0h8mhzd'},
+    {'id': 110, 'name': 'Rhinoceros', 'freebase_id': '/m/03d443'},
+    {'id': 111, 'name': 'Beehive', 'freebase_id': '/m/01gllr'},
+    {'id': 112, 'name': 'Cupboard', 'freebase_id': '/m/0642b4'},
+    {'id': 113, 'name': 'Chicken', 'freebase_id': '/m/09b5t'},
+    {'id': 114, 'name': 'Man', 'freebase_id': '/m/04yx4'},
+    {'id': 115, 'name': 'Blue jay', 'freebase_id': '/m/01f8m5'},
+    {'id': 116, 'name': 'Cucumber', 'freebase_id': '/m/015x4r'},
+    {'id': 117, 'name': 'Balloon', 'freebase_id': '/m/01j51'},
+    {'id': 118, 'name': 'Kite', 'freebase_id': '/m/02zt3'},
+    {'id': 119, 'name': 'Fireplace', 'freebase_id': '/m/03tw93'},
+    {'id': 120, 'name': 'Lantern', 'freebase_id': '/m/01jfsr'},
+    {'id': 121, 'name': 'Missile', 'freebase_id': '/m/04ylt'},
+    {'id': 122, 'name': 'Book', 'freebase_id': '/m/0bt_c3'},
+    {'id': 123, 'name': 'Spoon', 'freebase_id': '/m/0cmx8'},
+    {'id': 124, 'name': 'Grapefruit', 'freebase_id': '/m/0hqkz'},
+    {'id': 125, 'name': 'Squirrel', 'freebase_id': '/m/071qp'},
+    {'id': 126, 'name': 'Orange', 'freebase_id': '/m/0cyhj_'},
+    {'id': 127, 'name': 'Coat', 'freebase_id': '/m/01xygc'},
+    {'id': 128, 'name': 'Punching bag', 'freebase_id': '/m/0420v5'},
+    {'id': 129, 'name': 'Zebra', 'freebase_id': '/m/0898b'},
+    {'id': 130, 'name': 'Billboard', 'freebase_id': '/m/01knjb'},
+    {'id': 131, 'name': 'Bicycle', 'freebase_id': '/m/0199g'},
+    {'id': 132, 'name': 'Door handle', 'freebase_id': '/m/03c7gz'},
+    {'id': 133, 'name': 'Mechanical fan', 'freebase_id': '/m/02x984l'},
+    {'id': 134, 'name': 'Ring binder', 'freebase_id': '/m/04zwwv'},
+    {'id': 135, 'name': 'Table', 'freebase_id': '/m/04bcr3'},
+    {'id': 136, 'name': 'Parrot', 'freebase_id': '/m/0gv1x'},
+    {'id': 137, 'name': 'Sock', 'freebase_id': '/m/01nq26'},
+    {'id': 138, 'name': 'Vase', 'freebase_id': '/m/02s195'},
+    {'id': 139, 'name': 'Weapon', 'freebase_id': '/m/083kb'},
+    {'id': 140, 'name': 'Shotgun', 'freebase_id': '/m/06nrc'},
+    {'id': 141, 'name': 'Glasses', 'freebase_id': '/m/0jyfg'},
+    {'id': 142, 'name': 'Seahorse', 'freebase_id': '/m/0nybt'},
+    {'id': 143, 'name': 'Belt', 'freebase_id': '/m/0176mf'},
+    {'id': 144, 'name': 'Watercraft', 'freebase_id': '/m/01rzcn'},
+    {'id': 145, 'name': 'Window', 'freebase_id': '/m/0d4v4'},
+    {'id': 146, 'name': 'Giraffe', 'freebase_id': '/m/03bk1'},
+    {'id': 147, 'name': 'Lion', 'freebase_id': '/m/096mb'},
+    {'id': 148, 'name': 'Tire', 'freebase_id': '/m/0h9mv'},
+    {'id': 149, 'name': 'Vehicle', 'freebase_id': '/m/07yv9'},
+    {'id': 150, 'name': 'Canoe', 'freebase_id': '/m/0ph39'},
+    {'id': 151, 'name': 'Tie', 'freebase_id': '/m/01rkbr'},
+    {'id': 152, 'name': 'Shelf', 'freebase_id': '/m/0gjbg72'},
+    {'id': 153, 'name': 'Picture frame', 'freebase_id': '/m/06z37_'},
+    {'id': 154, 'name': 'Printer', 'freebase_id': '/m/01m4t'},
+    {'id': 155, 'name': 'Human leg', 'freebase_id': '/m/035r7c'},
+    {'id': 156, 'name': 'Boat', 'freebase_id': '/m/019jd'},
+    {'id': 157, 'name': 'Slow cooker', 'freebase_id': '/m/02tsc9'},
+    {'id': 158, 'name': 'Croissant', 'freebase_id': '/m/015wgc'},
+    {'id': 159, 'name': 'Candle', 'freebase_id': '/m/0c06p'},
+    {'id': 160, 'name': 'Pancake', 'freebase_id': '/m/01dwwc'},
+    {'id': 161, 'name': 'Pillow', 'freebase_id': '/m/034c16'},
+    {'id': 162, 'name': 'Coin', 'freebase_id': '/m/0242l'},
+    {'id': 163, 'name': 'Stretcher', 'freebase_id': '/m/02lbcq'},
+    {'id': 164, 'name': 'Sandal', 'freebase_id': '/m/03nfch'},
+    {'id': 165, 'name': 'Woman', 'freebase_id': '/m/03bt1vf'},
+    {'id': 166, 'name': 'Stairs', 'freebase_id': '/m/01lynh'},
+    {'id': 167, 'name': 'Harpsichord', 'freebase_id': '/m/03q5t'},
+    {'id': 168, 'name': 'Stool', 'freebase_id': '/m/0fqt361'},
+    {'id': 169, 'name': 'Bus', 'freebase_id': '/m/01bjv'},
+    {'id': 170, 'name': 'Suitcase', 'freebase_id': '/m/01s55n'},
+    {'id': 171, 'name': 'Human mouth', 'freebase_id': '/m/0283dt1'},
+    {'id': 172, 'name': 'Juice', 'freebase_id': '/m/01z1kdw'},
+    {'id': 173, 'name': 'Skull', 'freebase_id': '/m/016m2d'},
+    {'id': 174, 'name': 'Door', 'freebase_id': '/m/02dgv'},
+    {'id': 175, 'name': 'Violin', 'freebase_id': '/m/07y_7'},
+    {'id': 176, 'name': 'Chopsticks', 'freebase_id': '/m/01_5g'},
+    {'id': 177, 'name': 'Digital clock', 'freebase_id': '/m/06_72j'},
+    {'id': 178, 'name': 'Sunflower', 'freebase_id': '/m/0ftb8'},
+    {'id': 179, 'name': 'Leopard', 'freebase_id': '/m/0c29q'},
+    {'id': 180, 'name': 'Bell pepper', 'freebase_id': '/m/0jg57'},
+    {'id': 181, 'name': 'Harbor seal', 'freebase_id': '/m/02l8p9'},
+    {'id': 182, 'name': 'Snake', 'freebase_id': '/m/078jl'},
+    {'id': 183, 'name': 'Sewing machine', 'freebase_id': '/m/0llzx'},
+    {'id': 184, 'name': 'Goose', 'freebase_id': '/m/0dbvp'},
+    {'id': 185, 'name': 'Helicopter', 'freebase_id': '/m/09ct_'},
+    {'id': 186, 'name': 'Seat belt', 'freebase_id': '/m/0dkzw'},
+    {'id': 187, 'name': 'Coffee cup', 'freebase_id': '/m/02p5f1q'},
+    {'id': 188, 'name': 'Microwave oven', 'freebase_id': '/m/0fx9l'},
+    {'id': 189, 'name': 'Hot dog', 'freebase_id': '/m/01b9xk'},
+    {'id': 190, 'name': 'Countertop', 'freebase_id': '/m/0b3fp9'},
+    {'id': 191, 'name': 'Serving tray', 'freebase_id': '/m/0h8n27j'},
+    {'id': 192, 'name': 'Dog bed', 'freebase_id': '/m/0h8n6f9'},
+    {'id': 193, 'name': 'Beer', 'freebase_id': '/m/01599'},
+    {'id': 194, 'name': 'Sunglasses', 'freebase_id': '/m/017ftj'},
+    {'id': 195, 'name': 'Golf ball', 'freebase_id': '/m/044r5d'},
+    {'id': 196, 'name': 'Waffle', 'freebase_id': '/m/01dwsz'},
+    {'id': 197, 'name': 'Palm tree', 'freebase_id': '/m/0cdl1'},
+    {'id': 198, 'name': 'Trumpet', 'freebase_id': '/m/07gql'},
+    {'id': 199, 'name': 'Ruler', 'freebase_id': '/m/0hdln'},
+    {'id': 200, 'name': 'Helmet', 'freebase_id': '/m/0zvk5'},
+    {'id': 201, 'name': 'Ladder', 'freebase_id': '/m/012w5l'},
+    {'id': 202, 'name': 'Office building', 'freebase_id': '/m/021sj1'},
+    {'id': 203, 'name': 'Tablet computer', 'freebase_id': '/m/0bh9flk'},
+    {'id': 204, 'name': 'Toilet paper', 'freebase_id': '/m/09gtd'},
+    {'id': 205, 'name': 'Pomegranate', 'freebase_id': '/m/0jwn_'},
+    {'id': 206, 'name': 'Skirt', 'freebase_id': '/m/02wv6h6'},
+    {'id': 207, 'name': 'Gas stove', 'freebase_id': '/m/02wv84t'},
+    {'id': 208, 'name': 'Cookie', 'freebase_id': '/m/021mn'},
+    {'id': 209, 'name': 'Cart', 'freebase_id': '/m/018p4k'},
+    {'id': 210, 'name': 'Raven', 'freebase_id': '/m/06j2d'},
+    {'id': 211, 'name': 'Egg', 'freebase_id': '/m/033cnk'},
+    {'id': 212, 'name': 'Burrito', 'freebase_id': '/m/01j3zr'},
+    {'id': 213, 'name': 'Goat', 'freebase_id': '/m/03fwl'},
+    {'id': 214, 'name': 'Kitchen knife', 'freebase_id': '/m/058qzx'},
+    {'id': 215, 'name': 'Skateboard', 'freebase_id': '/m/06_fw'},
+    {'id': 216, 'name': 'Salt and pepper shakers', 'freebase_id': '/m/02x8cch'},
+    {'id': 217, 'name': 'Lynx', 'freebase_id': '/m/04g2r'},
+    {'id': 218, 'name': 'Boot', 'freebase_id': '/m/01b638'},
+    {'id': 219, 'name': 'Platter', 'freebase_id': '/m/099ssp'},
+    {'id': 220, 'name': 'Ski', 'freebase_id': '/m/071p9'},
+    {'id': 221, 'name': 'Swimwear', 'freebase_id': '/m/01gkx_'},
+    {'id': 222, 'name': 'Swimming pool', 'freebase_id': '/m/0b_rs'},
+    {'id': 223, 'name': 'Drinking straw', 'freebase_id': '/m/03v5tg'},
+    {'id': 224, 'name': 'Wrench', 'freebase_id': '/m/01j5ks'},
+    {'id': 225, 'name': 'Drum', 'freebase_id': '/m/026t6'},
+    {'id': 226, 'name': 'Ant', 'freebase_id': '/m/0_k2'},
+    {'id': 227, 'name': 'Human ear', 'freebase_id': '/m/039xj_'},
+    {'id': 228, 'name': 'Headphones', 'freebase_id': '/m/01b7fy'},
+    {'id': 229, 'name': 'Fountain', 'freebase_id': '/m/0220r2'},
+    {'id': 230, 'name': 'Bird', 'freebase_id': '/m/015p6'},
+    {'id': 231, 'name': 'Jeans', 'freebase_id': '/m/0fly7'},
+    {'id': 232, 'name': 'Television', 'freebase_id': '/m/07c52'},
+    {'id': 233, 'name': 'Crab', 'freebase_id': '/m/0n28_'},
+    {'id': 234, 'name': 'Microphone', 'freebase_id': '/m/0hg7b'},
+    {'id': 235, 'name': 'Home appliance', 'freebase_id': '/m/019dx1'},
+    {'id': 236, 'name': 'Snowplow', 'freebase_id': '/m/04vv5k'},
+    {'id': 237, 'name': 'Beetle', 'freebase_id': '/m/020jm'},
+    {'id': 238, 'name': 'Artichoke', 'freebase_id': '/m/047v4b'},
+    {'id': 239, 'name': 'Jet ski', 'freebase_id': '/m/01xs3r'},
+    {'id': 240, 'name': 'Stationary bicycle', 'freebase_id': '/m/03kt2w'},
+    {'id': 241, 'name': 'Human hair', 'freebase_id': '/m/03q69'},
+    {'id': 242, 'name': 'Brown bear', 'freebase_id': '/m/01dxs'},
+    {'id': 243, 'name': 'Starfish', 'freebase_id': '/m/01h8tj'},
+    {'id': 244, 'name': 'Fork', 'freebase_id': '/m/0dt3t'},
+    {'id': 245, 'name': 'Lobster', 'freebase_id': '/m/0cjq5'},
+    {'id': 246, 'name': 'Corded phone', 'freebase_id': '/m/0h8lkj8'},
+    {'id': 247, 'name': 'Drink', 'freebase_id': '/m/0271t'},
+    {'id': 248, 'name': 'Saucer', 'freebase_id': '/m/03q5c7'},
+    {'id': 249, 'name': 'Carrot', 'freebase_id': '/m/0fj52s'},
+    {'id': 250, 'name': 'Insect', 'freebase_id': '/m/03vt0'},
+    {'id': 251, 'name': 'Clock', 'freebase_id': '/m/01x3z'},
+    {'id': 252, 'name': 'Castle', 'freebase_id': '/m/0d5gx'},
+    {'id': 253, 'name': 'Tennis racket', 'freebase_id': '/m/0h8my_4'},
+    {'id': 254, 'name': 'Ceiling fan', 'freebase_id': '/m/03ldnb'},
+    {'id': 255, 'name': 'Asparagus', 'freebase_id': '/m/0cjs7'},
+    {'id': 256, 'name': 'Jaguar', 'freebase_id': '/m/0449p'},
+    {'id': 257, 'name': 'Musical instrument', 'freebase_id': '/m/04szw'},
+    {'id': 258, 'name': 'Train', 'freebase_id': '/m/07jdr'},
+    {'id': 259, 'name': 'Cat', 'freebase_id': '/m/01yrx'},
+    {'id': 260, 'name': 'Rifle', 'freebase_id': '/m/06c54'},
+    {'id': 261, 'name': 'Dumbbell', 'freebase_id': '/m/04h8sr'},
+    {'id': 262, 'name': 'Mobile phone', 'freebase_id': '/m/050k8'},
+    {'id': 263, 'name': 'Taxi', 'freebase_id': '/m/0pg52'},
+    {'id': 264, 'name': 'Shower', 'freebase_id': '/m/02f9f_'},
+    {'id': 265, 'name': 'Pitcher', 'freebase_id': '/m/054fyh'},
+    {'id': 266, 'name': 'Lemon', 'freebase_id': '/m/09k_b'},
+    {'id': 267, 'name': 'Invertebrate', 'freebase_id': '/m/03xxp'},
+    {'id': 268, 'name': 'Turkey', 'freebase_id': '/m/0jly1'},
+    {'id': 269, 'name': 'High heels', 'freebase_id': '/m/06k2mb'},
+    {'id': 270, 'name': 'Bust', 'freebase_id': '/m/04yqq2'},
+    {'id': 271, 'name': 'Elephant', 'freebase_id': '/m/0bwd_0j'},
+    {'id': 272, 'name': 'Scarf', 'freebase_id': '/m/02h19r'},
+    {'id': 273, 'name': 'Barrel', 'freebase_id': '/m/02zn6n'},
+    {'id': 274, 'name': 'Trombone', 'freebase_id': '/m/07c6l'},
+    {'id': 275, 'name': 'Pumpkin', 'freebase_id': '/m/05zsy'},
+    {'id': 276, 'name': 'Box', 'freebase_id': '/m/025dyy'},
+    {'id': 277, 'name': 'Tomato', 'freebase_id': '/m/07j87'},
+    {'id': 278, 'name': 'Frog', 'freebase_id': '/m/09ld4'},
+    {'id': 279, 'name': 'Bidet', 'freebase_id': '/m/01vbnl'},
+    {'id': 280, 'name': 'Human face', 'freebase_id': '/m/0dzct'},
+    {'id': 281, 'name': 'Houseplant', 'freebase_id': '/m/03fp41'},
+    {'id': 282, 'name': 'Van', 'freebase_id': '/m/0h2r6'},
+    {'id': 283, 'name': 'Shark', 'freebase_id': '/m/0by6g'},
+    {'id': 284, 'name': 'Ice cream', 'freebase_id': '/m/0cxn2'},
+    {'id': 285, 'name': 'Swim cap', 'freebase_id': '/m/04tn4x'},
+    {'id': 286, 'name': 'Falcon', 'freebase_id': '/m/0f6wt'},
+    {'id': 287, 'name': 'Ostrich', 'freebase_id': '/m/05n4y'},
+    {'id': 288, 'name': 'Handgun', 'freebase_id': '/m/0gxl3'},
+    {'id': 289, 'name': 'Whiteboard', 'freebase_id': '/m/02d9qx'},
+    {'id': 290, 'name': 'Lizard', 'freebase_id': '/m/04m9y'},
+    {'id': 291, 'name': 'Pasta', 'freebase_id': '/m/05z55'},
+    {'id': 292, 'name': 'Snowmobile', 'freebase_id': '/m/01x3jk'},
+    {'id': 293, 'name': 'Light bulb', 'freebase_id': '/m/0h8l4fh'},
+    {'id': 294, 'name': 'Window blind', 'freebase_id': '/m/031b6r'},
+    {'id': 295, 'name': 'Muffin', 'freebase_id': '/m/01tcjp'},
+    {'id': 296, 'name': 'Pretzel', 'freebase_id': '/m/01f91_'},
+    {'id': 297, 'name': 'Computer monitor', 'freebase_id': '/m/02522'},
+    {'id': 298, 'name': 'Horn', 'freebase_id': '/m/0319l'},
+    {'id': 299, 'name': 'Furniture', 'freebase_id': '/m/0c_jw'},
+    {'id': 300, 'name': 'Sandwich', 'freebase_id': '/m/0l515'},
+    {'id': 301, 'name': 'Fox', 'freebase_id': '/m/0306r'},
+    {'id': 302, 'name': 'Convenience store', 'freebase_id': '/m/0crjs'},
+    {'id': 303, 'name': 'Fish', 'freebase_id': '/m/0ch_cf'},
+    {'id': 304, 'name': 'Fruit', 'freebase_id': '/m/02xwb'},
+    {'id': 305, 'name': 'Earrings', 'freebase_id': '/m/01r546'},
+    {'id': 306, 'name': 'Curtain', 'freebase_id': '/m/03rszm'},
+    {'id': 307, 'name': 'Grape', 'freebase_id': '/m/0388q'},
+    {'id': 308, 'name': 'Sofa bed', 'freebase_id': '/m/03m3pdh'},
+    {'id': 309, 'name': 'Horse', 'freebase_id': '/m/03k3r'},
+    {'id': 310, 'name': 'Luggage and bags', 'freebase_id': '/m/0hf58v5'},
+    {'id': 311, 'name': 'Desk', 'freebase_id': '/m/01y9k5'},
+    {'id': 312, 'name': 'Crutch', 'freebase_id': '/m/05441v'},
+    {'id': 313, 'name': 'Bicycle helmet', 'freebase_id': '/m/03p3bw'},
+    {'id': 314, 'name': 'Tick', 'freebase_id': '/m/0175cv'},
+    {'id': 315, 'name': 'Airplane', 'freebase_id': '/m/0cmf2'},
+    {'id': 316, 'name': 'Canary', 'freebase_id': '/m/0ccs93'},
+    {'id': 317, 'name': 'Spatula', 'freebase_id': '/m/02d1br'},
+    {'id': 318, 'name': 'Watch', 'freebase_id': '/m/0gjkl'},
+    {'id': 319, 'name': 'Lily', 'freebase_id': '/m/0jqgx'},
+    {'id': 320, 'name': 'Kitchen appliance', 'freebase_id': '/m/0h99cwc'},
+    {'id': 321, 'name': 'Filing cabinet', 'freebase_id': '/m/047j0r'},
+    {'id': 322, 'name': 'Aircraft', 'freebase_id': '/m/0k5j'},
+    {'id': 323, 'name': 'Cake stand', 'freebase_id': '/m/0h8n6ft'},
+    {'id': 324, 'name': 'Candy', 'freebase_id': '/m/0gm28'},
+    {'id': 325, 'name': 'Sink', 'freebase_id': '/m/0130jx'},
+    {'id': 326, 'name': 'Mouse', 'freebase_id': '/m/04rmv'},
+    {'id': 327, 'name': 'Wine', 'freebase_id': '/m/081qc'},
+    {'id': 328, 'name': 'Wheelchair', 'freebase_id': '/m/0qmmr'},
+    {'id': 329, 'name': 'Goldfish', 'freebase_id': '/m/03fj2'},
+    {'id': 330, 'name': 'Refrigerator', 'freebase_id': '/m/040b_t'},
+    {'id': 331, 'name': 'French fries', 'freebase_id': '/m/02y6n'},
+    {'id': 332, 'name': 'Drawer', 'freebase_id': '/m/0fqfqc'},
+    {'id': 333, 'name': 'Treadmill', 'freebase_id': '/m/030610'},
+    {'id': 334, 'name': 'Picnic basket', 'freebase_id': '/m/07kng9'},
+    {'id': 335, 'name': 'Dice', 'freebase_id': '/m/029b3'},
+    {'id': 336, 'name': 'Cabbage', 'freebase_id': '/m/0fbw6'},
+    {'id': 337, 'name': 'Football helmet', 'freebase_id': '/m/07qxg_'},
+    {'id': 338, 'name': 'Pig', 'freebase_id': '/m/068zj'},
+    {'id': 339, 'name': 'Person', 'freebase_id': '/m/01g317'},
+    {'id': 340, 'name': 'Shorts', 'freebase_id': '/m/01bfm9'},
+    {'id': 341, 'name': 'Gondola', 'freebase_id': '/m/02068x'},
+    {'id': 342, 'name': 'Honeycomb', 'freebase_id': '/m/0fz0h'},
+    {'id': 343, 'name': 'Doughnut', 'freebase_id': '/m/0jy4k'},
+    {'id': 344, 'name': 'Chest of drawers', 'freebase_id': '/m/05kyg_'},
+    {'id': 345, 'name': 'Land vehicle', 'freebase_id': '/m/01prls'},
+    {'id': 346, 'name': 'Bat', 'freebase_id': '/m/01h44'},
+    {'id': 347, 'name': 'Monkey', 'freebase_id': '/m/08pbxl'},
+    {'id': 348, 'name': 'Dagger', 'freebase_id': '/m/02gzp'},
+    {'id': 349, 'name': 'Tableware', 'freebase_id': '/m/04brg2'},
+    {'id': 350, 'name': 'Human foot', 'freebase_id': '/m/031n1'},
+    {'id': 351, 'name': 'Mug', 'freebase_id': '/m/02jvh9'},
+    {'id': 352, 'name': 'Alarm clock', 'freebase_id': '/m/046dlr'},
+    {'id': 353, 'name': 'Pressure cooker', 'freebase_id': '/m/0h8ntjv'},
+    {'id': 354, 'name': 'Human hand', 'freebase_id': '/m/0k65p'},
+    {'id': 355, 'name': 'Tortoise', 'freebase_id': '/m/011k07'},
+    {'id': 356, 'name': 'Baseball glove', 'freebase_id': '/m/03grzl'},
+    {'id': 357, 'name': 'Sword', 'freebase_id': '/m/06y5r'},
+    {'id': 358, 'name': 'Pear', 'freebase_id': '/m/061_f'},
+    {'id': 359, 'name': 'Miniskirt', 'freebase_id': '/m/01cmb2'},
+    {'id': 360, 'name': 'Traffic sign', 'freebase_id': '/m/01mqdt'},
+    {'id': 361, 'name': 'Girl', 'freebase_id': '/m/05r655'},
+    {'id': 362, 'name': 'Roller skates', 'freebase_id': '/m/02p3w7d'},
+    {'id': 363, 'name': 'Dinosaur', 'freebase_id': '/m/029tx'},
+    {'id': 364, 'name': 'Porch', 'freebase_id': '/m/04m6gz'},
+    {'id': 365, 'name': 'Human beard', 'freebase_id': '/m/015h_t'},
+    {'id': 366, 'name': 'Submarine sandwich', 'freebase_id': '/m/06pcq'},
+    {'id': 367, 'name': 'Screwdriver', 'freebase_id': '/m/01bms0'},
+    {'id': 368, 'name': 'Strawberry', 'freebase_id': '/m/07fbm7'},
+    {'id': 369, 'name': 'Wine glass', 'freebase_id': '/m/09tvcd'},
+    {'id': 370, 'name': 'Seafood', 'freebase_id': '/m/06nwz'},
+    {'id': 371, 'name': 'Racket', 'freebase_id': '/m/0dv9c'},
+    {'id': 372, 'name': 'Wheel', 'freebase_id': '/m/083wq'},
+    {'id': 373, 'name': 'Sea lion', 'freebase_id': '/m/0gd36'},
+    {'id': 374, 'name': 'Toy', 'freebase_id': '/m/0138tl'},
+    {'id': 375, 'name': 'Tea', 'freebase_id': '/m/07clx'},
+    {'id': 376, 'name': 'Tennis ball', 'freebase_id': '/m/05ctyq'},
+    {'id': 377, 'name': 'Waste container', 'freebase_id': '/m/0bjyj5'},
+    {'id': 378, 'name': 'Mule', 'freebase_id': '/m/0dbzx'},
+    {'id': 379, 'name': 'Cricket ball', 'freebase_id': '/m/02ctlc'},
+    {'id': 380, 'name': 'Pineapple', 'freebase_id': '/m/0fp6w'},
+    {'id': 381, 'name': 'Coconut', 'freebase_id': '/m/0djtd'},
+    {'id': 382, 'name': 'Doll', 'freebase_id': '/m/0167gd'},
+    {'id': 383, 'name': 'Coffee table', 'freebase_id': '/m/078n6m'},
+    {'id': 384, 'name': 'Snowman', 'freebase_id': '/m/0152hh'},
+    {'id': 385, 'name': 'Lavender', 'freebase_id': '/m/04gth'},
+    {'id': 386, 'name': 'Shrimp', 'freebase_id': '/m/0ll1f78'},
+    {'id': 387, 'name': 'Maple', 'freebase_id': '/m/0cffdh'},
+    {'id': 388, 'name': 'Cowboy hat', 'freebase_id': '/m/025rp__'},
+    {'id': 389, 'name': 'Goggles', 'freebase_id': '/m/02_n6y'},
+    {'id': 390, 'name': 'Rugby ball', 'freebase_id': '/m/0wdt60w'},
+    {'id': 391, 'name': 'Caterpillar', 'freebase_id': '/m/0cydv'},
+    {'id': 392, 'name': 'Poster', 'freebase_id': '/m/01n5jq'},
+    {'id': 393, 'name': 'Rocket', 'freebase_id': '/m/09rvcxw'},
+    {'id': 394, 'name': 'Organ', 'freebase_id': '/m/013y1f'},
+    {'id': 395, 'name': 'Saxophone', 'freebase_id': '/m/06ncr'},
+    {'id': 396, 'name': 'Traffic light', 'freebase_id': '/m/015qff'},
+    {'id': 397, 'name': 'Cocktail', 'freebase_id': '/m/024g6'},
+    {'id': 398, 'name': 'Plastic bag', 'freebase_id': '/m/05gqfk'},
+    {'id': 399, 'name': 'Squash', 'freebase_id': '/m/0dv77'},
+    {'id': 400, 'name': 'Mushroom', 'freebase_id': '/m/052sf'},
+    {'id': 401, 'name': 'Hamburger', 'freebase_id': '/m/0cdn1'},
+    {'id': 402, 'name': 'Light switch', 'freebase_id': '/m/03jbxj'},
+    {'id': 403, 'name': 'Parachute', 'freebase_id': '/m/0cyfs'},
+    {'id': 404, 'name': 'Teddy bear', 'freebase_id': '/m/0kmg4'},
+    {'id': 405, 'name': 'Winter melon', 'freebase_id': '/m/02cvgx'},
+    {'id': 406, 'name': 'Deer', 'freebase_id': '/m/09kx5'},
+    {'id': 407, 'name': 'Musical keyboard', 'freebase_id': '/m/057cc'},
+    {'id': 408, 'name': 'Plumbing fixture', 'freebase_id': '/m/02pkr5'},
+    {'id': 409, 'name': 'Scoreboard', 'freebase_id': '/m/057p5t'},
+    {'id': 410, 'name': 'Baseball bat', 'freebase_id': '/m/03g8mr'},
+    {'id': 411, 'name': 'Envelope', 'freebase_id': '/m/0frqm'},
+    {'id': 412, 'name': 'Adhesive tape', 'freebase_id': '/m/03m3vtv'},
+    {'id': 413, 'name': 'Briefcase', 'freebase_id': '/m/0584n8'},
+    {'id': 414, 'name': 'Paddle', 'freebase_id': '/m/014y4n'},
+    {'id': 415, 'name': 'Bow and arrow', 'freebase_id': '/m/01g3x7'},
+    {'id': 416, 'name': 'Telephone', 'freebase_id': '/m/07cx4'},
+    {'id': 417, 'name': 'Sheep', 'freebase_id': '/m/07bgp'},
+    {'id': 418, 'name': 'Jacket', 'freebase_id': '/m/032b3c'},
+    {'id': 419, 'name': 'Boy', 'freebase_id': '/m/01bl7v'},
+    {'id': 420, 'name': 'Pizza', 'freebase_id': '/m/0663v'},
+    {'id': 421, 'name': 'Otter', 'freebase_id': '/m/0cn6p'},
+    {'id': 422, 'name': 'Office supplies', 'freebase_id': '/m/02rdsp'},
+    {'id': 423, 'name': 'Couch', 'freebase_id': '/m/02crq1'},
+    {'id': 424, 'name': 'Cello', 'freebase_id': '/m/01xqw'},
+    {'id': 425, 'name': 'Bull', 'freebase_id': '/m/0cnyhnx'},
+    {'id': 426, 'name': 'Camel', 'freebase_id': '/m/01x_v'},
+    {'id': 427, 'name': 'Ball', 'freebase_id': '/m/018xm'},
+    {'id': 428, 'name': 'Duck', 'freebase_id': '/m/09ddx'},
+    {'id': 429, 'name': 'Whale', 'freebase_id': '/m/084zz'},
+    {'id': 430, 'name': 'Shirt', 'freebase_id': '/m/01n4qj'},
+    {'id': 431, 'name': 'Tank', 'freebase_id': '/m/07cmd'},
+    {'id': 432, 'name': 'Motorcycle', 'freebase_id': '/m/04_sv'},
+    {'id': 433, 'name': 'Accordion', 'freebase_id': '/m/0mkg'},
+    {'id': 434, 'name': 'Owl', 'freebase_id': '/m/09d5_'},
+    {'id': 435, 'name': 'Porcupine', 'freebase_id': '/m/0c568'},
+    {'id': 436, 'name': 'Sun hat', 'freebase_id': '/m/02wbtzl'},
+    {'id': 437, 'name': 'Nail', 'freebase_id': '/m/05bm6'},
+    {'id': 438, 'name': 'Scissors', 'freebase_id': '/m/01lsmm'},
+    {'id': 439, 'name': 'Swan', 'freebase_id': '/m/0dftk'},
+    {'id': 440, 'name': 'Lamp', 'freebase_id': '/m/0dtln'},
+    {'id': 441, 'name': 'Crown', 'freebase_id': '/m/0nl46'},
+    {'id': 442, 'name': 'Piano', 'freebase_id': '/m/05r5c'},
+    {'id': 443, 'name': 'Sculpture', 'freebase_id': '/m/06msq'},
+    {'id': 444, 'name': 'Cheetah', 'freebase_id': '/m/0cd4d'},
+    {'id': 445, 'name': 'Oboe', 'freebase_id': '/m/05kms'},
+    {'id': 446, 'name': 'Tin can', 'freebase_id': '/m/02jnhm'},
+    {'id': 447, 'name': 'Mango', 'freebase_id': '/m/0fldg'},
+    {'id': 448, 'name': 'Tripod', 'freebase_id': '/m/073bxn'},
+    {'id': 449, 'name': 'Oven', 'freebase_id': '/m/029bxz'},
+    {'id': 450, 'name': 'Mouse', 'freebase_id': '/m/020lf'},
+    {'id': 451, 'name': 'Barge', 'freebase_id': '/m/01btn'},
+    {'id': 452, 'name': 'Coffee', 'freebase_id': '/m/02vqfm'},
+    {'id': 453, 'name': 'Snowboard', 'freebase_id': '/m/06__v'},
+    {'id': 454, 'name': 'Common fig', 'freebase_id': '/m/043nyj'},
+    {'id': 455, 'name': 'Salad', 'freebase_id': '/m/0grw1'},
+    {'id': 456, 'name': 'Marine invertebrates', 'freebase_id': '/m/03hl4l9'},
+    {'id': 457, 'name': 'Umbrella', 'freebase_id': '/m/0hnnb'},
+    {'id': 458, 'name': 'Kangaroo', 'freebase_id': '/m/04c0y'},
+    {'id': 459, 'name': 'Human arm', 'freebase_id': '/m/0dzf4'},
+    {'id': 460, 'name': 'Measuring cup', 'freebase_id': '/m/07v9_z'},
+    {'id': 461, 'name': 'Snail', 'freebase_id': '/m/0f9_l'},
+    {'id': 462, 'name': 'Loveseat', 'freebase_id': '/m/0703r8'},
+    {'id': 463, 'name': 'Suit', 'freebase_id': '/m/01xyhv'},
+    {'id': 464, 'name': 'Teapot', 'freebase_id': '/m/01fh4r'},
+    {'id': 465, 'name': 'Bottle', 'freebase_id': '/m/04dr76w'},
+    {'id': 466, 'name': 'Alpaca', 'freebase_id': '/m/0pcr'},
+    {'id': 467, 'name': 'Kettle', 'freebase_id': '/m/03s_tn'},
+    {'id': 468, 'name': 'Trousers', 'freebase_id': '/m/07mhn'},
+    {'id': 469, 'name': 'Popcorn', 'freebase_id': '/m/01hrv5'},
+    {'id': 470, 'name': 'Centipede', 'freebase_id': '/m/019h78'},
+    {'id': 471, 'name': 'Spider', 'freebase_id': '/m/09kmb'},
+    {'id': 472, 'name': 'Sparrow', 'freebase_id': '/m/0h23m'},
+    {'id': 473, 'name': 'Plate', 'freebase_id': '/m/050gv4'},
+    {'id': 474, 'name': 'Bagel', 'freebase_id': '/m/01fb_0'},
+    {'id': 475, 'name': 'Personal care', 'freebase_id': '/m/02w3_ws'},
+    {'id': 476, 'name': 'Apple', 'freebase_id': '/m/014j1m'},
+    {'id': 477, 'name': 'Brassiere', 'freebase_id': '/m/01gmv2'},
+    {'id': 478, 'name': 'Bathroom cabinet', 'freebase_id': '/m/04y4h8h'},
+    {'id': 479, 'name': 'studio couch', 'freebase_id': '/m/026qbn5'},
+    {'id': 480, 'name': 'Computer keyboard', 'freebase_id': '/m/01m2v'},
+    {'id': 481, 'name': 'Table tennis racket', 'freebase_id': '/m/05_5p_0'},
+    {'id': 482, 'name': 'Sushi', 'freebase_id': '/m/07030'},
+    {'id': 483, 'name': 'Cabinetry', 'freebase_id': '/m/01s105'},
+    {'id': 484, 'name': 'Street light', 'freebase_id': '/m/033rq4'},
+    {'id': 485, 'name': 'Towel', 'freebase_id': '/m/0162_1'},
+    {'id': 486, 'name': 'Nightstand', 'freebase_id': '/m/02z51p'},
+    {'id': 487, 'name': 'Rabbit', 'freebase_id': '/m/06mf6'},
+    {'id': 488, 'name': 'Dolphin', 'freebase_id': '/m/02hj4'},
+    {'id': 489, 'name': 'Dog', 'freebase_id': '/m/0bt9lr'},
+    {'id': 490, 'name': 'Jug', 'freebase_id': '/m/08hvt4'},
+    {'id': 491, 'name': 'Wok', 'freebase_id': '/m/084rd'},
+    {'id': 492, 'name': 'Fire hydrant', 'freebase_id': '/m/01pns0'},
+    {'id': 493, 'name': 'Human eye', 'freebase_id': '/m/014sv8'},
+    {'id': 494, 'name': 'Skyscraper', 'freebase_id': '/m/079cl'},
+    {'id': 495, 'name': 'Backpack', 'freebase_id': '/m/01940j'},
+    {'id': 496, 'name': 'Potato', 'freebase_id': '/m/05vtc'},
+    {'id': 497, 'name': 'Paper towel', 'freebase_id': '/m/02w3r3'},
+    {'id': 498, 'name': 'Lifejacket', 'freebase_id': '/m/054xkw'},
+    {'id': 499, 'name': 'Bicycle wheel', 'freebase_id': '/m/01bqk0'},
+    {'id': 500, 'name': 'Toilet', 'freebase_id': '/m/09g1w'},
+]
+
+
+def _get_builtin_metadata(cats):
+    id_to_name = {x['id']: x['name'] for x in cats}
+    thing_dataset_id_to_contiguous_id = {i + 1: i for i in range(len(cats))}
+    thing_classes = [x['name'] for x in sorted(cats, key=lambda x: x['id'])]
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+_PREDEFINED_SPLITS_OID = {
+    # cat threshold: 500, 1500: r 170, c 151, f 179
+    "oid_train": ("oid/images/", "oid/annotations/oid_challenge_2019_train_bbox.json"),
+    # "expanded" duplicates annotations to their father classes based on the official
+    #   hierarchy. This is used in the official evaulation protocol.
+    #   https://storage.googleapis.com/openimages/web/evaluation.html
+    "oid_val_expanded": ("oid/images/validation/", "oid/annotations/oid_challenge_2019_val_expanded.json"),
+    "oid_val_expanded_rare": ("oid/images/validation/", "oid/annotations/oid_challenge_2019_val_expanded_rare.json"),
+}
+
+
+for key, (image_root, json_file) in _PREDEFINED_SPLITS_OID.items():
+    register_oid_instances(
+        key,
+        _get_builtin_metadata(categories),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

detic/data/datasets/register_oid.py

@@ -0,0 +1,122 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# Modified by Xingyi Zhou from https://github.com/facebookresearch/detectron2/blob/master/detectron2/data/datasets/coco.py
+import copy
+import io
+import logging
+import contextlib
+import os
+import datetime
+import json
+import numpy as np
+
+from PIL import Image
+
+from fvcore.common.timer import Timer
+from fvcore.common.file_io import PathManager, file_lock
+from detectron2.structures import BoxMode, PolygonMasks, Boxes
+from detectron2.data import DatasetCatalog, MetadataCatalog
+
+logger = logging.getLogger(__name__)
+
+"""
+This file contains functions to register a COCO-format dataset to the DatasetCatalog.
+"""
+
+__all__ = ["register_coco_instances", "register_coco_panoptic_separated"]
+
+
+
+def register_oid_instances(name, metadata, json_file, image_root):
+    """
+    """
+    # 1. register a function which returns dicts
+    DatasetCatalog.register(name, lambda: load_coco_json_mem_efficient(
+        json_file, image_root, name))
+
+    # 2. Optionally, add metadata about this dataset,
+    # since they might be useful in evaluation, visualization or logging
+    MetadataCatalog.get(name).set(
+        json_file=json_file, image_root=image_root, evaluator_type="oid", **metadata
+    )
+
+
+def load_coco_json_mem_efficient(json_file, image_root, dataset_name=None, extra_annotation_keys=None):
+    """
+    Actually not mem efficient
+    """
+    from pycocotools.coco import COCO
+
+    timer = Timer()
+    json_file = PathManager.get_local_path(json_file)
+    with contextlib.redirect_stdout(io.StringIO()):
+        coco_api = COCO(json_file)
+    if timer.seconds() > 1:
+        logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds()))
+
+    id_map = None
+    if dataset_name is not None:
+        meta = MetadataCatalog.get(dataset_name)
+        cat_ids = sorted(coco_api.getCatIds())
+        cats = coco_api.loadCats(cat_ids)
+        # The categories in a custom json file may not be sorted.
+        thing_classes = [c["name"] for c in sorted(cats, key=lambda x: x["id"])]
+        meta.thing_classes = thing_classes
+
+        if not (min(cat_ids) == 1 and max(cat_ids) == len(cat_ids)):
+            if "coco" not in dataset_name:
+                logger.warning(
+                    """
+                    Category ids in annotations are not in [1, #categories]! We'll apply a mapping for you.
+                    """
+                )
+        id_map = {v: i for i, v in enumerate(cat_ids)}
+        meta.thing_dataset_id_to_contiguous_id = id_map
+
+    # sort indices for reproducible results
+    img_ids = sorted(coco_api.imgs.keys())
+    imgs = coco_api.loadImgs(img_ids)
+    logger.info("Loaded {} images in COCO format from {}".format(len(imgs), json_file))
+
+    dataset_dicts = []
+
+    ann_keys = ["iscrowd", "bbox", "category_id"] + (extra_annotation_keys or [])
+
+    for img_dict in imgs:
+        record = {}
+        record["file_name"] = os.path.join(image_root, img_dict["file_name"])
+        record["height"] = img_dict["height"]
+        record["width"] = img_dict["width"]
+        image_id = record["image_id"] = img_dict["id"]
+        anno_dict_list = coco_api.imgToAnns[image_id]
+        if 'neg_category_ids' in img_dict:
+            record['neg_category_ids'] = \
+                [id_map[x] for x in img_dict['neg_category_ids']]
+
+        objs = []
+        for anno in anno_dict_list:
+            assert anno["image_id"] == image_id
+
+            assert anno.get("ignore", 0) == 0
+
+            obj = {key: anno[key] for key in ann_keys if key in anno}
+
+            segm = anno.get("segmentation", None)
+            if segm:  # either list[list[float]] or dict(RLE)
+                if not isinstance(segm, dict):
+                    # filter out invalid polygons (< 3 points)
+                    segm = [poly for poly in segm if len(poly) % 2 == 0 and len(poly) >= 6]
+                    if len(segm) == 0:
+                        num_instances_without_valid_segmentation += 1
+                        continue  # ignore this instance
+                obj["segmentation"] = segm
+
+            obj["bbox_mode"] = BoxMode.XYWH_ABS
+
+            if id_map:
+                obj["category_id"] = id_map[obj["category_id"]]
+            objs.append(obj)
+        record["annotations"] = objs
+        dataset_dicts.append(record)
+    
+    del coco_api
+    return dataset_dicts