val_utils/scripts/run_burst.py

""" run_burst.py

The example commands given below expect the following folder structure:

- data
    - gt
        - burst
            - {val,test}
                - all_classes
                    - all_classes.json (filename is irrelevant)
    - trackers
        - burst
            - exemplar_guided
                - {val,test}
                    - my_tracking_method
                        - data
                            - results.json  (filename is irrelevant)
            - class_guided
                - {val,test}
                    - my_other_tracking_method
                        - data
                            - results.json (filename is irrelevant)

Run example:

1) Exemplar-guided tasks (all three tasks share the same eval logic):
run_burst.py --USE_PARALLEL True --EXEMPLAR_GUIDED True --GT_FOLDER ../data/gt/burst/{val,test}/all_classes --TRACKERS_FOLDER ../data/trackers/burst/exemplar_guided/{val,test}

2) Class-guided tasks (common class and long-tail):
run_burst.py --USE_PARALLEL FTrue --EXEMPLAR_GUIDED False --GT_FOLDER ../data/gt/burst/{val,test}/all_classes --TRACKERS_FOLDER ../data/trackers/burst/class_guided/{val,test}

3) Refer to run_burst_ow.py for open world evaluation

Command Line Arguments: Defaults, # Comments
    Eval arguments:
        'USE_PARALLEL': False,
        'NUM_PARALLEL_CORES': 8,
        'BREAK_ON_ERROR': True,
        'PRINT_RESULTS': True,
        'PRINT_ONLY_COMBINED': False,
        'PRINT_CONFIG': True,
        'TIME_PROGRESS': True,
        'OUTPUT_SUMMARY': True,
        'OUTPUT_DETAILED': True,
        'PLOT_CURVES': True,
    Dataset arguments:
        'GT_FOLDER': os.path.join(code_path, 'data/gt/burst/val'),  # Location of GT data
        'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/burst/class-guided/'),  # Trackers location
        'OUTPUT_FOLDER': None,  # Where to save eval results (if None, same as TRACKERS_FOLDER)
        'TRACKERS_TO_EVAL': None,  # Filenames of trackers to eval (if None, all in folder)
        'CLASSES_TO_EVAL': None,  # Classes to eval (if None, all classes)
        'SPLIT_TO_EVAL': 'training',  # Valid: 'training', 'val'
        'PRINT_CONFIG': True,  # Whether to print current config
        'TRACKER_SUB_FOLDER': 'data',  # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
        'OUTPUT_SUB_FOLDER': '',  # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
        'TRACKER_DISPLAY_NAMES': None,  # Names of trackers to display, if None: TRACKERS_TO_EVAL
        'MAX_DETECTIONS': 300,  # Number of maximal allowed detections per image (0 for unlimited)
    Metric arguments:
        'METRICS': ['HOTA', 'CLEAR', 'Identity', 'TrackMAP']
"""

import sys
import os
import argparse
from tabulate import tabulate
from multiprocessing import freeze_support

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
import trackeval  # noqa: E402


def main():
    freeze_support()

    # Command line interface:
    default_eval_config = trackeval.Evaluator.get_default_eval_config()
    default_eval_config['PRINT_ONLY_COMBINED'] = True
    default_eval_config['DISPLAY_LESS_PROGRESS'] = True
    default_eval_config['PLOT_CURVES'] = False
    default_eval_config["OUTPUT_DETAILED"] = False
    default_eval_config["PRINT_RESULTS"] = False
    default_eval_config["OUTPUT_SUMMARY"] = False

    default_dataset_config = trackeval.datasets.BURST.get_default_dataset_config()

    # default_metrics_config = {'METRICS': ['HOTA', 'CLEAR', 'Identity', 'TrackMAP']}
    # default_metrics_config = {'METRICS': ['HOTA']}
    default_metrics_config = {'METRICS': ['HOTA', 'TrackMAP']}
    config = {**default_eval_config, **default_dataset_config, **default_metrics_config}  # Merge default configs
    parser = argparse.ArgumentParser()
    for setting in config.keys():
        if type(config[setting]) == list or type(config[setting]) == type(None):
            parser.add_argument("--" + setting, nargs='+')
        else:
            parser.add_argument("--" + setting)
    args = parser.parse_args().__dict__
    for setting in args.keys():
        if args[setting] is not None:
            if type(config[setting]) == type(True):
                if args[setting] == 'True':
                    x = True
                elif args[setting] == 'False':
                    x = False
                else:
                    raise Exception('Command line parameter ' + setting + 'must be True or False')
            elif type(config[setting]) == type(1):
                x = int(args[setting])
            elif type(args[setting]) == type(None):
                x = None
            else:
                x = args[setting]
            config[setting] = x
    eval_config = {k: v for k, v in config.items() if k in default_eval_config.keys()}
    dataset_config = {k: v for k, v in config.items() if k in default_dataset_config.keys()}
    metrics_config = {k: v for k, v in config.items() if k in default_metrics_config.keys()}

    # Run code
    evaluator = trackeval.Evaluator(eval_config)
    dataset_list = [trackeval.datasets.BURST(dataset_config)]
    metrics_list = []
    for metric in [trackeval.metrics.TrackMAP, trackeval.metrics.CLEAR, trackeval.metrics.Identity,
                   trackeval.metrics.HOTA]:
        if metric.get_name() in metrics_config['METRICS']:
            metrics_list.append(metric())
    if len(metrics_list) == 0:
        raise Exception('No metrics selected for evaluation')
    output_res, output_msg = evaluator.evaluate(dataset_list, metrics_list, show_progressbar=True)

    class_name_to_id = {x['name']: x['id'] for x in dataset_list[0].gt_data['categories']}
    known_list = [4, 13, 1038, 544, 1057, 34, 35, 36, 41, 45, 58, 60, 579, 1091, 1097, 1099, 78, 79, 81, 91, 1115,
                  1117, 95, 1122, 99, 1132, 621, 1135, 625, 118, 1144, 126, 642, 1155, 133, 1162, 139, 154, 174, 185,
                  699, 1215, 714, 717, 1229, 211, 729, 221, 229, 747, 235, 237, 779, 276, 805, 299, 829, 852, 347,
                  371, 382, 896, 392, 926, 937, 428, 429, 961, 452, 979, 980, 982, 475, 480, 993, 1001, 502, 1018]

    row_labels = ("HOTA", "DetA", "AssA", "AP")
    trackers = list(output_res['BURST'].keys())
    print("\n")

    def average_metric(m):
        return round(100*sum(m) / len(m), 2)

    for tracker in trackers:
        res = output_res['BURST'][tracker]['COMBINED_SEQ']
        all_names = [x for x in res.keys() if (x != 'cls_comb_cls_av') and (x != 'cls_comb_det_av')]

        class_split_names = {
            "All": [x for x in res.keys() if (x != 'cls_comb_cls_av') and (x != 'cls_comb_det_av')],
            "Common": [x for x in all_names if class_name_to_id[x] in known_list],
            "Uncommon": [x for x in all_names if class_name_to_id[x] not in known_list]
        }

        # table columns: 'all', 'common', 'uncommon'
        # table rows: HOTA, AssA, DetA, mAP
        table_data = []

        for row_label in row_labels:
            row = [row_label]
            for split_name in ["All", "Common", "Uncommon"]:
                split_classes = class_split_names[split_name]

                if row_label == "AP":
                    row.append(average_metric([res[c]['TrackMAP']["AP_all"].mean() for c in split_classes]))
                else:
                    row.append(average_metric([res[c]['HOTA'][row_label].mean() for c in split_classes]))

            table_data.append(row)

        print(f"Results for Tracker: {tracker}\n")
        print(tabulate(table_data, ["Metric", "All", "Common", "Uncommon"]))


if __name__ == '__main__':
    main()