diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..bbb792eb712bc1bf0c55285ccde671d1c39fe5c6 --- /dev/null +++ b/app.py @@ -0,0 +1,115 @@ +""" + +Original Algorithm: +- https://github.com/GreenCUBIC/AudiogramDigitization + +Source: +- huggingface app + - https://huggingface.co/spaces/aravinds1811/neural-style-transfer/blob/main/app.py + - https://huggingface.co/spaces/keras-io/ocr-for-captcha/blob/main/app.py + - https://huggingface.co/spaces/hugginglearners/image-style-transfer/blob/main/app.py + - https://tmabraham.github.io/blog/gradio_hf_spaces_tutorial +- huggingface push + - https://huggingface.co/welcome +""" +import os +import sys +from pathlib import Path + +from PIL import Image +from matplotlib.offsetbox import OffsetImage, AnnotationBbox +import matplotlib.pyplot as plt +import pandas as pd +import numpy as np +import gradio as gr + +sys.path.append(os.path.join(os.path.dirname(__file__), "src")) +from digitizer.digitization import generate_partial_annotation, extract_thresholds + +EXAMPLES_PATH = Path('./examples') + + +max_length = 5 +img_width = 200 +img_height = 50 + + +def load_image(path, zoom=1): + return OffsetImage(plt.imread(path), zoom=zoom) + +def plot_audiogram(digital_result): + thresholds = pd.DataFrame(digital_result) + + # Figure + fig = plt.figure() + ax = fig.add_subplot(111) + + # x axis + axis = [250, 500, 1000, 2000, 4000, 8000, 16000] + ax.set_xscale('log') + ax.xaxis.tick_top() + ax.xaxis.set_major_formatter(plt.FuncFormatter('{:.0f}'.format)) + ax.set_xlabel('Frequency (Hz)') + ax.xaxis.set_label_position('top') + ax.set_xlim(125,16000) + plt.xticks(axis) + + # y axis + ax.set_ylim(-20, 120) + ax.invert_yaxis() + ax.set_ylabel('Threshold (dB HL)') + + plt.grid() + + for conduction in ("air", "bone"): + for masking in (True, False): + for ear in ("left", "right"): + symbol_name = f"{ear}_{conduction}_{'unmasked' if not masking else 'masked'}" + selection = thresholds[(thresholds.conduction == conduction) & (thresholds.ear == ear) & (thresholds.masking == masking)] + selection = selection.sort_values("frequency") + + # Plot the symbols + for i, threshold in selection.iterrows(): + ab = AnnotationBbox(load_image(f"src/digitizer/assets/symbols/{symbol_name}.png", zoom=0.1), (threshold.frequency, threshold.threshold), frameon=False) + ax.add_artist(ab) + + # Add joining line for air conduction thresholds + if conduction == "air": + plt.plot(selection.frequency, selection.threshold, color="red" if ear == "right" else "blue", linewidth=0.5) + + return plt.gcf() + # Save audiogram plot to nparray + # get image as np.array + # canvas = plt.gca().figure.canvas + # canvas.draw() + # image = np.frombuffer(canvas.tostring_rgb(), dtype=np.uint8) + # return Image.fromarray(image) + + +# Function for Audiogram Digit Recognition +def audiogram_digit_recognition(img_path): + digital_result = extract_thresholds(img_path, gpu=False) + return plot_audiogram(digital_result) + + +output = gr.Plot() +examples = [f'{EXAMPLES_PATH}/audiogram_example01.png'] + +iface = gr.Interface( + fn=audiogram_digit_recognition, + inputs = gr.inputs.Image(type='filepath'), + outputs = output , #"image", + title=" AudiogramDigitization", + description = "facilitate the digitization of audiology reports based on pytorch", + article = "Algorithm Authors: Francois Charih \ + and James R. Green . \ + Based on the AudiogramDigitization github repo", + examples = examples, + allow_flagging='never', + cache_examples=False, +) + + +iface.launch( + enable_queue=True, debug=False, inbrowser=False +) diff --git a/examples/audiogram_example01.png b/examples/audiogram_example01.png new file mode 100644 index 0000000000000000000000000000000000000000..4c261d18900a292da571b1257ea34b96a09a2a84 Binary files /dev/null and b/examples/audiogram_example01.png differ diff --git a/models/audiograms/audiograms_detection.yaml b/models/audiograms/audiograms_detection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b369cccdca004559e575a17dac5ef7e2c989e3fb --- /dev/null +++ b/models/audiograms/audiograms_detection.yaml @@ -0,0 +1,9 @@ +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +train: models/audiograms/dataset/images/train/ +val: models/audiograms/dataset/images/validation/ + +# number of classes +nc: 1 + +# class names +names: ["AUDIOGRAM"] diff --git a/models/audiograms/audiograms_detection_test.yaml b/models/audiograms/audiograms_detection_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8781b21c225aedb09e02013631c07c026cbdf54 --- /dev/null +++ b/models/audiograms/audiograms_detection_test.yaml @@ -0,0 +1,8 @@ +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +val: data/audiograms/test/images/ + +# number of classes +nc: 1 + +# class names +names: ["AUDIOGRAM"] diff --git a/models/audiograms/format_dataset.py b/models/audiograms/format_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..175eefad0c153780c6f615541af01d81441194f0 --- /dev/null +++ b/models/audiograms/format_dataset.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020 Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" +from typing import List +from types import SimpleNamespace +import argparse, os, json, shutil +from tqdm import tqdm +import os.path as path +import numpy as np +from PIL import Image + +def extract_audiograms(annotation: dict, image: Image) -> List[tuple]: + """Extracts the bounding boxes of audiograms into a tuple compatible + the YOLOv5 format. + + Parameters + ---------- + annotation : dict + A dictionary containing the annotations for the audiograms in a report. + + image : Image + The image in PIL format corresponding to the annotation. + + Returns + ------- + tuple + A tuple of the form + (class index, x_center, y_center, width, height) where all coordinates + and dimensions are normalized to the width/height of the image. + """ + audiogram_label_tuples = [] + image_width, image_height = image.size + for audiogram in annotation: + bounding_box = audiogram["boundingBox"] + x_center = (bounding_box["x"] + bounding_box["width"] / 2) / image_width + y_center = (bounding_box["y"] + bounding_box["height"] / 2) / image_height + box_width = bounding_box["width"] / image_width + box_height = bounding_box["height"] / image_width + audiogram_label_tuples.append((0, x_center, y_center, box_width, box_height)) + return audiogram_label_tuples + +def create_yolov5_file(bboxes: List[tuple], filename: str): + # Turn the bounding boxes into a string with a bounding box + # on each line + file_content = "\n".join([ + f"{bbox[0]} {bbox[1]} {bbox[2]} {bbox[3]} {bbox[4]}" + for bbox in bboxes + ]) + + # Save to a file + with open(filename, "w") as output_file: + output_file.write(file_content) + +def create_directory_structure(data_dir: str): + try: + shutil.rmtree(path.join(data_dir)) + except: + pass + os.mkdir(path.join(data_dir)) + os.mkdir(path.join(data_dir, "images")) + os.mkdir(path.join(data_dir, "images", "train")) + os.mkdir(path.join(data_dir, "images", "validation")) + os.mkdir(path.join(data_dir, "labels")) + os.mkdir(path.join(data_dir, "labels", "train")) + os.mkdir(path.join(data_dir, "labels", "validation")) + +def all_labels_valid(labels: List[tuple]): + for label in labels: + for value in label[1:]: + if value < 0 or value > 1: + return False + return True + +def main(args: SimpleNamespace): + # Find all the JSON files in the input directory + report_ids = [ + filename.rstrip(".json") + for filename in os.listdir(path.join(args.annotations_dir)) + if filename.endswith(".json") + and path.exists(path.join(args.images_dir, filename.rstrip(".json") + ".jpg")) + ] + + # Shuffle + np.random.seed(seed=42) # for reproducibility of the shuffle + np.random.shuffle(report_ids) + + # Create the directory structure in which the images and annotations + # are to be stored + create_directory_structure(args.data_dir) + + # Iterate through the report ids, extract the annotations in YOLOv5 format + # and place the file in the correct directory, and the image in the correct + # directory. + for i, report_id in enumerate(tqdm(report_ids)): + # Decide if the image is going into the training set or validation set + directory = ( + "train" if i < args.train_frac * len(report_ids) else "validation" + ) + + # Load the annotation` + annotation_content = open( + path.join(args.annotations_dir, f"{report_id}.json") + ) + annotation = json.load(annotation_content) + + # Open the corresponding image to get its dimensions + image = Image.open(os.path.join(args.images_dir, f"{report_id}.jpg")) + width, height = image.size + + # Audiogram labels + audiogram_labels = extract_audiograms(annotation, image) + + if not all_labels_valid(audiogram_labels): + continue + + create_yolov5_file( + audiogram_labels, + path.join(args.data_dir, "labels", directory, f"{report_id}.txt") + ) + image.save( + path.join(args.data_dir, "images", directory, f"{report_id}.jpg") + ) + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description=( + "Script that formats the training set for transfer learning via " + "the YOLOv5 model." + )) + parser.add_argument("-d", "--data_dir", type=str, required=True, help=( + "Path to the directory containing the data. It should have 3 " + "subfolders named `images`, `annotations` and `labels`." + )) + parser.add_argument("-a", "--annotations_dir", type=str, required=True, help=( + "Path to the directory containing the annotations in the JSON format." + )) + parser.add_argument("-i", "--images_dir", type=str, required=True, help=( + "Path to the directory containing the images." + )) + parser.add_argument("-f", "--train_frac", type=float, required=True, help=( + "Fraction of images to be used for training. (e.g. 0.8)" + )) + args = parser.parse_args() + + main(args) + diff --git a/models/audiograms/latest/hyp.yaml b/models/audiograms/latest/hyp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eb1f5e4c9197952a4e0670d6bd50caca8a1ef56 --- /dev/null +++ b/models/audiograms/latest/hyp.yaml @@ -0,0 +1,23 @@ +lr0: 0.01 +lrf: 0.2 +momentum: 0.937 +weight_decay: 0.0005 +giou: 0.05 +cls: 0.5 +cls_pw: 1.0 +obj: 1.0 +obj_pw: 1.0 +iou_t: 0.2 +anchor_t: 4.0 +fl_gamma: 0.0 +hsv_h: 0.015 +hsv_s: 0.7 +hsv_v: 0.4 +degrees: 0.0 +translate: 0.1 +scale: 0.5 +shear: 0.0 +perspective: 0.0 +flipud: 0.0 +fliplr: 0.5 +mixup: 0.0 diff --git a/models/audiograms/latest/labels.png b/models/audiograms/latest/labels.png new file mode 100644 index 0000000000000000000000000000000000000000..8007caabd330af534e58b113a2615269e6a02ac5 Binary files /dev/null and b/models/audiograms/latest/labels.png differ diff --git a/models/audiograms/latest/opt.yaml b/models/audiograms/latest/opt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6109708e9cb9514dbc64acf78891e4b110b4ae --- /dev/null +++ b/models/audiograms/latest/opt.yaml @@ -0,0 +1,30 @@ +weights: yolov5/weights/yolov5s.pt +cfg: yolov5/models/yolov5s.yaml +data: ./data/dataset.yaml +hyp: ./yolov5/data/hyp.scratch.yaml +epochs: 50 +batch_size: 16 +img_size: +- 1024 +- 1024 +rect: true +resume: false +nosave: false +notest: false +noautoanchor: false +evolve: false +bucket: '' +cache_images: false +image_weights: false +name: '' +device: '0' +multi_scale: false +single_cls: false +adam: false +sync_bn: false +local_rank: -1 +logdir: runs/ +workers: 8 +total_batch_size: 16 +world_size: 1 +global_rank: -1 diff --git a/models/audiograms/latest/results.png b/models/audiograms/latest/results.png new file mode 100644 index 0000000000000000000000000000000000000000..70b6befb80396acd09f627cdcc0befadd3b4ff12 Binary files /dev/null and b/models/audiograms/latest/results.png differ diff --git a/models/audiograms/latest/results.txt b/models/audiograms/latest/results.txt new file mode 100644 index 0000000000000000000000000000000000000000..e76e012b4153edac62e46b86dc01fd7a96f3cb66 --- /dev/null +++ b/models/audiograms/latest/results.txt @@ -0,0 +1,50 @@ + 0/49 7.24G 0.03451 0.01234 0 0.04685 18 768 0.386 0.8519 0.6314 0.2382 0.01455 0.007371 0 + 1/49 7.25G 0.02255 0.005412 0 0.02796 18 768 0.7806 0.9831 0.9762 0.7314 0.0121 0.003704 0 + 2/49 7.25G 0.01706 0.003746 0 0.0208 18 768 0.8866 0.9896 0.9793 0.741 0.009516 0.002951 0 + 3/49 7.25G 0.01726 0.003265 0 0.02052 18 768 0.8734 0.9961 0.9858 0.691 0.01019 0.002468 0 + 4/49 7.25G 0.01645 0.003014 0 0.01946 18 768 0.8789 0.9961 0.9885 0.8278 0.009626 0.002319 0 + 5/49 7.25G 0.01519 0.002852 0 0.01805 18 768 0.9051 0.9961 0.984 0.778 0.01027 0.002376 0 + 6/49 7.25G 0.01463 0.002948 0 0.01758 18 768 0.9223 0.9922 0.9857 0.8341 0.008192 0.002504 0 + 7/49 7.25G 0.01365 0.0028 0 0.01645 18 768 0.7791 0.9935 0.9828 0.8224 0.007701 0.002257 0 + 8/49 7.25G 0.01187 0.002558 0 0.01443 18 768 0.8548 0.9987 0.9807 0.843 0.007083 0.002018 0 + 9/49 7.25G 0.01161 0.002421 0 0.01403 18 768 0.9653 0.9961 0.9884 0.8421 0.007059 0.001885 0 + 10/49 7.25G 0.01053 0.002392 0 0.01292 18 768 0.9588 0.9974 0.9875 0.8607 0.005953 0.001821 0 + 11/49 7.25G 0.009697 0.002255 0 0.01195 18 768 0.9591 0.9935 0.9851 0.8387 0.006067 0.001839 0 + 12/49 7.25G 0.009541 0.002229 0 0.01177 18 768 0.9619 0.9987 0.9872 0.8725 0.005808 0.0017 0 + 13/49 7.25G 0.009704 0.002185 0 0.01189 18 768 0.9693 0.9961 0.9907 0.8767 0.005385 0.001728 0 + 14/49 7.25G 0.009079 0.002169 0 0.01125 18 768 0.9691 0.9974 0.9841 0.8531 0.006121 0.001715 0 + 15/49 7.25G 0.008323 0.002161 0 0.01048 18 768 0.9579 0.9948 0.9882 0.8838 0.004668 0.001631 0 + 16/49 7.25G 0.008653 0.002083 0 0.01074 18 768 0.9768 0.9987 0.9913 0.8845 0.004903 0.001578 0 + 17/49 7.25G 0.007987 0.002011 0 0.009997 18 768 0.9786 0.9974 0.9895 0.8873 0.005501 0.001565 0 + 18/49 7.25G 0.008903 0.001963 0 0.01087 18 768 0.9506 0.9987 0.9913 0.8784 0.006132 0.0016 0 + 19/49 7.25G 0.008407 0.001971 0 0.01038 18 768 0.9722 0.9999 0.9908 0.8825 0.004867 0.001558 0 + 20/49 7.25G 0.007909 0.001966 0 0.009875 18 768 0.9649 0.9974 0.9905 0.8781 0.005032 0.001563 0 + 21/49 7.25G 0.00755 0.001911 0 0.00946 18 768 0.9733 1 0.9916 0.8879 0.005532 0.001542 0 + 22/49 7.25G 0.007531 0.001892 0 0.009423 18 768 0.9359 1 0.9881 0.8821 0.005925 0.001577 0 + 23/49 7.25G 0.007707 0.001834 0 0.009541 18 768 0.9757 1 0.9881 0.8889 0.004511 0.001455 0 + 24/49 7.25G 0.00729 0.00182 0 0.00911 18 768 0.9608 1 0.9889 0.8925 0.004974 0.001477 0 + 25/49 7.25G 0.007106 0.001769 0 0.008875 18 768 0.9686 0.9987 0.9907 0.8887 0.004586 0.001471 0 + 26/49 7.25G 0.00679 0.001756 0 0.008546 18 768 0.9862 0.9974 0.993 0.8934 0.004579 0.001438 0 + 27/49 7.25G 0.006621 0.001754 0 0.008375 18 768 0.9812 1 0.993 0.8953 0.004354 0.001446 0 + 28/49 7.25G 0.006727 0.001672 0 0.008399 18 768 0.9821 1 0.9919 0.8892 0.004686 0.0014 0 + 29/49 7.25G 0.006892 0.001684 0 0.008577 18 768 0.9817 0.9987 0.993 0.8939 0.004496 0.001401 0 + 30/49 7.25G 0.006629 0.001684 0 0.008313 18 768 0.9834 1 0.9932 0.8978 0.004209 0.001381 0 + 31/49 7.25G 0.006299 0.001594 0 0.007892 18 768 0.973 1 0.9916 0.8962 0.004526 0.001369 0 + 32/49 7.25G 0.006212 0.00162 0 0.007831 18 768 0.9842 1 0.9909 0.8979 0.004059 0.001345 0 + 33/49 7.25G 0.006012 0.001572 0 0.007583 18 768 0.9817 1 0.9905 0.8964 0.004181 0.001342 0 + 34/49 7.25G 0.006232 0.001605 0 0.007837 18 768 0.9831 1 0.9922 0.9055 0.004233 0.001335 0 + 35/49 7.25G 0.005956 0.001566 0 0.007522 18 768 0.9866 1 0.9924 0.9064 0.004158 0.001314 0 + 36/49 7.25G 0.005979 0.001521 0 0.0075 18 768 0.9813 1 0.9915 0.8951 0.004402 0.00133 0 + 37/49 7.25G 0.005544 0.001476 0 0.00702 18 768 0.9859 1 0.9931 0.906 0.004128 0.001315 0 + 38/49 7.25G 0.00582 0.0015 0 0.007321 18 768 0.9808 1 0.9931 0.9066 0.004205 0.00132 0 + 39/49 7.25G 0.005607 0.001494 0 0.007101 18 768 0.9867 1 0.9926 0.9073 0.004047 0.001323 0 + 40/49 7.25G 0.005674 0.001459 0 0.007133 18 768 0.9863 1 0.9923 0.9064 0.004046 0.001307 0 + 41/49 7.25G 0.005712 0.001452 0 0.007165 18 768 0.9863 1 0.9924 0.9051 0.00421 0.001297 0 + 42/49 7.25G 0.005655 0.001459 0 0.007113 18 768 0.9823 1 0.9931 0.9089 0.004065 0.001296 0 + 43/49 7.25G 0.005481 0.001445 0 0.006926 18 768 0.9847 0.9987 0.9933 0.91 0.004256 0.001298 0 + 44/49 7.25G 0.005456 0.001474 0 0.00693 18 768 0.9844 0.9987 0.9938 0.9111 0.004057 0.001314 0 + 45/49 7.25G 0.005749 0.00145 0 0.0072 18 768 0.9863 0.9987 0.9938 0.9076 0.004 0.001296 0 + 46/49 7.25G 0.005401 0.001397 0 0.006798 18 768 0.985 1 0.9937 0.9114 0.004002 0.001283 0 + 47/49 7.25G 0.005416 0.001401 0 0.006818 18 768 0.9839 1 0.9936 0.9074 0.004271 0.001285 0 + 48/49 7.25G 0.005382 0.001397 0 0.00678 18 768 0.9857 0.9987 0.9939 0.9101 0.00399 0.00127 0 + 49/49 7.25G 0.00526 0.001375 0 0.006635 18 768 0.9754 1 0.994 0.9117 0.004094 0.001291 0 diff --git a/models/audiograms/latest/test_batch0_gt.jpg b/models/audiograms/latest/test_batch0_gt.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dfa48f930a8969eff1b862658ad9d17e30dab7f2 Binary files /dev/null and b/models/audiograms/latest/test_batch0_gt.jpg differ diff --git a/models/audiograms/latest/test_batch0_pred.jpg b/models/audiograms/latest/test_batch0_pred.jpg new file mode 100644 index 0000000000000000000000000000000000000000..47e485937c383d89656672a371260b7e7b1d9a26 Binary files /dev/null and b/models/audiograms/latest/test_batch0_pred.jpg differ diff --git a/models/audiograms/latest/train_batch0.jpg b/models/audiograms/latest/train_batch0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4102ff52892cadc3ed3831eadb2cae99f6c65772 Binary files /dev/null and b/models/audiograms/latest/train_batch0.jpg differ diff --git a/models/audiograms/latest/train_batch1.jpg b/models/audiograms/latest/train_batch1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e4aeea2f229e42c99db6095109723eb331a9cf71 Binary files /dev/null and b/models/audiograms/latest/train_batch1.jpg differ diff --git a/models/audiograms/latest/train_batch2.jpg b/models/audiograms/latest/train_batch2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9240275dd36ea926c09ab0d30514288310d52694 Binary files /dev/null and b/models/audiograms/latest/train_batch2.jpg differ diff --git a/models/audiograms/yolov5s.yaml b/models/audiograms/yolov5s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b157bdd5792ee98c4bb17996f906e1a6d0533cb --- /dev/null +++ b/models/audiograms/yolov5s.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 1 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/labels/format_dataset.py b/models/labels/format_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..2d7e19b7f43d37d418c1a2796d26b5c669eda928 --- /dev/null +++ b/models/labels/format_dataset.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020 Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" +from typing import List +from types import SimpleNamespace +import argparse, os, json, shutil +from tqdm import tqdm +import os.path as path +import numpy as np +from PIL import Image + +LABEL_CLASS_INDICES = { + "250": 0, + ".25": 1, + "500": 2, + ".5": 3, + "1000": 4, + "1": 5, + "1K": 6, + "2000": 7, + "2": 8, + "2K": 9, + "4000": 10, + "4": 11, + "4K": 12, + "8000": 13, + "8": 14, + "8K": 15, + "0": 16, + "20": 17, + "40": 18, + "60": 19, + "80": 20, + "100": 21, + "120": 22 +} + +def extract_labels(annotation: dict, image: Image) -> List[tuple]: + """Extracts the bounding boxes of labels into a tuple compatible + the YOLOv5 format. + + Parameters + ---------- + annotation : dict + A dictionary containing the annotations for the audiograms in a report. + + image : Image + The image in PIL format corresponding to the annotation. + + Returns + ------- + tuple + A tuple of the form + (class index, x_center, y_center, width, height) where all coordinates + and dimensions are normalized to the width/height of the image. + """ + label_label_tuples = [] + image_width, image_height = image.size + for audiogram in annotation: + for label in audiogram["labels"]: + bounding_box = label["boundingBox"] + x_center = (bounding_box["x"] + bounding_box["width"] / 2) / image_width + y_center = (bounding_box["y"] + bounding_box["height"] / 2) / image_height + box_width = bounding_box["width"] / image_width + box_height = bounding_box["height"] / image_width + try: + label_label_tuples.append((LABEL_CLASS_INDICES[label["value"]], x_center, y_center, box_width, box_height)) + except: + continue + return label_label_tuples + +def create_directory_structure(data_dir: str): + try: + shutil.rmtree(path.join(data_dir)) + except: + pass + os.mkdir(path.join(data_dir)) + os.mkdir(path.join(data_dir, "images")) + os.mkdir(path.join(data_dir, "images", "train")) + os.mkdir(path.join(data_dir, "images", "validation")) + os.mkdir(path.join(data_dir, "labels")) + os.mkdir(path.join(data_dir, "labels", "train")) + os.mkdir(path.join(data_dir, "labels", "validation")) + +def create_yolov5_file(bboxes: List[tuple], filename: str): + # Turn the bounding boxes into a string with a bounding box + # on each line + file_content = "\n".join([ + f"{bbox[0]} {bbox[1]} {bbox[2]} {bbox[3]} {bbox[4]}" + for bbox in bboxes + ]) + + # Save to a file + with open(filename, "w") as output_file: + output_file.write(file_content) + +def all_labels_valid(labels: List[tuple]): + for label in labels: + for value in label[1:]: + if value < 0 or value > 1: + return False + return True + + +def main(args: SimpleNamespace): + + # Find all the JSON files in the input directory + report_ids = [ + filename.rstrip(".json") + for filename in os.listdir(path.join(args.annotations_dir)) + if filename.endswith(".json") + and path.exists(path.join(args.images_dir, filename.rstrip(".json") + ".jpg")) + ] + + # Shuffle + np.random.seed(seed=42) # for reproducibility of the shuffle + np.random.shuffle(report_ids) + + # Create the directory structure in which the images and annotations + # are to be stored + create_directory_structure(args.data_dir) + + # Iterate through the report ids, extract the annotations in YOLOv5 format + # and place the file in the correct directory, and the image in the correct + # directory. + for i, report_id in enumerate(tqdm(report_ids)): + + # Decide if the image is going into the training set or validation set + directory = ( + "train" if i < args.train_frac * len(report_ids) else "validation" + ) + + # Load the annotation` + annotation_content = open( + path.join(args.annotations_dir, f"{report_id}.json") + ) + + image = Image.open(os.path.join(args.images_dir, f"{report_id}.jpg")) + + annotation = json.load(annotation_content) + bounding_boxes = extract_labels(annotation, image) + + if not all_labels_valid(bounding_boxes): + continue + + # Open the corresponding image to get its dimensions + image = Image.open(os.path.join(args.images_dir, f"{report_id}.jpg")) + + create_yolov5_file( + bounding_boxes, + path.join(args.data_dir, "labels", directory, f"{report_id}.txt") + ) + image.save( + path.join(args.data_dir, "images", directory, f"{report_id}.jpg") + ) + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description=( + "Script that formats the training set for transfer learning of labels detection via " + "the YOLOv5 model." + )) + parser.add_argument("-d", "--data_dir", type=str, required=True, help=( + "Path to the directory where the training set should be created." + )) + parser.add_argument("-a", "--annotations_dir", type=str, required=True, help=( + "Path to the directory containing the annotations in the JSON format." + )) + parser.add_argument("-i", "--images_dir", type=str, required=True, help=( + "Path to the directory containing the images." + )) + parser.add_argument("-f", "--train_frac", type=float, required=True, help=( + "Fraction of images to be used for training. (e.g. 0.8)" + )) + args = parser.parse_args() + + main(args) + diff --git a/models/labels/labels_detection.yaml b/models/labels/labels_detection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9aec7886d3cfb224d87769a66bd8a2f8b0158a8 --- /dev/null +++ b/models/labels/labels_detection.yaml @@ -0,0 +1,32 @@ +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +train: models/labels/dataset/images/train/ +val: models/labels/dataset/images/validation/ + +# number of classes +nc: 21 + +# class names +names: [ +#frequencies + "250", + ".25", + "500", + ".5", + "1000", + "1", + "1K", + "2000", + "2", + "2K", + "4000", + "4", + "4K", + "8000", + "8", + "8K", + "20", + "40", + "60", + "80", + "100", +] \ No newline at end of file diff --git a/models/labels/labels_detection_test.yaml b/models/labels/labels_detection_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94876a24139d3600389ca43b364d476504cb3caf --- /dev/null +++ b/models/labels/labels_detection_test.yaml @@ -0,0 +1,32 @@ +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +val: data/labels/test/images/ + + +# number of classes +nc: 21 + +# class names +names: [ +#frequencies + "250", + ".25", + "500", + ".5", + "1000", + "1", + "1K", + "2000", + "2", + "2K", + "4000", + "4", + "4K", + "8000", + "8", + "8K", + "20", + "40", + "60", + "80", + "100", +] diff --git a/models/labels/latest/hyp.yaml b/models/labels/latest/hyp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eb1f5e4c9197952a4e0670d6bd50caca8a1ef56 --- /dev/null +++ b/models/labels/latest/hyp.yaml @@ -0,0 +1,23 @@ +lr0: 0.01 +lrf: 0.2 +momentum: 0.937 +weight_decay: 0.0005 +giou: 0.05 +cls: 0.5 +cls_pw: 1.0 +obj: 1.0 +obj_pw: 1.0 +iou_t: 0.2 +anchor_t: 4.0 +fl_gamma: 0.0 +hsv_h: 0.015 +hsv_s: 0.7 +hsv_v: 0.4 +degrees: 0.0 +translate: 0.1 +scale: 0.5 +shear: 0.0 +perspective: 0.0 +flipud: 0.0 +fliplr: 0.5 +mixup: 0.0 diff --git a/models/labels/latest/labels.png b/models/labels/latest/labels.png new file mode 100644 index 0000000000000000000000000000000000000000..e28f3adeb3bb1f4169abd6d7b1100107db7c8002 Binary files /dev/null and b/models/labels/latest/labels.png differ diff --git a/models/labels/latest/opt.yaml b/models/labels/latest/opt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f4a25335851db8064db7ccf636a2582f750edd5 --- /dev/null +++ b/models/labels/latest/opt.yaml @@ -0,0 +1,30 @@ +weights: src/digitizer/yolov5/weights/yolov5s.pt +cfg: models/labels/yolov5s.yaml +data: models/labels/labels_detection.yaml +hyp: models/labels/latest/hyp.yaml +epochs: 100 +batch_size: 16 +img_size: +- 1024 +- 1024 +img_weights: false +rect: true +resume: false +nosave: false +notest: false +noautoanchor: false +evolve: false +bucket: '' +cache_images: false +name: '' +device: '0' +multi_scale: false +single_cls: false +adam: false +sync_bn: false +local_rank: -1 +logdir: models/labels/model_2022-02-13T23:42:43.525128 +workers: 8 +total_batch_size: 16 +world_size: 1 +global_rank: -1 diff --git a/models/labels/latest/results.png b/models/labels/latest/results.png new file mode 100644 index 0000000000000000000000000000000000000000..056498db6620da6ff77fa1ed6601cccfdf4cb4fe Binary files /dev/null and b/models/labels/latest/results.png differ diff --git a/models/labels/latest/results.txt b/models/labels/latest/results.txt new file mode 100644 index 0000000000000000000000000000000000000000..423b44decbcb7cddfa8fbe4215147f36579c926f --- /dev/null +++ b/models/labels/latest/results.txt @@ -0,0 +1,100 @@ + 0/99 6.21G 0.1343 0.03079 0.08276 0.2479 11 768 0 0 7.281e-07 7.281e-08 0.1162 0.04079 0.07292 + 1/99 7.53G 0.1242 0.03164 0.07888 0.2347 21 768 0 0 6.771e-07 1.354e-07 0.108 0.04562 0.07046 + 2/99 7.54G 0.1144 0.03571 0.07596 0.2261 11 768 0 0 1.755e-05 3.077e-06 0.103 0.04908 0.06943 + 3/99 7.54G 0.0974 0.03939 0.07176 0.2086 21 768 0 0 0.002906 0.0006505 0.08903 0.05595 0.06616 + 4/99 7.54G 0.08186 0.04142 0.06885 0.1921 21 768 0.06358 0.04671 0.009447 0.002094 0.07272 0.05272 0.0647 + 5/99 7.54G 0.07081 0.03785 0.06656 0.1752 16 768 0.0354 0.1326 0.02577 0.007177 0.07022 0.04713 0.06211 + 6/99 7.54G 0.06693 0.03364 0.06371 0.1643 21 768 0.08782 0.1313 0.0288 0.00777 0.06502 0.04441 0.05983 + 7/99 7.54G 0.07509 0.03042 0.06239 0.1679 21 768 0.09319 0.08421 0.03582 0.009998 0.07345 0.04032 0.05778 + 8/99 7.54G 0.07271 0.03005 0.06178 0.1645 21 768 0.03703 0.08961 0.02776 0.007952 0.07044 0.03962 0.05648 + 9/99 7.54G 0.06547 0.03006 0.06104 0.1566 11 768 0.0528 0.1506 0.05014 0.01864 0.06365 0.0403 0.05501 + 10/99 7.54G 0.06298 0.02965 0.05742 0.15 21 768 0.1122 0.1863 0.04733 0.01125 0.0575 0.03968 0.05371 + 11/99 7.54G 0.06355 0.02845 0.05799 0.15 21 768 0.02575 0.2244 0.0501 0.01571 0.06196 0.03754 0.05244 + 12/99 7.54G 0.06407 0.02723 0.05621 0.1475 15 768 0.09019 0.2198 0.07907 0.02394 0.05992 0.03758 0.05161 + 13/99 7.54G 0.06065 0.02711 0.05451 0.1423 21 768 0.1057 0.2525 0.08484 0.02365 0.05797 0.03694 0.05046 + 14/99 7.54G 0.06177 0.02657 0.05381 0.1421 15 768 0.121 0.2575 0.1265 0.04451 0.04893 0.03792 0.04975 + 15/99 7.54G 0.05702 0.02646 0.05244 0.1359 21 768 0.1032 0.17 0.05749 0.0109 0.0623 0.03332 0.04756 + 16/99 7.54G 0.05945 0.0258 0.05168 0.1369 21 768 0.0545 0.3034 0.1181 0.03307 0.05174 0.03514 0.04698 + 17/99 7.54G 0.05679 0.02512 0.049 0.1309 21 768 0.07869 0.2468 0.08314 0.01848 0.06121 0.03403 0.04614 + 18/99 7.54G 0.05769 0.02475 0.04614 0.1286 21 768 0.1255 0.3201 0.1718 0.05578 0.0461 0.03457 0.04471 + 19/99 7.54G 0.05541 0.02479 0.04628 0.1265 16 768 0.0921 0.3097 0.1579 0.04787 0.05198 0.0329 0.0437 + 20/99 7.54G 0.05562 0.02399 0.04531 0.1249 21 768 0.1884 0.2838 0.2334 0.09286 0.04295 0.03469 0.04319 + 21/99 7.54G 0.05902 0.02642 0.0444 0.1298 21 768 0.08879 0.3126 0.1348 0.04382 0.05467 0.03429 0.0427 + 22/99 7.54G 0.05915 0.02351 0.04213 0.1248 21 768 0.1733 0.2864 0.1863 0.05512 0.0518 0.03256 0.04173 + 23/99 7.54G 0.05873 0.02303 0.04268 0.1244 21 768 0.1824 0.2923 0.2162 0.08527 0.04832 0.03428 0.04204 + 24/99 7.54G 0.05814 0.02323 0.04174 0.1231 21 768 0.2552 0.2882 0.2324 0.08491 0.04881 0.03281 0.04108 + 25/99 7.54G 0.0542 0.02279 0.04018 0.1172 21 768 0.1997 0.3405 0.2469 0.09714 0.04412 0.03141 0.03916 + 26/99 7.54G 0.0565 0.02218 0.03958 0.1183 21 768 0.182 0.3361 0.264 0.1002 0.04624 0.03252 0.03926 + 27/99 7.54G 0.05618 0.02221 0.03895 0.1173 21 768 0.1902 0.3454 0.2715 0.1179 0.04296 0.03168 0.0382 + 28/99 7.54G 0.05488 0.02226 0.03859 0.1157 16 768 0.2451 0.3586 0.2861 0.1166 0.04555 0.03129 0.03693 + 29/99 7.54G 0.05608 0.02212 0.0377 0.1159 21 768 0.2855 0.3915 0.3077 0.1391 0.04094 0.03167 0.03626 + 30/99 7.54G 0.05181 0.02179 0.03597 0.1096 21 768 0.1752 0.4288 0.3178 0.1396 0.04052 0.03098 0.03552 + 31/99 7.54G 0.05332 0.02198 0.03573 0.111 21 768 0.1629 0.4209 0.3078 0.142 0.04067 0.03126 0.03505 + 32/99 7.54G 0.05079 0.02127 0.03426 0.1063 16 768 0.2647 0.4019 0.3205 0.1273 0.04228 0.0313 0.03469 + 33/99 7.54G 0.0494 0.02121 0.03287 0.1035 16 768 0.2514 0.4279 0.3248 0.1422 0.04237 0.03096 0.03346 + 34/99 7.54G 0.05187 0.02088 0.03268 0.1054 21 768 0.1209 0.4301 0.1953 0.05969 0.06217 0.03133 0.03224 + 35/99 7.54G 0.05873 0.02148 0.03242 0.1126 21 768 0.2324 0.4623 0.3573 0.1543 0.04246 0.0305 0.03147 + 36/99 7.54G 0.05398 0.02115 0.031 0.1061 21 768 0.2775 0.3886 0.3195 0.1042 0.05145 0.02996 0.03089 + 37/99 7.54G 0.05629 0.02095 0.03121 0.1084 21 768 0.2928 0.4967 0.3996 0.191 0.03951 0.03025 0.03075 + 38/99 7.54G 0.05136 0.02113 0.03045 0.1029 16 768 0.2254 0.4235 0.2982 0.08752 0.05017 0.02989 0.02975 + 39/99 7.54G 0.05178 0.02102 0.03069 0.1035 18 768 0.2775 0.5149 0.4013 0.178 0.0392 0.02998 0.03019 + 40/99 7.54G 0.05023 0.02042 0.02886 0.09951 21 768 0.303 0.5104 0.4054 0.1795 0.03967 0.03003 0.03028 + 41/99 7.54G 0.05038 0.02051 0.02829 0.09918 21 768 0.3075 0.4837 0.3993 0.1842 0.03969 0.03067 0.03043 + 42/99 7.54G 0.04899 0.0214 0.02776 0.09815 21 768 0.263 0.5256 0.3875 0.1401 0.04635 0.02994 0.02939 + 43/99 7.54G 0.05134 0.02092 0.02657 0.09883 21 768 0.3034 0.5298 0.4287 0.1998 0.03948 0.02972 0.02878 + 44/99 7.54G 0.04718 0.02077 0.02645 0.0944 21 768 0.28 0.5414 0.3957 0.1446 0.04332 0.02932 0.02855 + 45/99 7.54G 0.04808 0.02024 0.0247 0.09302 21 768 0.3138 0.583 0.4504 0.2132 0.0362 0.02902 0.02797 + 46/99 7.54G 0.04708 0.01997 0.02575 0.0928 16 768 0.3195 0.5838 0.4504 0.2127 0.0376 0.02911 0.02815 + 47/99 7.54G 0.0468 0.01994 0.02458 0.09131 21 768 0.2988 0.5071 0.389 0.1299 0.04774 0.0288 0.02772 + 48/99 7.54G 0.04817 0.01988 0.02352 0.09157 21 768 0.344 0.5396 0.452 0.2008 0.03903 0.02954 0.0275 + 49/99 7.54G 0.04504 0.01988 0.02324 0.08816 21 768 0.3524 0.5616 0.4528 0.1937 0.03945 0.02953 0.02615 + 50/99 7.54G 0.04447 0.01977 0.0229 0.08714 21 768 0.331 0.5934 0.468 0.2189 0.03849 0.02878 0.0256 + 51/99 7.54G 0.04404 0.01981 0.02222 0.08607 21 768 0.3857 0.6196 0.4866 0.2108 0.03942 0.02866 0.02547 + 52/99 7.54G 0.04531 0.01942 0.02096 0.08569 21 768 0.3871 0.6105 0.4906 0.2136 0.04076 0.02887 0.02571 + 53/99 7.54G 0.04653 0.01982 0.02123 0.08759 16 768 0.4044 0.6181 0.5071 0.2409 0.03791 0.029 0.02615 + 54/99 7.54G 0.0447 0.0197 0.02206 0.08646 19 768 0.3528 0.5916 0.4848 0.1903 0.04306 0.02868 0.02565 + 55/99 7.54G 0.04564 0.01984 0.02034 0.08581 15 768 0.3561 0.6169 0.5131 0.253 0.03622 0.02889 0.02553 + 56/99 7.54G 0.04329 0.01954 0.02013 0.08295 21 768 0.3473 0.5974 0.4971 0.1924 0.0421 0.02892 0.02461 + 57/99 7.54G 0.04636 0.02079 0.02197 0.08912 21 768 0.4038 0.6035 0.5051 0.2287 0.03948 0.02927 0.02535 + 58/99 7.54G 0.04345 0.02027 0.0214 0.08512 21 768 0.3235 0.5661 0.4657 0.1647 0.04387 0.0298 0.02548 + 59/99 7.54G 0.04443 0.02014 0.02079 0.08536 16 768 0.3375 0.6146 0.5142 0.2345 0.03623 0.02925 0.02474 + 60/99 7.54G 0.03998 0.01952 0.01991 0.07941 21 768 0.3307 0.61 0.529 0.2253 0.03825 0.02906 0.02412 + 61/99 7.54G 0.04457 0.01975 0.01874 0.08306 16 768 0.4053 0.6245 0.529 0.248 0.03673 0.02878 0.02379 + 62/99 7.54G 0.04051 0.01926 0.01898 0.07875 21 768 0.3944 0.6172 0.5391 0.2481 0.03759 0.02885 0.02325 + 63/99 7.54G 0.04078 0.0187 0.02034 0.07983 15 768 0.4068 0.5992 0.542 0.2405 0.03861 0.02906 0.02271 + 64/99 7.54G 0.04194 0.01933 0.01778 0.07905 21 768 0.4722 0.6268 0.5746 0.2853 0.03541 0.02915 0.02239 + 65/99 7.54G 0.04337 0.0192 0.01906 0.08163 15 768 0.4341 0.5795 0.5289 0.1987 0.04128 0.02902 0.02316 + 66/99 7.54G 0.04128 0.01879 0.01742 0.07749 21 768 0.4445 0.6163 0.5647 0.2619 0.03696 0.02837 0.0229 + 67/99 7.54G 0.03981 0.01887 0.01654 0.07521 21 768 0.429 0.6053 0.5656 0.2208 0.04042 0.02861 0.02219 + 68/99 7.54G 0.04166 0.01899 0.01693 0.07757 21 768 0.4739 0.6238 0.5834 0.2604 0.0365 0.02828 0.02249 + 69/99 7.54G 0.03884 0.01799 0.01664 0.07348 21 768 0.4778 0.6301 0.5943 0.2901 0.03459 0.02814 0.02168 + 70/99 7.54G 0.03924 0.01881 0.01617 0.07422 16 768 0.4868 0.6183 0.5752 0.2273 0.03829 0.02862 0.02152 + 71/99 7.54G 0.04033 0.01877 0.01641 0.0755 16 768 0.5021 0.6403 0.5829 0.2733 0.03533 0.02833 0.02202 + 72/99 7.54G 0.0377 0.01832 0.01589 0.0719 21 768 0.5 0.6262 0.5632 0.2426 0.03893 0.02869 0.02321 + 73/99 7.54G 0.04055 0.01906 0.01569 0.0753 21 768 0.5057 0.634 0.587 0.2631 0.03679 0.02905 0.02234 + 74/99 7.54G 0.03823 0.01857 0.01563 0.07243 21 768 0.5011 0.628 0.593 0.2608 0.03783 0.02941 0.02249 + 75/99 7.54G 0.03925 0.01877 0.01487 0.07289 21 768 0.5047 0.6283 0.5913 0.2768 0.0361 0.02957 0.0222 + 76/99 7.54G 0.03875 0.01839 0.01479 0.07193 16 768 0.4463 0.599 0.5736 0.2188 0.0413 0.02932 0.02138 + 77/99 7.54G 0.03812 0.01841 0.01509 0.07162 21 768 0.4509 0.6188 0.6065 0.2811 0.03619 0.02859 0.02097 + 78/99 7.54G 0.03728 0.01822 0.01423 0.06973 21 768 0.4983 0.6299 0.6103 0.2777 0.0357 0.02857 0.02159 + 79/99 7.54G 0.03603 0.01836 0.01353 0.06792 21 768 0.4991 0.6373 0.6051 0.2815 0.03612 0.02848 0.02253 + 80/99 7.54G 0.03777 0.01839 0.01383 0.06999 21 768 0.513 0.6229 0.5937 0.263 0.03608 0.02879 0.02299 + 81/99 7.54G 0.03728 0.01836 0.01279 0.06842 21 768 0.514 0.6186 0.5827 0.2583 0.03696 0.02926 0.02275 + 82/99 7.54G 0.03713 0.01859 0.01255 0.06827 21 768 0.5083 0.6211 0.5856 0.2528 0.03705 0.02936 0.02303 + 83/99 7.54G 0.03855 0.0181 0.01291 0.06957 21 768 0.4544 0.6131 0.6127 0.2791 0.0352 0.02904 0.02008 + 84/99 7.54G 0.03822 0.01856 0.01275 0.06954 21 768 0.5615 0.6229 0.598 0.2552 0.0372 0.02896 0.02022 + 85/99 7.54G 0.03704 0.01768 0.01312 0.06783 11 768 0.5044 0.6311 0.6029 0.2736 0.03652 0.02857 0.01919 + 86/99 7.54G 0.03856 0.01819 0.01323 0.06998 16 768 0.5074 0.635 0.5867 0.2737 0.03738 0.0284 0.02141 + 87/99 7.54G 0.03727 0.01812 0.01262 0.06801 21 768 0.499 0.6212 0.59 0.2568 0.03755 0.02851 0.01913 + 88/99 7.54G 0.03734 0.01828 0.01226 0.06787 21 768 0.4615 0.6179 0.5588 0.2472 0.03796 0.02839 0.02268 + 89/99 7.54G 0.03812 0.01831 0.01117 0.0676 21 768 0.4669 0.614 0.5726 0.2468 0.03694 0.02835 0.02175 + 90/99 7.54G 0.0374 0.01776 0.01215 0.06732 21 768 0.4729 0.6058 0.5507 0.2349 0.03751 0.02848 0.02446 + 91/99 7.54G 0.0367 0.018 0.01198 0.06668 21 768 0.4756 0.5957 0.5559 0.2342 0.03869 0.02888 0.02457 + 92/99 7.54G 0.03697 0.01809 0.01169 0.06676 21 768 0.5018 0.5968 0.5633 0.2202 0.03954 0.02893 0.02166 + 93/99 7.54G 0.03864 0.01795 0.01148 0.06806 21 768 0.5179 0.6163 0.5839 0.2593 0.03644 0.02868 0.02088 + 94/99 7.54G 0.03675 0.01788 0.01188 0.06651 21 768 0.5553 0.6344 0.6026 0.2615 0.03735 0.02848 0.01651 + 95/99 7.54G 0.03669 0.01775 0.01129 0.06573 21 768 0.4619 0.6223 0.6031 0.2763 0.03487 0.02825 0.01765 + 96/99 7.54G 0.03597 0.01817 0.01073 0.06487 21 768 0.4948 0.6346 0.6214 0.2883 0.0345 0.02828 0.01605 + 97/99 7.54G 0.03609 0.01831 0.01066 0.06506 21 768 0.6066 0.6309 0.5749 0.246 0.03606 0.02862 0.02427 + 98/99 7.54G 0.03695 0.01775 0.01158 0.06628 21 768 0.5525 0.629 0.6034 0.2737 0.0357 0.02861 0.01991 + 99/99 7.54G 0.03731 0.01756 0.0107 0.06556 21 768 0.5626 0.6219 0.5937 0.2479 0.03711 0.02889 0.02241 diff --git a/models/labels/latest/train_batch0.jpg b/models/labels/latest/train_batch0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5a1d78958db93ecdc39fdb55006e33a0dc73073f Binary files /dev/null and b/models/labels/latest/train_batch0.jpg differ diff --git a/models/labels/latest/train_batch1.jpg b/models/labels/latest/train_batch1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bdaa9e0d844702784e6697c9bc62fd63a631dfda Binary files /dev/null and b/models/labels/latest/train_batch1.jpg differ diff --git a/models/labels/latest/train_batch2.jpg b/models/labels/latest/train_batch2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d31b862e6348f3078a6bb6649757fc377ceacfb7 Binary files /dev/null and b/models/labels/latest/train_batch2.jpg differ diff --git a/models/labels/yolov5s.yaml b/models/labels/yolov5s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9459a81d2312da665c7c89f4d5b0842f134cb15e --- /dev/null +++ b/models/labels/yolov5s.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 23 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/symbols/format_dataset.py b/models/symbols/format_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..64c6ba4758381164bf225a3669e9b050bccf28d2 --- /dev/null +++ b/models/symbols/format_dataset.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020 Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" +from typing import List +from types import SimpleNamespace +import argparse, os, json, shutil +from tqdm import tqdm +import os.path as path +import numpy as np +from PIL import Image + +# The different types of symbols that appear on audiograms +SYMBOL_CLASS_INDICES = { + "AIR_UNMASKED_LEFT": 0, + "AIR_UNMASKED_RIGHT": 1, + "AIR_MASKED_LEFT": 2, + "AIR_MASKED_RIGHT": 3, + "BONE_UNMASKED_LEFT": 4, + "BONE_UNMASKED_RIGHT": 5, + "BONE_MASKED_LEFT": 6, + "BONE_MASKED_RIGHT": 7, +} + +def extract_audiograms(annotation: dict, image: Image) -> List[tuple]: + """Extracts the bounding boxes of audiograms into a tuple compatible + the YOLOv5 format. + + Parameters + ---------- + annotation : dict + A dictionary containing the annotations for the audiograms in a report. + + image : Image + The image in PIL format corresponding to the annotation. + + Returns + ------- + tuple + A tuple of the form + (class index, x_center, y_center, width, height) where all coordinates + and dimensions are normalized to the width/height of the image. + """ + audiogram_label_tuples = [] + image_width, image_height = image.size + for audiogram in annotation: + bounding_box = audiogram["boundingBox"] + x_center = (bounding_box["x"] + bounding_box["width"] / 2) / image_width + y_center = (bounding_box["y"] + bounding_box["height"] / 2) / image_height + box_width = bounding_box["width"] / image_width + box_height = bounding_box["height"] / image_width + audiogram_label_tuples.append((0, x_center, y_center, box_width, box_height)) + return audiogram_label_tuples + +def extract_symbols(annotation: dict, image: Image) -> List[tuple]: + """Extracts the bounding boxes of the symbols into tuples + compatible the YOLOv5 format. + + Parameters + ---------- + annotation : dict + A dictionary containing the annotations for the audiograms in a report. + + image: Image + The corresponding image. + + Returns + ------- + List[List[tuple]] + A list of lists of tuples, where the inner lists correspond to the different + audiograms that may appear in the report and the tuples are the symbols. + """ + symbol_labels_tuples = [] + for audiogram in annotation: + left = audiogram["boundingBox"]["x"] + top = audiogram["boundingBox"]["y"] + image_width = audiogram["boundingBox"]["width"] + image_height = audiogram["boundingBox"]["height"] + audiogram_labels = [] + for symbol in audiogram["symbols"]: + bounding_box = symbol["boundingBox"] + x_center = (bounding_box["x"] - left + bounding_box["width"] / 2) / image_width + y_center = (bounding_box["y"] - top + bounding_box["height"] / 2) / image_height + box_width = bounding_box["width"] / image_width + box_height = bounding_box["height"] / image_width + audiogram_labels.append((SYMBOL_CLASS_INDICES[symbol["measurementType"]], x_center, y_center, box_width, box_height)) + symbol_labels_tuples.append(audiogram_labels) + return symbol_labels_tuples + +def create_yolov5_file(bboxes: List[tuple], filename: str): + # Turn the bounding boxes into a string with a bounding box + # on each line + file_content = "\n".join([ + f"{bbox[0]} {bbox[1]} {bbox[2]} {bbox[3]} {bbox[4]}" + for bbox in bboxes + ]) + + # Save to a file + with open(filename, "w") as output_file: + output_file.write(file_content) + +def create_directory_structure(data_dir: str): + try: + shutil.rmtree(path.join(data_dir)) + except: + pass + os.mkdir(path.join(data_dir)) + os.mkdir(path.join(data_dir, "images")) + os.mkdir(path.join(data_dir, "images", "train")) + os.mkdir(path.join(data_dir, "images", "validation")) + os.mkdir(path.join(data_dir, "labels")) + os.mkdir(path.join(data_dir, "labels", "train")) + os.mkdir(path.join(data_dir, "labels", "validation")) + + +def all_labels_valid(labels: List[tuple]): + for label in labels: + for value in label[1:]: + if value < 0 or value > 1: + return False + return True + +def main(args: SimpleNamespace): + # Find all the JSON files in the input directory + report_ids = [ + filename.rstrip(".json") + for filename in os.listdir(path.join(args.annotations_dir)) + if filename.endswith(".json") + and path.exists(path.join(args.images_dir, filename.rstrip(".json") + ".jpg")) + ] + + # Shuffle + np.random.seed(seed=42) # for reproducibility of the shuffle + np.random.shuffle(report_ids) + + # Create the directory structure in which the images and annotations + # are to be stored + create_directory_structure(args.data_dir) + + # Iterate through the report ids, extract the annotations in YOLOv5 format + # and place the file in the correct directory, and the image in the correct + # directory. + for i, report_id in enumerate(tqdm(report_ids)): + # Decide if the image is going into the training set or validation set + directory = ( + "train" if i < args.train_frac * len(report_ids) else "validation" + ) + + # Load the annotation` + annotation_content = open( + path.join(args.annotations_dir, f"{report_id}.json") + ) + annotation = json.load(annotation_content) + + # Open the corresponding image to get its dimensions + image = Image.open(os.path.join(args.images_dir, f"{report_id}.jpg")) + width, height = image.size + + # Audiogram labels + audiogram_labels = extract_audiograms(annotation, image) + + if not all_labels_valid(audiogram_labels): + continue + + # Symbol labels + symbol_labels = extract_symbols(annotation, image) + for i, plot_symbols in enumerate(symbol_labels): + if not all_labels_valid(plot_symbols): + continue + x1 = annotation[i]["boundingBox"]["x"] + y1 = annotation[i]["boundingBox"]["y"] + x2 = annotation[i]["boundingBox"]["x"] + annotation[i]["boundingBox"]["width"] + y2 = annotation[i]["boundingBox"]["y"] + annotation[i]["boundingBox"]["height"] + create_yolov5_file( + plot_symbols, + path.join(args.data_dir, "labels", directory, f"{report_id}_{i+1}.txt") + ) + image.crop((x1, y1, x2, y2)).save( + path.join(args.data_dir, "images", directory, f"{report_id}_{i+1}.jpg") + ) + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description=( + "Script that formats the training set for transfer learning via " + "the YOLOv5 model." + )) + parser.add_argument("-d", "--data_dir", type=str, required=True, help=( + "Path to the directory containing the data. It should have 3 " + "subfolders named `images`, `annotations` and `labels`." + )) + parser.add_argument("-a", "--annotations_dir", type=str, required=True, help=( + "Path to the directory containing the annotations in the JSON format." + )) + parser.add_argument("-i", "--images_dir", type=str, required=True, help=( + "Path to the directory containing the images." + )) + parser.add_argument("-f", "--train_frac", type=float, required=True, help=( + "Fraction of images to be used for training. (e.g. 0.8)" + )) + args = parser.parse_args() + + main(args) + diff --git a/models/symbols/latest/hyp.yaml b/models/symbols/latest/hyp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eb1f5e4c9197952a4e0670d6bd50caca8a1ef56 --- /dev/null +++ b/models/symbols/latest/hyp.yaml @@ -0,0 +1,23 @@ +lr0: 0.01 +lrf: 0.2 +momentum: 0.937 +weight_decay: 0.0005 +giou: 0.05 +cls: 0.5 +cls_pw: 1.0 +obj: 1.0 +obj_pw: 1.0 +iou_t: 0.2 +anchor_t: 4.0 +fl_gamma: 0.0 +hsv_h: 0.015 +hsv_s: 0.7 +hsv_v: 0.4 +degrees: 0.0 +translate: 0.1 +scale: 0.5 +shear: 0.0 +perspective: 0.0 +flipud: 0.0 +fliplr: 0.5 +mixup: 0.0 diff --git a/models/symbols/latest/labels.png b/models/symbols/latest/labels.png new file mode 100644 index 0000000000000000000000000000000000000000..22e2425186eed916d69573aec0e264f9b75a95e1 Binary files /dev/null and b/models/symbols/latest/labels.png differ diff --git a/models/symbols/latest/opt.yaml b/models/symbols/latest/opt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41c892343325b9306d85e7ff4fdd8dabb86883e1 --- /dev/null +++ b/models/symbols/latest/opt.yaml @@ -0,0 +1,30 @@ +weights: yolov5/weights/yolov5s.pt +cfg: yolov5/models/yolov5s.yaml +data: symbol_detection.yaml +hyp: ./yolov5/data/hyp.scratch.yaml +epochs: 50 +batch_size: 16 +img_size: +- 1024 +- 1024 +rect: true +resume: false +nosave: false +notest: false +noautoanchor: false +evolve: false +bucket: '' +cache_images: false +image_weights: false +name: '' +device: '0' +multi_scale: false +single_cls: false +adam: false +sync_bn: false +local_rank: -1 +logdir: runs/ +workers: 8 +total_batch_size: 16 +world_size: 1 +global_rank: -1 diff --git a/models/symbols/latest/results.png b/models/symbols/latest/results.png new file mode 100644 index 0000000000000000000000000000000000000000..9dfad5a0b47a0ee02f24cc42d2a2ddc526b8f56c Binary files /dev/null and b/models/symbols/latest/results.png differ diff --git a/models/symbols/latest/results.txt b/models/symbols/latest/results.txt new file mode 100644 index 0000000000000000000000000000000000000000..e940691905c15191ac08a8fd33ea854be965a0ff --- /dev/null +++ b/models/symbols/latest/results.txt @@ -0,0 +1,50 @@ + 0/49 7.49G 0.07765 0.05224 0.04646 0.1763 7 576 0.04767 0.2132 0.1067 0.0261 0.0709 0.04132 0.03329 + 1/49 7.53G 0.0574 0.03939 0.03081 0.1276 7 576 0.1938 0.3598 0.2717 0.06994 0.05514 0.03799 0.02433 + 2/49 7.53G 0.05059 0.03685 0.02426 0.1117 7 576 0.2269 0.3609 0.2415 0.05457 0.05992 0.03802 0.0213 + 3/49 7.53G 0.04991 0.03578 0.02254 0.1082 7 576 0.2634 0.3865 0.2704 0.0603 0.06173 0.03594 0.02003 + 4/49 7.53G 0.04831 0.03514 0.02124 0.1047 6 576 0.2995 0.5582 0.3703 0.107 0.05316 0.03657 0.01871 + 5/49 7.53G 0.04502 0.03474 0.01945 0.09921 7 576 0.2936 0.5627 0.3964 0.11 0.05526 0.03665 0.01701 + 6/49 7.53G 0.04389 0.03427 0.01799 0.09615 7 576 0.2852 0.5298 0.3742 0.1009 0.0553 0.0355 0.01604 + 7/49 7.53G 0.04235 0.03403 0.01705 0.09343 7 576 0.2816 0.4995 0.3463 0.08569 0.05838 0.03602 0.01547 + 8/49 7.53G 0.0421 0.03383 0.01663 0.09256 7 576 0.4374 0.5792 0.4381 0.1241 0.05197 0.03575 0.01499 + 9/49 7.53G 0.04017 0.03327 0.01569 0.08912 7 576 0.4495 0.6056 0.4835 0.141 0.05317 0.03561 0.01422 + 10/49 7.53G 0.03991 0.0332 0.01515 0.08826 7 576 0.5538 0.5598 0.4569 0.1217 0.05581 0.03582 0.01345 + 11/49 7.53G 0.03979 0.03298 0.01464 0.08741 7 576 0.4092 0.633 0.5494 0.1704 0.05083 0.03614 0.01397 + 12/49 7.53G 0.03885 0.03274 0.01418 0.08577 7 576 0.4398 0.6878 0.5847 0.1786 0.05148 0.03602 0.01281 + 13/49 7.53G 0.03938 0.0328 0.01332 0.0855 7 576 0.4823 0.7566 0.6708 0.2135 0.05016 0.03501 0.01239 + 14/49 7.53G 0.03869 0.0325 0.0126 0.08379 7 576 0.5036 0.7224 0.6623 0.2015 0.05224 0.03578 0.01118 + 15/49 7.53G 0.03856 0.03249 0.0121 0.08314 7 576 0.4471 0.7152 0.6042 0.1691 0.05259 0.03617 0.01143 + 16/49 7.53G 0.03844 0.03263 0.01164 0.08271 7 576 0.4748 0.7556 0.6721 0.2094 0.0505 0.03606 0.01165 + 17/49 7.53G 0.03824 0.03244 0.01134 0.08202 7 576 0.4454 0.8108 0.6782 0.2079 0.05038 0.03572 0.01091 + 18/49 7.53G 0.0378 0.03225 0.01097 0.08103 7 576 0.4437 0.7971 0.6736 0.1934 0.052 0.03567 0.01057 + 19/49 7.53G 0.03797 0.03225 0.01065 0.08087 7 576 0.437 0.7995 0.6595 0.202 0.04957 0.03569 0.01094 + 20/49 7.53G 0.03772 0.03216 0.01053 0.08041 7 576 0.4631 0.8147 0.713 0.2205 0.04931 0.03605 0.01022 + 21/49 7.53G 0.03769 0.03196 0.01036 0.08001 7 576 0.4799 0.8187 0.7363 0.2399 0.04898 0.03545 0.0103 + 22/49 7.53G 0.03721 0.0318 0.01005 0.07905 7 576 0.4458 0.8343 0.7107 0.2057 0.05059 0.03609 0.009893 + 23/49 7.53G 0.0373 0.03188 0.01003 0.0792 7 576 0.462 0.8368 0.7502 0.2398 0.04766 0.03542 0.01006 + 24/49 7.53G 0.03706 0.03175 0.009794 0.07861 7 576 0.4662 0.848 0.7574 0.2366 0.04742 0.03576 0.01004 + 25/49 7.53G 0.037 0.03154 0.009702 0.07824 7 576 0.4671 0.8519 0.7534 0.2438 0.04799 0.03574 0.00978 + 26/49 7.53G 0.03664 0.03155 0.009522 0.07771 7 576 0.4553 0.8399 0.7258 0.2114 0.04911 0.03574 0.009557 + 27/49 7.53G 0.03663 0.03134 0.009434 0.07741 6 576 0.4576 0.8515 0.7464 0.2495 0.046 0.03549 0.01006 + 28/49 7.53G 0.03649 0.03125 0.009326 0.07707 7 576 0.4592 0.8557 0.7699 0.2423 0.04658 0.03556 0.009594 + 29/49 7.53G 0.03631 0.03119 0.009154 0.07665 7 576 0.4649 0.8459 0.7688 0.2489 0.04703 0.03547 0.009588 + 30/49 7.53G 0.03638 0.03128 0.009051 0.07672 7 576 0.4404 0.8472 0.7481 0.2368 0.04745 0.03581 0.01008 + 31/49 7.53G 0.03626 0.03109 0.009029 0.07638 5 576 0.4682 0.8503 0.7648 0.2715 0.0447 0.03521 0.009697 + 32/49 7.53G 0.03619 0.03078 0.008949 0.07592 7 576 0.4727 0.8656 0.7748 0.2627 0.04556 0.03557 0.009528 + 33/49 7.53G 0.03582 0.03089 0.008738 0.07544 7 576 0.4715 0.8527 0.7524 0.2595 0.04565 0.03552 0.009686 + 34/49 7.53G 0.03597 0.03074 0.008762 0.07547 6 576 0.4748 0.8733 0.7652 0.2553 0.04547 0.03571 0.009658 + 35/49 7.53G 0.03585 0.03071 0.00865 0.07521 7 576 0.5028 0.8653 0.7807 0.2915 0.04359 0.03525 0.009554 + 36/49 7.53G 0.03551 0.03053 0.008564 0.07461 6 576 0.4858 0.8738 0.7824 0.2899 0.04349 0.0353 0.009391 + 37/49 7.53G 0.03535 0.03063 0.008459 0.07443 7 576 0.4847 0.8664 0.7892 0.2833 0.04479 0.03553 0.009333 + 38/49 7.53G 0.03561 0.03047 0.008462 0.07454 7 576 0.4791 0.8654 0.7652 0.2747 0.04421 0.03549 0.009689 + 39/49 7.53G 0.03544 0.03056 0.008445 0.07445 7 576 0.4997 0.8772 0.7985 0.3108 0.04193 0.0351 0.009489 + 40/49 7.53G 0.03495 0.03029 0.008376 0.07361 7 576 0.4959 0.8783 0.8073 0.3197 0.04251 0.03527 0.009663 + 41/49 7.53G 0.03491 0.03027 0.008229 0.0734 6 576 0.4818 0.8804 0.7962 0.3107 0.043 0.03533 0.009612 + 42/49 7.53G 0.03491 0.03008 0.008287 0.07328 7 576 0.5063 0.8718 0.8105 0.3125 0.0428 0.03537 0.009683 + 43/49 7.53G 0.03472 0.03013 0.008238 0.07309 6 576 0.5203 0.8777 0.81 0.3442 0.04114 0.03507 0.009757 + 44/49 7.53G 0.03416 0.02998 0.008148 0.07229 7 576 0.4981 0.8905 0.8147 0.3268 0.04171 0.03498 0.009373 + 45/49 7.53G 0.03424 0.03003 0.008063 0.07233 7 576 0.5018 0.8824 0.8067 0.3253 0.04174 0.03512 0.009696 + 46/49 7.53G 0.03433 0.02986 0.008007 0.07219 7 576 0.5216 0.8748 0.8155 0.3331 0.04193 0.03524 0.009595 + 47/49 7.53G 0.03398 0.0299 0.00801 0.07189 7 576 0.5127 0.8726 0.8061 0.3483 0.04056 0.03489 0.009472 + 48/49 7.53G 0.03383 0.02974 0.008026 0.0716 7 576 0.5021 0.8772 0.8126 0.3428 0.04086 0.035 0.009722 + 49/49 7.53G 0.03385 0.02983 0.00794 0.07161 6 576 0.5133 0.878 0.8072 0.3366 0.04112 0.03492 0.009682 diff --git a/models/symbols/latest/test_batch0_gt.jpg b/models/symbols/latest/test_batch0_gt.jpg new file mode 100644 index 0000000000000000000000000000000000000000..914bb8e1673d9a1719bf157329c3dc91e9df51e7 Binary files /dev/null and b/models/symbols/latest/test_batch0_gt.jpg differ diff --git a/models/symbols/latest/test_batch0_pred.jpg b/models/symbols/latest/test_batch0_pred.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7faf13d1428e11d5fa5587885212cbbeca412e26 Binary files /dev/null and b/models/symbols/latest/test_batch0_pred.jpg differ diff --git a/models/symbols/latest/train_batch0.jpg b/models/symbols/latest/train_batch0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e8c5458b55926994d260f6236e049d6c99cb7b1b Binary files /dev/null and b/models/symbols/latest/train_batch0.jpg differ diff --git a/models/symbols/latest/train_batch1.jpg b/models/symbols/latest/train_batch1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0cf54a825879ec61750f006e253a9cf0947f51cc Binary files /dev/null and b/models/symbols/latest/train_batch1.jpg differ diff --git a/models/symbols/latest/train_batch2.jpg b/models/symbols/latest/train_batch2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1330b2d86082480da1319cfe144a95f7e2630cc0 Binary files /dev/null and b/models/symbols/latest/train_batch2.jpg differ diff --git a/models/symbols/symbols_detection.yaml b/models/symbols/symbols_detection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a669b06cb74c37ba1cae23bf84980c0bba46613d --- /dev/null +++ b/models/symbols/symbols_detection.yaml @@ -0,0 +1,18 @@ +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +train: models/symbols/dataset/images/train/ +val: models/symbols/dataset/images/validation/ + +# number of classes +nc: 8 + +# class names +names: [ + "AIR_UNMASKED_LEFT", + "AIR_UNMASKED_RIGHT", + "AIR_MASKED_LEFT", + "AIR_MASKED_RIGHT", + "BONE_UNMASKED_LEFT", + "BONE_UNMASKED_RIGHT", + "BONE_MASKED_LEFT", + "BONE_MASKED_RIGHT" +] diff --git a/models/symbols/symbols_detection_test.yaml b/models/symbols/symbols_detection_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a06e634c561f53891aa15bce060f4f93b69d317 --- /dev/null +++ b/models/symbols/symbols_detection_test.yaml @@ -0,0 +1,17 @@ +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +val: data/symbols/test/images/ + +# number of classes +nc: 8 + +# class names +names: [ + "AIR_UNMASKED_LEFT", + "AIR_UNMASKED_RIGHT", + "AIR_MASKED_LEFT", + "AIR_MASKED_RIGHT", + "BONE_UNMASKED_LEFT", + "BONE_UNMASKED_RIGHT", + "BONE_MASKED_LEFT", + "BONE_MASKED_RIGHT" +] diff --git a/models/symbols/yolov5s.yaml b/models/symbols/yolov5s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5959a41b83643890314b0fe3ac964bfea8a6f6e9 --- /dev/null +++ b/models/symbols/yolov5s.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 8 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..23fad7dcc3504607f6895c4c24859b2be1f54295 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +matplotlib +pandas +Pillow +pytesseract +PyYAML +requests +scipy +toml +torch==1.9.0 +torchvision==0.10.0 +tqdm +typed-ast +typing-extensions +opencv-python +gradio \ No newline at end of file diff --git a/src/__pycache__/interfaces.cpython-38.pyc b/src/__pycache__/interfaces.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37562d644644767aec4120a727267511b4df8c10 Binary files /dev/null and b/src/__pycache__/interfaces.cpython-38.pyc differ diff --git a/src/digitize_report.py b/src/digitize_report.py new file mode 100755 index 0000000000000000000000000000000000000000..b5b468f547175a5cf09f460af4776ebf9b153263 --- /dev/null +++ b/src/digitize_report.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +import json +import os + +from tqdm import tqdm + +from digitizer.digitization import generate_partial_annotation, extract_thresholds + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description=("Digitization script " + "for the conversion of audiology reports to JSON documents.")) + parser.add_argument("-i", "--input", type=str, required=True, + help=("Path to the audiology report (or directory) to be digitized.")) + parser.add_argument("-o", "--output_dir", type=str, required=False, + help="Path to the directory in which the result is to be saved (file will have same base name as input file, but with .json extension). If not provided, result is printed to the console.") + parser.add_argument("-a", "--annotation_mode", action="store_true", + help="Whether the script should be run in `annotation mode`, i.e. return results similar in format to those of a human-made annotation. If not given, a list of thresholds is computed.") + parser.add_argument("-g", "--gpu", action="store_true", + help="Use the GPU.") + args = parser.parse_args() + + input_files = [] + if os.path.isfile(args.input): + input_files += [os.path.abspath(args.input)] + else: + input_files += [os.path.join(args.input, filename) for filename in os.listdir(args.input)] + + with tqdm(total=len(input_files)) as pbar: + for input_file in input_files: + pbar.set_description(f"{os.path.basename(input_file)}") + + result = None + if args.annotation_mode: + result = generate_partial_annotation(input_file, gpu=args.gpu) + else: + result = extract_thresholds(input_file, gpu=args.gpu) + + result_as_string = json.dumps(result, indent=4, separators=(',', ': ')) + + if args.output_dir: + predictions_filename = os.path.basename(input_file).split(".")[0] + ".json" + with open(os.path.join(args.output_dir, predictions_filename), "w") as ofile: + ofile.write(result_as_string) + else: + print(result_as_string) + + pbar.update(1) # increment the progress bar diff --git a/src/digitizer/__pycache__/digitization.cpython-38.pyc b/src/digitizer/__pycache__/digitization.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..097ef2ecade81bb6359cffeaee0950d9f8afdde7 Binary files /dev/null and b/src/digitizer/__pycache__/digitization.cpython-38.pyc differ diff --git a/src/digitizer/assets/fonts/arial.ttf b/src/digitizer/assets/fonts/arial.ttf new file mode 100644 index 0000000000000000000000000000000000000000..ab68fb197d4479b3b6dec6e85bd5cbaf433a87c5 Binary files /dev/null and b/src/digitizer/assets/fonts/arial.ttf differ diff --git a/src/digitizer/assets/symbols/left_air_masked.png b/src/digitizer/assets/symbols/left_air_masked.png new file mode 100644 index 0000000000000000000000000000000000000000..5b166d29f270da971c71f1107c118fc38829f9e7 Binary files /dev/null and b/src/digitizer/assets/symbols/left_air_masked.png differ diff --git a/src/digitizer/assets/symbols/left_air_masked.svg b/src/digitizer/assets/symbols/left_air_masked.svg new file mode 100644 index 0000000000000000000000000000000000000000..9bf450f4ad05a5379b0acdee450cecd22d3bf91b --- /dev/null +++ b/src/digitizer/assets/symbols/left_air_masked.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/digitizer/assets/symbols/left_air_unmasked.png b/src/digitizer/assets/symbols/left_air_unmasked.png new file mode 100644 index 0000000000000000000000000000000000000000..eddaca93418c76d788ee5d1162f525c7216a4307 Binary files /dev/null and b/src/digitizer/assets/symbols/left_air_unmasked.png differ diff --git a/src/digitizer/assets/symbols/left_air_unmasked.svg b/src/digitizer/assets/symbols/left_air_unmasked.svg new file mode 100644 index 0000000000000000000000000000000000000000..19275e6293d7b2a8e4d58b821b1cfce1e484351c --- /dev/null +++ b/src/digitizer/assets/symbols/left_air_unmasked.svg @@ -0,0 +1,10 @@ + + + + + + + diff --git a/src/digitizer/assets/symbols/left_bone_masked.png b/src/digitizer/assets/symbols/left_bone_masked.png new file mode 100644 index 0000000000000000000000000000000000000000..6050693d9e3f314350f84a05f042a95733cc977a Binary files /dev/null and b/src/digitizer/assets/symbols/left_bone_masked.png differ diff --git a/src/digitizer/assets/symbols/left_bone_masked.svg b/src/digitizer/assets/symbols/left_bone_masked.svg new file mode 100644 index 0000000000000000000000000000000000000000..08279a36ecc4016a405a0225ca62b58e89ca9e98 --- /dev/null +++ b/src/digitizer/assets/symbols/left_bone_masked.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/digitizer/assets/symbols/left_bone_unmasked.png b/src/digitizer/assets/symbols/left_bone_unmasked.png new file mode 100644 index 0000000000000000000000000000000000000000..7b6ff69fb8f0408fb36bd9a094162167d3c1c58d Binary files /dev/null and b/src/digitizer/assets/symbols/left_bone_unmasked.png differ diff --git a/src/digitizer/assets/symbols/left_bone_unmasked.svg b/src/digitizer/assets/symbols/left_bone_unmasked.svg new file mode 100644 index 0000000000000000000000000000000000000000..024748fcbfa62735c31b075df50f333177779531 --- /dev/null +++ b/src/digitizer/assets/symbols/left_bone_unmasked.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/digitizer/assets/symbols/right_air_masked.png b/src/digitizer/assets/symbols/right_air_masked.png new file mode 100644 index 0000000000000000000000000000000000000000..70fc89e549e8ebc05febaa699fdb01382963da74 Binary files /dev/null and b/src/digitizer/assets/symbols/right_air_masked.png differ diff --git a/src/digitizer/assets/symbols/right_air_masked.svg b/src/digitizer/assets/symbols/right_air_masked.svg new file mode 100644 index 0000000000000000000000000000000000000000..79241929e4b017871894cae17177df1ee3e74cbc --- /dev/null +++ b/src/digitizer/assets/symbols/right_air_masked.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/digitizer/assets/symbols/right_air_unmasked.png b/src/digitizer/assets/symbols/right_air_unmasked.png new file mode 100644 index 0000000000000000000000000000000000000000..1a489cd07b7f2347a3751d94def7ae661430eacc Binary files /dev/null and b/src/digitizer/assets/symbols/right_air_unmasked.png differ diff --git a/src/digitizer/assets/symbols/right_air_unmasked.svg b/src/digitizer/assets/symbols/right_air_unmasked.svg new file mode 100644 index 0000000000000000000000000000000000000000..eed58575b0613f7a2e106353092f0478e0c1d288 --- /dev/null +++ b/src/digitizer/assets/symbols/right_air_unmasked.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/digitizer/assets/symbols/right_bone_masked.png b/src/digitizer/assets/symbols/right_bone_masked.png new file mode 100644 index 0000000000000000000000000000000000000000..08860d09b8eeeec7d0848d51a5d94199c75b0663 Binary files /dev/null and b/src/digitizer/assets/symbols/right_bone_masked.png differ diff --git a/src/digitizer/assets/symbols/right_bone_masked.svg b/src/digitizer/assets/symbols/right_bone_masked.svg new file mode 100644 index 0000000000000000000000000000000000000000..31aadcaa8710c816efc14f6d4c9955c6139e3259 --- /dev/null +++ b/src/digitizer/assets/symbols/right_bone_masked.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/digitizer/assets/symbols/right_bone_unmasked.png b/src/digitizer/assets/symbols/right_bone_unmasked.png new file mode 100644 index 0000000000000000000000000000000000000000..e895cecca1c49fcab30eea0134f8d7906890bf87 Binary files /dev/null and b/src/digitizer/assets/symbols/right_bone_unmasked.png differ diff --git a/src/digitizer/assets/symbols/right_bone_unmasked.svg b/src/digitizer/assets/symbols/right_bone_unmasked.svg new file mode 100644 index 0000000000000000000000000000000000000000..5d1db4a50957e92c0c69facfbd15d3070ce71e17 --- /dev/null +++ b/src/digitizer/assets/symbols/right_bone_unmasked.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/digitizer/digitization.py b/src/digitizer/digitization.py new file mode 100644 index 0000000000000000000000000000000000000000..502c09763b9723c99f2a25c250ed7e1c64c0e801 --- /dev/null +++ b/src/digitizer/digitization.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +import pathlib +import json +import os +import subprocess as sp +import tempfile +from typing import List, Callable + +from tqdm import tqdm +import numpy as np + +from interfaces import AudiogramDict, AudiogramAnnotationDict, ThresholdDict +from digitizer.report_components.grid import Grid +from digitizer.report_components.label import Label +from digitizer.report_components.symbol import Symbol +from digitizer.report_components.report import Report +import utils.audiology as Audiology +from utils.geometry import compute_rotation_angle, apply_rotation + +DIR = os.path.join(pathlib.Path(__file__).parent.absolute(), "..") # current directory + +def detect_audiograms(filepath: str, weights: str, device: str = "cpu") -> List[AudiogramDict]: + """Runs the audiogram detector. + + The detector is run as a subprocess. + + Parameters + ---------- + filepath : str + Path to the image on which the detector is to be run. + weights : str + Path to the file holding the weights of the neural network (detector). + device : str + "cpu" or "gpu" + + Returns + ------- + List[AudiogramDict] + The AudiogramDict corresponding to the audiograms detected in the report. + """ + subprocess = sp.Popen([ + "python3", + f"{os.path.join(DIR, 'digitizer/yolov5/detect_audiograms.py')}", + "--source", f"{filepath}", + "--weights", weights, + "--device", device + ], stdout=sp.PIPE) # TODO timeout should be an environment variable + output = subprocess.stdout.read().decode("utf-8") + audiograms = json.loads(output.split("$$$")[1]) + return audiograms + +def detect_labels(filepath: str, weights: str, audiogram_coordinates: dict, correction_angle: float, device: str = "cpu") -> List[Label]: + """Runs the label detector. + + The detector is run as a subprocess. + + Parameters + ---------- + filepath : str + Path to the image on which the detector is to be run. + audiogram_coordinates: dict + The coordinates of the audiogram { "x": int, "y": int } needed to convert the label locations + with respect to the top-left corner of the bounding audiogram to relative to the top-left corner + of the report. + correction_angle: float + The correction angle in degrees that was applied to the audiogram, so that it can be reversed to + get the coordinates of the label with respect to the top-left corner of the original unrotated report. + weights : str + Path to the file holding the weights of the neural network (detector). + device : str + "cpu" or "gpu" + + Returns + ------- + List[Label] + A list of Label objects (NOT LabelDict). + """ + subprocess = sp.Popen([ + "python3", + os.path.join(DIR, "digitizer/yolov5/detect_labels.py"), + "--source", f"{filepath}", + "--weights", weights, + "--device", device + ], stdout=sp.PIPE) + output = subprocess.stdout.read().decode("utf-8") + parsed = json.loads(output.split("$$$")[1]) + label_dicts = parsed + labels = [Label(label, audiogram_coordinates, correction_angle) for label in parsed] + return labels + +def detect_symbols(filepath: str, weights: str, audiogram_coordinates: dict, correction_angle: float, device: str = "cpu") -> List[Symbol]: + """Runs the symbol detector. + + The detector is run as a subprocess. + + Parameters + ---------- + filepath : str + Path to the image on which the detector is to be run. + audiogram_coordinates: dict + The coordinates of the audiogram { "x": int, "y": int } needed to convert the label locations + with respect to the top-left corner of the bounding audiogram to relative to the top-left corner + of the report. + correction_angle: float + The correction angle in degrees that was applied to the audiogram, so that it can be reversed to + get the coordinates of the label with respect to the top-left corner of the original unrotated report. + weights : str + Path to the file holding the weights of the neural network (detector). + device : str + "cpu" or "gpu" + + Returns + ------- + List[Label] + A list of Symbol objects (NOT SymbolDict). + """ + subprocess = sp.Popen([ + "python3", + os.path.join(DIR, "digitizer/yolov5/detect_symbols.py"), + "--source", filepath, + "--weights", weights, + "--device", device + ], stdout=sp.PIPE) + + output = json.loads(subprocess.stdout.read().decode("utf-8").split("$$$")[1]) + symbols = [Symbol(detection, audiogram_coordinates, correction_angle) for detection in output] + return symbols + +def detect_components(filepath: str, gpu: bool = False) -> List: + """Invokes the object detectors. + + Parameters + ---------- + filepath : str + Path to the image. + gpu : bool + Whether the GPU should be used (default: False). + + Returns + ------- + List + A list (of length 0, 1 or 2) of the form + [ + { "audiogram": AudiogramDict, "labels": List[Label], "symbols": List[Symbol] }, # plot 1 + { "audiogram": AudiogramDict, "labels": List[Label], "symbols": List[Symbol] } # plot 2 + ] + """ + + components = [] + + # Detect audiograms within the report + audiogram_model_weights_path = os.path.join(DIR, "..", "models/audiograms/latest/weights/best.pt") + audiograms = detect_audiograms(f"{filepath}", audiogram_model_weights_path) + + # If no audiogram is detected, return... + if len(audiograms) == 0: + return components + + # Iterate through every audiogram in the report + for i, audiogram in enumerate(audiograms): + components.append({}) + + # Load the report + report = Report(filename=filepath) + + # Generate a cropped version of the report around the detected audiogram + report = report.crop( + audiogram["boundingBox"]["x"], + audiogram["boundingBox"]["y"], + audiogram["boundingBox"]["x"] + audiogram["boundingBox"]["width"], + audiogram["boundingBox"]["y"] + audiogram["boundingBox"]["height"] + ) + + # Create a temporary file + cropped_file = tempfile.NamedTemporaryFile(suffix=".jpg") + + # Correct for rotation + lines = report.detect_lines(threshold=200) + perpendicular_lines = [ + line for line in lines + if line.has_a_perpendicular_line(lines) + and (abs(line.get_angle() - 90) < 10 + or abs(line.get_angle()) < 10) + ] + correction_angle = compute_rotation_angle(perpendicular_lines) + audiogram["correctionAngle"] = correction_angle + report = report.rotate(correction_angle) + report.save(cropped_file.name) + + audiogram_coordinates = { + "x": audiogram["boundingBox"]["x"], + "y": audiogram["boundingBox"]["y"] + } + + components[i]["audiogram"] = audiogram + + labels_model_weights_path = os.path.join(DIR, "..", "models/labels/latest/weights/best.pt") + components[i]["labels"] = detect_labels(cropped_file.name, labels_model_weights_path, audiogram_coordinates, correction_angle) + symbols_model_weights_path = os.path.join(DIR, "..", "models/symbols/latest/weights/best.pt") + components[i]["symbols"] = detect_symbols(cropped_file.name, symbols_model_weights_path, audiogram_coordinates, correction_angle) + + return components + +def generate_partial_annotation(filepath: str, gpu: bool = False) -> List[AudiogramAnnotationDict]: + """Generates a seed annotation to be completed in the nihl portal. + + It is ``partial`` because it does not locate the corners of the audiogram. + + Parameters + ---------- + filepath : str + Path to the file for which an initial annotation is to b + gpu : bool + Whether the gpu should be used. + + Returns + ------- + List[AudiogramAnnotationDict] + An Annotation dict. + """ + components = detect_components(filepath, gpu=gpu) + audiograms = [] + for i in range(len(components)): + audiogram = components[i]["audiogram"] + audiogram["labels"] = [label.to_dict() for label in components[i]["labels"]] + audiogram["symbols"] = [symbol.to_dict() for symbol in components[i]["symbols"]] + audiogram["corners"] = [] # these are not located by the algorithm + audiograms.append(audiogram) + return audiograms + +def extract_thresholds(filepath: str, gpu: bool = False) -> List[ThresholdDict]: + """Extracts the thresholds from the report. + + parameters + ---------- + filepath : str + Path to the file for which an initial annotation is to b + gpu : bool + Whether the gpu should be used. + + Returns + ------- + list[ThresholdDict] + A list of thresholds. + """ + components = detect_components(filepath, gpu=gpu) + + thresholds = [] + + # For each audiogram, extract the thresholds and append them to the + # thresholds list + for i in range(len(components)): + audiogram = components[i]["audiogram"] + labels = components[i]["labels"] + symbols = components[i]["symbols"] + + report = Report(filename=filepath) + report = report.crop( + audiogram["boundingBox"]["x"], + audiogram["boundingBox"]["y"], + audiogram["boundingBox"]["x"] + audiogram["boundingBox"]["width"], + audiogram["boundingBox"]["y"] + audiogram["boundingBox"]["height"] + ) + report = report.rotate(audiogram["correctionAngle"]) + + try: + grid = Grid(report, labels) + except Exception as e: + continue + + thresholds += [{ + "ear": symbol.ear, + "conduction": symbol.conduction, + "masking": symbol.masking, + "measurementType": Audiology.stringify_measurement(symbol.to_dict()), + "frequency": grid.get_snapped_frequency(symbol), + "threshold": grid.get_snapped_threshold(symbol), + "response": True # IMPORTANT: assume that a response was obtain for measurements + } + for symbol in symbols + ] + return thresholds + +def get_correction_angle(corners: List[dict]) -> float: + """Computes the rotation angle that must be applied based on + corner coordinates to get an unrotated audiogram. + + Parameters + ---------- + corners : List[dict] + A list of corners. + + Returns + ------- + float + The rotation angle that must be applied to correct for the rotation + of the audiogram. + """ + # sort the corners + corners = sorted(corners, key=lambda c: c["y"]) + top_corners = sorted(corners[2:], key=lambda c: c["x"]) + bottom_corners = sorted(corners[0:2], key=lambda c: c["x"]) + + # Find the rotation angle based on the top_corners 2 corners + dx1 = top_corners[1]["x"] - top_corners[0]["x"] + dy1 = top_corners[1]["y"] - top_corners[0]["y"] + angle1 = np.arcsin(abs(dy1)/abs(dx1)) + + # Repeat for the bottom_corners angles + dx2 = bottom_corners[1]["x"] - bottom_corners[0]["x"] + dy2 = bottom_corners[1]["y"] - bottom_corners[0]["y"] + angle2 = np.arcsin(abs(dy2)/abs(dx2)) + + return np.sign(dy1) * np.mean([angle1, angle2]) + +def get_conversion_maps(corners: List[dict]) -> List[Callable]: + """Computes the functions that map pixel coordinates to frequency-threshold coordinates + and vice versa. + + Parameters + ---------- + corners : List[dict] + The audiogram corners. + + Returns + ------- + List[Callable] + A list of lambda functions. These functions all accept a single float argument. + They are in the following order. + 1. pixel->frequency + 2. pixel->threshold + 3. frequency->pixel + 4. threshold->pixel + """ + + # For x axis + y_sorted_corners = sorted(corners, key=lambda c: c["y"]) + top_corners = sorted(y_sorted_corners[0:2], key=lambda c: c["x"]) + o_max = Audiology.frequency_to_octave(top_corners[1]["frequency"]) # max octave + x_max = top_corners[1]["x"] # max pixel value + o_min = Audiology.frequency_to_octave(top_corners[0]["frequency"]) # min octave + x_min = top_corners[0]["x"] + frequency_map = lambda p: Audiology.octave_to_frequency(o_min + (o_max - o_min)*(p - x_min)/(x_max - x_min)) + inverse_frequency_map = lambda f: x_min + (Audiology.frequency_to_octave(f) - o_min)*(x_max - x_min)/(o_max - o_min) + + # For y axis + x_sorted_corners = sorted(corners, key=lambda c: c["x"]) + left_corners = sorted(x_sorted_corners[0:2], key=lambda c: c["y"]) + t_max = left_corners[1]["threshold"] # max threshold + y_max = left_corners[1]["y"] # max pixel value + t_min = left_corners[0]["threshold"] + y_min = left_corners[0]["y"] + threshold_map = lambda p: t_min + (t_max - t_min)*(p - y_min)/(y_max - y_min) + inverse_threshold_map = lambda t: y_min + (t - t_min)*(y_max - y_min)/(t_max - t_min) + + return [frequency_map, threshold_map, inverse_frequency_map, inverse_threshold_map] + +def annotation_to_thresholds(audiograms: dict) -> List[ThresholdDict]: + """Extracts the thresholds from an annotation. + + Parameters + ---------- + audiograms : dict + An annotation. + + Returns + ------- + List[ThresholdDict] + A list of thresholds + """ + combined_thresholds = [] + for audiogram in audiograms: + correction_angle = get_correction_angle(audiogram["corners"]) + corners = [apply_rotation(corner, correction_angle) for corner in audiogram["corners"]] + frequency_map, threshold_map, inverse_frequency_map, inverse_threshold_map = get_conversion_maps(corners) + + thresholds: List[ThresholdDict] = [] + for symbol in audiogram["symbols"]: + symbol_center = { + "x": symbol["boundingBox"]["x"] + symbol["boundingBox"]["width"] / 2, + "y": symbol["boundingBox"]["y"] + symbol["boundingBox"]["height"] / 2, + } + symbol = { **symbol, "boundingBox": symbol_center } + new_symbol = {**symbol, "boundingBox": apply_rotation(symbol["boundingBox"], correction_angle) } + bounding_box = new_symbol["boundingBox"] + ear = "left" if "left" in new_symbol["measurementType"].lower() else "right" + conduction = "air" if "air" in new_symbol["measurementType"].lower() else "bone" + masking = False if "unmasked" in new_symbol["measurementType"].lower() else True + if conduction == "air": + frequency = Audiology.round_frequency(frequency_map(bounding_box["x"])) + else: + frequency = Audiology.round_frequency_bone(frequency_map(bounding_box["x"]), ear) + threshold = Audiology.round_threshold(threshold_map(bounding_box["y"])) + + thresholds.append({ + "ear": ear, + "conduction": conduction, + "masking": masking, + "frequency": frequency, + "threshold": threshold, + "response": True, # IMPORTANT: assume that a response was measured for threshold + "measurementType": f"{conduction}_{'MASKED' if masking else 'UNMASKED'}_{ear}".upper() + }) + + combined_thresholds += thresholds + + return combined_thresholds diff --git a/src/digitizer/interfaces.py b/src/digitizer/interfaces.py new file mode 100644 index 0000000000000000000000000000000000000000..78e1f741814dea35c37c882f30bdc721657de536 --- /dev/null +++ b/src/digitizer/interfaces.py @@ -0,0 +1,63 @@ +from typing import List, Optional +from typing_extensions import TypedDict + +class Threshold(TypedDict): + """Represents a hearing threshold (measurement). + """ + frequency: int + threshold: int + ear: str + masking: bool + conduction: str + measurementType: str + +class BoundingBox(TypedDict): + """Represents the dictionary holding the minimum information + for a bounding box. + """ + x: int + y: int + width: int + height: int + +class AudiogramDict(TypedDict): + """Represents the dictionary for an audiogram as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + confidence: Optional[float] + +class LabelDict(TypedDict): + """Represents the dictionary for a label as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + value: str + confidence: Optional[float] + +class SymbolDict(TypedDict): + """Represents the dictionary for a symbol as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + measurementType: str + confidence: Optional[float] + +class CornerDict(TypedDict): + """Represents a corner, as annotated. + """ + frequency: int + threshold: int + position: TypedDict("PositionDict", { "horizontal": str, "vertical": str }) + x: float + y: float + +class AudiogramAnnotationDict(TypedDict): + """Represents an audiogram as structured within an annotation. + """ + confidence: Optional[float] + correctionAngle: Optional[float] + boundingBox: BoundingBox + corners: List[CornerDict] + labels: List[LabelDict] + symbols: List[SymbolDict] diff --git a/src/digitizer/mypy.ini b/src/digitizer/mypy.ini new file mode 100644 index 0000000000000000000000000000000000000000..a7edf875e238777c434f33a949cbf7e21ec9563d --- /dev/null +++ b/src/digitizer/mypy.ini @@ -0,0 +1,9 @@ + +[mypy-numpy] +ignore_missing_imports = True + +[mypy-matplotlib.*] +ignore_missing_imports = True + +[mypy-PIL.*] +ignore_missing_imports = True diff --git a/src/digitizer/report_components/__pycache__/grid.cpython-38.pyc b/src/digitizer/report_components/__pycache__/grid.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8b9cea4465906087ea2e312ed04b130cc1b24b1 Binary files /dev/null and b/src/digitizer/report_components/__pycache__/grid.cpython-38.pyc differ diff --git a/src/digitizer/report_components/__pycache__/label.cpython-38.pyc b/src/digitizer/report_components/__pycache__/label.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89bebbf0ad379bf6167b6327a42e10b499b9d3d2 Binary files /dev/null and b/src/digitizer/report_components/__pycache__/label.cpython-38.pyc differ diff --git a/src/digitizer/report_components/__pycache__/line.cpython-38.pyc b/src/digitizer/report_components/__pycache__/line.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c0dd40f0c6653e1c1b7131628da9f0df095fd91 Binary files /dev/null and b/src/digitizer/report_components/__pycache__/line.cpython-38.pyc differ diff --git a/src/digitizer/report_components/__pycache__/report.cpython-38.pyc b/src/digitizer/report_components/__pycache__/report.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99d3f69d02345fb91244fc66c59321e7d7d986a4 Binary files /dev/null and b/src/digitizer/report_components/__pycache__/report.cpython-38.pyc differ diff --git a/src/digitizer/report_components/__pycache__/symbol.cpython-38.pyc b/src/digitizer/report_components/__pycache__/symbol.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d62645742ed1f07ed78047e148f0e5d5cbff237 Binary files /dev/null and b/src/digitizer/report_components/__pycache__/symbol.cpython-38.pyc differ diff --git a/src/digitizer/report_components/grid.py b/src/digitizer/report_components/grid.py new file mode 100644 index 0000000000000000000000000000000000000000..64a85981f3690b2749801d0bf0729be23ea88915 --- /dev/null +++ b/src/digitizer/report_components/grid.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import List +from PIL import ImageDraw +from digitizer.report_components.line import Line +from digitizer.report_components.label import Label +from digitizer.report_components.symbol import Symbol +import utils.audiology as Audiology +from utils.exceptions import InsufficientLinesException + +class Grid(object): + + def __init__(self, report, labels, threshold=150): + lines = report.detect_lines(threshold=threshold) + lines = [line for line in lines if line.is_vertical() or line.is_horizontal()] + frequency_labels = [label for label in labels if label.is_frequency()] + threshold_labels = [label for label in labels if label.is_threshold()] + + if len(lines) == 0 or \ + all([line.is_vertical() for line in lines]) \ + or all([line.is_horizontal() for line in lines]): + raise InsufficientLinesException() + + x_lines = [label.find_closest_line(lines) for label in frequency_labels] + x_pixels = [line[0].get_x() for line in x_lines] + x_frequency = [label.get_value() for label in frequency_labels] + self.x_distances = [line[1] for line in x_lines] + + y_lines = [label.find_closest_line(lines) for label in threshold_labels] + y_pixels = [line[0].get_y() for line in y_lines] + y_threshold = [label.get_value() for label in threshold_labels] + self.y_distances = [line[1] for line in y_lines] + + x_points = sorted(list(zip(x_pixels, x_frequency)), key=lambda p: p[0]) + y_points = sorted(list(zip(y_pixels, y_threshold)), key=lambda p: p[0]) + + # Take the first and last points for the octaves (frequencies) + o_max = Audiology.frequency_to_octave(x_points[-1][1]) # max octave + x_max = x_points[-1][0] # max pixel value + o_min = Audiology.frequency_to_octave(x_points[0][1]) + x_min = x_points[0][0] + + # Take the first and last points for the thresholds + t_max = y_points[-1][1] # max threshold + y_max = y_points[-1][0] # max pixel value + t_min = y_points[0][1] + y_min = y_points[0][0] + + if x_min == x_max or y_max == y_min: + raise InsufficientLinesException() + + # Derive the forward and reverse mapping functions via simple linear + # interpolation using the **OCTAVE SCALE** (which is linear), because + # the frequency scale is logarithmic. + self.pixel_freq_map = lambda p: Audiology.octave_to_frequency(o_min + (o_max - o_min)*(p - x_min)/(x_max - x_min)) + self.freq_pixel_map = lambda f: x_min + (Audiology.frequency_to_octave(f) - o_min)*(x_max - x_min)/(o_max - o_min) + + # Linear interpolation can be applied directly to the thresholds, + # because the threshold axis is linear. + self.pixel_threshold_map = lambda p: t_min + (t_max - t_min)*(p - y_min)/(y_max - y_min) + self.threshold_pixel_map = lambda t: y_min + (t - t_min)*(y_max - y_min)/(t_max - t_min) + + def get_x(self, frequency: float) -> int: + """Given a frequency value, returns the x coordinate predicted by the + grid. + + Parameters + ---------- + frequency : float + The frequency value whose x-position on the image is to be determined. + + Returns + ------- + int + The x position (in pixels) of the frequency, as predicted by the grid. + """ + return self.freq_pixel_map(frequency) + + def get_frequency(self, symbol: Symbol) -> float: + """Returns the frequency of the symbol. + + Parameters + ---------- + symbol : Symbol + The symbol whose frequency is to be extracted using the computed grid. + + Returns + ------- + float + The frequency value (in Hz). + """ + return self.pixel_freq_map(symbol.get_center()["x"]) + + + def get_snapped_frequency(self, symbol: Symbol, epsilon: float = 0.15) -> float: + """Returns the frequency of the symbol, snapped to the nearest + commonly recorded frequency (all octaves and select semi-octaves). + + Parameters + ---------- + symbol : Symbol + The symbol whose frequency is to be extracted using the computed grid. + + epsilon: float + Distance (in octaves) below which the bone threshold is snapped to + the nearest frequency as opposed to shifted to the nearest threshold + in the direction of the corresponding ear. + + Returns + ------- + int + The `snapped-to-the-grid` frequency value (in Hz). + """ + if symbol.conduction == "air": + return Audiology.round_frequency(self.pixel_freq_map(symbol.get_center()["x"])) + else: + return Audiology.round_frequency_bone(self.pixel_freq_map(symbol.get_center()["x"]), symbol.ear, epsilon=epsilon) + + + def get_y(self, threshold: float) -> int: + """Given a threshold value, returns the y coordinate predicted by the + grid. + + Parameters + ---------- + threshold : float + The threshold value whose y-position on the image is to be determined. + + Returns + ------- + int + The y position (in pixels) of the threshold, as predicted by the grid. + """ + return self.threshold_pixel_map(threshold) + + + def get_threshold(self, symbol: Symbol) -> int: + """Returns the threshold of the symbol. + + Parameters + ---------- + symbol : Symbol + The symbol whose threshold is to be extracted using the computed grid. + + Returns + ------- + int + The threshold value. + """ + return self.pixel_threshold_map(symbol.get_center()["y"]) + + + def get_snapped_threshold(self, symbol: Symbol) -> int: + """Returns the threshold of the symbol, snapped to the nearest 5dB. + + Parameters + ---------- + symbol : Symbol + The symbol whose threshold is to be extracted using the computed grid. + + Returns + ------- + int + The `snapped-to-the-grid` threshold value. + """ + return Audiology.round_threshold(self.pixel_threshold_map(symbol.get_center()["y"])) + + def draw( + self, + image_drawer: ImageDraw, + frequency_range: List[int] = [125, 8000], + threshold_range: List[int] = [-10, 120], + color: str = "rgb(255,0,0)" + ): + """Draws the calculated grid on the provided image. + + Parameters + ---------- + image : PIL.ImageDraw + The `ImageDraw` object with which the grid is to be drawn. + + frequency_range : [int, int] + The minimum and maximum value of the frequencies to be included + (default: [250, 8000]). + + threshold_range : [int, int] + The minimum and maximum value of the threshold to be included + (default: [-10, 120]). + + color: str + Color of the grid as a string of the form =`rgb(R,G,B)`. + """ + lines = [] + for freq in Audiology.OCTAVE_FREQS_HZ: + x = self.get_x(freq) + y1 = self.get_y(threshold_range[0]) + y2 = self.get_y(threshold_range[1]) + line = Line(x, y1, x, y2, color=color, label=freq) + line.draw(image_drawer) + + for threshold in Audiology.THRESHOLDS: + x1 = self.get_x(frequency_range[0]) + x2 = self.get_x(frequency_range[1]) + y = self.get_y(threshold) + line = Line(x1, y, x2, y, color=color, label=threshold) + line.draw(image_drawer) diff --git a/src/digitizer/report_components/label.py b/src/digitizer/report_components/label.py new file mode 100644 index 0000000000000000000000000000000000000000..a7ac17fdcea88b96be45751d72925bc916358ca7 --- /dev/null +++ b/src/digitizer/report_components/label.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" +from typing import List, Optional, Type +import PIL.ImageDraw +import numpy as np + +from interfaces import LabelDict +from utils.geometry import get_bounding_box_relative_to_original_report +import utils.audiology as Audiology + +class Label(object): + + def __init__(self, label_dict: dict, audiogram_coordinates: dict, correction_angle: float): + bbox = label_dict["boundingBox"] + self.p1 = { + "x": bbox["x"], + "y": bbox["y"] + } + + self.p2 = { + "x": bbox["x"] + bbox["width"], + "y": bbox["y"] + bbox["height"] + } + + self.dimensions = { + "width": bbox["width"], + "height": bbox["height"] + } + + self.text = label_dict["text"] + + self.absolute_bounding_box = get_bounding_box_relative_to_original_report(bbox, audiogram_coordinates, correction_angle) + + def draw(self, canvas: PIL.ImageDraw): + """Draws the label on the canvas (image) passed. + + Parameters + ---------- + canvas : PIL.ImageDraw + The PIL.ImageDraw on which the labels are to be displayed. + """ + color = "rgb(255,0,0)" if self.is_frequency() else "rgb(0,0,255)" + canvas.rectangle( + (self.p1["x"], self.p1["y"], self.p2["x"], self.p2["y"]), + outline=color, + width=3 + ) + canvas.text((self.p1["x"], self.p1["y"] - 10), str(self.get_value()), fill=color) + + def get_type(self) -> str: + """Returns the type of label. + + Returns + ------- + str + The type of label (`threshold` or `frequency`). + """ + + if self.is_frequency(): + return "frequency" + elif self.is_threshold(): + return "threshold" + else: + return None + + def get_value(self) -> int: + """Returns the numerical value of the label. + + Returns + ------- + int + The numerical value of the label (in dB if threshold, or in Hz if frequency). + """ + if not self.is_frequency() and not self.is_threshold(): + raise "Attempted to get the value of a label which is not a frequency or threshold." + + raw_value = float(self.text.lower()\ + .rstrip("hz")\ + .rstrip("h")\ + .rstrip("khz")\ + .rstrip("k")) + + # If the value extracted is < 100 and corresponds to one of the + # standard frequency values, the value is in kHz, which we can + # convert to Hz. + if self.is_frequency() and raw_value < 100: + return raw_value * 1000 + + return raw_value + + + def is_frequency(self) -> bool: + """Checks if the label corresponds to a frequency. + + Returns + ------- + bool + True if the label corresponds to a frequency, False otherwise. + """ + if not isinstance(self.text, str): + return False + + try: + stripped_label = self.text.lower()\ + .rstrip("hz")\ + .rstrip("h")\ + .rstrip("khz")\ + .rstrip("k") + frequency_label = float(stripped_label) + return frequency_label in Audiology.OCTAVE_FREQS_HZ \ + or frequency_label in Audiology.OCTAVE_FREQS_KHZ + except ValueError: + return False # label cannot be converted to a float + + def is_threshold(self) -> bool: + """Checks if the label corresponds to a threshold. + + Returns + ------- + bool + True if the label corresponds to a threshold, False otherwise. + """ + try: + value = int(self.text) + return value in list(range(-10, 130, 10)) + except ValueError: + return False + + def get_area(self) -> int: + """Computes the area of the label's bounding box. + + Returns + ------- + int + The area of the label's bounding box in pixels squared. + """ + return self.dimensions["height"] * self.dimensions["width"] + + def overlaps_vertically_with(self, label: "Label") -> bool: + """Checks of the label overlaps vertically with the label passed. + + Returns + ------- + bool + True if the labels overlap and False otherwise. + """ + return (self.p1["y"] >= label.p1["y"] and self.p1["y"] <= label.p2["y"]) \ + or (self.p2["y"] >= label.p1["y"] and self.p2["y"] <= label.p2["y"]) \ + or (label.p1["y"] >= self.p1["y"] and label.p1["y"] <= self.p2["y"]) \ + or (label.p2["y"] >= self.p1["y"] and label.p2["y"] <= self.p2["y"]) + + def overlaps_horizontally_with(self, label: "Label") -> bool: + """Checks of the label overlaps horizontally with the label passed. + + Returns + ------- + bool + True if the labels overlap and False otherwise. + """ + return (self.p1["x"] >= label.p1["x"] and self.p1["x"] <= label.p2["x"]) \ + or (self.p2["x"] >= label.p1["x"] and self.p2["x"] <= label.p2["x"]) \ + or (label.p1["x"] >= self.p1["x"] and label.p1["x"] <= self.p2["x"]) \ + or (label.p2["x"] >= self.p1["x"] and label.p2["x"] <= self.p2["x"]) + + def overlaps_with(self, label: "Label") -> bool: + """Checks of the label overlaps vertically OR horizontally with the label passed. + + Returns + ------- + bool + True if the labels overlap and False otherwise. + """ + return self.overlaps_vertically_with(label) and self.overlaps_horizontally_with(label) + + def encompasses_x_value(self, x: int) -> bool: + """Checks of the the pixel value of x pass is encompassed in the label's x range. + + Returns + ------- + bool + True if x is in the label's x range and False otherwise. + """ + return x >= self.p1["x"] and x <= self.p2["x"] + + def encompasses_y_value(self, y: int) -> bool: + """Checks of the the pixel value of y pass is encompassed in the label's y range. + + Returns + ------- + bool + True if y is in the label's y range and False otherwise. + """ + return y >= self.p1["y"] and y <= self.p2["y"] + + def get_center(self) -> dict: + """Returns the center of the label's bounding box. + + Returns + ------- + dict + A dictionary describing the center of the label's bounding box + of the form { "x": int, "y": int }. + """ + center = { + "x": int((self.p1["x"] + self.p2["x"]) / 2), + "y": int((self.p1["y"] + self.p2["y"]) / 2) + } + return center + + + def find_closest_line(self, lines: List["Line"]) -> "Line": + """Find the closest line to the label. + + If the label corresponds to a frequency, the line is vertical, + otherwise it is a horizontal line. + + Parameters + ---------- + lines : List[Line] + The set of lines detected in the audiogram image. + + Returns + ------- + Line + The closest line. + """ + if self.is_threshold(): + lines = [line for line in lines if line.is_horizontal()] + closest_line_distance = 100000 + closest_line_index = None + distances = [] + for i, line in enumerate(lines): + distance = abs(line.get_y() - self.get_center()["y"]) + distances.append(distance) + if distance < closest_line_distance: + closest_line_index = i + closest_line_distance = distance + return lines[closest_line_index], distances[closest_line_index] + + elif self.is_frequency(): + lines = [line for line in lines if line.is_vertical()] + closest_line_distance = 100000 + closest_line_index = None + distances = [] + for i, line in enumerate(lines): + distance = abs(line.get_x() - self.get_center()["x"]) + distances.append(distance) + if distance < closest_line_distance: + closest_line_index = i + closest_line_distance = distance + return lines[closest_line_index], distances[closest_line_index] + else: + raise "Error: Tried to find the closest line to a label that corresponds neither to a frequency nor a threshold." + + def to_dict(self) -> dict: + """Returns the label as a dictionary. + + More thorough description of the function here. + + Returns + ------- + dict + The label as a dictionary with the keys `boundingBox` and `value`. + + """ + return { + "boundingBox": self.absolute_bounding_box, + "value": self.text + } + + + def __str__(self): + return f"Textbox(x={self.p1['x']}, y={self.p1['y']}, text={self.text})" + + def __repr__(self): + return self.__str__() diff --git a/src/digitizer/report_components/label.pyi b/src/digitizer/report_components/label.pyi new file mode 100644 index 0000000000000000000000000000000000000000..41a797c8f657db5396258802d91ba99ac00f0a48 --- /dev/null +++ b/src/digitizer/report_components/label.pyi @@ -0,0 +1,35 @@ +from typing_extensions import TypedDict + +class Threshold(TypedDict): + """Represents a hearing threshold (measurement). + """ + frequency: int + threshold: int + ear: str + masking: bool + conduction: str + +class BoundingBox(TypedDict): + """Represents the dictionary holding the minimum information + for a bounding box. + """ + x: int + y: int + width: int + height: int + +class LabelDict(TypedDict): + """Represents the dictionary for a label as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + text: str + confidence: float + +class SymbolDict(TypedDict): + """Represents the dictionary for a symbol as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + measurementType: str + confidence: float diff --git a/src/digitizer/report_components/line.py b/src/digitizer/report_components/line.py new file mode 100644 index 0000000000000000000000000000000000000000..5f53d654ff1908e88ee394c92cdd38640b8b3047 --- /dev/null +++ b/src/digitizer/report_components/line.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import List + +import PIL.ImageDraw +import numpy as np + +class Line(object): + + def __init__(self, x1, y1, x2, y2, color="rgb(255,0,0)", label=None): + self.p1 = { "x": x1, "y": y1 } + self.p2 = { "x": x2, "y": y2 } + self.color = color + self.label = label + + def get_angle(self) -> float: + """Returns the angle of the line in degrees in the range [0, 180[. + + Returns + ------- + float + The angle of the line in degrees. + """ + dx = self.p2["x"] - self.p1["x"] + dy = self.p2["y"] - self.p1["y"] + angle = np.degrees(np.arctan2(abs(dy), abs(dx))) + if self.p2["y"] - self.p1["y"] < 0: + angle = 180 - angle + return angle + + + def has_a_perpendicular_line(self, lines: List["Line"], tolerance: float = 1) -> bool: + """Checks if a line has at least one perpendicular line among a list + of lines. + + Parameters + ---------- + lines : List[Line] + A list of lines. + + tolerance : float + A difference of `tolerance` from a 90 degrees angle between the two lines + is still considered perpendicular. + + Returns + ------- + bool + `True` if the line has a perpendicular line in the list, `False` otherwise. + + """ + assert tolerance >= 0 + + line_angle1 = self.get_angle() + for other_line in lines: + line_angle2 = other_line.get_angle() + angle = abs(line_angle1 - line_angle2) + if abs(angle - 90) < tolerance: + return True + return False + + def get_x(self) -> int: + """Return the middle x pixel coordinate of a vertical line. + + Only applicable to vertical lines. + + Returns + ------- + int + The middle x coordinate of a vertical line. + """ + assert self.is_vertical() + return int((self.p1["x"] + self.p2["x"]) / 2) + + def get_y(self) -> int: + """Return the middle y pixel coordinate of a horizontal line. + + Only applicable to horizontal lines. + + Returns + ------- + int + The middle y coordinate of a horizontal line. + """ + assert self.is_horizontal() + return (self.p1["y"] + self.p2["y"]) / 2 + + def is_vertical(self, tolerance: float = 1) -> bool: + """Returns true if the line is vertical. + + Parameters + ---------- + tolerance : float + A deviation of `tolerance` degrees from the vertical line is still + considered vertical. + + Returns + ------- + bool + True if the line is vertical, False otherwise. + """ + assert tolerance >= 0 + + angle = self.get_angle() + return angle <= (-90 + tolerance) or angle >= (90 - tolerance) + + def is_horizontal(self, tolerance: float = 1) -> bool: + """Returns true if the line is horizontal. + + Parameters + ---------- + tolerance : float + A deviation of `tolerance` degrees from the horizontal line is still + considered horizontal. + + Returns + ------- + bool + True if the line is horiontal, False otherwise. + """ + assert tolerance >= 0 + + angle = self.get_angle() + return angle >= -tolerance and angle <= tolerance + + def crosses_label(self, labels: List["Label"]) -> bool: + """Checks if the line intersects one of the labels passed. + + Parameters + ---------- + labels : List[Label] + A list of labels that the line could possibly cross. + + Returns + ------- + bool + True if the line crosses at least one Label among the ones provided, + False otherwise. + """ + for label in labels: + if self.is_vertical(): + x = (self.p1["x"] + self.p2["x"]) / 2 + if x >= label.p1["x"] and x <= label.p2["x"]: + return True + elif self.is_horizontal(): + y = (self.p1["y"] + self.p2["y"]) / 2 + if y >= label.p1["y"] and y <= label.p2["y"]: + return True + return False + + + def draw(self, canvas: PIL.ImageDraw): + """Draws the line on the canvas (image) passed. + + Parameters + ---------- + canvas : PIL.ImageDraw + The PIL.ImageDraw on which the line is to be displayed. + """ + + canvas.line([ + (self.p1["x"], self.p1["y"]), + (self.p2["x"], self.p2["y"]) + ], width=3, fill=self.color) + + if self.label: + canvas.text((self.p1["x"] + 5, self.p1["y"]), str(self.label), fill=self.color) diff --git a/src/digitizer/report_components/report.py b/src/digitizer/report_components/report.py new file mode 100644 index 0000000000000000000000000000000000000000..f9f153577921a5ff9fb3797bc39881981da435fb --- /dev/null +++ b/src/digitizer/report_components/report.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" +from typing import Optional, List +import os + +from PIL import Image, ImageDraw, ImageFont +import numpy as np +import cv2 + +from .label import Label +from .line import Line +from .grid import Grid + +class Report(object): + + def __init__(self, filename: Optional[str] = None, image: Optional[Image.Image] = None): + assert not (filename and image) and bool(filename) != bool(image) + if filename: + self.filename = filename + self.pil_image = Image.open(filename) + else: + self.filename = None + self.pil_image = image + + def rescale(self, factor: float) -> "Report": + """Creates a new Report that has been resized. + + Parameters + ---------- + factor : float + The resize factor. + + Returns + ------- + Report + A new Report that has be rescaled. + """ + return Report(image=self.pil_image.resize(int(self.pil_image.width * factor), int(self.pil_image.height * factor))) + + def rotate(self, angle: float) -> "Report": + """Creates a new Report that has been rotated. + + Parameters + ---------- + angle : float + The rotation (in degrees) to apply (CCW). + + Returns + ------- + Report + A new Report that has be rotated. + """ + return Report(image=self.pil_image.rotate(angle, center=(0,0), fillcolor="rgb(255,255,255)")) + + def crop(self, x1: int, y1: int, x2: int, y2: int) -> "Report": + """Creates a new cropped Report. + + Parameters + ---------- + x1: int + The x pixel coordinate of the top-left corner. + y1: int + The y pixel coordinate of the top-left corner. + x2: int + The x pixel coordinate of the bottom-right corner. + y2: int + The y pixel coordinate of the bottom-right corner. + + Returns + ------- + Report + A new cropped Report. + """ + return Report(image=self.pil_image.crop((x1, y1, x2, y2))) + + def show( + self, + labels: Optional[List[Label]] = [], + lines: Optional[List[Line]] = [], + grids: Optional[List[Grid]] = [], + points: Optional[List[tuple]] = [], + title: Optional[str] = None, + filename: Optional[str] = None + ): + """Displays the audiogram on the screen, and saves it to a file + if `filename` parameter is provided. + + Parameters + ---------- + labels : List[Label] + A list of labels to show (default: []). + lines : List[Lines] + A list of lines to show (default: []). + grids : List[Grid] + A list of grids to show (default: []). + points : List[dict] + A list of points of the form { "x": int, "y": int } to show (default: []). + title: str + The title of the plot + filename: Optional[str] + The path to where the image should be save. Image is not saved + if no filename is provided (default: None). + """ + labeled_copy = self.pil_image.copy().convert("RGB") + drawing = ImageDraw.Draw(labeled_copy) + fontpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "assets", "fonts", "Arial.ttf") + font = None + try: + font = ImageFont.truetype(fontpath, 32) + except: + font = ImageFont.truetype("DejaVuSans.ttf", 32) + + for label in labels: + label.draw(drawing) + + for line in lines: + line.draw(drawing) + + for grid in grids: + grid.draw(drawing) + + for point in points: + r = 5 # radius + drawing.ellipse([(point[0] - r, point[1] - r), (point[0] + r, point[1] + r)], fill="rgb(0,0,255)") + + if title: + drawing.text((self.pil_image.size[0]/2, 50), title, font=font, align="center", fill="rgb(53,155,232)") + + if filename: + labeled_copy.save(filename) + + labeled_copy.show() + + def save( + self, + filename: str, + labels: Optional[List[Label]] = [], + lines: Optional[List[Line]] = [], + grids: Optional[List[Grid]] = [], + title: Optional[str] = None, + ): + """Saves the report to a file along with annotations for the + provided report elements. + + Parameters + ---------- + filename: str + The path to where the image should be save. + labels : List[Label] + A list of labels to show (default: []). + lines : List[Lines] + A list of lines to show (default: []). + grids : List[Grid] + A list of grids to show (default: []). + title: str + The title of the plot + """ + labeled_copy = self.pil_image.copy().convert("RGB") + drawing = ImageDraw.Draw(labeled_copy) + fontpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "assets", "Arial.ttf") + font = None + try: + font = ImageFont.truetype(fontpath, 32) + except: + font = ImageFont.truetype("DejaVuSans.ttf", 32) + + + for label in labels: + label.draw(drawing) + + for line in lines: + line.draw(drawing) + + for grid in grids: + grid.draw(drawing) + + if title: + drawing.text((self.pil_image.size[0]/2, 50), title, font=font, align="center", fill="rgb(53,155,232)") + + labeled_copy.save(filename) + + def get_image(self, resize_factor: float = 1) -> Image: + """Returns a copy of the PIL image representing the report. + + Parameters + ---------- + resize_factor : float + The resize factor of the image sought (default: 1). + + Returns + ------- + PIL.Image + A copy of the image at the resize factor provided. + """ + + return self.pil_image.resize( + (int(self.pil_image.size[0] * resize_factor), + int(self.pil_image.size[1] * resize_factor)) + ) + + def detect_lines(self, threshold=250) -> List[Line]: + """Detects lines in the report using the Hough Transform. + + For details, see: https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_imgproc/py_houghlines/py_houghlines.html + + Parameters + ---------- + threshold : int + The threshold above which a line is detected. See documentation for OpenCV's HoughLine function for details. + + Returns + ------- + List[Line] + A list of lines detected in the report. + """ + gray = np.array(self.get_image()) + edges = cv2.Canny(gray, 150, 300, apertureSize = 3) + lines = cv2.HoughLines(edges, 1, np.pi/180, threshold, None, 0, 0) + + if lines is None: + return [] + + lines_list = [] + + for line in lines: + for l in (line[0],): + rho = l[0] + theta = l[1] + a = np.cos(theta) + b = np.sin(theta) + x0 = a*rho + y0 = b*rho + x1 = int(x0 + 1000*(-b)) + y1 = int(y0 + 1000*(a)) + x2 = int(x0 - 1000*(-b)) + y2 = int(y0 - 1000*(a)) + lines_list.append(Line(x1, y1, x2, y2)) + + return lines_list diff --git a/src/digitizer/report_components/symbol.py b/src/digitizer/report_components/symbol.py new file mode 100644 index 0000000000000000000000000000000000000000..7e9c72567677cbdba98447b1fa6fc9a14e0686f0 --- /dev/null +++ b/src/digitizer/report_components/symbol.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import List, Optional, Type +import PIL.ImageDraw + +from interfaces import SymbolDict +from .line import Line +from .label import Label +import utils.audiology as Audiology +from utils.geometry import get_bounding_box_relative_to_original_report + +class Symbol(object): + + def __init__(self, symbol_dict: dict, audiogram_coordinates: dict, correction_angle: float): + bbox = symbol_dict["boundingBox"] + self.p1 = { + "x": bbox["x"], + "y": bbox["y"] + } + + self.p2 = { + "x": bbox["x"] + bbox["width"], + "y": bbox["y"] + bbox["height"] + } + + self.dimensions = { + "width": bbox["width"], + "height": bbox["height"] + } + + self.absolute_bounding_box = get_bounding_box_relative_to_original_report(bbox, audiogram_coordinates, correction_angle) + + self.ear = "left" if "left" in symbol_dict["measurementType"].lower() else "right" + self.masking = False if "unmasked" in symbol_dict["measurementType"].lower() else True + self.conduction = "air" if "air" in symbol_dict["measurementType"].lower() else "bone" + self.measurement_type = symbol_dict["measurementType"] + self.confidence = symbol_dict["confidence"] + + def draw(self, canvas: PIL.ImageDraw): + """Draws the symbol's bounding box on the canvas (image) passed. + + Parameters + ---------- + canvas : PIL.ImageDraw + The PIL.ImageDraw on which the symbol is to be displayed. + """ + color = "rgb(255,0,0)" if self.is_frequency() else "rgb(0,0,255)" + canvas.rectangle( + (self.p1["x"], self.p1["y"], self.p2["x"], self.p2["y"]), + outline=color, + width=3 + ) + canvas.text((self.p1["x"], self.p1["y"] - 10), str(self.get_value()), fill=color) + + def get_center(self): + """Returns the center of the symbol's bounding box. + + Returns + ------- + dict + A dictionary describing the center of the symbol's bounding box + of the form { "x": int, "y": int }. + """ + center = { + "x": (self.p1["x"] + self.p2["x"]) / 2, + "y": (self.p1["y"] + self.p2["y"]) / 2 + } + return center + + def to_dict(self) -> dict: + """Serializes the symbol to a dictionary. + + Returns + ------- + dict + A dictionary representing the symbol. + """ + return { + "boundingBox": self.absolute_bounding_box, + "ear": self.ear, + "conduction": self.conduction, + "masking": self.masking, + "confidence": self.confidence, + "response": True, + "measurementType": self.measurement_type + } + + def __str__(self): + return f"Threshold(ear={self.ear}, conduction={self.conduction})" + + def __repr__(self): + return self.__str__() diff --git a/src/digitizer/yolov5/Dockerfile b/src/digitizer/yolov5/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..afc494a8b51232afaebd94eb3d1afca18d2d6c0b --- /dev/null +++ b/src/digitizer/yolov5/Dockerfile @@ -0,0 +1,52 @@ +# Start FROM Nvidia PyTorch image https://ngc.nvidia.com/catalog/containers/nvidia:pytorch +FROM nvcr.io/nvidia/pytorch:20.08-py3 + +# Install dependencies +RUN pip install --upgrade pip +# COPY requirements.txt . +# RUN pip install -r requirements.txt +RUN pip install gsutil + +# Create working directory +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Copy contents +COPY . /usr/src/app + +# Copy weights +#RUN python3 -c "from models import *; \ +#attempt_download('weights/yolov5s.pt'); \ +#attempt_download('weights/yolov5m.pt'); \ +#attempt_download('weights/yolov5l.pt')" + + +# --------------------------------------------------- Extras Below --------------------------------------------------- + +# Build and Push +# t=ultralytics/yolov5:latest && sudo docker build -t $t . && sudo docker push $t +# for v in {300..303}; do t=ultralytics/coco:v$v && sudo docker build -t $t . && sudo docker push $t; done + +# Pull and Run +# t=ultralytics/yolov5:latest && sudo docker pull $t && sudo docker run -it --ipc=host $t + +# Pull and Run with local directory access +# t=ultralytics/yolov5:latest && sudo docker pull $t && sudo docker run -it --ipc=host --gpus all -v "$(pwd)"/coco:/usr/src/coco $t + +# Kill all +# sudo docker kill "$(sudo docker ps -q)" + +# Kill all image-based +# sudo docker kill $(sudo docker ps -a -q --filter ancestor=ultralytics/yolov5:latest) + +# Bash into running container +# sudo docker container exec -it ba65811811ab bash + +# Bash into stopped container +# sudo docker commit 092b16b25c5b usr/resume && sudo docker run -it --gpus all --ipc=host -v "$(pwd)"/coco:/usr/src/coco --entrypoint=sh usr/resume + +# Send weights to GCP +# python -c "from utils.general import *; strip_optimizer('runs/exp0_*/weights/best.pt', 'tmp.pt')" && gsutil cp tmp.pt gs://*.pt + +# Clean up +# docker system prune -a --volumes diff --git a/src/digitizer/yolov5/LICENSE b/src/digitizer/yolov5/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9e419e042146a2ce2e354202d4f7d8e4a3d66f31 --- /dev/null +++ b/src/digitizer/yolov5/LICENSE @@ -0,0 +1,674 @@ +GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. \ No newline at end of file diff --git a/src/digitizer/yolov5/README.md b/src/digitizer/yolov5/README.md new file mode 100755 index 0000000000000000000000000000000000000000..9e625061ecd4eccebd097b935cf63fef459b952d --- /dev/null +++ b/src/digitizer/yolov5/README.md @@ -0,0 +1,130 @@ + + +  + +![CI CPU testing](https://github.com/ultralytics/yolov5/workflows/CI%20CPU%20testing/badge.svg) + +This repository represents Ultralytics open-source research into future object detection methods, and incorporates our lessons learned and best practices evolved over training thousands of models on custom client datasets with our previous YOLO repository https://github.com/ultralytics/yolov3. **All code and models are under active development, and are subject to modification or deletion without notice.** Use at your own risk. + +** GPU Speed measures end-to-end time per image averaged over 5000 COCO val2017 images using a V100 GPU with batch size 32, and includes image preprocessing, PyTorch FP16 inference, postprocessing and NMS. EfficientDet data from [google/automl](https://github.com/google/automl) at batch size 8. + +- **August 13, 2020**: [v3.0 release](https://github.com/ultralytics/yolov5/releases/tag/v3.0): nn.Hardswish() activations, data autodownload, native AMP. +- **July 23, 2020**: [v2.0 release](https://github.com/ultralytics/yolov5/releases/tag/v2.0): improved model definition, training and mAP. +- **June 22, 2020**: [PANet](https://arxiv.org/abs/1803.01534) updates: new heads, reduced parameters, improved speed and mAP [364fcfd](https://github.com/ultralytics/yolov5/commit/364fcfd7dba53f46edd4f04c037a039c0a287972). +- **June 19, 2020**: [FP16](https://pytorch.org/docs/stable/nn.html#torch.nn.Module.half) as new default for smaller checkpoints and faster inference [d4c6674](https://github.com/ultralytics/yolov5/commit/d4c6674c98e19df4c40e33a777610a18d1961145). +- **June 9, 2020**: [CSP](https://github.com/WongKinYiu/CrossStagePartialNetworks) updates: improved speed, size, and accuracy (credit to @WongKinYiu for CSP). +- **May 27, 2020**: Public release. YOLOv5 models are SOTA among all known YOLO implementations. +- **April 1, 2020**: Start development of future compound-scaled [YOLOv3](https://github.com/ultralytics/yolov3)/[YOLOv4](https://github.com/AlexeyAB/darknet)-based PyTorch models. + + +## Pretrained Checkpoints + +| Model | APval | APtest | AP50 | SpeedGPU | FPSGPU || params | FLOPS | +|---------- |------ |------ |------ | -------- | ------| ------ |------ | :------: | +| [YOLOv5s](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | 37.0 | 37.0 | 56.2 | **2.4ms** | **416** || 7.5M | 13.2B +| [YOLOv5m](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | 44.3 | 44.3 | 63.2 | 3.4ms | 294 || 21.8M | 39.4B +| [YOLOv5l](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | 47.7 | 47.7 | 66.5 | 4.4ms | 227 || 47.8M | 88.1B +| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | **49.2** | **49.2** | **67.7** | 6.9ms | 145 || 89.0M | 166.4B +| | | | | | || | +| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0) + TTA|**50.8**| **50.8** | **68.9** | 25.5ms | 39 || 89.0M | 354.3B +| | | | | | || | +| [YOLOv3-SPP](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | 45.6 | 45.5 | 65.2 | 4.5ms | 222 || 63.0M | 118.0B + +** APtest denotes COCO [test-dev2017](http://cocodataset.org/#upload) server results, all other AP results in the table denote val2017 accuracy. +** All AP numbers are for single-model single-scale without ensemble or test-time augmentation. **Reproduce** by `python test.py --data coco.yaml --img 640 --conf 0.001` +** SpeedGPU measures end-to-end time per image averaged over 5000 COCO val2017 images using a GCP [n1-standard-16](https://cloud.google.com/compute/docs/machine-types#n1_standard_machine_types) instance with one V100 GPU, and includes image preprocessing, PyTorch FP16 image inference at --batch-size 32 --img-size 640, postprocessing and NMS. Average NMS time included in this chart is 1-2ms/img. **Reproduce** by `python test.py --data coco.yaml --img 640 --conf 0.1` +** All checkpoints are trained to 300 epochs with default settings and hyperparameters (no autoaugmentation). +** Test Time Augmentation ([TTA](https://github.com/ultralytics/yolov5/issues/303)) runs at 3 image sizes. **Reproduce** by `python test.py --data coco.yaml --img 832 --augment` + +## Requirements + +Python 3.8 or later with all [requirements.txt](https://github.com/ultralytics/yolov5/blob/master/requirements.txt) dependencies installed, including `torch>=1.6`. To install run: +```bash +$ pip install -r requirements.txt +``` + + +## Tutorials + +* [Train Custom Data](https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data) +* [Multi-GPU Training](https://github.com/ultralytics/yolov5/issues/475) +* [PyTorch Hub](https://github.com/ultralytics/yolov5/issues/36) +* [ONNX and TorchScript Export](https://github.com/ultralytics/yolov5/issues/251) +* [Test-Time Augmentation (TTA)](https://github.com/ultralytics/yolov5/issues/303) +* [Model Ensembling](https://github.com/ultralytics/yolov5/issues/318) +* [Model Pruning/Sparsity](https://github.com/ultralytics/yolov5/issues/304) +* [Hyperparameter Evolution](https://github.com/ultralytics/yolov5/issues/607) +* [TensorRT Deployment](https://github.com/wang-xinyu/tensorrtx) + + +## Environments + +YOLOv5 may be run in any of the following up-to-date verified environments (with all dependencies including [CUDA](https://developer.nvidia.com/cuda)/[CUDNN](https://developer.nvidia.com/cudnn), [Python](https://www.python.org/) and [PyTorch](https://pytorch.org/) preinstalled): + +- **Google Colab Notebook** with free GPU: Open In Colab +- **Kaggle Notebook** with free GPU: [https://www.kaggle.com/ultralytics/yolov5](https://www.kaggle.com/ultralytics/yolov5) +- **Google Cloud** Deep Learning VM. See [GCP Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/GCP-Quickstart) +- **Docker Image** https://hub.docker.com/r/ultralytics/yolov5. See [Docker Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/Docker-Quickstart) ![Docker Pulls](https://img.shields.io/docker/pulls/ultralytics/yolov5?logo=docker) + + +## Inference + +Inference can be run on most common media formats. Model [checkpoints](https://drive.google.com/open?id=1Drs_Aiu7xx6S-ix95f9kNsA6ueKRpN2J) are downloaded automatically if available. Results are saved to `./inference/output`. +```bash +$ python detect.py --source 0 # webcam + file.jpg # image + file.mp4 # video + path/ # directory + path/*.jpg # glob + rtsp://170.93.143.139/rtplive/470011e600ef003a004ee33696235daa # rtsp stream + http://112.50.243.8/PLTV/88888888/224/3221225900/1.m3u8 # http stream +``` + +To run inference on examples in the `./inference/images` folder: + +```bash +$ python detect.py --source ./inference/images/ --weights yolov5s.pt --conf 0.4 + +Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.4, device='', fourcc='mp4v', half=False, img_size=640, iou_thres=0.5, output='inference/output', save_txt=False, source='./inference/images/', view_img=False, weights='yolov5s.pt') +Using CUDA device0 _CudaDeviceProperties(name='Tesla P100-PCIE-16GB', total_memory=16280MB) + +Downloading https://drive.google.com/uc?export=download&id=1R5T6rIyy3lLwgFXNms8whc-387H0tMQO as yolov5s.pt... Done (2.6s) + +image 1/2 inference/images/bus.jpg: 640x512 3 persons, 1 buss, Done. (0.009s) +image 2/2 inference/images/zidane.jpg: 384x640 2 persons, 2 ties, Done. (0.009s) +Results saved to /content/yolov5/inference/output +``` + + + + +## Training + +Download [COCO](https://github.com/ultralytics/yolov5/blob/master/data/scripts/get_coco.sh) and run command below. Training times for YOLOv5s/m/l/x are 2/4/6/8 days on a single V100 (multi-GPU times faster). Use the largest `--batch-size` your GPU allows (batch sizes shown for 16 GB devices). +```bash +$ python train.py --data coco.yaml --cfg yolov5s.yaml --weights '' --batch-size 64 + yolov5m 40 + yolov5l 24 + yolov5x 16 +``` + + + +## Citation + +[![DOI](https://zenodo.org/badge/264818686.svg)](https://zenodo.org/badge/latestdoi/264818686) + + +## About Us + +Ultralytics is a U.S.-based particle physics and AI startup with over 6 years of expertise supporting government, academic and business clients. We offer a wide range of vision AI services, spanning from simple expert advice up to delivery of fully customized, end-to-end production solutions, including: +- **Cloud-based AI** systems operating on **hundreds of HD video streams in realtime.** +- **Edge AI** integrated into custom iOS and Android apps for realtime **30 FPS video inference.** +- **Custom data training**, hyperparameter evolution, and model exportation to any destination. + +For business inquiries and professional support requests please visit us at https://www.ultralytics.com. + + +## Contact + +**Issues should be raised directly in the repository.** For business inquiries or professional support requests please visit https://www.ultralytics.com or email Glenn Jocher at glenn.jocher@ultralytics.com. diff --git a/src/digitizer/yolov5/detect.py b/src/digitizer/yolov5/detect.py new file mode 100644 index 0000000000000000000000000000000000000000..f5558bab4edf5757a3f212a2312092e554418186 --- /dev/null +++ b/src/digitizer/yolov5/detect.py @@ -0,0 +1,201 @@ +import argparse +import json +import os +import platform +import shutil +import time +from pathlib import Path + +import cv2 +import torch +import torch.backends.cudnn as cudnn +from numpy import random + +from models.experimental import attempt_load +from utils.datasets import LoadStreams, LoadImages +from utils.general import ( + check_img_size, non_max_suppression, apply_classifier, scale_coords, + xyxy2xywh, plot_one_box, strip_optimizer, set_logging) +from utils.torch_utils import select_device, load_classifier, time_synchronized + + +def detect(save_img=False): + out, source, weights, view_img, save_txt, imgsz = \ + opt.output, opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size + webcam = source.isnumeric() or source.startswith('rtsp') or source.startswith('http') or source.endswith('.txt') + + # Initialize + set_logging() + device = select_device(opt.device) + if os.path.exists(out): + shutil.rmtree(out) # delete output folder + os.makedirs(out) # make new output folder + half = device.type != 'cpu' # half precision only supported on CUDA + + # Load model + model = attempt_load(weights, map_location=device) # load FP32 model + imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size + if half: + model.half() # to FP16 + + # Second-stage classifier + classify = False + if classify: + modelc = load_classifier(name='resnet101', n=2) # initialize + modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']) # load weights + modelc.to(device).eval() + + # Set Dataloader + vid_path, vid_writer = None, None + if webcam: + view_img = True + cudnn.benchmark = True # set True to speed up constant image size inference + dataset = LoadStreams(source, img_size=imgsz) + else: + save_img = True + dataset = LoadImages(source, img_size=imgsz) + + # Get names and colors + names = model.module.names if hasattr(model, 'module') else model.names + colors = [[random.randint(0, 255) for _ in range(3)] for _ in range(len(names))] + + # Run inference + t0 = time.time() + img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img + _ = model(img.half() if half else img) if device.type != 'cpu' else None # run once + for path, img, im0s, vid_cap in dataset: + img = torch.from_numpy(img).to(device) + img = img.half() if half else img.float() # uint8 to fp16/32 + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + if img.ndimension() == 3: + img = img.unsqueeze(0) + + # Inference + t1 = time_synchronized() + pred = model(img, augment=opt.augment)[0] + + # Apply NMS + pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms) + t2 = time_synchronized() + + # Apply Classifier + if classify: + pred = apply_classifier(pred, modelc, img, im0s) + + # Process detections + for i, det in enumerate(pred): # detections per image + if webcam: # batch_size >= 1 + p, s, im0 = path[i], '%g: ' % i, im0s[i].copy() + else: + p, s, im0 = path, '', im0s + + save_path = str(Path(out) / Path(p).name) + txt_path = str(Path(out) / Path(p).stem) + ('_%g' % dataset.frame if dataset.mode == 'video' else '') + s += '%gx%g ' % img.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + if det is not None and len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, -1].unique(): + n = (det[:, -1] == c).sum() # detections per class + s += '%g %ss, ' % (n, names[int(c)]) # add to string + + # Write results + #for *xyxy, conf, cls in reversed(det): + # if save_txt: # Write to file + # xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + # with open(txt_path + '.txt', 'a') as f: + # f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format + + + symbols = [] + for *xyxy, conf, cls in reversed(det): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4))).view(-1).tolist() # ADDED BY FRANCOIS + symbols.append({ + "measurementType": names[int(cls)], + "boundingBox": { + "x": int(xywh[0] - xywh[2]/2), + "y": int(xywh[1] - xywh[3]/2), + "width": int(xywh[2]), + "height": int(xywh[3]) + }, + "confidence": float(conf) + }) + if save_txt: # Write to file + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + with open(txt_path + '.txt', 'a') as f: + f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format + + if save_img or view_img: # Add bbox to image + label = '%s %.2f' % (names[int(cls)], conf) + plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=3) + print("\n$$$") + print(json.dumps(symbols)) + print("$$$\n") + + + if save_img or view_img: # Add bbox to image + label = '%s %.2f' % (names[int(cls)], conf) + plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=3) + + # Print time (inference + NMS) + print('%sDone. (%.3fs)' % (s, t2 - t1)) + + # Stream results + if view_img: + cv2.imshow(p, im0) + if cv2.waitKey(1) == ord('q'): # q to quit + raise StopIteration + + # Save results (image with detections) + if save_img: + if dataset.mode == 'images': + cv2.imwrite(save_path, im0) + else: + if vid_path != save_path: # new video + vid_path = save_path + if isinstance(vid_writer, cv2.VideoWriter): + vid_writer.release() # release previous video writer + + fourcc = 'mp4v' # output video codec + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*fourcc), fps, (w, h)) + vid_writer.write(im0) + + if save_txt or save_img: + print('Results saved to %s' % Path(out)) + if platform.system() == 'Darwin' and not opt.update: # MacOS + os.system('open ' + save_path) + + print('Done. (%.3fs)' % (time.time() - t0)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') + parser.add_argument('--source', type=str, default='inference/images', help='source') # file/folder, 0 for webcam + parser.add_argument('--output', type=str, default='inference/output', help='output folder') # output folder + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.4, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.5, help='IOU threshold for NMS') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='display results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--update', action='store_true', help='update all models') + opt = parser.parse_args() + print(opt) + + with torch.no_grad(): + if opt.update: # update all models (to fix SourceChangeWarning) + for opt.weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']: + detect() + strip_optimizer(opt.weights) + else: + detect() diff --git a/src/digitizer/yolov5/detect_audiograms.py b/src/digitizer/yolov5/detect_audiograms.py new file mode 100644 index 0000000000000000000000000000000000000000..5b33b3f6bf0fe96a6760dd35717f4a8b4d6ad1ce --- /dev/null +++ b/src/digitizer/yolov5/detect_audiograms.py @@ -0,0 +1,155 @@ +import argparse +import json +import os +import platform +import shutil +import time +from pathlib import Path + +import cv2 +import torch +import torch.backends.cudnn as cudnn +from numpy import random + +from models.experimental import attempt_load +from utils.datasets import LoadStreams, LoadImages +from utils.general import ( + check_img_size, non_max_suppression, apply_classifier, scale_coords, + xyxy2xywh, plot_one_box, strip_optimizer, set_logging) +from utils.torch_utils import select_device, load_classifier, time_synchronized + + +def detect(save_img=False): + + results = [] + + out, source, weights, view_img, save_txt, imgsz = \ + opt.output, opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size + webcam = source.isnumeric() or source.startswith('rtsp') or source.startswith('http') or source.endswith('.txt') + + # Initialize + set_logging() + device = select_device(opt.device) + half = device.type != 'cpu' # half precision only supported on CUDA + + # Load model + model = attempt_load(weights, map_location=device) # load FP32 model + imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size + if half: + model.half() # to FP16 + + # Second-stage classifier + classify = False + if classify: + modelc = load_classifier(name='resnet101', n=2) # initialize + modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']) # load weights + modelc.to(device).eval() + + # Set Dataloader + vid_path, vid_writer = None, None + if webcam: + view_img = True + cudnn.benchmark = True # set True to speed up constant image size inference + dataset = LoadStreams(source, img_size=imgsz) + else: + save_img = True + dataset = LoadImages(source, img_size=imgsz) + + # Get names and colors + names = model.module.names if hasattr(model, 'module') else model.names + colors = [[random.randint(0, 255) for _ in range(3)] for _ in range(len(names))] + + # Run inference + t0 = time.time() + img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img + _ = model(img.half() if half else img) if device.type != 'cpu' else None # run once + + for path, img, im0s, vid_cap in dataset: + img = torch.from_numpy(img).to(device) + img = img.half() if half else img.float() # uint8 to fp16/32 + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + if img.ndimension() == 3: + img = img.unsqueeze(0) + + # Inference + t1 = time_synchronized() + pred = model(img, augment=opt.augment)[0] + + # Apply NMS + pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms) + t2 = time_synchronized() + + # Apply Classifier + if classify: + pred = apply_classifier(pred, modelc, img, im0s) + + # Process detections + for i, det in enumerate(pred): # detections per image + if webcam: # batch_size >= 1 + p, s, im0 = path[i], '%g: ' % i, im0s[i].copy() + else: + p, s, im0 = path, '', im0s + + save_path = str(Path(out) / Path(p).name) + txt_path = str(Path(out) / Path(p).stem) + ('_%g' % dataset.frame if dataset.mode == 'video' else '') + s += '%gx%g ' % img.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + + if det is not None and len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, -1].unique(): + n = (det[:, -1] == c).sum() # detections per class + s += '%g %ss, ' % (n, names[int(c)]) # add to string + + # Write results + #for *xyxy, conf, cls in reversed(det): + # if save_txt: # Write to file + # xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + # with open(txt_path + '.txt', 'a') as f: + # f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format + + + for *xyxy, conf, cls in reversed(det): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4))).view(-1).tolist() # ADDED BY FRANCOIS + results.append({ + "boundingBox": { + "x": int(xywh[0] - xywh[2]/2), + "y": int(xywh[1] - xywh[3]/2), + "width": int(xywh[2]), + "height": int(xywh[3]) + }, + "confidence": float(conf) + }) + + print("\n$$$") + print(json.dumps(results)) + print("$$$\n") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') + parser.add_argument('--source', type=str, default='inference/images', help='source') # file/folder, 0 for webcam + parser.add_argument('--output', type=str, default='inference/output', help='output folder') # output folder + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.4, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.5, help='IOU threshold for NMS') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='display results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--update', action='store_true', help='update all models') + opt = parser.parse_args() + print(opt) + + with torch.no_grad(): + if opt.update: # update all models (to fix SourceChangeWarning) + for opt.weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']: + detect() + strip_optimizer(opt.weights) + else: + detect() diff --git a/src/digitizer/yolov5/detect_labels.py b/src/digitizer/yolov5/detect_labels.py new file mode 100644 index 0000000000000000000000000000000000000000..8fd0f4ad15f3395ea458ef9df480312ca7977581 --- /dev/null +++ b/src/digitizer/yolov5/detect_labels.py @@ -0,0 +1,153 @@ +import argparse +import json +import os +import platform +import shutil +import time +from pathlib import Path + +import cv2 +import torch +import torch.backends.cudnn as cudnn +from numpy import random + +from models.experimental import attempt_load +from utils.datasets import LoadStreams, LoadImages +from utils.general import ( + check_img_size, non_max_suppression, apply_classifier, scale_coords, + xyxy2xywh, plot_one_box, strip_optimizer, set_logging) +from utils.torch_utils import select_device, load_classifier, time_synchronized + + +def detect(save_img=False): + results = [] + out, source, weights, view_img, save_txt, imgsz = \ + opt.output, opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size + webcam = source.isnumeric() or source.startswith('rtsp') or source.startswith('http') or source.endswith('.txt') + + # Initialize + set_logging() + device = select_device(opt.device) + half = device.type != 'cpu' # half precision only supported on CUDA + + # Load model + model = attempt_load(weights, map_location=device) # load FP32 model + imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size + if half: + model.half() # to FP16 + + # Second-stage classifier + classify = False + if classify: + modelc = load_classifier(name='resnet101', n=2) # initialize + modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']) # load weights + modelc.to(device).eval() + + # Set Dataloader + vid_path, vid_writer = None, None + if webcam: + view_img = True + cudnn.benchmark = True # set True to speed up constant image size inference + dataset = LoadStreams(source, img_size=imgsz) + else: + save_img = True + dataset = LoadImages(source, img_size=imgsz) + + # Get names and colors + names = model.module.names if hasattr(model, 'module') else model.names + colors = [[random.randint(0, 255) for _ in range(3)] for _ in range(len(names))] + + # Run inference + t0 = time.time() + img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img + _ = model(img.half() if half else img) if device.type != 'cpu' else None # run once + for path, img, im0s, vid_cap in dataset: + img = torch.from_numpy(img).to(device) + img = img.half() if half else img.float() # uint8 to fp16/32 + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + if img.ndimension() == 3: + img = img.unsqueeze(0) + + # Inference + t1 = time_synchronized() + pred = model(img, augment=opt.augment)[0] + + # Apply NMS + pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms) + t2 = time_synchronized() + + # Apply Classifier + if classify: + pred = apply_classifier(pred, modelc, img, im0s) + + # Process detections + for i, det in enumerate(pred): # detections per image + if webcam: # batch_size >= 1 + p, s, im0 = path[i], '%g: ' % i, im0s[i].copy() + else: + p, s, im0 = path, '', im0s + + save_path = str(Path(out) / Path(p).name) + txt_path = str(Path(out) / Path(p).stem) + ('_%g' % dataset.frame if dataset.mode == 'video' else '') + s += '%gx%g ' % img.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + if det is not None and len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, -1].unique(): + n = (det[:, -1] == c).sum() # detections per class + s += '%g %ss, ' % (n, names[int(c)]) # add to string + + # Write results + #for *xyxy, conf, cls in reversed(det): + # if save_txt: # Write to file + # xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + # with open(txt_path + '.txt', 'a') as f: + # f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format + + + for *xyxy, conf, cls in reversed(det): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4))).view(-1).tolist() # ADDED BY FRANCOIS + results.append({ + "text": names[int(cls)], + "boundingBox": { + "x": int(xywh[0] - xywh[2]/2), + "y": int(xywh[1] - xywh[3]/2), + "width": int(xywh[2]), + "height": int(xywh[3]) + }, + "confidence": float(conf) + }) + + print("\n$$$") + print(json.dumps(results)) + print("$$$\n") + return results + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') + parser.add_argument('--source', type=str, default='inference/images', help='source') # file/folder, 0 for webcam + parser.add_argument('--output', type=str, default='inference/output', help='output folder') # output folder + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.4, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.5, help='IOU threshold for NMS') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='display results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--update', action='store_true', help='update all models') + opt = parser.parse_args() + print(opt) + + with torch.no_grad(): + if opt.update: # update all models (to fix SourceChangeWarning) + for opt.weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']: + detect() + strip_optimizer(opt.weights) + else: + detect() diff --git a/src/digitizer/yolov5/detect_symbols.py b/src/digitizer/yolov5/detect_symbols.py new file mode 100644 index 0000000000000000000000000000000000000000..16626d7b1b57f4a5940d5cd242119856bfda2e31 --- /dev/null +++ b/src/digitizer/yolov5/detect_symbols.py @@ -0,0 +1,153 @@ +import argparse +import json +import os +import platform +import shutil +import time +from pathlib import Path + +import cv2 +import torch +import torch.backends.cudnn as cudnn +from numpy import random + +from models.experimental import attempt_load +from utils.datasets import LoadStreams, LoadImages +from utils.general import ( + check_img_size, non_max_suppression, apply_classifier, scale_coords, + xyxy2xywh, plot_one_box, strip_optimizer, set_logging) +from utils.torch_utils import select_device, load_classifier, time_synchronized + + +def detect(save_img=False): + results = [] + out, source, weights, view_img, save_txt, imgsz = \ + opt.output, opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size + webcam = source.isnumeric() or source.startswith('rtsp') or source.startswith('http') or source.endswith('.txt') + + # Initialize + set_logging() + device = select_device(opt.device) + half = device.type != 'cpu' # half precision only supported on CUDA + + # Load model + model = attempt_load(weights, map_location=device) # load FP32 model + imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size + if half: + model.half() # to FP16 + + # Second-stage classifier + classify = False + if classify: + modelc = load_classifier(name='resnet101', n=2) # initialize + modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']) # load weights + modelc.to(device).eval() + + # Set Dataloader + vid_path, vid_writer = None, None + if webcam: + view_img = True + cudnn.benchmark = True # set True to speed up constant image size inference + dataset = LoadStreams(source, img_size=imgsz) + else: + save_img = True + dataset = LoadImages(source, img_size=imgsz) + + # Get names and colors + names = model.module.names if hasattr(model, 'module') else model.names + colors = [[random.randint(0, 255) for _ in range(3)] for _ in range(len(names))] + + # Run inference + t0 = time.time() + img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img + _ = model(img.half() if half else img) if device.type != 'cpu' else None # run once + for path, img, im0s, vid_cap in dataset: + img = torch.from_numpy(img).to(device) + img = img.half() if half else img.float() # uint8 to fp16/32 + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + if img.ndimension() == 3: + img = img.unsqueeze(0) + + # Inference + t1 = time_synchronized() + pred = model(img, augment=opt.augment)[0] + + # Apply NMS + pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms) + t2 = time_synchronized() + + # Apply Classifier + if classify: + pred = apply_classifier(pred, modelc, img, im0s) + + # Process detections + for i, det in enumerate(pred): # detections per image + if webcam: # batch_size >= 1 + p, s, im0 = path[i], '%g: ' % i, im0s[i].copy() + else: + p, s, im0 = path, '', im0s + + save_path = str(Path(out) / Path(p).name) + txt_path = str(Path(out) / Path(p).stem) + ('_%g' % dataset.frame if dataset.mode == 'video' else '') + s += '%gx%g ' % img.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + if det is not None and len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, -1].unique(): + n = (det[:, -1] == c).sum() # detections per class + s += '%g %ss, ' % (n, names[int(c)]) # add to string + + # Write results + #for *xyxy, conf, cls in reversed(det): + # if save_txt: # Write to file + # xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + # with open(txt_path + '.txt', 'a') as f: + # f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format + + + for *xyxy, conf, cls in reversed(det): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4))).view(-1).tolist() # ADDED BY FRANCOIS + results.append({ + "measurementType": names[int(cls)], + "noResponse": False, + "boundingBox": { + "x": int(xywh[0] - xywh[2]/2), + "y": int(xywh[1] - xywh[3]/2), + "width": int(xywh[2]), + "height": int(xywh[3]) + }, + "confidence": float(conf) + }) + + print("\n$$$") + print(json.dumps(results)) + print("$$$\n") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') + parser.add_argument('--source', type=str, default='inference/images', help='source') # file/folder, 0 for webcam + parser.add_argument('--output', type=str, default='inference/output', help='output folder') # output folder + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.4, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.5, help='IOU threshold for NMS') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='display results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--update', action='store_true', help='update all models') + opt = parser.parse_args() + print(opt) + + with torch.no_grad(): + if opt.update: # update all models (to fix SourceChangeWarning) + for opt.weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']: + detect() + strip_optimizer(opt.weights) + else: + detect() diff --git a/src/digitizer/yolov5/hubconf.py b/src/digitizer/yolov5/hubconf.py new file mode 100644 index 0000000000000000000000000000000000000000..98416a5b8563d9ee37a74257ae58daeb7c7b100a --- /dev/null +++ b/src/digitizer/yolov5/hubconf.py @@ -0,0 +1,99 @@ +"""File for accessing YOLOv5 via PyTorch Hub https://pytorch.org/hub/ + +Usage: + import torch + model = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True, channels=3, classes=80) +""" + +dependencies = ['torch', 'yaml'] +import os + +import torch + +from models.yolo import Model +from utils.google_utils import attempt_download + + +def create(name, pretrained, channels, classes): + """Creates a specified YOLOv5 model + + Arguments: + name (str): name of model, i.e. 'yolov5s' + pretrained (bool): load pretrained weights into the model + channels (int): number of input channels + classes (int): number of model classes + + Returns: + pytorch model + """ + config = os.path.join(os.path.dirname(__file__), 'models', '%s.yaml' % name) # model.yaml path + try: + model = Model(config, channels, classes) + if pretrained: + ckpt = '%s.pt' % name # checkpoint filename + attempt_download(ckpt) # download if not found locally + state_dict = torch.load(ckpt, map_location=torch.device('cpu'))['model'].float().state_dict() # to FP32 + state_dict = {k: v for k, v in state_dict.items() if model.state_dict()[k].shape == v.shape} # filter + model.load_state_dict(state_dict, strict=False) # load + return model + + except Exception as e: + help_url = 'https://github.com/ultralytics/yolov5/issues/36' + s = 'Cache maybe be out of date, deleting cache and retrying may solve this. See %s for help.' % help_url + raise Exception(s) from e + + +def yolov5s(pretrained=False, channels=3, classes=80): + """YOLOv5-small model from https://github.com/ultralytics/yolov5 + + Arguments: + pretrained (bool): load pretrained weights into the model, default=False + channels (int): number of input channels, default=3 + classes (int): number of model classes, default=80 + + Returns: + pytorch model + """ + return create('yolov5s', pretrained, channels, classes) + + +def yolov5m(pretrained=False, channels=3, classes=80): + """YOLOv5-medium model from https://github.com/ultralytics/yolov5 + + Arguments: + pretrained (bool): load pretrained weights into the model, default=False + channels (int): number of input channels, default=3 + classes (int): number of model classes, default=80 + + Returns: + pytorch model + """ + return create('yolov5m', pretrained, channels, classes) + + +def yolov5l(pretrained=False, channels=3, classes=80): + """YOLOv5-large model from https://github.com/ultralytics/yolov5 + + Arguments: + pretrained (bool): load pretrained weights into the model, default=False + channels (int): number of input channels, default=3 + classes (int): number of model classes, default=80 + + Returns: + pytorch model + """ + return create('yolov5l', pretrained, channels, classes) + + +def yolov5x(pretrained=False, channels=3, classes=80): + """YOLOv5-xlarge model from https://github.com/ultralytics/yolov5 + + Arguments: + pretrained (bool): load pretrained weights into the model, default=False + channels (int): number of input channels, default=3 + classes (int): number of model classes, default=80 + + Returns: + pytorch model + """ + return create('yolov5x', pretrained, channels, classes) diff --git a/src/digitizer/yolov5/inference/images/bus.jpg b/src/digitizer/yolov5/inference/images/bus.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b43e311165c785f000eb7493ff8fb662d06a3f83 Binary files /dev/null and b/src/digitizer/yolov5/inference/images/bus.jpg differ diff --git a/src/digitizer/yolov5/inference/images/zidane.jpg b/src/digitizer/yolov5/inference/images/zidane.jpg new file mode 100644 index 0000000000000000000000000000000000000000..92d72ea124760ce5dbf9425e3aa8f371e7481328 Binary files /dev/null and b/src/digitizer/yolov5/inference/images/zidane.jpg differ diff --git a/src/digitizer/yolov5/models/__init__.py b/src/digitizer/yolov5/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/digitizer/yolov5/models/common.py b/src/digitizer/yolov5/models/common.py new file mode 100644 index 0000000000000000000000000000000000000000..e8c07f4db657882a92434420be439339ea43f532 --- /dev/null +++ b/src/digitizer/yolov5/models/common.py @@ -0,0 +1,118 @@ +# This file contains modules common to various models +import math + +import torch +import torch.nn as nn + + +def autopad(k, p=None): # kernel, padding + # Pad to 'same' + if p is None: + p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad + return p + + +def DWConv(c1, c2, k=1, s=1, act=True): + # Depthwise convolution + return Conv(c1, c2, k, s, g=math.gcd(c1, c2), act=act) + + +class Conv(nn.Module): + # Standard convolution + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups + super(Conv, self).__init__() + self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False) + self.bn = nn.BatchNorm2d(c2) + self.act = nn.Hardswish() if act else nn.Identity() + + def forward(self, x): + return self.act(self.bn(self.conv(x))) + + def fuseforward(self, x): + return self.act(self.conv(x)) + + +class Bottleneck(nn.Module): + # Standard bottleneck + def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super(Bottleneck, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c2, 3, 1, g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class BottleneckCSP(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(BottleneckCSP, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False) + self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False) + self.cv4 = Conv(2 * c_, c2, 1, 1) + self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3) + self.act = nn.LeakyReLU(0.1, inplace=True) + self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + y1 = self.cv3(self.m(self.cv1(x))) + y2 = self.cv2(x) + return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1)))) + + +class SPP(nn.Module): + # Spatial pyramid pooling layer used in YOLOv3-SPP + def __init__(self, c1, c2, k=(5, 9, 13)): + super(SPP, self).__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1) + self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k]) + + def forward(self, x): + x = self.cv1(x) + return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1)) + + +class Focus(nn.Module): + # Focus wh information into c-space + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups + super(Focus, self).__init__() + self.conv = Conv(c1 * 4, c2, k, s, p, g, act) + + def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2) + return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)) + + +class Concat(nn.Module): + # Concatenate a list of tensors along dimension + def __init__(self, dimension=1): + super(Concat, self).__init__() + self.d = dimension + + def forward(self, x): + return torch.cat(x, self.d) + + +class Flatten(nn.Module): + # Use after nn.AdaptiveAvgPool2d(1) to remove last 2 dimensions + @staticmethod + def forward(x): + return x.view(x.size(0), -1) + + +class Classify(nn.Module): + # Classification head, i.e. x(b,c1,20,20) to x(b,c2) + def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups + super(Classify, self).__init__() + self.aap = nn.AdaptiveAvgPool2d(1) # to x(b,c1,1,1) + self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False) # to x(b,c2,1,1) + self.flat = Flatten() + + def forward(self, x): + z = torch.cat([self.aap(y) for y in (x if isinstance(x, list) else [x])], 1) # cat if list + return self.flat(self.conv(z)) # flatten to x(b,c2) diff --git a/src/digitizer/yolov5/models/experimental.py b/src/digitizer/yolov5/models/experimental.py new file mode 100644 index 0000000000000000000000000000000000000000..1b99ce47db535cdcb567adc612056a2f625ea122 --- /dev/null +++ b/src/digitizer/yolov5/models/experimental.py @@ -0,0 +1,145 @@ +# This file contains experimental modules + +import numpy as np +import torch +import torch.nn as nn + +from models.common import Conv, DWConv +from utils.google_utils import attempt_download + + +class CrossConv(nn.Module): + # Cross Convolution Downsample + def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False): + # ch_in, ch_out, kernel, stride, groups, expansion, shortcut + super(CrossConv, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, (1, k), (1, s)) + self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class C3(nn.Module): + # Cross Convolution CSP + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(C3, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False) + self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False) + self.cv4 = Conv(2 * c_, c2, 1, 1) + self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3) + self.act = nn.LeakyReLU(0.1, inplace=True) + self.m = nn.Sequential(*[CrossConv(c_, c_, 3, 1, g, 1.0, shortcut) for _ in range(n)]) + + def forward(self, x): + y1 = self.cv3(self.m(self.cv1(x))) + y2 = self.cv2(x) + return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1)))) + + +class Sum(nn.Module): + # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070 + def __init__(self, n, weight=False): # n: number of inputs + super(Sum, self).__init__() + self.weight = weight # apply weights boolean + self.iter = range(n - 1) # iter object + if weight: + self.w = nn.Parameter(-torch.arange(1., n) / 2, requires_grad=True) # layer weights + + def forward(self, x): + y = x[0] # no weight + if self.weight: + w = torch.sigmoid(self.w) * 2 + for i in self.iter: + y = y + x[i + 1] * w[i] + else: + for i in self.iter: + y = y + x[i + 1] + return y + + +class GhostConv(nn.Module): + # Ghost Convolution https://github.com/huawei-noah/ghostnet + def __init__(self, c1, c2, k=1, s=1, g=1, act=True): # ch_in, ch_out, kernel, stride, groups + super(GhostConv, self).__init__() + c_ = c2 // 2 # hidden channels + self.cv1 = Conv(c1, c_, k, s, g, act) + self.cv2 = Conv(c_, c_, 5, 1, c_, act) + + def forward(self, x): + y = self.cv1(x) + return torch.cat([y, self.cv2(y)], 1) + + +class GhostBottleneck(nn.Module): + # Ghost Bottleneck https://github.com/huawei-noah/ghostnet + def __init__(self, c1, c2, k, s): + super(GhostBottleneck, self).__init__() + c_ = c2 // 2 + self.conv = nn.Sequential(GhostConv(c1, c_, 1, 1), # pw + DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw + GhostConv(c_, c2, 1, 1, act=False)) # pw-linear + self.shortcut = nn.Sequential(DWConv(c1, c1, k, s, act=False), + Conv(c1, c2, 1, 1, act=False)) if s == 2 else nn.Identity() + + def forward(self, x): + return self.conv(x) + self.shortcut(x) + + +class MixConv2d(nn.Module): + # Mixed Depthwise Conv https://arxiv.org/abs/1907.09595 + def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True): + super(MixConv2d, self).__init__() + groups = len(k) + if equal_ch: # equal c_ per group + i = torch.linspace(0, groups - 1E-6, c2).floor() # c2 indices + c_ = [(i == g).sum() for g in range(groups)] # intermediate channels + else: # equal weight.numel() per group + b = [c2] + [0] * groups + a = np.eye(groups + 1, groups, k=-1) + a -= np.roll(a, 1, axis=1) + a *= np.array(k) ** 2 + a[0] = 1 + c_ = np.linalg.lstsq(a, b, rcond=None)[0].round() # solve for equal weight indices, ax = b + + self.m = nn.ModuleList([nn.Conv2d(c1, int(c_[g]), k[g], s, k[g] // 2, bias=False) for g in range(groups)]) + self.bn = nn.BatchNorm2d(c2) + self.act = nn.LeakyReLU(0.1, inplace=True) + + def forward(self, x): + return x + self.act(self.bn(torch.cat([m(x) for m in self.m], 1))) + + +class Ensemble(nn.ModuleList): + # Ensemble of models + def __init__(self): + super(Ensemble, self).__init__() + + def forward(self, x, augment=False): + y = [] + for module in self: + y.append(module(x, augment)[0]) + # y = torch.stack(y).max(0)[0] # max ensemble + # y = torch.cat(y, 1) # nms ensemble + y = torch.stack(y).mean(0) # mean ensemble + return y, None # inference, train output + + +def attempt_load(weights, map_location=None): + # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a + model = Ensemble() + for w in weights if isinstance(weights, list) else [weights]: + attempt_download(w) + model.append(torch.load(w, map_location=map_location)['model'].float().fuse().eval()) # load FP32 model + + if len(model) == 1: + return model[-1] # return model + else: + print('Ensemble created with %s\n' % weights) + for k in ['names', 'stride']: + setattr(model, k, getattr(model[-1], k)) + return model # return ensemble diff --git a/src/digitizer/yolov5/models/export.py b/src/digitizer/yolov5/models/export.py new file mode 100644 index 0000000000000000000000000000000000000000..168d24d50a5a6b9937c868497f3854ca85fd2d9b --- /dev/null +++ b/src/digitizer/yolov5/models/export.py @@ -0,0 +1,84 @@ +"""Exports a YOLOv5 *.pt model to ONNX and TorchScript formats + +Usage: + $ export PYTHONPATH="$PWD" && python models/export.py --weights ./weights/yolov5s.pt --img 640 --batch 1 +""" + +import argparse + +import torch +import torch.nn as nn + +from models.common import Conv +from models.experimental import attempt_load +from utils.activations import Hardswish +from utils.general import set_logging + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default='./yolov5s.pt', help='weights path') # from yolov5/models/ + parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='image size') # height, width + parser.add_argument('--batch-size', type=int, default=1, help='batch size') + opt = parser.parse_args() + opt.img_size *= 2 if len(opt.img_size) == 1 else 1 # expand + print(opt) + set_logging() + + # Input + img = torch.zeros((opt.batch_size, 3, *opt.img_size)) # image size(1,3,320,192) iDetection + + # Load PyTorch model + model = attempt_load(opt.weights, map_location=torch.device('cpu')) # load FP32 model + + # Update model + for k, m in model.named_modules(): + m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatability + if isinstance(m, Conv) and isinstance(m.act, nn.Hardswish): + m.act = Hardswish() # assign activation + # if isinstance(m, Detect): + # m.forward = m.forward_export # assign forward (optional) + model.model[-1].export = True # set Detect() layer export=True + y = model(img) # dry run + + # TorchScript export + try: + print('\nStarting TorchScript export with torch %s...' % torch.__version__) + f = opt.weights.replace('.pt', '.torchscript.pt') # filename + ts = torch.jit.trace(model, img) + ts.save(f) + print('TorchScript export success, saved as %s' % f) + except Exception as e: + print('TorchScript export failure: %s' % e) + + # ONNX export + try: + import onnx + + print('\nStarting ONNX export with onnx %s...' % onnx.__version__) + f = opt.weights.replace('.pt', '.onnx') # filename + torch.onnx.export(model, img, f, verbose=False, opset_version=12, input_names=['images'], + output_names=['classes', 'boxes'] if y is None else ['output']) + + # Checks + onnx_model = onnx.load(f) # load onnx model + onnx.checker.check_model(onnx_model) # check onnx model + # print(onnx.helper.printable_graph(onnx_model.graph)) # print a human readable model + print('ONNX export success, saved as %s' % f) + except Exception as e: + print('ONNX export failure: %s' % e) + + # CoreML export + try: + import coremltools as ct + + print('\nStarting CoreML export with coremltools %s...' % ct.__version__) + # convert model from torchscript and apply pixel scaling as per detect.py + model = ct.convert(ts, inputs=[ct.ImageType(name='images', shape=img.shape, scale=1 / 255.0, bias=[0, 0, 0])]) + f = opt.weights.replace('.pt', '.mlmodel') # filename + model.save(f) + print('CoreML export success, saved as %s' % f) + except Exception as e: + print('CoreML export failure: %s' % e) + + # Finish + print('\nExport complete. Visualize with https://github.com/lutzroeder/netron.') diff --git a/src/digitizer/yolov5/models/hub/yolov3-spp.yaml b/src/digitizer/yolov5/models/hub/yolov3-spp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6cadd9fb708450399aeb3e436f6e55da8e1e8c1 --- /dev/null +++ b/src/digitizer/yolov5/models/hub/yolov3-spp.yaml @@ -0,0 +1,51 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# darknet53 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, Bottleneck, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, Bottleneck, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, Bottleneck, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, Bottleneck, [1024]], # 10 + ] + +# YOLOv3-SPP head +head: + [[-1, 1, Bottleneck, [1024, False]], + [-1, 1, SPP, [512, [5, 9, 13]]], + [-1, 1, Conv, [1024, 3, 1]], + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [1024, 3, 1]], # 15 (P5/32-large) + + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], # 22 (P4/16-medium) + + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P3 + [-1, 1, Bottleneck, [256, False]], + [-1, 2, Bottleneck, [256, False]], # 27 (P3/8-small) + + [[27, 22, 15], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/src/digitizer/yolov5/models/hub/yolov5-fpn.yaml b/src/digitizer/yolov5/models/hub/yolov5-fpn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d2fae10bb22d0a4cd8b3c92912080a042f56320 --- /dev/null +++ b/src/digitizer/yolov5/models/hub/yolov5-fpn.yaml @@ -0,0 +1,42 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, Bottleneck, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 6, BottleneckCSP, [1024]], # 9 + ] + +# YOLOv5 FPN head +head: + [[-1, 3, BottleneckCSP, [1024, False]], # 10 (P5/32-large) + + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Conv, [512, 1, 1]], + [-1, 3, BottleneckCSP, [512, False]], # 14 (P4/16-medium) + + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 1, Conv, [256, 1, 1]], + [-1, 3, BottleneckCSP, [256, False]], # 18 (P3/8-small) + + [[18, 14, 10], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/src/digitizer/yolov5/models/hub/yolov5-panet.yaml b/src/digitizer/yolov5/models/hub/yolov5-panet.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ed05ddc5d0356b8d75ecd88314fdfa1c579e105 --- /dev/null +++ b/src/digitizer/yolov5/models/hub/yolov5-panet.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [116,90, 156,198, 373,326] # P5/32 + - [30,61, 62,45, 59,119] # P4/16 + - [10,13, 16,30, 33,23] # P3/8 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 PANet head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P5, P4, P3) + ] diff --git a/src/digitizer/yolov5/models/yolo.py b/src/digitizer/yolov5/models/yolo.py new file mode 100644 index 0000000000000000000000000000000000000000..f1c3c3f9084aca7b4726d245065b02ad809adbad --- /dev/null +++ b/src/digitizer/yolov5/models/yolo.py @@ -0,0 +1,264 @@ +import argparse +import logging +import math +from copy import deepcopy +from pathlib import Path + +import torch +import torch.nn as nn + +from models.common import Conv, Bottleneck, SPP, DWConv, Focus, BottleneckCSP, Concat +from models.experimental import MixConv2d, CrossConv, C3 +from utils.general import check_anchor_order, make_divisible, check_file, set_logging +from utils.torch_utils import ( + time_synchronized, fuse_conv_and_bn, model_info, scale_img, initialize_weights, select_device) + +logger = logging.getLogger(__name__) + + +class Detect(nn.Module): + stride = None # strides computed during build + export = False # onnx export + + def __init__(self, nc=80, anchors=(), ch=()): # detection layer + super(Detect, self).__init__() + self.nc = nc # number of classes + self.no = nc + 5 # number of outputs per anchor + self.nl = len(anchors) # number of detection layers + self.na = len(anchors[0]) // 2 # number of anchors + self.grid = [torch.zeros(1)] * self.nl # init grid + a = torch.tensor(anchors).float().view(self.nl, -1, 2) + self.register_buffer('anchors', a) # shape(nl,na,2) + self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2) + self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv + + def forward(self, x): + # x = x.copy() # for profiling + z = [] # inference output + self.training |= self.export + for i in range(self.nl): + x[i] = self.m[i](x[i]) # conv + bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85) + x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous() + + if not self.training: # inference + if self.grid[i].shape[2:4] != x[i].shape[2:4]: + self.grid[i] = self._make_grid(nx, ny).to(x[i].device) + + y = x[i].sigmoid() + y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i].to(x[i].device)) * self.stride[i] # xy + y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh + z.append(y.view(bs, -1, self.no)) + + return x if self.training else (torch.cat(z, 1), x) + + @staticmethod + def _make_grid(nx=20, ny=20): + yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)]) + return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float() + + +class Model(nn.Module): + def __init__(self, cfg='yolov5s.yaml', ch=3, nc=None): # model, input channels, number of classes + super(Model, self).__init__() + if isinstance(cfg, dict): + self.yaml = cfg # model dict + else: # is *.yaml + import yaml # for torch hub + self.yaml_file = Path(cfg).name + with open(cfg) as f: + self.yaml = yaml.load(f, Loader=yaml.FullLoader) # model dict + + # Define model + if nc and nc != self.yaml['nc']: + print('Overriding %s nc=%g with nc=%g' % (cfg, self.yaml['nc'], nc)) + self.yaml['nc'] = nc # override yaml value + self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist, ch_out + # print([x.shape for x in self.forward(torch.zeros(1, ch, 64, 64))]) + + # Build strides, anchors + m = self.model[-1] # Detect() + if isinstance(m, Detect): + s = 128 # 2x min stride + m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward + m.anchors /= m.stride.view(-1, 1, 1) + check_anchor_order(m) + self.stride = m.stride + self._initialize_biases() # only run once + # print('Strides: %s' % m.stride.tolist()) + + # Init weights, biases + initialize_weights(self) + self.info() + print('') + + def forward(self, x, augment=False, profile=False): + if augment: + img_size = x.shape[-2:] # height, width + s = [1, 0.83, 0.67] # scales + f = [None, 3, None] # flips (2-ud, 3-lr) + y = [] # outputs + for si, fi in zip(s, f): + xi = scale_img(x.flip(fi) if fi else x, si) + yi = self.forward_once(xi)[0] # forward + # cv2.imwrite('img%g.jpg' % s, 255 * xi[0].numpy().transpose((1, 2, 0))[:, :, ::-1]) # save + yi[..., :4] /= si # de-scale + if fi == 2: + yi[..., 1] = img_size[0] - yi[..., 1] # de-flip ud + elif fi == 3: + yi[..., 0] = img_size[1] - yi[..., 0] # de-flip lr + y.append(yi) + return torch.cat(y, 1), None # augmented inference, train + else: + return self.forward_once(x, profile) # single-scale inference, train + + def forward_once(self, x, profile=False): + y, dt = [], [] # outputs + for m in self.model: + if m.f != -1: # if not from previous layer + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers + + if profile: + try: + import thop + o = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 # FLOPS + except: + o = 0 + t = time_synchronized() + for _ in range(10): + _ = m(x) + dt.append((time_synchronized() - t) * 100) + print('%10.1f%10.0f%10.1fms %-40s' % (o, m.np, dt[-1], m.type)) + + x = m(x) # run + y.append(x if m.i in self.save else None) # save output + + if profile: + print('%.1fms total' % sum(dt)) + return x + + def _initialize_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency + # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1. + m = self.model[-1] # Detect() module + for mi, s in zip(m.m, m.stride): # from + b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85) + b[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image) + b[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls + mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + def _print_biases(self): + m = self.model[-1] # Detect() module + for mi in m.m: # from + b = mi.bias.detach().view(m.na, -1).T # conv.bias(255) to (3,85) + print(('%6g Conv2d.bias:' + '%10.3g' * 6) % (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean())) + + # def _print_weights(self): + # for m in self.model.modules(): + # if type(m) is Bottleneck: + # print('%10.3g' % (m.w.detach().sigmoid() * 2)) # shortcut weights + + def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers + print('Fusing layers... ') + for m in self.model.modules(): + if type(m) is Conv: + m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatability + m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv + delattr(m, 'bn') # remove batchnorm + m.forward = m.fuseforward # update forward + self.info() + return self + + def info(self, verbose=False): # print model information + model_info(self, verbose) + + +def parse_model(d, ch): # model_dict, input_channels(3) + logger.info('\n%3s%18s%3s%10s %-40s%-30s' % ('', 'from', 'n', 'params', 'module', 'arguments')) + anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple'] + na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors + no = na * (nc + 5) # number of outputs = anchors * (classes + 5) + + layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out + for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args + m = eval(m) if isinstance(m, str) else m # eval strings + for j, a in enumerate(args): + try: + args[j] = eval(a) if isinstance(a, str) else a # eval strings + except: + pass + + n = max(round(n * gd), 1) if n > 1 else n # depth gain + if m in [nn.Conv2d, Conv, Bottleneck, SPP, DWConv, MixConv2d, Focus, CrossConv, BottleneckCSP, C3]: + c1, c2 = ch[f], args[0] + + # Normal + # if i > 0 and args[0] != no: # channel expansion factor + # ex = 1.75 # exponential (default 2.0) + # e = math.log(c2 / ch[1]) / math.log(2) + # c2 = int(ch[1] * ex ** e) + # if m != Focus: + + c2 = make_divisible(c2 * gw, 8) if c2 != no else c2 + + # Experimental + # if i > 0 and args[0] != no: # channel expansion factor + # ex = 1 + gw # exponential (default 2.0) + # ch1 = 32 # ch[1] + # e = math.log(c2 / ch1) / math.log(2) # level 1-n + # c2 = int(ch1 * ex ** e) + # if m != Focus: + # c2 = make_divisible(c2, 8) if c2 != no else c2 + + args = [c1, c2, *args[1:]] + if m in [BottleneckCSP, C3]: + args.insert(2, n) + n = 1 + elif m is nn.BatchNorm2d: + args = [ch[f]] + elif m is Concat: + c2 = sum([ch[-1 if x == -1 else x + 1] for x in f]) + elif m is Detect: + args.append([ch[x + 1] for x in f]) + if isinstance(args[1], int): # number of anchors + args[1] = [list(range(args[1] * 2))] * len(f) + else: + c2 = ch[f] + + m_ = nn.Sequential(*[m(*args) for _ in range(n)]) if n > 1 else m(*args) # module + t = str(m)[8:-2].replace('__main__.', '') # module type + np = sum([x.numel() for x in m_.parameters()]) # number params + m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params + logger.info('%3s%18s%3s%10.0f %-40s%-30s' % (i, f, n, np, t, args)) # print + save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist + layers.append(m_) + ch.append(c2) + return nn.Sequential(*layers), sorted(save) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--cfg', type=str, default='yolov5s.yaml', help='model.yaml') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + opt = parser.parse_args() + opt.cfg = check_file(opt.cfg) # check file + set_logging() + device = select_device(opt.device) + + # Create model + model = Model(opt.cfg).to(device) + model.train() + + # Profile + # img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 640, 640).to(device) + # y = model(img, profile=True) + + # ONNX export + # model.model[-1].export = True + # torch.onnx.export(model, img, opt.cfg.replace('.yaml', '.onnx'), verbose=True, opset_version=11) + + # Tensorboard + # from torch.utils.tensorboard import SummaryWriter + # tb_writer = SummaryWriter() + # print("Run 'tensorboard --logdir=models/runs' to view tensorboard at http://localhost:6006/") + # tb_writer.add_graph(model.model, img) # add model to tensorboard + # tb_writer.add_image('test', img[0], dataformats='CWH') # add model to tensorboard diff --git a/src/digitizer/yolov5/models/yolov5l.yaml b/src/digitizer/yolov5/models/yolov5l.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13095541703f98efb6a7703a75586e7384962096 --- /dev/null +++ b/src/digitizer/yolov5/models/yolov5l.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/src/digitizer/yolov5/models/yolov5m.yaml b/src/digitizer/yolov5/models/yolov5m.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb50a713f2ffa6183ffb95630af2668e939f6792 --- /dev/null +++ b/src/digitizer/yolov5/models/yolov5m.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 0.67 # model depth multiple +width_multiple: 0.75 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/src/digitizer/yolov5/models/yolov5s.yaml b/src/digitizer/yolov5/models/yolov5s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bec4529294e01c917df62b220c5de53fbc6c4e3 --- /dev/null +++ b/src/digitizer/yolov5/models/yolov5s.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/src/digitizer/yolov5/models/yolov5x.yaml b/src/digitizer/yolov5/models/yolov5x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9676402040ce2e171ea28813f1f150bead0c2b5b --- /dev/null +++ b/src/digitizer/yolov5/models/yolov5x.yaml @@ -0,0 +1,48 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.33 # model depth multiple +width_multiple: 1.25 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Focus, [64, 3]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, BottleneckCSP, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 9, BottleneckCSP, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, BottleneckCSP, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 1, SPP, [1024, [5, 9, 13]]], + [-1, 3, BottleneckCSP, [1024, False]], # 9 + ] + +# YOLOv5 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, BottleneckCSP, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, BottleneckCSP, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, BottleneckCSP, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, BottleneckCSP, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/src/digitizer/yolov5/requirements.txt b/src/digitizer/yolov5/requirements.txt new file mode 100755 index 0000000000000000000000000000000000000000..50efbd1019659e539d9b783f99e1af42ef434a39 --- /dev/null +++ b/src/digitizer/yolov5/requirements.txt @@ -0,0 +1,27 @@ +# pip install -r requirements.txt + +# base ---------------------------------------- +Cython +matplotlib>=3.2.2 +numpy>=1.18.5 +opencv-python>=4.1.2 +pillow +PyYAML>=5.3 +scipy>=1.4.1 +tensorboard>=2.2 +torch>=1.6.0 +torchvision>=0.7.0 +tqdm>=4.41.0 + +# coco ---------------------------------------- +# pycocotools>=2.0 + +# export -------------------------------------- +# packaging # for coremltools +# coremltools==4.0b3 +# onnx>=1.7.0 +# scikit-learn==0.19.2 # for coreml quantization + +# extras -------------------------------------- +# thop # FLOPS computation +# seaborn # plotting diff --git a/src/digitizer/yolov5/test.py b/src/digitizer/yolov5/test.py new file mode 100644 index 0000000000000000000000000000000000000000..3bcbbd86f442fce0bedb5b10a7fd0ede6c44e979 --- /dev/null +++ b/src/digitizer/yolov5/test.py @@ -0,0 +1,295 @@ +import argparse +import glob +import json +import os +import shutil +from pathlib import Path + +import numpy as np +import torch +import yaml +from tqdm import tqdm + +from models.experimental import attempt_load +from utils.datasets import create_dataloader +from utils.general import ( + coco80_to_coco91_class, check_dataset, check_file, check_img_size, compute_loss, non_max_suppression, scale_coords, + xyxy2xywh, clip_coords, plot_images, xywh2xyxy, box_iou, output_to_target, ap_per_class, set_logging) +from utils.torch_utils import select_device, time_synchronized + + +def test(data, + weights=None, + batch_size=16, + imgsz=640, + conf_thres=0.001, + iou_thres=0.6, # for NMS + save_json=False, + single_cls=False, + augment=False, + verbose=False, + model=None, + dataloader=None, + save_dir='', + merge=False, + save_txt=False): + # Initialize/load model and set device + training = model is not None + if training: # called by train.py + device = next(model.parameters()).device # get model device + + else: # called directly + set_logging() + device = select_device(opt.device, batch_size=batch_size) + merge, save_txt = opt.merge, opt.save_txt # use Merge NMS, save *.txt labels + if save_txt: + out = Path('inference/output') + if os.path.exists(out): + shutil.rmtree(out) # delete output folder + os.makedirs(out) # make new output folder + + # Remove previous + for f in glob.glob(str(Path(save_dir) / 'test_batch*.jpg')): + os.remove(f) + + # Load model + model = attempt_load(weights, map_location=device) # load FP32 model + imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size + + # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99 + # if device.type != 'cpu' and torch.cuda.device_count() > 1: + # model = nn.DataParallel(model) + + # Half + half = device.type != 'cpu' # half precision only supported on CUDA + if half: + model.half() + + # Configure + model.eval() + with open(data) as f: + data = yaml.load(f, Loader=yaml.FullLoader) # model dict + check_dataset(data) # check + nc = 1 if single_cls else int(data['nc']) # number of classes + iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for mAP@0.5:0.95 + niou = iouv.numel() + + # Dataloader + if not training: + img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img + _ = model(img.half() if half else img) if device.type != 'cpu' else None # run once + path = data['test'] if opt.task == 'test' else data['val'] # path to val/test images + dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, + hyp=None, augment=False, cache=False, pad=0.5, rect=True)[0] + + seen = 0 + names = model.names if hasattr(model, 'names') else model.module.names + coco91class = coco80_to_coco91_class() + s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') + p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. + loss = torch.zeros(3, device=device) + jdict, stats, ap, ap_class = [], [], [], [] + for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): + img = img.to(device, non_blocking=True) + img = img.half() if half else img.float() # uint8 to fp16/32 + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + targets = targets.to(device) + nb, _, height, width = img.shape # batch size, channels, height, width + whwh = torch.Tensor([width, height, width, height]).to(device) + + # Disable gradients + with torch.no_grad(): + # Run model + t = time_synchronized() + inf_out, train_out = model(img, augment=augment) # inference and training outputs + t0 += time_synchronized() - t + + # Compute loss + if training: # if model has loss hyperparameters + loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # GIoU, obj, cls + + # Run NMS + t = time_synchronized() + output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge) + t1 += time_synchronized() - t + + # Statistics per image + for si, pred in enumerate(output): + labels = targets[targets[:, 0] == si, 1:] + nl = len(labels) + tcls = labels[:, 0].tolist() if nl else [] # target class + seen += 1 + + if pred is None: + if nl: + stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) + continue + + # Append to text file + if save_txt: + gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh + x = pred.clone() + x[:, :4] = scale_coords(img[si].shape[1:], x[:, :4], shapes[si][0], shapes[si][1]) # to original + for *xyxy, conf, cls in x: + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + with open(str(out / Path(paths[si]).stem) + '.txt', 'a') as f: + f.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format + + # Clip boxes to image bounds + clip_coords(pred, (height, width)) + + # Append to pycocotools JSON dictionary + if save_json: + # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... + image_id = Path(paths[si]).stem + box = pred[:, :4].clone() # xyxy + scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape + box = xyxy2xywh(box) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + for p, b in zip(pred.tolist(), box.tolist()): + jdict.append({'image_id': int(image_id) if image_id.isnumeric() else image_id, + 'category_id': coco91class[int(p[5])], + 'bbox': [round(x, 3) for x in b], + 'score': round(p[4], 5)}) + + # Assign all predictions as incorrect + correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) + if nl: + detected = [] # target indices + tcls_tensor = labels[:, 0] + + # target boxes + tbox = xywh2xyxy(labels[:, 1:5]) * whwh + + # Per target class + for cls in torch.unique(tcls_tensor): + ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices + pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices + + # Search for detections + if pi.shape[0]: + # Prediction to target ious + ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1) # best ious, indices + + # Append detections + detected_set = set() + for j in (ious > iouv[0]).nonzero(as_tuple=False): + d = ti[i[j]] # detected target + if d.item() not in detected_set: + detected_set.add(d.item()) + detected.append(d) + correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn + if len(detected) == nl: # all targets already located in image + break + + # Append statistics (correct, conf, pcls, tcls) + stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) + + # Plot images + if batch_i < 1: + f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i) # filename + plot_images(img, targets, paths, str(f), names) # ground truth + f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i) + plot_images(img, output_to_target(output, width, height), paths, str(f), names) # predictions + + # Compute statistics + stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy + if len(stats) and stats[0].any(): + p, r, ap, f1, ap_class = ap_per_class(*stats) + p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(1) # [P, R, AP@0.5, AP@0.5:0.95] + mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() + nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class + else: + nt = torch.zeros(1) + + # Print results + pf = '%20s' + '%12.3g' * 6 # print format + print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) + + # Print results per class + if verbose and nc > 1 and len(stats): + for i, c in enumerate(ap_class): + print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) + + # Print speeds + t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple + if not training: + print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) + + # Save JSON + if save_json and len(jdict): + f = 'detections_val2017_%s_results.json' % \ + (weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '') # filename + print('\nCOCO mAP with pycocotools... saving %s...' % f) + with open(f, 'w') as file: + json.dump(jdict, file) + + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + + imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files] + cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api + cocoDt = cocoGt.loadRes(f) # initialize COCO pred api + cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') + cocoEval.params.imgIds = imgIds # image IDs to evaluate + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + map, map50 = cocoEval.stats[:2] # update results (mAP@0.5:0.95, mAP@0.5) + except Exception as e: + print('ERROR: pycocotools unable to run: %s' % e) + + # Return results + model.float() # for training + maps = np.zeros(nc) + map + for i, c in enumerate(ap_class): + maps[c] = ap[i] + return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(prog='test.py') + parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') + parser.add_argument('--data', type=str, default='data/coco128.yaml', help='*.data path') + parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch') + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.65, help='IOU threshold for NMS') + parser.add_argument('--save-json', action='store_true', help='save a cocoapi-compatible JSON results file') + parser.add_argument('--task', default='val', help="'val', 'test', 'study'") + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--merge', action='store_true', help='use Merge NMS') + parser.add_argument('--verbose', action='store_true', help='report mAP by class') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + opt = parser.parse_args() + opt.save_json |= opt.data.endswith('coco.yaml') + opt.data = check_file(opt.data) # check file + print(opt) + + if opt.task in ['val', 'test']: # run normally + test(opt.data, + opt.weights, + opt.batch_size, + opt.img_size, + opt.conf_thres, + opt.iou_thres, + opt.save_json, + opt.single_cls, + opt.augment, + opt.verbose) + + elif opt.task == 'study': # run over a range of settings and save/plot + for weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']: + f = 'study_%s_%s.txt' % (Path(opt.data).stem, Path(weights).stem) # filename to save to + x = list(range(320, 800, 64)) # x axis + y = [] # y axis + for i in x: # img-size + print('\nRunning %s point %s...' % (f, i)) + r, _, t = test(opt.data, weights, opt.batch_size, i, opt.conf_thres, opt.iou_thres, opt.save_json) + y.append(r + t) # results and times + np.savetxt(f, y, fmt='%10.4g') # save + os.system('zip -r study.zip study_*.txt') + # utils.general.plot_study_txt(f, x) # plot diff --git a/src/digitizer/yolov5/train.py b/src/digitizer/yolov5/train.py new file mode 100644 index 0000000000000000000000000000000000000000..a556200838b6187b7d5befa42403291f1fb2f78b --- /dev/null +++ b/src/digitizer/yolov5/train.py @@ -0,0 +1,534 @@ +import argparse +import logging +import math +import os +import random +import shutil +import time +from pathlib import Path + +import numpy as np +import torch.distributed as dist +import torch.nn.functional as F +import torch.optim as optim +import torch.optim.lr_scheduler as lr_scheduler +import torch.utils.data +import yaml +from torch.cuda import amp +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.tensorboard import SummaryWriter +from tqdm import tqdm + +import test # import test.py to get mAP after each epoch +from models.yolo import Model +from utils.datasets import create_dataloader +from utils.general import ( + torch_distributed_zero_first, labels_to_class_weights, plot_labels, check_anchors, labels_to_image_weights, + compute_loss, plot_images, fitness, strip_optimizer, plot_results, get_latest_run, check_dataset, check_file, + check_git_status, check_img_size, increment_dir, print_mutation, plot_evolution, set_logging) +from utils.google_utils import attempt_download +from utils.torch_utils import init_seeds, ModelEMA, select_device, intersect_dicts + +logger = logging.getLogger(__name__) + + +def train(hyp, opt, device, tb_writer=None): + logger.info(f'Hyperparameters {hyp}') + log_dir = Path(tb_writer.log_dir) if tb_writer else Path(opt.logdir) / 'evolve' # logging directory + wdir = log_dir / 'weights' # weights directory + os.makedirs(wdir, exist_ok=True) + last = wdir / 'last.pt' + best = wdir / 'best.pt' + results_file = str(log_dir / 'results.txt') + epochs, batch_size, total_batch_size, weights, rank = \ + opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank + + # Save run settings + with open(log_dir / 'hyp.yaml', 'w') as f: + yaml.dump(hyp, f, sort_keys=False) + with open(log_dir / 'opt.yaml', 'w') as f: + yaml.dump(vars(opt), f, sort_keys=False) + + # Configure + cuda = device.type != 'cpu' + init_seeds(2 + rank) + with open(opt.data) as f: + data_dict = yaml.load(f, Loader=yaml.FullLoader) # data dict + with torch_distributed_zero_first(rank): + check_dataset(data_dict) # check + train_path = data_dict['train'] + test_path = data_dict['val'] + nc, names = (1, ['item']) if opt.single_cls else (int(data_dict['nc']), data_dict['names']) # number classes, names + assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check + + # Model + pretrained = weights.endswith('.pt') + if pretrained: + with torch_distributed_zero_first(rank): + attempt_download(weights) # download if not found locally + ckpt = torch.load(weights, map_location=device) # load checkpoint + # if hyp['anchors']: + # ckpt['model'].yaml['anchors'] = round(hyp['anchors']) # force autoanchor + model = Model(opt.cfg or ckpt['model'].yaml, ch=3, nc=nc).to(device) # create + exclude = ['anchor'] if opt.cfg else [] # exclude keys + state_dict = ckpt['model'].float().state_dict() # to FP32 + state_dict = intersect_dicts(state_dict, model.state_dict(), exclude=exclude) # intersect + model.load_state_dict(state_dict, strict=False) # load + logger.info('Transferred %g/%g items from %s' % (len(state_dict), len(model.state_dict()), weights)) # report + else: + model = Model(opt.cfg, ch=3, nc=nc).to(device) # create + + # Freeze + freeze = ['', ] # parameter names to freeze (full or partial) + if any(freeze): + for k, v in model.named_parameters(): + if any(x in k for x in freeze): + print('freezing %s' % k) + v.requires_grad = False + + # Optimizer + nbs = 64 # nominal batch size + accumulate = max(round(nbs / total_batch_size), 1) # accumulate loss before optimizing + hyp['weight_decay'] *= total_batch_size * accumulate / nbs # scale weight_decay + + pg0, pg1, pg2 = [], [], [] # optimizer parameter groups + for k, v in model.named_parameters(): + v.requires_grad = True + if '.bias' in k: + pg2.append(v) # biases + elif '.weight' in k and '.bn' not in k: + pg1.append(v) # apply weight decay + else: + pg0.append(v) # all else + + if opt.adam: + optimizer = optim.Adam(pg0, lr=hyp['lr0'], betas=(hyp['momentum'], 0.999)) # adjust beta1 to momentum + else: + optimizer = optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True) + + optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay + optimizer.add_param_group({'params': pg2}) # add pg2 (biases) + logger.info('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0))) + del pg0, pg1, pg2 + + # Scheduler https://arxiv.org/pdf/1812.01187.pdf + # https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#OneCycleLR + lf = lambda x: ((1 + math.cos(x * math.pi / epochs)) / 2) * (1 - hyp['lrf']) + hyp['lrf'] # cosine + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) + # plot_lr_scheduler(optimizer, scheduler, epochs) + + # Resume + start_epoch, best_fitness = 0, 0.0 + if pretrained: + # Optimizer + if ckpt['optimizer'] is not None: + optimizer.load_state_dict(ckpt['optimizer']) + best_fitness = ckpt['best_fitness'] + + # Results + if ckpt.get('training_results') is not None: + with open(results_file, 'w') as file: + file.write(ckpt['training_results']) # write results.txt + + # Epochs + start_epoch = ckpt['epoch'] + 1 + if opt.resume: + assert start_epoch > 0, '%s training to %g epochs is finished, nothing to resume.' % (weights, epochs) + shutil.copytree(wdir, wdir.parent / f'weights_backup_epoch{start_epoch - 1}') # save previous weights + if epochs < start_epoch: + logger.info('%s has been trained for %g epochs. Fine-tuning for %g additional epochs.' % + (weights, ckpt['epoch'], epochs)) + epochs += ckpt['epoch'] # finetune additional epochs + + del ckpt, state_dict + + # Image sizes + gs = int(max(model.stride)) # grid size (max stride) + imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size] # verify imgsz are gs-multiples + + # DP mode + if cuda and rank == -1 and torch.cuda.device_count() > 1: + model = torch.nn.DataParallel(model) + + # SyncBatchNorm + if opt.sync_bn and cuda and rank != -1: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device) + logger.info('Using SyncBatchNorm()') + + # Exponential moving average + ema = ModelEMA(model) if rank in [-1, 0] else None + + # DDP mode + if cuda and rank != -1: + model = DDP(model, device_ids=[opt.local_rank], output_device=(opt.local_rank)) + + # Trainloader + dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt, + hyp=hyp, augment=True, cache=opt.cache_images, rect=opt.rect, rank=rank, + world_size=opt.world_size, workers=opt.workers) + mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class + nb = len(dataloader) # number of batches + assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Possible class labels are 0-%g' % (mlc, nc, opt.data, nc - 1) + + # Testloader + if rank in [-1, 0]: + ema.updates = start_epoch * nb // accumulate # set EMA updates + testloader = create_dataloader(test_path, imgsz_test, total_batch_size, gs, opt, + hyp=hyp, augment=False, cache=opt.cache_images, rect=True, rank=-1, + world_size=opt.world_size, workers=opt.workers)[0] # only runs on process 0 + + # Model parameters + hyp['cls'] *= nc / 80. # scale coco-tuned hyp['cls'] to current dataset + model.nc = nc # attach number of classes to model + model.hyp = hyp # attach hyperparameters to model + model.gr = 1.0 # giou loss ratio (obj_loss = 1.0 or giou) + model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) # attach class weights + model.names = names + + # Class frequency + if rank in [-1, 0]: + labels = np.concatenate(dataset.labels, 0) + c = torch.tensor(labels[:, 0]) # classes + # cf = torch.bincount(c.long(), minlength=nc) + 1. + # model._initialize_biases(cf.to(device)) + plot_labels(labels, save_dir=log_dir) + if tb_writer: + # tb_writer.add_hparams(hyp, {}) # causes duplicate https://github.com/ultralytics/yolov5/pull/384 + tb_writer.add_histogram('classes', c, 0) + + # Check anchors + if not opt.noautoanchor: + check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz) + + # Start training + t0 = time.time() + nw = max(3 * nb, 1e3) # number of warmup iterations, max(3 epochs, 1k iterations) + # nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training + maps = np.zeros(nc) # mAP per class + results = (0, 0, 0, 0, 0, 0, 0) # 'P', 'R', 'mAP', 'F1', 'val GIoU', 'val Objectness', 'val Classification' + scheduler.last_epoch = start_epoch - 1 # do not move + scaler = amp.GradScaler(enabled=cuda) + logger.info('Image sizes %g train, %g test' % (imgsz, imgsz_test)) + logger.info('Using %g dataloader workers' % dataloader.num_workers) + logger.info('Starting training for %g epochs...' % epochs) + # torch.autograd.set_detect_anomaly(True) + for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------ + model.train() + + # Update image weights (optional) + if opt.img_weights: + # Generate indices + if rank in [-1, 0]: + cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 # class weights + iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights + dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx + # Broadcast if DDP + if rank != -1: + indices = (torch.tensor(dataset.indices) if rank == 0 else torch.zeros(dataset.n)).int() + dist.broadcast(indices, 0) + if rank != 0: + dataset.indices = indices.cpu().numpy() + + # Update mosaic border + # b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs) + # dataset.mosaic_border = [b - imgsz, -b] # height, width borders + + mloss = torch.zeros(4, device=device) # mean losses + if rank != -1: + dataloader.sampler.set_epoch(epoch) + pbar = enumerate(dataloader) + logger.info(('\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'GIoU', 'obj', 'cls', 'total', 'targets', 'img_size')) + if rank in [-1, 0]: + pbar = tqdm(pbar, total=nb) # progress bar + optimizer.zero_grad() + for i, (imgs, targets, paths, _) in pbar: # batch ------------------------------------------------------------- + ni = i + nb * epoch # number integrated batches (since train start) + imgs = imgs.to(device, non_blocking=True).float() / 255.0 # uint8 to float32, 0-255 to 0.0-1.0 + + # Warmup + if ni <= nw: + xi = [0, nw] # x interp + # model.gr = np.interp(ni, xi, [0.0, 1.0]) # giou loss ratio (obj_loss = 1.0 or giou) + accumulate = max(1, np.interp(ni, xi, [1, nbs / total_batch_size]).round()) + for j, x in enumerate(optimizer.param_groups): + # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0 + x['lr'] = np.interp(ni, xi, [0.1 if j == 2 else 0.0, x['initial_lr'] * lf(epoch)]) + if 'momentum' in x: + x['momentum'] = np.interp(ni, xi, [0.9, hyp['momentum']]) + + # Multi-scale + if opt.multi_scale: + sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size + sf = sz / max(imgs.shape[2:]) # scale factor + if sf != 1: + ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple) + imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False) + + # Forward + with amp.autocast(enabled=cuda): + pred = model(imgs) # forward + loss, loss_items = compute_loss(pred, targets.to(device), model) # loss scaled by batch_size + if rank != -1: + loss *= opt.world_size # gradient averaged between devices in DDP mode + + # Backward + scaler.scale(loss).backward() + + # Optimize + if ni % accumulate == 0: + scaler.step(optimizer) # optimizer.step + scaler.update() + optimizer.zero_grad() + if ema: + ema.update(model) + + # Print + if rank in [-1, 0]: + mloss = (mloss * i + loss_items) / (i + 1) # update mean losses + mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0) # (GB) + s = ('%10s' * 2 + '%10.4g' * 6) % ( + '%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1]) + pbar.set_description(s) + + # Plot + if ni < 3: + f = str(log_dir / ('train_batch%g.jpg' % ni)) # filename + result = plot_images(images=imgs, targets=targets, paths=paths, fname=f) + if tb_writer and result is not None: + tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch) + # tb_writer.add_graph(model, imgs) # add model to tensorboard + + # end batch ------------------------------------------------------------------------------------------------ + + # Scheduler + lr = [x['lr'] for x in optimizer.param_groups] # for tensorboard + scheduler.step() + + # DDP process 0 or single-GPU + if rank in [-1, 0]: + # mAP + if ema: + ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'gr', 'names', 'stride']) + final_epoch = epoch + 1 == epochs + if not opt.notest or final_epoch: # Calculate mAP + results, maps, times = test.test(opt.data, + batch_size=total_batch_size, + imgsz=imgsz_test, + model=ema.ema, + single_cls=opt.single_cls, + dataloader=testloader, + save_dir=log_dir) + + # Write + with open(results_file, 'a') as f: + f.write(s + '%10.4g' * 7 % results + '\n') # P, R, mAP, F1, test_losses=(GIoU, obj, cls) + if len(opt.name) and opt.bucket: + os.system('gsutil cp %s gs://%s/results/results%s.txt' % (results_file, opt.bucket, opt.name)) + + # Tensorboard + if tb_writer: + tags = ['train/giou_loss', 'train/obj_loss', 'train/cls_loss', # train loss + 'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95', + 'val/giou_loss', 'val/obj_loss', 'val/cls_loss', # val loss + 'x/lr0', 'x/lr1', 'x/lr2'] # params + for x, tag in zip(list(mloss[:-1]) + list(results) + lr, tags): + tb_writer.add_scalar(tag, x, epoch) + + # Update best mAP + fi = fitness(np.array(results).reshape(1, -1)) # fitness_i = weighted combination of [P, R, mAP, F1] + if fi > best_fitness: + best_fitness = fi + + # Save model + save = (not opt.nosave) or (final_epoch and not opt.evolve) + if save: + with open(results_file, 'r') as f: # create checkpoint + ckpt = {'epoch': epoch, + 'best_fitness': best_fitness, + 'training_results': f.read(), + 'model': ema.ema, + 'optimizer': None if final_epoch else optimizer.state_dict()} + + # Save last, best and delete + torch.save(ckpt, last) + if best_fitness == fi: + torch.save(ckpt, best) + del ckpt + # end epoch ---------------------------------------------------------------------------------------------------- + # end training + + if rank in [-1, 0]: + # Strip optimizers + n = ('_' if len(opt.name) and not opt.name.isnumeric() else '') + opt.name + fresults, flast, fbest = 'results%s.txt' % n, wdir / f'last{n}.pt', wdir / f'best{n}.pt' + for f1, f2 in zip([wdir / 'last.pt', wdir / 'best.pt', 'results.txt'], [flast, fbest, fresults]): + if os.path.exists(f1): + os.rename(f1, f2) # rename + if str(f2).endswith('.pt'): # is *.pt + strip_optimizer(f2) # strip optimizer + os.system('gsutil cp %s gs://%s/weights' % (f2, opt.bucket)) if opt.bucket else None # upload + # Finish + if not opt.evolve: + plot_results(save_dir=log_dir) # save as results.png + logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) + + dist.destroy_process_group() if rank not in [-1, 0] else None + torch.cuda.empty_cache() + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default='yolov5s.pt', help='initial weights path') + parser.add_argument('--cfg', type=str, default='', help='model.yaml path') + parser.add_argument('--data', type=str, default='data/coco128.yaml', help='data.yaml path') + parser.add_argument('--hyp', type=str, default='', help='hyperparameters path, i.e. data/hyp.scratch.yaml') + parser.add_argument('--epochs', type=int, default=300) + parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs') + parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='[train, test] image sizes') + parser.add_argument('--img-weights', action='store_true', help='use weighted image selection for training') + parser.add_argument('--rect', action='store_true', help='rectangular training') + parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') + parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') + parser.add_argument('--notest', action='store_true', help='only test final epoch') + parser.add_argument('--noautoanchor', action='store_true', help='disable autoanchor check') + parser.add_argument('--evolve', action='store_true', help='evolve hyperparameters') + parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') + parser.add_argument('--cache-images', action='store_true', help='cache images for faster training') + parser.add_argument('--name', default='', help='renames results.txt to results_name.txt if supplied') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') + parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset') + parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer') + parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') + parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify') + parser.add_argument('--logdir', type=str, default='runs/', help='logging directory') + parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers') + opt = parser.parse_args() + + # Set DDP variables + opt.total_batch_size = opt.batch_size + opt.world_size = int(os.environ['WORLD_SIZE']) if 'WORLD_SIZE' in os.environ else 1 + opt.global_rank = int(os.environ['RANK']) if 'RANK' in os.environ else -1 + set_logging(opt.global_rank) + if opt.global_rank in [-1, 0]: + check_git_status() + + # Resume + if opt.resume: # resume an interrupted run + ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path + log_dir = Path(ckpt).parent.parent # runs/exp0 + assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist' + with open(log_dir / 'opt.yaml') as f: + opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader)) # replace + opt.cfg, opt.weights, opt.resume = '', ckpt, True + logger.info('Resuming training from %s' % ckpt) + + else: + opt.hyp = opt.hyp or ('data/hyp.finetune.yaml' if opt.weights else 'data/hyp.scratch.yaml') + opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files + assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' + opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test) + log_dir = increment_dir(Path(opt.logdir) / 'exp', opt.name) # runs/exp1 + + device = select_device(opt.device, batch_size=opt.batch_size) + + # DDP mode + if opt.local_rank != -1: + assert torch.cuda.device_count() > opt.local_rank + torch.cuda.set_device(opt.local_rank) + device = torch.device('cuda', opt.local_rank) + dist.init_process_group(backend='nccl', init_method='env://') # distributed backend + assert opt.batch_size % opt.world_size == 0, '--batch-size must be multiple of CUDA device count' + opt.batch_size = opt.total_batch_size // opt.world_size + + logger.info(opt) + with open(opt.hyp) as f: + hyp = yaml.load(f, Loader=yaml.FullLoader) # load hyps + + # Train + if not opt.evolve: + tb_writer = None + if opt.global_rank in [-1, 0]: + logger.info('Start Tensorboard with "tensorboard --logdir %s", view at http://localhost:6006/' % opt.logdir) + tb_writer = SummaryWriter(log_dir=log_dir) # runs/exp0 + + train(hyp, opt, device, tb_writer) + + # Evolve hyperparameters (optional) + else: + # Hyperparameter evolution metadata (mutation scale 0-1, lower_limit, upper_limit) + meta = {'lr0': (1, 1e-5, 1e-1), # initial learning rate (SGD=1E-2, Adam=1E-3) + 'lrf': (1, 0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) + 'momentum': (0.1, 0.6, 0.98), # SGD momentum/Adam beta1 + 'weight_decay': (1, 0.0, 0.001), # optimizer weight decay + 'giou': (1, 0.02, 0.2), # GIoU loss gain + 'cls': (1, 0.2, 4.0), # cls loss gain + 'cls_pw': (1, 0.5, 2.0), # cls BCELoss positive_weight + 'obj': (1, 0.2, 4.0), # obj loss gain (scale with pixels) + 'obj_pw': (1, 0.5, 2.0), # obj BCELoss positive_weight + 'iou_t': (0, 0.1, 0.7), # IoU training threshold + 'anchor_t': (1, 2.0, 8.0), # anchor-multiple threshold + # 'anchors': (1, 2.0, 10.0), # anchors per output grid (0 to ignore) + 'fl_gamma': (0, 0.0, 2.0), # focal loss gamma (efficientDet default gamma=1.5) + 'hsv_h': (1, 0.0, 0.1), # image HSV-Hue augmentation (fraction) + 'hsv_s': (1, 0.0, 0.9), # image HSV-Saturation augmentation (fraction) + 'hsv_v': (1, 0.0, 0.9), # image HSV-Value augmentation (fraction) + 'degrees': (1, 0.0, 45.0), # image rotation (+/- deg) + 'translate': (1, 0.0, 0.9), # image translation (+/- fraction) + 'scale': (1, 0.0, 0.9), # image scale (+/- gain) + 'shear': (1, 0.0, 10.0), # image shear (+/- deg) + 'perspective': (0, 0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 + 'flipud': (1, 0.0, 1.0), # image flip up-down (probability) + 'fliplr': (0, 0.0, 1.0), # image flip left-right (probability) + 'mixup': (1, 0.0, 1.0)} # image mixup (probability) + + assert opt.local_rank == -1, 'DDP mode not implemented for --evolve' + opt.notest, opt.nosave = True, True # only test/save final epoch + # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices + yaml_file = Path('runs/evolve/hyp_evolved.yaml') # save best result here + if opt.bucket: + os.system('gsutil cp gs://%s/evolve.txt .' % opt.bucket) # download evolve.txt if exists + + for _ in range(100): # generations to evolve + if os.path.exists('evolve.txt'): # if evolve.txt exists: select best hyps and mutate + # Select parent(s) + parent = 'single' # parent selection method: 'single' or 'weighted' + x = np.loadtxt('evolve.txt', ndmin=2) + n = min(5, len(x)) # number of previous results to consider + x = x[np.argsort(-fitness(x))][:n] # top n mutations + w = fitness(x) - fitness(x).min() # weights + if parent == 'single' or len(x) == 1: + # x = x[random.randint(0, n - 1)] # random selection + x = x[random.choices(range(n), weights=w)[0]] # weighted selection + elif parent == 'weighted': + x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination + + # Mutate + mp, s = 0.9, 0.2 # mutation probability, sigma + npr = np.random + npr.seed(int(time.time())) + g = np.array([x[0] for x in meta.values()]) # gains 0-1 + ng = len(meta) + v = np.ones(ng) + while all(v == 1): # mutate until a change occurs (prevent duplicates) + v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0) + for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300) + hyp[k] = float(x[i + 7] * v[i]) # mutate + + # Constrain to limits + for k, v in meta.items(): + hyp[k] = max(hyp[k], v[1]) # lower limit + hyp[k] = min(hyp[k], v[2]) # upper limit + hyp[k] = round(hyp[k], 5) # significant digits + + # Train mutation + results = train(hyp.copy(), opt, device) + + # Write mutation results + print_mutation(hyp.copy(), results, yaml_file, opt.bucket) + + # Plot results + plot_evolution(yaml_file) + print('Hyperparameter evolution complete. Best results saved as: %s\nCommand to train a new model with these ' + 'hyperparameters: $ python train.py --hyp %s' % (yaml_file, yaml_file)) diff --git a/src/digitizer/yolov5/utils/__init__.py b/src/digitizer/yolov5/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/digitizer/yolov5/utils/__pycache__/__init__.cpython-38.pyc b/src/digitizer/yolov5/utils/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6138894c5fd91cd886c74685cee4e5ab0c1f750b Binary files /dev/null and b/src/digitizer/yolov5/utils/__pycache__/__init__.cpython-38.pyc differ diff --git a/src/digitizer/yolov5/utils/__pycache__/datasets.cpython-38.pyc b/src/digitizer/yolov5/utils/__pycache__/datasets.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6c7f6eb39ec878b16bcab3a441bcf07d405b4a4 Binary files /dev/null and b/src/digitizer/yolov5/utils/__pycache__/datasets.cpython-38.pyc differ diff --git a/src/digitizer/yolov5/utils/__pycache__/general.cpython-38.pyc b/src/digitizer/yolov5/utils/__pycache__/general.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ca99ce61f473bf075071fe5009542db46009d67 Binary files /dev/null and b/src/digitizer/yolov5/utils/__pycache__/general.cpython-38.pyc differ diff --git a/src/digitizer/yolov5/utils/__pycache__/google_utils.cpython-38.pyc b/src/digitizer/yolov5/utils/__pycache__/google_utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..beced12063c935669e95a79a4c11fd72b14d28c4 Binary files /dev/null and b/src/digitizer/yolov5/utils/__pycache__/google_utils.cpython-38.pyc differ diff --git a/src/digitizer/yolov5/utils/__pycache__/torch_utils.cpython-38.pyc b/src/digitizer/yolov5/utils/__pycache__/torch_utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..748449ef8755dc0993108ae47a4376dc90bb372a Binary files /dev/null and b/src/digitizer/yolov5/utils/__pycache__/torch_utils.cpython-38.pyc differ diff --git a/src/digitizer/yolov5/utils/activations.py b/src/digitizer/yolov5/utils/activations.py new file mode 100644 index 0000000000000000000000000000000000000000..162cb9fc3e87b71e8dc53729020f56c73c8922d5 --- /dev/null +++ b/src/digitizer/yolov5/utils/activations.py @@ -0,0 +1,70 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +# Swish https://arxiv.org/pdf/1905.02244.pdf --------------------------------------------------------------------------- +class Swish(nn.Module): # + @staticmethod + def forward(x): + return x * torch.sigmoid(x) + + +class Hardswish(nn.Module): # export-friendly version of nn.Hardswish() + @staticmethod + def forward(x): + # return x * F.hardsigmoid(x) # for torchscript and CoreML + return x * F.hardtanh(x + 3, 0., 6.) / 6. # for torchscript, CoreML and ONNX + + +class MemoryEfficientSwish(nn.Module): + class F(torch.autograd.Function): + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return x * torch.sigmoid(x) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + sx = torch.sigmoid(x) + return grad_output * (sx * (1 + x * (1 - sx))) + + def forward(self, x): + return self.F.apply(x) + + +# Mish https://github.com/digantamisra98/Mish -------------------------------------------------------------------------- +class Mish(nn.Module): + @staticmethod + def forward(x): + return x * F.softplus(x).tanh() + + +class MemoryEfficientMish(nn.Module): + class F(torch.autograd.Function): + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x))) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + sx = torch.sigmoid(x) + fx = F.softplus(x).tanh() + return grad_output * (fx + x * sx * (1 - fx * fx)) + + def forward(self, x): + return self.F.apply(x) + + +# FReLU https://arxiv.org/abs/2007.11824 ------------------------------------------------------------------------------- +class FReLU(nn.Module): + def __init__(self, c1, k=3): # ch_in, kernel + super().__init__() + self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1) + self.bn = nn.BatchNorm2d(c1) + + def forward(self, x): + return torch.max(x, self.bn(self.conv(x))) diff --git a/src/digitizer/yolov5/utils/datasets.py b/src/digitizer/yolov5/utils/datasets.py new file mode 100755 index 0000000000000000000000000000000000000000..d6c220f3a8209cf43058beabdef921d07d9aab88 --- /dev/null +++ b/src/digitizer/yolov5/utils/datasets.py @@ -0,0 +1,911 @@ +import glob +import math +import os +import random +import shutil +import time +from pathlib import Path +from threading import Thread + +import cv2 +import numpy as np +import torch +from PIL import Image, ExifTags +from torch.utils.data import Dataset +from tqdm import tqdm + +from utils.general import xyxy2xywh, xywh2xyxy, torch_distributed_zero_first + +help_url = 'https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data' +img_formats = ['.bmp', '.jpg', '.jpeg', '.png', '.tif', '.tiff', '.dng'] +vid_formats = ['.mov', '.avi', '.mp4', '.mpg', '.mpeg', '.m4v', '.wmv', '.mkv'] + +# Get orientation exif tag +for orientation in ExifTags.TAGS.keys(): + if ExifTags.TAGS[orientation] == 'Orientation': + break + + +def get_hash(files): + # Returns a single hash value of a list of files + return sum(os.path.getsize(f) for f in files if os.path.isfile(f)) + + +def exif_size(img): + # Returns exif-corrected PIL size + s = img.size # (width, height) + try: + rotation = dict(img._getexif().items())[orientation] + if rotation == 6: # rotation 270 + s = (s[1], s[0]) + elif rotation == 8: # rotation 90 + s = (s[1], s[0]) + except: + pass + + return s + + +def create_dataloader(path, imgsz, batch_size, stride, opt, hyp=None, augment=False, cache=False, pad=0.0, rect=False, + rank=-1, world_size=1, workers=8): + # Make sure only the first process in DDP process the dataset first, and the following others can use the cache. + with torch_distributed_zero_first(rank): + dataset = LoadImagesAndLabels(path, imgsz, batch_size, + augment=augment, # augment images + hyp=hyp, # augmentation hyperparameters + rect=rect, # rectangular training + cache_images=cache, + single_cls=opt.single_cls, + stride=int(stride), + pad=pad, + rank=rank) + + batch_size = min(batch_size, len(dataset)) + nw = min([os.cpu_count() // world_size, batch_size if batch_size > 1 else 0, workers]) # number of workers + train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) if rank != -1 else None + dataloader = torch.utils.data.DataLoader(dataset, + batch_size=batch_size, + num_workers=nw, + sampler=train_sampler, + pin_memory=True, + collate_fn=LoadImagesAndLabels.collate_fn) + return dataloader, dataset + + +class LoadImages: # for inference + def __init__(self, path, img_size=640): + p = str(Path(path)) # os-agnostic + p = os.path.abspath(p) # absolute path + if '*' in p: + files = sorted(glob.glob(p)) # glob + elif os.path.isdir(p): + files = sorted(glob.glob(os.path.join(p, '*.*'))) # dir + elif os.path.isfile(p): + files = [p] # files + else: + raise Exception('ERROR: %s does not exist' % p) + + images = [x for x in files if os.path.splitext(x)[-1].lower() in img_formats] + videos = [x for x in files if os.path.splitext(x)[-1].lower() in vid_formats] + ni, nv = len(images), len(videos) + + self.img_size = img_size + self.files = images + videos + self.nf = ni + nv # number of files + self.video_flag = [False] * ni + [True] * nv + self.mode = 'images' + if any(videos): + self.new_video(videos[0]) # new video + else: + self.cap = None + assert self.nf > 0, 'No images or videos found in %s. Supported formats are:\nimages: %s\nvideos: %s' % \ + (p, img_formats, vid_formats) + + def __iter__(self): + self.count = 0 + return self + + def __next__(self): + if self.count == self.nf: + raise StopIteration + path = self.files[self.count] + + if self.video_flag[self.count]: + # Read video + self.mode = 'video' + ret_val, img0 = self.cap.read() + if not ret_val: + self.count += 1 + self.cap.release() + if self.count == self.nf: # last video + raise StopIteration + else: + path = self.files[self.count] + self.new_video(path) + ret_val, img0 = self.cap.read() + + self.frame += 1 + print('video %g/%g (%g/%g) %s: ' % (self.count + 1, self.nf, self.frame, self.nframes, path), end='') + + else: + # Read image + self.count += 1 + img0 = cv2.imread(path) # BGR + assert img0 is not None, 'Image Not Found ' + path + print('image %g/%g %s: ' % (self.count, self.nf, path), end='') + + # Padded resize + img = letterbox(img0, new_shape=self.img_size)[0] + + # Convert + img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + img = np.ascontiguousarray(img) + + # cv2.imwrite(path + '.letterbox.jpg', 255 * img.transpose((1, 2, 0))[:, :, ::-1]) # save letterbox image + return path, img, img0, self.cap + + def new_video(self, path): + self.frame = 0 + self.cap = cv2.VideoCapture(path) + self.nframes = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + def __len__(self): + return self.nf # number of files + + +class LoadWebcam: # for inference + def __init__(self, pipe=0, img_size=640): + self.img_size = img_size + + if pipe == '0': + pipe = 0 # local camera + # pipe = 'rtsp://192.168.1.64/1' # IP camera + # pipe = 'rtsp://username:password@192.168.1.64/1' # IP camera with login + # pipe = 'rtsp://170.93.143.139/rtplive/470011e600ef003a004ee33696235daa' # IP traffic camera + # pipe = 'http://wmccpinetop.axiscam.net/mjpg/video.mjpg' # IP golf camera + + # https://answers.opencv.org/question/215996/changing-gstreamer-pipeline-to-opencv-in-pythonsolved/ + # pipe = '"rtspsrc location="rtsp://username:password@192.168.1.64/1" latency=10 ! appsink' # GStreamer + + # https://answers.opencv.org/question/200787/video-acceleration-gstremer-pipeline-in-videocapture/ + # https://stackoverflow.com/questions/54095699/install-gstreamer-support-for-opencv-python-package # install help + # pipe = "rtspsrc location=rtsp://root:root@192.168.0.91:554/axis-media/media.amp?videocodec=h264&resolution=3840x2160 protocols=GST_RTSP_LOWER_TRANS_TCP ! rtph264depay ! queue ! vaapih264dec ! videoconvert ! appsink" # GStreamer + + self.pipe = pipe + self.cap = cv2.VideoCapture(pipe) # video capture object + self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 3) # set buffer size + + def __iter__(self): + self.count = -1 + return self + + def __next__(self): + self.count += 1 + if cv2.waitKey(1) == ord('q'): # q to quit + self.cap.release() + cv2.destroyAllWindows() + raise StopIteration + + # Read frame + if self.pipe == 0: # local camera + ret_val, img0 = self.cap.read() + img0 = cv2.flip(img0, 1) # flip left-right + else: # IP camera + n = 0 + while True: + n += 1 + self.cap.grab() + if n % 30 == 0: # skip frames + ret_val, img0 = self.cap.retrieve() + if ret_val: + break + + # Print + assert ret_val, 'Camera Error %s' % self.pipe + img_path = 'webcam.jpg' + print('webcam %g: ' % self.count, end='') + + # Padded resize + img = letterbox(img0, new_shape=self.img_size)[0] + + # Convert + img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + img = np.ascontiguousarray(img) + + return img_path, img, img0, None + + def __len__(self): + return 0 + + +class LoadStreams: # multiple IP or RTSP cameras + def __init__(self, sources='streams.txt', img_size=640): + self.mode = 'images' + self.img_size = img_size + + if os.path.isfile(sources): + with open(sources, 'r') as f: + sources = [x.strip() for x in f.read().splitlines() if len(x.strip())] + else: + sources = [sources] + + n = len(sources) + self.imgs = [None] * n + self.sources = sources + for i, s in enumerate(sources): + # Start the thread to read frames from the video stream + print('%g/%g: %s... ' % (i + 1, n, s), end='') + cap = cv2.VideoCapture(eval(s) if s.isnumeric() else s) + assert cap.isOpened(), 'Failed to open %s' % s + w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = cap.get(cv2.CAP_PROP_FPS) % 100 + _, self.imgs[i] = cap.read() # guarantee first frame + thread = Thread(target=self.update, args=([i, cap]), daemon=True) + print(' success (%gx%g at %.2f FPS).' % (w, h, fps)) + thread.start() + print('') # newline + + # check for common shapes + s = np.stack([letterbox(x, new_shape=self.img_size)[0].shape for x in self.imgs], 0) # inference shapes + self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal + if not self.rect: + print('WARNING: Different stream shapes detected. For optimal performance supply similarly-shaped streams.') + + def update(self, index, cap): + # Read next stream frame in a daemon thread + n = 0 + while cap.isOpened(): + n += 1 + # _, self.imgs[index] = cap.read() + cap.grab() + if n == 4: # read every 4th frame + _, self.imgs[index] = cap.retrieve() + n = 0 + time.sleep(0.01) # wait time + + def __iter__(self): + self.count = -1 + return self + + def __next__(self): + self.count += 1 + img0 = self.imgs.copy() + if cv2.waitKey(1) == ord('q'): # q to quit + cv2.destroyAllWindows() + raise StopIteration + + # Letterbox + img = [letterbox(x, new_shape=self.img_size, auto=self.rect)[0] for x in img0] + + # Stack + img = np.stack(img, 0) + + # Convert + img = img[:, :, :, ::-1].transpose(0, 3, 1, 2) # BGR to RGB, to bsx3x416x416 + img = np.ascontiguousarray(img) + + return self.sources, img, img0, None + + def __len__(self): + return 0 # 1E12 frames = 32 streams at 30 FPS for 30 years + + +class LoadImagesAndLabels(Dataset): # for training/testing + def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, rect=False, image_weights=False, + cache_images=False, single_cls=False, stride=32, pad=0.0, rank=-1): + try: + f = [] # image files + for p in path if isinstance(path, list) else [path]: + p = str(Path(p)) # os-agnostic + parent = str(Path(p).parent) + os.sep + if os.path.isfile(p): # file + with open(p, 'r') as t: + t = t.read().splitlines() + f += [x.replace('./', parent) if x.startswith('./') else x for x in t] # local to global path + elif os.path.isdir(p): # folder + f += glob.iglob(p + os.sep + '*.*') + else: + raise Exception('%s does not exist' % p) + self.img_files = sorted( + [x.replace('/', os.sep) for x in f if os.path.splitext(x)[-1].lower() in img_formats]) + except Exception as e: + raise Exception('Error loading data from %s: %s\nSee %s' % (path, e, help_url)) + + n = len(self.img_files) + assert n > 0, 'No images found in %s. See %s' % (path, help_url) + bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index + nb = bi[-1] + 1 # number of batches + + self.n = n # number of images + self.batch = bi # batch index of image + self.img_size = img_size + self.augment = augment + self.hyp = hyp + self.image_weights = image_weights + self.rect = False if image_weights else rect + self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training) + self.mosaic_border = [-img_size // 2, -img_size // 2] + self.stride = stride + + # Define labels + self.label_files = [x.replace('images', 'labels').replace(os.path.splitext(x)[-1], '.txt') for x in + self.img_files] + + # Check cache + cache_path = str(Path(self.label_files[0]).parent) + '.cache' # cached labels + if os.path.isfile(cache_path): + cache = torch.load(cache_path) # load + if cache['hash'] != get_hash(self.label_files + self.img_files): # dataset changed + cache = self.cache_labels(cache_path) # re-cache + else: + cache = self.cache_labels(cache_path) # cache + + # Get labels + labels, shapes = zip(*[cache[x] for x in self.img_files]) + self.shapes = np.array(shapes, dtype=np.float64) + self.labels = list(labels) + + # Rectangular Training https://github.com/ultralytics/yolov3/issues/232 + if self.rect: + # Sort by aspect ratio + s = self.shapes # wh + ar = s[:, 1] / s[:, 0] # aspect ratio + irect = ar.argsort() + self.img_files = [self.img_files[i] for i in irect] + self.label_files = [self.label_files[i] for i in irect] + self.labels = [self.labels[i] for i in irect] + self.shapes = s[irect] # wh + ar = ar[irect] + + # Set training image shapes + shapes = [[1, 1]] * nb + for i in range(nb): + ari = ar[bi == i] + mini, maxi = ari.min(), ari.max() + if maxi < 1: + shapes[i] = [maxi, 1] + elif mini > 1: + shapes[i] = [1, 1 / mini] + + self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(np.int) * stride + + # Cache labels + create_datasubset, extract_bounding_boxes, labels_loaded = False, False, False + nm, nf, ne, ns, nd = 0, 0, 0, 0, 0 # number missing, found, empty, datasubset, duplicate + pbar = enumerate(self.label_files) + if rank in [-1, 0]: + pbar = tqdm(pbar) + for i, file in pbar: + l = self.labels[i] # label + if l is not None and l.shape[0]: + assert l.shape[1] == 5, '> 5 label columns: %s' % file + assert (l >= 0).all(), 'negative labels: %s' % file + assert (l[:, 1:] <= 1).all(), 'non-normalized or out of bounds coordinate labels: %s' % file + if np.unique(l, axis=0).shape[0] < l.shape[0]: # duplicate rows + nd += 1 # print('WARNING: duplicate rows in %s' % self.label_files[i]) # duplicate rows + if single_cls: + l[:, 0] = 0 # force dataset into single-class mode + self.labels[i] = l + nf += 1 # file found + + # Create subdataset (a smaller dataset) + if create_datasubset and ns < 1E4: + if ns == 0: + create_folder(path='./datasubset') + os.makedirs('./datasubset/images') + exclude_classes = 43 + if exclude_classes not in l[:, 0]: + ns += 1 + # shutil.copy(src=self.img_files[i], dst='./datasubset/images/') # copy image + with open('./datasubset/images.txt', 'a') as f: + f.write(self.img_files[i] + '\n') + + # Extract object detection boxes for a second stage classifier + if extract_bounding_boxes: + p = Path(self.img_files[i]) + img = cv2.imread(str(p)) + h, w = img.shape[:2] + for j, x in enumerate(l): + f = '%s%sclassifier%s%g_%g_%s' % (p.parent.parent, os.sep, os.sep, x[0], j, p.name) + if not os.path.exists(Path(f).parent): + os.makedirs(Path(f).parent) # make new output folder + + b = x[1:] * [w, h, w, h] # box + b[2:] = b[2:].max() # rectangle to square + b[2:] = b[2:] * 1.3 + 30 # pad + b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int) + + b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image + b[[1, 3]] = np.clip(b[[1, 3]], 0, h) + assert cv2.imwrite(f, img[b[1]:b[3], b[0]:b[2]]), 'Failure extracting classifier boxes' + else: + ne += 1 # print('empty labels for image %s' % self.img_files[i]) # file empty + # os.system("rm '%s' '%s'" % (self.img_files[i], self.label_files[i])) # remove + + if rank in [-1, 0]: + pbar.desc = 'Scanning labels %s (%g found, %g missing, %g empty, %g duplicate, for %g images)' % ( + cache_path, nf, nm, ne, nd, n) + if nf == 0: + s = 'WARNING: No labels found in %s. See %s' % (os.path.dirname(file) + os.sep, help_url) + print(s) + assert not augment, '%s. Can not train without labels.' % s + + # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM) + self.imgs = [None] * n + if cache_images: + gb = 0 # Gigabytes of cached images + pbar = tqdm(range(len(self.img_files)), desc='Caching images') + self.img_hw0, self.img_hw = [None] * n, [None] * n + for i in pbar: # max 10k images + self.imgs[i], self.img_hw0[i], self.img_hw[i] = load_image(self, i) # img, hw_original, hw_resized + gb += self.imgs[i].nbytes + pbar.desc = 'Caching images (%.1fGB)' % (gb / 1E9) + + def cache_labels(self, path='labels.cache'): + # Cache dataset labels, check images and read shapes + x = {} # dict + pbar = tqdm(zip(self.img_files, self.label_files), desc='Scanning images', total=len(self.img_files)) + for (img, label) in pbar: + try: + l = [] + image = Image.open(img) + image.verify() # PIL verify + # _ = io.imread(img) # skimage verify (from skimage import io) + shape = exif_size(image) # image size + assert (shape[0] > 9) & (shape[1] > 9), 'image size <10 pixels' + if os.path.isfile(label): + with open(label, 'r') as f: + l = np.array([x.split() for x in f.read().splitlines()], dtype=np.float32) # labels + if len(l) == 0: + l = np.zeros((0, 5), dtype=np.float32) + x[img] = [l, shape] + except Exception as e: + x[img] = [None, None] + print('WARNING: %s: %s' % (img, e)) + + x['hash'] = get_hash(self.label_files + self.img_files) + torch.save(x, path) # save for next time + return x + + def __len__(self): + return len(self.img_files) + + # def __iter__(self): + # self.count = -1 + # print('ran dataset iter') + # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF) + # return self + + def __getitem__(self, index): + if self.image_weights: + index = self.indices[index] + + hyp = self.hyp + if self.mosaic: + # Load mosaic + img, labels = load_mosaic(self, index) + shapes = None + + # MixUp https://arxiv.org/pdf/1710.09412.pdf + if random.random() < hyp['mixup']: + img2, labels2 = load_mosaic(self, random.randint(0, len(self.labels) - 1)) + r = np.random.beta(8.0, 8.0) # mixup ratio, alpha=beta=8.0 + img = (img * r + img2 * (1 - r)).astype(np.uint8) + labels = np.concatenate((labels, labels2), 0) + + else: + # Load image + img, (h0, w0), (h, w) = load_image(self, index) + + # Letterbox + shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape + img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment) + shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling + + # Load labels + labels = [] + x = self.labels[index] + if x.size > 0: + # Normalized xywh to pixel xyxy format + labels = x.copy() + labels[:, 1] = ratio[0] * w * (x[:, 1] - x[:, 3] / 2) + pad[0] # pad width + labels[:, 2] = ratio[1] * h * (x[:, 2] - x[:, 4] / 2) + pad[1] # pad height + labels[:, 3] = ratio[0] * w * (x[:, 1] + x[:, 3] / 2) + pad[0] + labels[:, 4] = ratio[1] * h * (x[:, 2] + x[:, 4] / 2) + pad[1] + + if self.augment: + # Augment imagespace + if not self.mosaic: + img, labels = random_perspective(img, labels, + degrees=hyp['degrees'], + translate=hyp['translate'], + scale=hyp['scale'], + shear=hyp['shear'], + perspective=hyp['perspective']) + + # Augment colorspace + augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v']) + + # Apply cutouts + # if random.random() < 0.9: + # labels = cutout(img, labels) + + nL = len(labels) # number of labels + if nL: + labels[:, 1:5] = xyxy2xywh(labels[:, 1:5]) # convert xyxy to xywh + labels[:, [2, 4]] /= img.shape[0] # normalized height 0-1 + labels[:, [1, 3]] /= img.shape[1] # normalized width 0-1 + + if self.augment: + # flip up-down + if random.random() < hyp['flipud']: + img = np.flipud(img) + if nL: + labels[:, 2] = 1 - labels[:, 2] + + # flip left-right + if random.random() < hyp['fliplr']: + img = np.fliplr(img) + if nL: + labels[:, 1] = 1 - labels[:, 1] + + labels_out = torch.zeros((nL, 6)) + if nL: + labels_out[:, 1:] = torch.from_numpy(labels) + + # Convert + img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + img = np.ascontiguousarray(img) + + return torch.from_numpy(img), labels_out, self.img_files[index], shapes + + @staticmethod + def collate_fn(batch): + img, label, path, shapes = zip(*batch) # transposed + for i, l in enumerate(label): + l[:, 0] = i # add target image index for build_targets() + return torch.stack(img, 0), torch.cat(label, 0), path, shapes + + +# Ancillary functions -------------------------------------------------------------------------------------------------- +def load_image(self, index): + # loads 1 image from dataset, returns img, original hw, resized hw + img = self.imgs[index] + if img is None: # not cached + path = self.img_files[index] + img = cv2.imread(path) # BGR + assert img is not None, 'Image Not Found ' + path + h0, w0 = img.shape[:2] # orig hw + r = self.img_size / max(h0, w0) # resize image to img_size + if r != 1: # always resize down, only resize up if training with augmentation + interp = cv2.INTER_AREA if r < 1 and not self.augment else cv2.INTER_LINEAR + img = cv2.resize(img, (int(w0 * r), int(h0 * r)), interpolation=interp) + return img, (h0, w0), img.shape[:2] # img, hw_original, hw_resized + else: + return self.imgs[index], self.img_hw0[index], self.img_hw[index] # img, hw_original, hw_resized + + +def augment_hsv(img, hgain=0.5, sgain=0.5, vgain=0.5): + r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains + hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV)) + dtype = img.dtype # uint8 + + x = np.arange(0, 256, dtype=np.int16) + lut_hue = ((x * r[0]) % 180).astype(dtype) + lut_sat = np.clip(x * r[1], 0, 255).astype(dtype) + lut_val = np.clip(x * r[2], 0, 255).astype(dtype) + + img_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))).astype(dtype) + cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed + + # Histogram equalization + # if random.random() < 0.2: + # for i in range(3): + # img[:, :, i] = cv2.equalizeHist(img[:, :, i]) + + +def load_mosaic(self, index): + # loads images in a mosaic + + labels4 = [] + s = self.img_size + yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border] # mosaic center x, y + indices = [index] + [random.randint(0, len(self.labels) - 1) for _ in range(3)] # 3 additional image indices + for i, index in enumerate(indices): + # Load image + img, _, (h, w) = load_image(self, index) + + # place img in img4 + if i == 0: # top left + img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) + x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) + elif i == 1: # top right + x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc + x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h + elif i == 2: # bottom left + x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) + x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h) + elif i == 3: # bottom right + x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) + x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) + + img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + padw = x1a - x1b + padh = y1a - y1b + + # Labels + x = self.labels[index] + labels = x.copy() + if x.size > 0: # Normalized xywh to pixel xyxy format + labels[:, 1] = w * (x[:, 1] - x[:, 3] / 2) + padw + labels[:, 2] = h * (x[:, 2] - x[:, 4] / 2) + padh + labels[:, 3] = w * (x[:, 1] + x[:, 3] / 2) + padw + labels[:, 4] = h * (x[:, 2] + x[:, 4] / 2) + padh + labels4.append(labels) + + # Concat/clip labels + if len(labels4): + labels4 = np.concatenate(labels4, 0) + # np.clip(labels4[:, 1:] - s / 2, 0, s, out=labels4[:, 1:]) # use with center crop + np.clip(labels4[:, 1:], 0, 2 * s, out=labels4[:, 1:]) # use with random_affine + + # Replicate + # img4, labels4 = replicate(img4, labels4) + + # Augment + # img4 = img4[s // 2: int(s * 1.5), s // 2:int(s * 1.5)] # center crop (WARNING, requires box pruning) + img4, labels4 = random_perspective(img4, labels4, + degrees=self.hyp['degrees'], + translate=self.hyp['translate'], + scale=self.hyp['scale'], + shear=self.hyp['shear'], + perspective=self.hyp['perspective'], + border=self.mosaic_border) # border to remove + + return img4, labels4 + + +def replicate(img, labels): + # Replicate labels + h, w = img.shape[:2] + boxes = labels[:, 1:].astype(int) + x1, y1, x2, y2 = boxes.T + s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels) + for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices + x1b, y1b, x2b, y2b = boxes[i] + bh, bw = y2b - y1b, x2b - x1b + yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y + x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh] + img[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0) + + return img, labels + + +def letterbox(img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True): + # Resize image to a 32-pixel-multiple rectangle https://github.com/ultralytics/yolov3/issues/232 + shape = img.shape[:2] # current shape [height, width] + if isinstance(new_shape, int): + new_shape = (new_shape, new_shape) + + # Scale ratio (new / old) + r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) + if not scaleup: # only scale down, do not scale up (for better test mAP) + r = min(r, 1.0) + + # Compute padding + ratio = r, r # width, height ratios + new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) + dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding + if auto: # minimum rectangle + dw, dh = np.mod(dw, 64), np.mod(dh, 64) # wh padding + elif scaleFill: # stretch + dw, dh = 0.0, 0.0 + new_unpad = (new_shape[1], new_shape[0]) + ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios + + dw /= 2 # divide padding into 2 sides + dh /= 2 + + if shape[::-1] != new_unpad: # resize + img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR) + top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) + left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) + img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border + return img, ratio, (dw, dh) + + +def random_perspective(img, targets=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0, border=(0, 0)): + # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10)) + # targets = [cls, xyxy] + + height = img.shape[0] + border[0] * 2 # shape(h,w,c) + width = img.shape[1] + border[1] * 2 + + # Center + C = np.eye(3) + C[0, 2] = -img.shape[1] / 2 # x translation (pixels) + C[1, 2] = -img.shape[0] / 2 # y translation (pixels) + + # Perspective + P = np.eye(3) + P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y) + P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x) + + # Rotation and Scale + R = np.eye(3) + a = random.uniform(-degrees, degrees) + # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations + s = random.uniform(1 - scale, 1 + scale) + # s = 2 ** random.uniform(-scale, scale) + R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s) + + # Shear + S = np.eye(3) + S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg) + S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg) + + # Translation + T = np.eye(3) + T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels) + T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels) + + # Combined rotation matrix + M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT + if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed + if perspective: + img = cv2.warpPerspective(img, M, dsize=(width, height), borderValue=(114, 114, 114)) + else: # affine + img = cv2.warpAffine(img, M[:2], dsize=(width, height), borderValue=(114, 114, 114)) + + # Visualize + # import matplotlib.pyplot as plt + # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel() + # ax[0].imshow(img[:, :, ::-1]) # base + # ax[1].imshow(img2[:, :, ::-1]) # warped + + # Transform label coordinates + n = len(targets) + if n: + # warp points + xy = np.ones((n * 4, 3)) + xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1 + xy = xy @ M.T # transform + if perspective: + xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8) # rescale + else: # affine + xy = xy[:, :2].reshape(n, 8) + + # create new boxes + x = xy[:, [0, 2, 4, 6]] + y = xy[:, [1, 3, 5, 7]] + xy = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T + + # # apply angle-based reduction of bounding boxes + # radians = a * math.pi / 180 + # reduction = max(abs(math.sin(radians)), abs(math.cos(radians))) ** 0.5 + # x = (xy[:, 2] + xy[:, 0]) / 2 + # y = (xy[:, 3] + xy[:, 1]) / 2 + # w = (xy[:, 2] - xy[:, 0]) * reduction + # h = (xy[:, 3] - xy[:, 1]) * reduction + # xy = np.concatenate((x - w / 2, y - h / 2, x + w / 2, y + h / 2)).reshape(4, n).T + + # clip boxes + xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width) + xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height) + + # filter candidates + i = box_candidates(box1=targets[:, 1:5].T * s, box2=xy.T) + targets = targets[i] + targets[:, 1:5] = xy[i] + + return img, targets + + +def box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.1): # box1(4,n), box2(4,n) + # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio + w1, h1 = box1[2] - box1[0], box1[3] - box1[1] + w2, h2 = box2[2] - box2[0], box2[3] - box2[1] + ar = np.maximum(w2 / (h2 + 1e-16), h2 / (w2 + 1e-16)) # aspect ratio + return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + 1e-16) > area_thr) & (ar < ar_thr) # candidates + + +def cutout(image, labels): + # Applies image cutout augmentation https://arxiv.org/abs/1708.04552 + h, w = image.shape[:2] + + def bbox_ioa(box1, box2): + # Returns the intersection over box2 area given box1, box2. box1 is 4, box2 is nx4. boxes are x1y1x2y2 + box2 = box2.transpose() + + # Get the coordinates of bounding boxes + b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3] + b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3] + + # Intersection area + inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * \ + (np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)).clip(0) + + # box2 area + box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + 1e-16 + + # Intersection over box2 area + return inter_area / box2_area + + # create random masks + scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction + for s in scales: + mask_h = random.randint(1, int(h * s)) + mask_w = random.randint(1, int(w * s)) + + # box + xmin = max(0, random.randint(0, w) - mask_w // 2) + ymin = max(0, random.randint(0, h) - mask_h // 2) + xmax = min(w, xmin + mask_w) + ymax = min(h, ymin + mask_h) + + # apply random color mask + image[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)] + + # return unobscured labels + if len(labels) and s > 0.03: + box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32) + ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area + labels = labels[ioa < 0.60] # remove >60% obscured labels + + return labels + + +def reduce_img_size(path='path/images', img_size=1024): # from utils.datasets import *; reduce_img_size() + # creates a new ./images_reduced folder with reduced size images of maximum size img_size + path_new = path + '_reduced' # reduced images path + create_folder(path_new) + for f in tqdm(glob.glob('%s/*.*' % path)): + try: + img = cv2.imread(f) + h, w = img.shape[:2] + r = img_size / max(h, w) # size ratio + if r < 1.0: + img = cv2.resize(img, (int(w * r), int(h * r)), interpolation=cv2.INTER_AREA) # _LINEAR fastest + fnew = f.replace(path, path_new) # .replace(Path(f).suffix, '.jpg') + cv2.imwrite(fnew, img) + except: + print('WARNING: image failure %s' % f) + + +def recursive_dataset2bmp(dataset='path/dataset_bmp'): # from utils.datasets import *; recursive_dataset2bmp() + # Converts dataset to bmp (for faster training) + formats = [x.lower() for x in img_formats] + [x.upper() for x in img_formats] + for a, b, files in os.walk(dataset): + for file in tqdm(files, desc=a): + p = a + '/' + file + s = Path(file).suffix + if s == '.txt': # replace text + with open(p, 'r') as f: + lines = f.read() + for f in formats: + lines = lines.replace(f, '.bmp') + with open(p, 'w') as f: + f.write(lines) + elif s in formats: # replace image + cv2.imwrite(p.replace(s, '.bmp'), cv2.imread(p)) + if s != '.bmp': + os.system("rm '%s'" % p) + + +def imagelist2folder(path='path/images.txt'): # from utils.datasets import *; imagelist2folder() + # Copies all the images in a text file (list of images) into a folder + create_folder(path[:-4]) + with open(path, 'r') as f: + for line in f.read().splitlines(): + os.system('cp "%s" %s' % (line, path[:-4])) + print(line) + + +def create_folder(path='./new'): + # Create folder + if os.path.exists(path): + shutil.rmtree(path) # delete output folder + os.makedirs(path) # make new output folder diff --git a/src/digitizer/yolov5/utils/evolve.sh b/src/digitizer/yolov5/utils/evolve.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd3344042fefa586aff053b344ed5d7f9e92a0bc --- /dev/null +++ b/src/digitizer/yolov5/utils/evolve.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Hyperparameter evolution commands (avoids CUDA memory leakage issues) +# Replaces train.py python generations 'for' loop with a bash 'for' loop + +# Start on 4-GPU machine +#for i in 0 1 2 3; do +# t=ultralytics/yolov5:test && sudo docker pull $t && sudo docker run -d --ipc=host --gpus all -v "$(pwd)"/VOC:/usr/src/VOC $t bash utils/evolve.sh $i +# sleep 60 # avoid simultaneous evolve.txt read/write +#done + +# Hyperparameter evolution commands +while true; do + python train.py --batch 64 --weights yolov5m.pt --data voc.yaml --img 512 --epochs 50 --evolve --bucket ult/voc --device $1 +done diff --git a/src/digitizer/yolov5/utils/general.py b/src/digitizer/yolov5/utils/general.py new file mode 100755 index 0000000000000000000000000000000000000000..fa5e93d90bd58122c98ccf59a4ec4eeea0651401 --- /dev/null +++ b/src/digitizer/yolov5/utils/general.py @@ -0,0 +1,1288 @@ +import glob +import logging +import math +import os +import platform +import random +import shutil +import subprocess +import time +from contextlib import contextmanager +from copy import copy +from pathlib import Path + +import cv2 +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +import yaml +from scipy.cluster.vq import kmeans +from scipy.signal import butter, filtfilt +from tqdm import tqdm + +from utils.torch_utils import init_seeds as init_torch_seeds +from utils.torch_utils import is_parallel + +# Set printoptions +torch.set_printoptions(linewidth=320, precision=5, profile='long') +np.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5 +matplotlib.rc('font', **{'size': 11}) + +# Prevent OpenCV from multithreading (to use PyTorch DataLoader) +cv2.setNumThreads(0) + + +@contextmanager +def torch_distributed_zero_first(local_rank: int): + """ + Decorator to make all processes in distributed training wait for each local_master to do something. + """ + if local_rank not in [-1, 0]: + torch.distributed.barrier() + yield + if local_rank == 0: + torch.distributed.barrier() + + +def set_logging(rank=-1): + logging.basicConfig( + format="%(message)s", + level=logging.INFO if rank in [-1, 0] else logging.WARN) + + +def init_seeds(seed=0): + random.seed(seed) + np.random.seed(seed) + init_torch_seeds(seed=seed) + + +def get_latest_run(search_dir='./runs'): + # Return path to most recent 'last.pt' in /runs (i.e. to --resume from) + last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True) + return max(last_list, key=os.path.getctime) if last_list else '' + + +def check_git_status(): + return + # Suggest 'git pull' if repo is out of date + if platform.system() in ['Linux', 'Darwin'] and not os.path.isfile('/.dockerenv'): + s = subprocess.check_output('if [ -d .git ]; then git fetch && git status -uno; fi', shell=True).decode('utf-8') + if 'Your branch is behind' in s: + print(s[s.find('Your branch is behind'):s.find('\n\n')] + '\n') + + +def check_img_size(img_size, s=32): + # Verify img_size is a multiple of stride s + new_size = make_divisible(img_size, int(s)) # ceil gs-multiple + if new_size != img_size: + print('WARNING: --img-size %g must be multiple of max stride %g, updating to %g' % (img_size, s, new_size)) + return new_size + + +def check_anchors(dataset, model, thr=4.0, imgsz=640): + # Check anchor fit to data, recompute if necessary + print('\nAnalyzing anchors... ', end='') + m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1] # Detect() + shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True) + scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1)) # augment scale + wh = torch.tensor(np.concatenate([l[:, 3:5] * s for s, l in zip(shapes * scale, dataset.labels)])).float() # wh + + def metric(k): # compute metric + r = wh[:, None] / k[None] + x = torch.min(r, 1. / r).min(2)[0] # ratio metric + best = x.max(1)[0] # best_x + aat = (x > 1. / thr).float().sum(1).mean() # anchors above threshold + bpr = (best > 1. / thr).float().mean() # best possible recall + return bpr, aat + + bpr, aat = metric(m.anchor_grid.clone().cpu().view(-1, 2)) + print('anchors/target = %.2f, Best Possible Recall (BPR) = %.4f' % (aat, bpr), end='') + if bpr < 0.98: # threshold to recompute + print('. Attempting to generate improved anchors, please wait...' % bpr) + na = m.anchor_grid.numel() // 2 # number of anchors + new_anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False) + new_bpr = metric(new_anchors.reshape(-1, 2))[0] + if new_bpr > bpr: # replace anchors + new_anchors = torch.tensor(new_anchors, device=m.anchors.device).type_as(m.anchors) + m.anchor_grid[:] = new_anchors.clone().view_as(m.anchor_grid) # for inference + m.anchors[:] = new_anchors.clone().view_as(m.anchors) / m.stride.to(m.anchors.device).view(-1, 1, 1) # loss + check_anchor_order(m) + print('New anchors saved to model. Update model *.yaml to use these anchors in the future.') + else: + print('Original anchors better than new anchors. Proceeding with original anchors.') + print('') # newline + + +def check_anchor_order(m): + # Check anchor order against stride order for YOLOv5 Detect() module m, and correct if necessary + a = m.anchor_grid.prod(-1).view(-1) # anchor area + da = a[-1] - a[0] # delta a + ds = m.stride[-1] - m.stride[0] # delta s + if da.sign() != ds.sign(): # same order + print('Reversing anchor order') + m.anchors[:] = m.anchors.flip(0) + m.anchor_grid[:] = m.anchor_grid.flip(0) + + +def check_file(file): + # Search for file if not found + if os.path.isfile(file) or file == '': + return file + else: + files = glob.glob('./**/' + file, recursive=True) # find file + assert len(files), 'File Not Found: %s' % file # assert file was found + return files[0] # return first file if multiple found + + +def check_dataset(dict): + # Download dataset if not found + val, s = dict.get('val'), dict.get('download') + if val and len(val): + val = [os.path.abspath(x) for x in (val if isinstance(val, list) else [val])] # val path + if not all(os.path.exists(x) for x in val): + print('\nWARNING: Dataset not found, nonexistant paths: %s' % [*val]) + if s and len(s): # download script + print('Downloading %s ...' % s) + if s.startswith('http') and s.endswith('.zip'): # URL + f = Path(s).name # filename + if platform.system() == 'Darwin': # avoid MacOS python requests certificate error + os.system('curl -L %s -o %s' % (s, f)) + else: + torch.hub.download_url_to_file(s, f) + r = os.system('unzip -q %s -d ../ && rm %s' % (f, f)) # unzip + else: # bash script + r = os.system(s) + print('Dataset autodownload %s\n' % ('success' if r == 0 else 'failure')) # analyze return value + else: + raise Exception('Dataset not found.') + + +def make_divisible(x, divisor): + # Returns x evenly divisble by divisor + return math.ceil(x / divisor) * divisor + + +def labels_to_class_weights(labels, nc=80): + # Get class weights (inverse frequency) from training labels + if labels[0] is None: # no labels loaded + return torch.Tensor() + + labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO + classes = labels[:, 0].astype(np.int) # labels = [class xywh] + weights = np.bincount(classes, minlength=nc) # occurences per class + + # Prepend gridpoint count (for uCE trianing) + # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image + # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start + + weights[weights == 0] = 1 # replace empty bins with 1 + weights = 1 / weights # number of targets per class + weights /= weights.sum() # normalize + return torch.from_numpy(weights) + + +def labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)): + # Produces image weights based on class mAPs + n = len(labels) + class_counts = np.array([np.bincount(labels[i][:, 0].astype(np.int), minlength=nc) for i in range(n)]) + image_weights = (class_weights.reshape(1, nc) * class_counts).sum(1) + # index = random.choices(range(n), weights=image_weights, k=1) # weight image sample + return image_weights + + +def coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper) + # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/ + # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\n') + # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\n') + # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco + # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet + x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90] + return x + + +def xyxy2xywh(x): + # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right + y = torch.zeros_like(x) if isinstance(x, torch.Tensor) else np.zeros_like(x) + y[:, 0] = (x[:, 0] + x[:, 2]) / 2 # x center + y[:, 1] = (x[:, 1] + x[:, 3]) / 2 # y center + y[:, 2] = x[:, 2] - x[:, 0] # width + y[:, 3] = x[:, 3] - x[:, 1] # height + return y + + +def xywh2xyxy(x): + # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + y = torch.zeros_like(x) if isinstance(x, torch.Tensor) else np.zeros_like(x) + y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x + y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y + y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x + y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y + return y + + +def scale_coords(img1_shape, coords, img0_shape, ratio_pad=None): + # Rescale coords (xyxy) from img1_shape to img0_shape + if ratio_pad is None: # calculate from img0_shape + gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new + pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + + coords[:, [0, 2]] -= pad[0] # x padding + coords[:, [1, 3]] -= pad[1] # y padding + coords[:, :4] /= gain + clip_coords(coords, img0_shape) + return coords + + +def clip_coords(boxes, img_shape): + # Clip bounding xyxy bounding boxes to image shape (height, width) + boxes[:, 0].clamp_(0, img_shape[1]) # x1 + boxes[:, 1].clamp_(0, img_shape[0]) # y1 + boxes[:, 2].clamp_(0, img_shape[1]) # x2 + boxes[:, 3].clamp_(0, img_shape[0]) # y2 + + +def ap_per_class(tp, conf, pred_cls, target_cls): + """ Compute the average precision, given the recall and precision curves. + Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. + # Arguments + tp: True positives (nparray, nx1 or nx10). + conf: Objectness value from 0-1 (nparray). + pred_cls: Predicted object classes (nparray). + target_cls: True object classes (nparray). + # Returns + The average precision as computed in py-faster-rcnn. + """ + + # Sort by objectness + i = np.argsort(-conf) + tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] + + # Find unique classes + unique_classes = np.unique(target_cls) + + # Create Precision-Recall curve and compute AP for each class + pr_score = 0.1 # score to evaluate P and R https://github.com/ultralytics/yolov3/issues/898 + s = [unique_classes.shape[0], tp.shape[1]] # number class, number iou thresholds (i.e. 10 for mAP0.5...0.95) + ap, p, r = np.zeros(s), np.zeros(s), np.zeros(s) + for ci, c in enumerate(unique_classes): + i = pred_cls == c + n_gt = (target_cls == c).sum() # Number of ground truth objects + n_p = i.sum() # Number of predicted objects + + if n_p == 0 or n_gt == 0: + continue + else: + # Accumulate FPs and TPs + fpc = (1 - tp[i]).cumsum(0) + tpc = tp[i].cumsum(0) + + # Recall + recall = tpc / (n_gt + 1e-16) # recall curve + r[ci] = np.interp(-pr_score, -conf[i], recall[:, 0]) # r at pr_score, negative x, xp because xp decreases + + # Precision + precision = tpc / (tpc + fpc) # precision curve + p[ci] = np.interp(-pr_score, -conf[i], precision[:, 0]) # p at pr_score + + # AP from recall-precision curve + for j in range(tp.shape[1]): + ap[ci, j] = compute_ap(recall[:, j], precision[:, j]) + + # Plot + # fig, ax = plt.subplots(1, 1, figsize=(5, 5)) + # ax.plot(recall, precision) + # ax.set_xlabel('Recall') + # ax.set_ylabel('Precision') + # ax.set_xlim(0, 1.01) + # ax.set_ylim(0, 1.01) + # fig.tight_layout() + # fig.savefig('PR_curve.png', dpi=300) + + # Compute F1 score (harmonic mean of precision and recall) + f1 = 2 * p * r / (p + r + 1e-16) + + return p, r, ap, f1, unique_classes.astype('int32') + + +def compute_ap(recall, precision): + """ Compute the average precision, given the recall and precision curves. + Source: https://github.com/rbgirshick/py-faster-rcnn. + # Arguments + recall: The recall curve (list). + precision: The precision curve (list). + # Returns + The average precision as computed in py-faster-rcnn. + """ + + # Append sentinel values to beginning and end + mrec = np.concatenate(([0.], recall, [min(recall[-1] + 1E-3, 1.)])) + mpre = np.concatenate(([0.], precision, [0.])) + + # Compute the precision envelope + mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) + + # Integrate area under curve + method = 'interp' # methods: 'continuous', 'interp' + if method == 'interp': + x = np.linspace(0, 1, 101) # 101-point interp (COCO) + ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate + else: # 'continuous' + i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve + + return ap + + +def bbox_iou(box1, box2, x1y1x2y2=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-9): + # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4 + box2 = box2.T + + # Get the coordinates of bounding boxes + if x1y1x2y2: # x1, y1, x2, y2 = box1 + b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3] + b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3] + else: # transform from xywh to xyxy + b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2 + b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2 + b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2 + b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2 + + # Intersection area + inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \ + (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0) + + # Union Area + w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps + w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps + union = w1 * h1 + w2 * h2 - inter + eps + + iou = inter / union + if GIoU or DIoU or CIoU: + cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width + ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height + if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1 + c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared + rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 + + (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared + if DIoU: + return iou - rho2 / c2 # DIoU + elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47 + v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2) + with torch.no_grad(): + alpha = v / ((1 + eps) - iou + v) + return iou - (rho2 / c2 + v * alpha) # CIoU + else: # GIoU https://arxiv.org/pdf/1902.09630.pdf + c_area = cw * ch + eps # convex area + return iou - (c_area - union) / c_area # GIoU + else: + return iou # IoU + + +def box_iou(box1, box2): + # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Arguments: + box1 (Tensor[N, 4]) + box2 (Tensor[M, 4]) + Returns: + iou (Tensor[N, M]): the NxM matrix containing the pairwise + IoU values for every element in boxes1 and boxes2 + """ + + def box_area(box): + # box = 4xn + return (box[2] - box[0]) * (box[3] - box[1]) + + area1 = box_area(box1.T) + area2 = box_area(box2.T) + + # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2) + inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2) + return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter) + + +def wh_iou(wh1, wh2): + # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2 + wh1 = wh1[:, None] # [N,1,2] + wh2 = wh2[None] # [1,M,2] + inter = torch.min(wh1, wh2).prod(2) # [N,M] + return inter / (wh1.prod(2) + wh2.prod(2) - inter) # iou = inter / (area1 + area2 - inter) + + +class FocalLoss(nn.Module): + # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5) + def __init__(self, loss_fcn, gamma=1.5, alpha=0.25): + super(FocalLoss, self).__init__() + self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss() + self.gamma = gamma + self.alpha = alpha + self.reduction = loss_fcn.reduction + self.loss_fcn.reduction = 'none' # required to apply FL to each element + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + # p_t = torch.exp(-loss) + # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability + + # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py + pred_prob = torch.sigmoid(pred) # prob from logits + p_t = true * pred_prob + (1 - true) * (1 - pred_prob) + alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha) + modulating_factor = (1.0 - p_t) ** self.gamma + loss *= alpha_factor * modulating_factor + + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + else: # 'none' + return loss + + +def smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441 + # return positive, negative label smoothing BCE targets + return 1.0 - 0.5 * eps, 0.5 * eps + + +class BCEBlurWithLogitsLoss(nn.Module): + # BCEwithLogitLoss() with reduced missing label effects. + def __init__(self, alpha=0.05): + super(BCEBlurWithLogitsLoss, self).__init__() + self.loss_fcn = nn.BCEWithLogitsLoss(reduction='none') # must be nn.BCEWithLogitsLoss() + self.alpha = alpha + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + pred = torch.sigmoid(pred) # prob from logits + dx = pred - true # reduce only missing label effects + # dx = (pred - true).abs() # reduce missing label and false label effects + alpha_factor = 1 - torch.exp((dx - 1) / (self.alpha + 1e-4)) + loss *= alpha_factor + return loss.mean() + + +def compute_loss(p, targets, model): # predictions, targets, model + device = targets.device + lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device) + tcls, tbox, indices, anchors = build_targets(p, targets, model) # targets + h = model.hyp # hyperparameters + + # Define criteria + BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['cls_pw']])).to(device) + BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['obj_pw']])).to(device) + + # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3 + cp, cn = smooth_BCE(eps=0.0) + + # Focal loss + g = h['fl_gamma'] # focal loss gamma + if g > 0: + BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g) + + # Losses + nt = 0 # number of targets + np = len(p) # number of outputs + balance = [4.0, 1.0, 0.4] if np == 3 else [4.0, 1.0, 0.4, 0.1] # P3-5 or P3-6 + for i, pi in enumerate(p): # layer index, layer predictions + b, a, gj, gi = indices[i] # image, anchor, gridy, gridx + tobj = torch.zeros_like(pi[..., 0], device=device) # target obj + + n = b.shape[0] # number of targets + if n: + nt += n # cumulative targets + ps = pi[b, a, gj, gi] # prediction subset corresponding to targets + + # Regression + pxy = ps[:, :2].sigmoid() * 2. - 0.5 + pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i] + pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box + giou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # giou(prediction, target) + lbox += (1.0 - giou).mean() # giou loss + + # Objectness + tobj[b, a, gj, gi] = (1.0 - model.gr) + model.gr * giou.detach().clamp(0).type(tobj.dtype) # giou ratio + + # Classification + if model.nc > 1: # cls loss (only if multiple classes) + t = torch.full_like(ps[:, 5:], cn, device=device) # targets + t[range(n), tcls[i]] = cp + lcls += BCEcls(ps[:, 5:], t) # BCE + + # Append targets to text file + # with open('targets.txt', 'a') as file: + # [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)] + + lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss + + s = 3 / np # output count scaling + lbox *= h['giou'] * s + lobj *= h['obj'] * s * (1.4 if np == 4 else 1.) + lcls *= h['cls'] * s + bs = tobj.shape[0] # batch size + + loss = lbox + lobj + lcls + return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach() + + +def build_targets(p, targets, model): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module + na, nt = det.na, targets.shape[0] # number of anchors, targets + tcls, tbox, indices, anch = [], [], [], [] + gain = torch.ones(7, device=targets.device) # normalized to gridspace gain + ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices + + g = 0.5 # bias + off = torch.tensor([[0, 0], + [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], device=targets.device).float() * g # offsets + + for i in range(det.nl): + anchors = det.anchors[i] + gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain + if nt: + # Matches + r = t[:, :, 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1. / r).max(2)[0] < model.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1. < g) & (gxy > 1.)).T + l, m = ((gxi % 1. < g) & (gxi > 1.)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + b, c = t[:, :2].long().T # image, class + gxy = t[:, 2:4] # grid xy + gwh = t[:, 4:6] # grid wh + gij = (gxy - offsets).long() + gi, gj = gij.T # grid xy indices + + # Append + a = t[:, 6].long() # anchor indices + indices.append((b, a, gj, gi)) # image, anchor, grid indices + tbox.append(torch.cat((gxy - gij, gwh), 1)) # box + anch.append(anchors[a]) # anchors + tcls.append(c) # class + + return tcls, tbox, indices, anch + + +def non_max_suppression(prediction, conf_thres=0.1, iou_thres=0.6, merge=False, classes=None, agnostic=False): + """Performs Non-Maximum Suppression (NMS) on inference results + + Returns: + detections with shape: nx6 (x1, y1, x2, y2, conf, cls) + """ + + nc = prediction[0].shape[1] - 5 # number of classes + xc = prediction[..., 4] > conf_thres # candidates + + # Settings + min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height + max_det = 300 # maximum number of detections per image + time_limit = 10.0 # seconds to quit after + redundant = True # require redundant detections + multi_label = nc > 1 # multiple labels per box (adds 0.5ms/img) + + t = time.time() + output = [None] * prediction.shape[0] + for xi, x in enumerate(prediction): # image index, image inference + # Apply constraints + # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height + x = x[xc[xi]] # confidence + + # If none remain process next image + if not x.shape[0]: + continue + + # Compute conf + x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf + + # Box (center x, center y, width, height) to (x1, y1, x2, y2) + box = xywh2xyxy(x[:, :4]) + + # Detections matrix nx6 (xyxy, conf, cls) + if multi_label: + i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T + x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1) + else: # best class only + conf, j = x[:, 5:].max(1, keepdim=True) + x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres] + + # Filter by class + if classes: + x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)] + + # Apply finite constraint + # if not torch.isfinite(x).all(): + # x = x[torch.isfinite(x).all(1)] + + # If none remain process next image + n = x.shape[0] # number of boxes + if not n: + continue + + # Sort by confidence + # x = x[x[:, 4].argsort(descending=True)] + + # Batched NMS + c = x[:, 5:6] * (0 if agnostic else max_wh) # classes + boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores + i = torch.ops.torchvision.nms(boxes, scores, iou_thres) + if i.shape[0] > max_det: # limit detections + i = i[:max_det] + if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean) + try: # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) + iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix + weights = iou * scores[None] # box weights + x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes + if redundant: + i = i[iou.sum(1) > 1] # require redundancy + except: # possible CUDA error https://github.com/ultralytics/yolov3/issues/1139 + print(x, i, x.shape, i.shape) + pass + + output[xi] = x[i] + if (time.time() - t) > time_limit: + break # time limit exceeded + + return output + + +def strip_optimizer(f='weights/best.pt', s=''): # from utils.general import *; strip_optimizer() + # Strip optimizer from 'f' to finalize training, optionally save as 's' + x = torch.load(f, map_location=torch.device('cpu')) + x['optimizer'] = None + x['training_results'] = None + x['epoch'] = -1 + x['model'].half() # to FP16 + for p in x['model'].parameters(): + p.requires_grad = False + torch.save(x, s or f) + mb = os.path.getsize(s or f) / 1E6 # filesize + print('Optimizer stripped from %s,%s %.1fMB' % (f, (' saved as %s,' % s) if s else '', mb)) + + +def coco_class_count(path='../coco/labels/train2014/'): + # Histogram of occurrences per class + nc = 80 # number classes + x = np.zeros(nc, dtype='int32') + files = sorted(glob.glob('%s/*.*' % path)) + for i, file in enumerate(files): + labels = np.loadtxt(file, dtype=np.float32).reshape(-1, 5) + x += np.bincount(labels[:, 0].astype('int32'), minlength=nc) + print(i, len(files)) + + +def coco_only_people(path='../coco/labels/train2017/'): # from utils.general import *; coco_only_people() + # Find images with only people + files = sorted(glob.glob('%s/*.*' % path)) + for i, file in enumerate(files): + labels = np.loadtxt(file, dtype=np.float32).reshape(-1, 5) + if all(labels[:, 0] == 0): + print(labels.shape[0], file) + + +def crop_images_random(path='../images/', scale=0.50): # from utils.general import *; crop_images_random() + # crops images into random squares up to scale fraction + # WARNING: overwrites images! + for file in tqdm(sorted(glob.glob('%s/*.*' % path))): + img = cv2.imread(file) # BGR + if img is not None: + h, w = img.shape[:2] + + # create random mask + a = 30 # minimum size (pixels) + mask_h = random.randint(a, int(max(a, h * scale))) # mask height + mask_w = mask_h # mask width + + # box + xmin = max(0, random.randint(0, w) - mask_w // 2) + ymin = max(0, random.randint(0, h) - mask_h // 2) + xmax = min(w, xmin + mask_w) + ymax = min(h, ymin + mask_h) + + # apply random color mask + cv2.imwrite(file, img[ymin:ymax, xmin:xmax]) + + +def coco_single_class_labels(path='../coco/labels/train2014/', label_class=43): + # Makes single-class coco datasets. from utils.general import *; coco_single_class_labels() + if os.path.exists('new/'): + shutil.rmtree('new/') # delete output folder + os.makedirs('new/') # make new output folder + os.makedirs('new/labels/') + os.makedirs('new/images/') + for file in tqdm(sorted(glob.glob('%s/*.*' % path))): + with open(file, 'r') as f: + labels = np.array([x.split() for x in f.read().splitlines()], dtype=np.float32) + i = labels[:, 0] == label_class + if any(i): + img_file = file.replace('labels', 'images').replace('txt', 'jpg') + labels[:, 0] = 0 # reset class to 0 + with open('new/images.txt', 'a') as f: # add image to dataset list + f.write(img_file + '\n') + with open('new/labels/' + Path(file).name, 'a') as f: # write label + for l in labels[i]: + f.write('%g %.6f %.6f %.6f %.6f\n' % tuple(l)) + shutil.copyfile(src=img_file, dst='new/images/' + Path(file).name.replace('txt', 'jpg')) # copy images + + +def kmean_anchors(path='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True): + """ Creates kmeans-evolved anchors from training dataset + + Arguments: + path: path to dataset *.yaml, or a loaded dataset + n: number of anchors + img_size: image size used for training + thr: anchor-label wh ratio threshold hyperparameter hyp['anchor_t'] used for training, default=4.0 + gen: generations to evolve anchors using genetic algorithm + + Return: + k: kmeans evolved anchors + + Usage: + from utils.general import *; _ = kmean_anchors() + """ + thr = 1. / thr + + def metric(k, wh): # compute metrics + r = wh[:, None] / k[None] + x = torch.min(r, 1. / r).min(2)[0] # ratio metric + # x = wh_iou(wh, torch.tensor(k)) # iou metric + return x, x.max(1)[0] # x, best_x + + def fitness(k): # mutation fitness + _, best = metric(torch.tensor(k, dtype=torch.float32), wh) + return (best * (best > thr).float()).mean() # fitness + + def print_results(k): + k = k[np.argsort(k.prod(1))] # sort small to large + x, best = metric(k, wh0) + bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr + print('thr=%.2f: %.4f best possible recall, %.2f anchors past thr' % (thr, bpr, aat)) + print('n=%g, img_size=%s, metric_all=%.3f/%.3f-mean/best, past_thr=%.3f-mean: ' % + (n, img_size, x.mean(), best.mean(), x[x > thr].mean()), end='') + for i, x in enumerate(k): + print('%i,%i' % (round(x[0]), round(x[1])), end=', ' if i < len(k) - 1 else '\n') # use in *.cfg + return k + + if isinstance(path, str): # *.yaml file + with open(path) as f: + data_dict = yaml.load(f, Loader=yaml.FullLoader) # model dict + from utils.datasets import LoadImagesAndLabels + dataset = LoadImagesAndLabels(data_dict['train'], augment=True, rect=True) + else: + dataset = path # dataset + + # Get label wh + shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True) + wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh + + # Filter + i = (wh0 < 3.0).any(1).sum() + if i: + print('WARNING: Extremely small objects found. ' + '%g of %g labels are < 3 pixels in width or height.' % (i, len(wh0))) + wh = wh0[(wh0 >= 2.0).any(1)] # filter > 2 pixels + + # Kmeans calculation + print('Running kmeans for %g anchors on %g points...' % (n, len(wh))) + s = wh.std(0) # sigmas for whitening + k, dist = kmeans(wh / s, n, iter=30) # points, mean distance + k *= s + wh = torch.tensor(wh, dtype=torch.float32) # filtered + wh0 = torch.tensor(wh0, dtype=torch.float32) # unflitered + k = print_results(k) + + # Plot + # k, d = [None] * 20, [None] * 20 + # for i in tqdm(range(1, 21)): + # k[i-1], d[i-1] = kmeans(wh / s, i) # points, mean distance + # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) + # ax = ax.ravel() + # ax[0].plot(np.arange(1, 21), np.array(d) ** 2, marker='.') + # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) # plot wh + # ax[0].hist(wh[wh[:, 0]<100, 0],400) + # ax[1].hist(wh[wh[:, 1]<100, 1],400) + # fig.tight_layout() + # fig.savefig('wh.png', dpi=200) + + # Evolve + npr = np.random + f, sh, mp, s = fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma + pbar = tqdm(range(gen), desc='Evolving anchors with Genetic Algorithm') # progress bar + for _ in pbar: + v = np.ones(sh) + while (v == 1).all(): # mutate until a change occurs (prevent duplicates) + v = ((npr.random(sh) < mp) * npr.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0) + kg = (k.copy() * v).clip(min=2.0) + fg = fitness(kg) + if fg > f: + f, k = fg, kg.copy() + pbar.desc = 'Evolving anchors with Genetic Algorithm: fitness = %.4f' % f + if verbose: + print_results(k) + + return print_results(k) + + +def print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''): + # Print mutation results to evolve.txt (for use with train.py --evolve) + a = '%10s' * len(hyp) % tuple(hyp.keys()) # hyperparam keys + b = '%10.3g' * len(hyp) % tuple(hyp.values()) # hyperparam values + c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) + print('\n%s\n%s\nEvolved fitness: %s\n' % (a, b, c)) + + if bucket: + os.system('gsutil cp gs://%s/evolve.txt .' % bucket) # download evolve.txt + + with open('evolve.txt', 'a') as f: # append result + f.write(c + b + '\n') + x = np.unique(np.loadtxt('evolve.txt', ndmin=2), axis=0) # load unique rows + x = x[np.argsort(-fitness(x))] # sort + np.savetxt('evolve.txt', x, '%10.3g') # save sort by fitness + + # Save yaml + for i, k in enumerate(hyp.keys()): + hyp[k] = float(x[0, i + 7]) + with open(yaml_file, 'w') as f: + results = tuple(x[0, :7]) + c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) + f.write('# Hyperparameter Evolution Results\n# Generations: %g\n# Metrics: ' % len(x) + c + '\n\n') + yaml.dump(hyp, f, sort_keys=False) + + if bucket: + os.system('gsutil cp evolve.txt %s gs://%s' % (yaml_file, bucket)) # upload + + +def apply_classifier(x, model, img, im0): + # applies a second stage classifier to yolo outputs + im0 = [im0] if isinstance(im0, np.ndarray) else im0 + for i, d in enumerate(x): # per image + if d is not None and len(d): + d = d.clone() + + # Reshape and pad cutouts + b = xyxy2xywh(d[:, :4]) # boxes + b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square + b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad + d[:, :4] = xywh2xyxy(b).long() + + # Rescale boxes from img_size to im0 size + scale_coords(img.shape[2:], d[:, :4], im0[i].shape) + + # Classes + pred_cls1 = d[:, 5].long() + ims = [] + for j, a in enumerate(d): # per item + cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])] + im = cv2.resize(cutout, (224, 224)) # BGR + # cv2.imwrite('test%i.jpg' % j, cutout) + + im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32 + im /= 255.0 # 0 - 255 to 0.0 - 1.0 + ims.append(im) + + pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction + x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections + + return x + + +def fitness(x): + # Returns fitness (for use with results.txt or evolve.txt) + w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95] + return (x[:, :4] * w).sum(1) + + +def output_to_target(output, width, height): + # Convert model output to target format [batch_id, class_id, x, y, w, h, conf] + if isinstance(output, torch.Tensor): + output = output.cpu().numpy() + + targets = [] + for i, o in enumerate(output): + if o is not None: + for pred in o: + box = pred[:4] + w = (box[2] - box[0]) / width + h = (box[3] - box[1]) / height + x = box[0] / width + w / 2 + y = box[1] / height + h / 2 + conf = pred[4] + cls = int(pred[5]) + + targets.append([i, cls, x, y, w, h, conf]) + + return np.array(targets) + + +def increment_dir(dir, comment=''): + # Increments a directory runs/exp1 --> runs/exp2_comment + n = 0 # number + dir = str(Path(dir)) # os-agnostic + d = sorted(glob.glob(dir + '*')) # directories + if len(d): + n = max([int(x[len(dir):x.find('_') if '_' in x else None]) for x in d]) + 1 # increment + return dir + str(n) + ('_' + comment if comment else '') + + +# Plotting functions --------------------------------------------------------------------------------------------------- +def hist2d(x, y, n=100): + # 2d histogram used in labels.png and evolve.png + xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n) + hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges)) + xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1) + yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1) + return np.log(hist[xidx, yidx]) + + +def butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5): + # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy + def butter_lowpass(cutoff, fs, order): + nyq = 0.5 * fs + normal_cutoff = cutoff / nyq + b, a = butter(order, normal_cutoff, btype='low', analog=False) + return b, a + + b, a = butter_lowpass(cutoff, fs, order=order) + return filtfilt(b, a, data) # forward-backward filter + + +def plot_one_box(x, img, color=None, label=None, line_thickness=None): + # Plots one bounding box on image img + tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness + color = color or [random.randint(0, 255) for _ in range(3)] + c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3])) + cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA) + if label: + tf = max(tl - 1, 1) # font thickness + t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0] + c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3 + cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled + cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA) + + +def plot_wh_methods(): # from utils.general import *; plot_wh_methods() + # Compares the two methods for width-height anchor multiplication + # https://github.com/ultralytics/yolov3/issues/168 + x = np.arange(-4.0, 4.0, .1) + ya = np.exp(x) + yb = torch.sigmoid(torch.from_numpy(x)).numpy() * 2 + + fig = plt.figure(figsize=(6, 3), dpi=150) + plt.plot(x, ya, '.-', label='YOLOv3') + plt.plot(x, yb ** 2, '.-', label='YOLOv5 ^2') + plt.plot(x, yb ** 1.6, '.-', label='YOLOv5 ^1.6') + plt.xlim(left=-4, right=4) + plt.ylim(bottom=0, top=6) + plt.xlabel('input') + plt.ylabel('output') + plt.grid() + plt.legend() + fig.tight_layout() + fig.savefig('comparison.png', dpi=200) + + +def plot_images(images, targets, paths=None, fname='images.jpg', names=None, max_size=640, max_subplots=16): + tl = 3 # line thickness + tf = max(tl - 1, 1) # font thickness + if os.path.isfile(fname): # do not overwrite + return None + + if isinstance(images, torch.Tensor): + images = images.cpu().float().numpy() + + if isinstance(targets, torch.Tensor): + targets = targets.cpu().numpy() + + # un-normalise + if np.max(images[0]) <= 1: + images *= 255 + + bs, _, h, w = images.shape # batch size, _, height, width + bs = min(bs, max_subplots) # limit plot images + ns = np.ceil(bs ** 0.5) # number of subplots (square) + + # Check if we should resize + scale_factor = max_size / max(h, w) + if scale_factor < 1: + h = math.ceil(scale_factor * h) + w = math.ceil(scale_factor * w) + + # Empty array for output + mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) + + # Fix class - colour map + prop_cycle = plt.rcParams['axes.prop_cycle'] + # https://stackoverflow.com/questions/51350872/python-from-color-name-to-rgb + hex2rgb = lambda h: tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4)) + color_lut = [hex2rgb(h) for h in prop_cycle.by_key()['color']] + + for i, img in enumerate(images): + if i == max_subplots: # if last batch has fewer images than we expect + break + + block_x = int(w * (i // ns)) + block_y = int(h * (i % ns)) + + img = img.transpose(1, 2, 0) + if scale_factor < 1: + img = cv2.resize(img, (w, h)) + + mosaic[block_y:block_y + h, block_x:block_x + w, :] = img + if len(targets) > 0: + image_targets = targets[targets[:, 0] == i] + boxes = xywh2xyxy(image_targets[:, 2:6]).T + classes = image_targets[:, 1].astype('int') + gt = image_targets.shape[1] == 6 # ground truth if no conf column + conf = None if gt else image_targets[:, 6] # check for confidence presence (gt vs pred) + + boxes[[0, 2]] *= w + boxes[[0, 2]] += block_x + boxes[[1, 3]] *= h + boxes[[1, 3]] += block_y + for j, box in enumerate(boxes.T): + cls = int(classes[j]) + color = color_lut[cls % len(color_lut)] + cls = names[cls] if names else cls + if gt or conf[j] > 0.3: # 0.3 conf thresh + label = '%s' % cls if gt else '%s %.1f' % (cls, conf[j]) + plot_one_box(box, mosaic, label=label, color=color, line_thickness=tl) + + # Draw image filename labels + if paths is not None: + label = os.path.basename(paths[i])[:40] # trim to 40 char + t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0] + cv2.putText(mosaic, label, (block_x + 5, block_y + t_size[1] + 5), 0, tl / 3, [220, 220, 220], thickness=tf, + lineType=cv2.LINE_AA) + + # Image border + cv2.rectangle(mosaic, (block_x, block_y), (block_x + w, block_y + h), (255, 255, 255), thickness=3) + + if fname is not None: + mosaic = cv2.resize(mosaic, (int(ns * w * 0.5), int(ns * h * 0.5)), interpolation=cv2.INTER_AREA) + cv2.imwrite(fname, cv2.cvtColor(mosaic, cv2.COLOR_BGR2RGB)) + + return mosaic + + +def plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''): + # Plot LR simulating training for full epochs + optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals + y = [] + for _ in range(epochs): + scheduler.step() + y.append(optimizer.param_groups[0]['lr']) + plt.plot(y, '.-', label='LR') + plt.xlabel('epoch') + plt.ylabel('LR') + plt.grid() + plt.xlim(0, epochs) + plt.ylim(0) + plt.tight_layout() + plt.savefig(Path(save_dir) / 'LR.png', dpi=200) + + +def plot_test_txt(): # from utils.general import *; plot_test() + # Plot test.txt histograms + x = np.loadtxt('test.txt', dtype=np.float32) + box = xyxy2xywh(x[:, :4]) + cx, cy = box[:, 0], box[:, 1] + + fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True) + ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0) + ax.set_aspect('equal') + plt.savefig('hist2d.png', dpi=300) + + fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True) + ax[0].hist(cx, bins=600) + ax[1].hist(cy, bins=600) + plt.savefig('hist1d.png', dpi=200) + + +def plot_targets_txt(): # from utils.general import *; plot_targets_txt() + # Plot targets.txt histograms + x = np.loadtxt('targets.txt', dtype=np.float32).T + s = ['x targets', 'y targets', 'width targets', 'height targets'] + fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True) + ax = ax.ravel() + for i in range(4): + ax[i].hist(x[i], bins=100, label='%.3g +/- %.3g' % (x[i].mean(), x[i].std())) + ax[i].legend() + ax[i].set_title(s[i]) + plt.savefig('targets.jpg', dpi=200) + + +def plot_study_txt(f='study.txt', x=None): # from utils.general import *; plot_study_txt() + # Plot study.txt generated by test.py + fig, ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True) + ax = ax.ravel() + + fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True) + for f in ['study/study_coco_yolov5%s.txt' % x for x in ['s', 'm', 'l', 'x']]: + y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T + x = np.arange(y.shape[1]) if x is None else np.array(x) + s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_inference (ms/img)', 't_NMS (ms/img)', 't_total (ms/img)'] + for i in range(7): + ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8) + ax[i].set_title(s[i]) + + j = y[3].argmax() + 1 + ax2.plot(y[6, :j], y[3, :j] * 1E2, '.-', linewidth=2, markersize=8, + label=Path(f).stem.replace('study_coco_', '').replace('yolo', 'YOLO')) + + ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5], + 'k.-', linewidth=2, markersize=8, alpha=.25, label='EfficientDet') + + ax2.grid() + ax2.set_xlim(0, 30) + ax2.set_ylim(28, 50) + ax2.set_yticks(np.arange(30, 55, 5)) + ax2.set_xlabel('GPU Speed (ms/img)') + ax2.set_ylabel('COCO AP val') + ax2.legend(loc='lower right') + plt.savefig('study_mAP_latency.png', dpi=300) + plt.savefig(f.replace('.txt', '.png'), dpi=300) + + +def plot_labels(labels, save_dir=''): + # plot dataset labels + c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes + nc = int(c.max() + 1) # number of classes + + fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True) + ax = ax.ravel() + ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8) + ax[0].set_xlabel('classes') + ax[1].scatter(b[0], b[1], c=hist2d(b[0], b[1], 90), cmap='jet') + ax[1].set_xlabel('x') + ax[1].set_ylabel('y') + ax[2].scatter(b[2], b[3], c=hist2d(b[2], b[3], 90), cmap='jet') + ax[2].set_xlabel('width') + ax[2].set_ylabel('height') + plt.savefig(Path(save_dir) / 'labels.png', dpi=200) + plt.close() + + # seaborn correlogram + try: + import seaborn as sns + import pandas as pd + x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height']) + sns.pairplot(x, corner=True, diag_kind='hist', kind='scatter', markers='o', + plot_kws=dict(s=3, edgecolor=None, linewidth=1, alpha=0.02), + diag_kws=dict(bins=50)) + plt.savefig(Path(save_dir) / 'labels_correlogram.png', dpi=200) + plt.close() + except Exception as e: + pass + + +def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.general import *; plot_evolution() + # Plot hyperparameter evolution results in evolve.txt + with open(yaml_file) as f: + hyp = yaml.load(f, Loader=yaml.FullLoader) + x = np.loadtxt('evolve.txt', ndmin=2) + f = fitness(x) + # weights = (f - f.min()) ** 2 # for weighted results + plt.figure(figsize=(10, 10), tight_layout=True) + matplotlib.rc('font', **{'size': 8}) + for i, (k, v) in enumerate(hyp.items()): + y = x[:, i + 7] + # mu = (y * weights).sum() / weights.sum() # best weighted result + mu = y[f.argmax()] # best single result + plt.subplot(5, 5, i + 1) + plt.scatter(y, f, c=hist2d(y, f, 20), cmap='viridis', alpha=.8, edgecolors='none') + plt.plot(mu, f.max(), 'k+', markersize=15) + plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters + if i % 5 != 0: + plt.yticks([]) + print('%15s: %.3g' % (k, mu)) + plt.savefig('evolve.png', dpi=200) + print('\nPlot saved as evolve.png') + + +def plot_results_overlay(start=0, stop=0): # from utils.general import *; plot_results_overlay() + # Plot training 'results*.txt', overlaying train and val losses + s = ['train', 'train', 'train', 'Precision', 'mAP@0.5', 'val', 'val', 'val', 'Recall', 'mAP@0.5:0.95'] # legends + t = ['GIoU', 'Objectness', 'Classification', 'P-R', 'mAP-F1'] # titles + for f in sorted(glob.glob('results*.txt') + glob.glob('../../Downloads/results*.txt')): + results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T + n = results.shape[1] # number of rows + x = range(start, min(stop, n) if stop else n) + fig, ax = plt.subplots(1, 5, figsize=(14, 3.5), tight_layout=True) + ax = ax.ravel() + for i in range(5): + for j in [i, i + 5]: + y = results[j, x] + ax[i].plot(x, y, marker='.', label=s[j]) + # y_smooth = butter_lowpass_filtfilt(y) + # ax[i].plot(x, np.gradient(y_smooth), marker='.', label=s[j]) + + ax[i].set_title(t[i]) + ax[i].legend() + ax[i].set_ylabel(f) if i == 0 else None # add filename + fig.savefig(f.replace('.txt', '.png'), dpi=200) + + +def plot_results(start=0, stop=0, bucket='', id=(), labels=(), + save_dir=''): # from utils.general import *; plot_results() + # Plot training 'results*.txt' as seen in https://github.com/ultralytics/yolov5#reproduce-our-training + fig, ax = plt.subplots(2, 5, figsize=(12, 6)) + ax = ax.ravel() + s = ['GIoU', 'Objectness', 'Classification', 'Precision', 'Recall', + 'val GIoU', 'val Objectness', 'val Classification', 'mAP@0.5', 'mAP@0.5:0.95'] + if bucket: + # os.system('rm -rf storage.googleapis.com') + # files = ['https://storage.googleapis.com/%s/results%g.txt' % (bucket, x) for x in id] + files = ['results%g.txt' % x for x in id] + c = ('gsutil cp ' + '%s ' * len(files) + '.') % tuple('gs://%s/results%g.txt' % (bucket, x) for x in id) + os.system(c) + else: + files = glob.glob(str(Path(save_dir) / 'results*.txt')) + glob.glob('../../Downloads/results*.txt') + for fi, f in enumerate(files): + try: + results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T + n = results.shape[1] # number of rows + x = range(start, min(stop, n) if stop else n) + for i in range(10): + y = results[i, x] + if i in [0, 1, 2, 5, 6, 7]: + y[y == 0] = np.nan # dont show zero loss values + # y /= y[0] # normalize + label = labels[fi] if len(labels) else Path(f).stem + ax[i].plot(x, y, marker='.', label=label, linewidth=1, markersize=6) + ax[i].set_title(s[i]) + # if i in [5, 6, 7]: # share train and val loss y axes + # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5]) + except Exception as e: + print('Warning: Plotting error for %s; %s' % (f, e)) + + fig.tight_layout() + ax[1].legend() + fig.savefig(Path(save_dir) / 'results.png', dpi=200) diff --git a/src/digitizer/yolov5/utils/google_utils.py b/src/digitizer/yolov5/utils/google_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5c60fa2e83e8bef9ede22a43ff748930a2276c03 --- /dev/null +++ b/src/digitizer/yolov5/utils/google_utils.py @@ -0,0 +1,118 @@ +# This file contains google utils: https://cloud.google.com/storage/docs/reference/libraries +# pip install --upgrade google-cloud-storage +# from google.cloud import storage + +import os +import platform +import time +from pathlib import Path + +import torch + + +def attempt_download(weights): + # Attempt to download pretrained weights if not found locally + weights = weights.strip().replace("'", '') + file = Path(weights).name + + msg = weights + ' missing, try downloading from https://github.com/ultralytics/yolov5/releases/' + models = ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt'] # available models + + if file in models and not os.path.isfile(weights): + # Google Drive + # d = {'yolov5s.pt': '1R5T6rIyy3lLwgFXNms8whc-387H0tMQO', + # 'yolov5m.pt': '1vobuEExpWQVpXExsJ2w-Mbf3HJjWkQJr', + # 'yolov5l.pt': '1hrlqD1Wdei7UT4OgT785BEk1JwnSvNEV', + # 'yolov5x.pt': '1mM8aZJlWTxOg7BZJvNUMrTnA2AbeCVzS'} + # r = gdrive_download(id=d[file], name=weights) if file in d else 1 + # if r == 0 and os.path.exists(weights) and os.path.getsize(weights) > 1E6: # check + # return + + try: # GitHub + url = 'https://github.com/ultralytics/yolov5/releases/download/v3.0/' + file + print('Downloading %s to %s...' % (url, weights)) + if platform.system() == 'Darwin': # avoid MacOS python requests certificate error + r = os.system('curl -L %s -o %s' % (url, weights)) + else: + torch.hub.download_url_to_file(url, weights) + assert os.path.exists(weights) and os.path.getsize(weights) > 1E6 # check + except Exception as e: # GCP + print('Download error: %s' % e) + url = 'https://storage.googleapis.com/ultralytics/yolov5/ckpt/' + file + print('Downloading %s to %s...' % (url, weights)) + r = os.system('curl -L %s -o %s' % (url, weights)) # torch.hub.download_url_to_file(url, weights) + finally: + if not (os.path.exists(weights) and os.path.getsize(weights) > 1E6): # check + os.remove(weights) if os.path.exists(weights) else None # remove partial downloads + print('ERROR: Download failure: %s' % msg) + print('') + return + + +def gdrive_download(id='1n_oKgR81BJtqk75b00eAjdv03qVCQn2f', name='coco128.zip'): + # Downloads a file from Google Drive. from utils.google_utils import *; gdrive_download() + t = time.time() + + print('Downloading https://drive.google.com/uc?export=download&id=%s as %s... ' % (id, name), end='') + os.remove(name) if os.path.exists(name) else None # remove existing + os.remove('cookie') if os.path.exists('cookie') else None + + # Attempt file download + out = "NUL" if platform.system() == "Windows" else "/dev/null" + os.system('curl -c ./cookie -s -L "drive.google.com/uc?export=download&id=%s" > %s ' % (id, out)) + if os.path.exists('cookie'): # large file + s = 'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm=%s&id=%s" -o %s' % (get_token(), id, name) + else: # small file + s = 'curl -s -L -o %s "drive.google.com/uc?export=download&id=%s"' % (name, id) + r = os.system(s) # execute, capture return + os.remove('cookie') if os.path.exists('cookie') else None + + # Error check + if r != 0: + os.remove(name) if os.path.exists(name) else None # remove partial + print('Download error ') # raise Exception('Download error') + return r + + # Unzip if archive + if name.endswith('.zip'): + print('unzipping... ', end='') + os.system('unzip -q %s' % name) # unzip + os.remove(name) # remove zip to free space + + print('Done (%.1fs)' % (time.time() - t)) + return r + + +def get_token(cookie="./cookie"): + with open(cookie) as f: + for line in f: + if "download" in line: + return line.split()[-1] + return "" + +# def upload_blob(bucket_name, source_file_name, destination_blob_name): +# # Uploads a file to a bucket +# # https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python +# +# storage_client = storage.Client() +# bucket = storage_client.get_bucket(bucket_name) +# blob = bucket.blob(destination_blob_name) +# +# blob.upload_from_filename(source_file_name) +# +# print('File {} uploaded to {}.'.format( +# source_file_name, +# destination_blob_name)) +# +# +# def download_blob(bucket_name, source_blob_name, destination_file_name): +# # Uploads a blob from a bucket +# storage_client = storage.Client() +# bucket = storage_client.get_bucket(bucket_name) +# blob = bucket.blob(source_blob_name) +# +# blob.download_to_filename(destination_file_name) +# +# print('Blob {} downloaded to {}.'.format( +# source_blob_name, +# destination_file_name)) diff --git a/src/digitizer/yolov5/utils/torch_utils.py b/src/digitizer/yolov5/utils/torch_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..52a1264bd9fb803bce1007636e05eaf9b2dbb29f --- /dev/null +++ b/src/digitizer/yolov5/utils/torch_utils.py @@ -0,0 +1,231 @@ +import logging +import math +import os +import time +from copy import deepcopy + +import torch +import torch.backends.cudnn as cudnn +import torch.nn as nn +import torch.nn.functional as F +import torchvision.models as models + +logger = logging.getLogger(__name__) + + +def init_seeds(seed=0): + torch.manual_seed(seed) + + # Speed-reproducibility tradeoff https://pytorch.org/docs/stable/notes/randomness.html + if seed == 0: # slower, more reproducible + cudnn.deterministic = True + cudnn.benchmark = False + else: # faster, less reproducible + cudnn.deterministic = False + cudnn.benchmark = True + + +def select_device(device='', batch_size=None): + # device = 'cpu' or '0' or '0,1,2,3' + cpu_request = device.lower() == 'cpu' + if device and not cpu_request: # if device requested other than 'cpu' + os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable + assert torch.cuda.is_available(), 'CUDA unavailable, invalid device %s requested' % device # check availablity + + cuda = False if cpu_request else torch.cuda.is_available() + if cuda: + c = 1024 ** 2 # bytes to MB + ng = torch.cuda.device_count() + if ng > 1 and batch_size: # check that batch_size is compatible with device_count + assert batch_size % ng == 0, 'batch-size %g not multiple of GPU count %g' % (batch_size, ng) + x = [torch.cuda.get_device_properties(i) for i in range(ng)] + s = 'Using CUDA ' + for i in range(0, ng): + if i == 1: + s = ' ' * len(s) + #logger.info("%sdevice%g _CudaDeviceProperties(name='%s', total_memory=%dMB)" % + #(s, i, x[i].name, x[i].total_memory / c)) + else: + pass + #logger.info('Using CPU') + + #logger.info('') # skip a line + return torch.device('cuda:0' if cuda else 'cpu') + + +def time_synchronized(): + torch.cuda.synchronize() if torch.cuda.is_available() else None + return time.time() + + +def is_parallel(model): + return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel) + + +def intersect_dicts(da, db, exclude=()): + # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values + return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape} + + +def initialize_weights(model): + for m in model.modules(): + t = type(m) + if t is nn.Conv2d: + pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif t is nn.BatchNorm2d: + m.eps = 1e-3 + m.momentum = 0.03 + elif t in [nn.LeakyReLU, nn.ReLU, nn.ReLU6]: + m.inplace = True + + +def find_modules(model, mclass=nn.Conv2d): + # Finds layer indices matching module class 'mclass' + return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)] + + +def sparsity(model): + # Return global model sparsity + a, b = 0., 0. + for p in model.parameters(): + a += p.numel() + b += (p == 0).sum() + return b / a + + +def prune(model, amount=0.3): + # Prune model to requested global sparsity + import torch.nn.utils.prune as prune + print('Pruning model... ', end='') + for name, m in model.named_modules(): + if isinstance(m, nn.Conv2d): + prune.l1_unstructured(m, name='weight', amount=amount) # prune + prune.remove(m, 'weight') # make permanent + print(' %.3g global sparsity' % sparsity(model)) + + +def fuse_conv_and_bn(conv, bn): + # https://tehnokv.com/posts/fusing-batchnorm-and-conv/ + with torch.no_grad(): + # init + fusedconv = nn.Conv2d(conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + bias=True).to(conv.weight.device) + + # prepare filters + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.size())) + + # prepare spatial bias + b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) + fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fusedconv + + +def model_info(model, verbose=False): + # Plots a line-by-line description of a PyTorch model + n_p = sum(x.numel() for x in model.parameters()) # number parameters + n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients + if verbose: + print('%5s %40s %9s %12s %20s %10s %10s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma')) + for i, (name, p) in enumerate(model.named_parameters()): + name = name.replace('module_list.', '') + print('%5g %40s %9s %12g %20s %10.3g %10.3g' % + (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std())) + + try: # FLOPS + from thop import profile + flops = profile(deepcopy(model), inputs=(torch.zeros(1, 3, 64, 64),), verbose=False)[0] / 1E9 * 2 + fs = ', %.1f GFLOPS' % (flops * 100) # 640x640 FLOPS + except: + fs = '' + + #logger.info( + # 'Model Summary: %g layers, %g parameters, %g gradients%s' % (len(list(model.parameters())), n_p, n_g, fs)) + + +def load_classifier(name='resnet101', n=2): + # Loads a pretrained model reshaped to n-class output + model = models.__dict__[name](pretrained=True) + + # Display model properties + input_size = [3, 224, 224] + input_space = 'RGB' + input_range = [0, 1] + mean = [0.485, 0.456, 0.406] + std = [0.229, 0.224, 0.225] + for x in ['input_size', 'input_space', 'input_range', 'mean', 'std']: + print(x + ' =', eval(x)) + + # Reshape output to n classes + filters = model.fc.weight.shape[1] + model.fc.bias = nn.Parameter(torch.zeros(n), requires_grad=True) + model.fc.weight = nn.Parameter(torch.zeros(n, filters), requires_grad=True) + model.fc.out_features = n + return model + + +def scale_img(img, ratio=1.0, same_shape=False): # img(16,3,256,416), r=ratio + # scales img(bs,3,y,x) by ratio + if ratio == 1.0: + return img + else: + h, w = img.shape[2:] + s = (int(h * ratio), int(w * ratio)) # new size + img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize + if not same_shape: # pad/crop img + gs = 32 # (pixels) grid size + h, w = [math.ceil(x * ratio / gs) * gs for x in (h, w)] + return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean + + +def copy_attr(a, b, include=(), exclude=()): + # Copy attributes from b to a, options to only include [...] and to exclude [...] + for k, v in b.__dict__.items(): + if (len(include) and k not in include) or k.startswith('_') or k in exclude: + continue + else: + setattr(a, k, v) + + +class ModelEMA: + """ Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models + Keep a moving average of everything in the model state_dict (parameters and buffers). + This is intended to allow functionality like + https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + A smoothed version of the weights is necessary for some training schemes to perform well. + This class is sensitive where it is initialized in the sequence of model init, + GPU assignment and distributed training wrappers. + """ + + def __init__(self, model, decay=0.9999, updates=0): + # Create EMA + self.ema = deepcopy(model.module if is_parallel(model) else model).eval() # FP32 EMA + # if next(model.parameters()).device.type != 'cpu': + # self.ema.half() # FP16 EMA + self.updates = updates # number of EMA updates + self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) # decay exponential ramp (to help early epochs) + for p in self.ema.parameters(): + p.requires_grad_(False) + + def update(self, model): + # Update EMA parameters + with torch.no_grad(): + self.updates += 1 + d = self.decay(self.updates) + + msd = model.module.state_dict() if is_parallel(model) else model.state_dict() # model state_dict + for k, v in self.ema.state_dict().items(): + if v.dtype.is_floating_point: + v *= d + v += (1. - d) * msd[k].detach() + + def update_attr(self, model, include=(), exclude=('process_group', 'reducer')): + # Update EMA attributes + copy_attr(self.ema, model, include, exclude) diff --git a/src/digitizer/yolov5/weights/download_weights.sh b/src/digitizer/yolov5/weights/download_weights.sh new file mode 100755 index 0000000000000000000000000000000000000000..206b7002aecaabdd0bbe1a721ff3a20860d0245d --- /dev/null +++ b/src/digitizer/yolov5/weights/download_weights.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Download common models + +python -c " +from utils.google_utils import *; +attempt_download('weights/yolov5s.pt'); +attempt_download('weights/yolov5m.pt'); +attempt_download('weights/yolov5l.pt'); +attempt_download('weights/yolov5x.pt') +" diff --git a/src/extract_ground_truth.py b/src/extract_ground_truth.py new file mode 100644 index 0000000000000000000000000000000000000000..af6a7de5d53000fed3d89ce19388693104dbcc9d --- /dev/null +++ b/src/extract_ground_truth.py @@ -0,0 +1,16 @@ +import json +import os + +from digitizer.digitization import annotation_to_thresholds + +if __name__ == "__main__": + for filename in os.listdir("../data/annotations_test"): + thresholds = annotation_to_thresholds(json.load(open(f"../data/annotations_test/{filename}"))) + thresholds_list = [] + for threshold in thresholds: + thresholds_list.append([threshold["ear"], threshold["conduction"], threshold["masking"], threshold["frequency"], threshold["threshold"]]) + + with open(f"../data/ground_truth/{filename.split('.')[0]}.csv", "w") as ofile: + ofile.write(f"ear,conduction,masking,frequency,threshold\n") + for t in thresholds_list: + ofile.write(f"{t[0]},{t[1]},{t[2]},{t[3]},{t[4]}\n") \ No newline at end of file diff --git a/src/interfaces.py b/src/interfaces.py new file mode 100644 index 0000000000000000000000000000000000000000..720c05c6997439cc8f2ba2fa6f0ae8991416d839 --- /dev/null +++ b/src/interfaces.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import List, Optional +from typing_extensions import TypedDict + +class ThresholdDict(TypedDict): + """Represents a hearing threshold (measurement). + """ + frequency: int + threshold: int + ear: str + masking: bool + conduction: str + measurementType: str + response: bool + +class BoundingBox(TypedDict): + """Represents the dictionary holding the minimum information + for a bounding box. + """ + x: int + y: int + width: int + height: int + +class AudiogramDict(TypedDict): + """Represents the dictionary for an audiogram as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + confidence: Optional[float] + +class LabelDict(TypedDict): + """Represents the dictionary for a label as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + value: str + confidence: Optional[float] + +class SymbolDict(TypedDict): + """Represents the dictionary for a symbol as extracted + by the Yolo model. + """ + boundingBox: BoundingBox + measurementType: str + confidence: Optional[float] + +class CornerDict(TypedDict): + """Represents a corner, as annotated. + """ + frequency: int + threshold: int + position: TypedDict("PositionDict", { "horizontal": str, "vertical": str }) + x: float + y: float + +class AudiogramAnnotationDict(TypedDict): + """Represents an audiogram as structured within an annotation. + """ + confidence: Optional[float] + correctionAngle: Optional[float] + boundingBox: BoundingBox + corners: List[CornerDict] + labels: List[LabelDict] + symbols: List[SymbolDict] + +class ClaimantProfileDict(TypedDict): + """Profile of the claimant. + """ + age: int + exposure: List[dict] # out of scope for me + thresholds: List[ThresholdDict] + +class CalculationsDict(TypedDict): + """Values calculated for the claim. + """ + bestEarPta: float + correctedBestEarPta: float + worstEarPta: float + correctedWorstEarPta: float + bestEarRabinowitzNotchIndex: Optional[float] + worstEarRabinowitzNotchIndex: Optional[float] + +class HearingLossCriteriaDict(TypedDict): + """Information related to the hearing loss for the claim. + Includes different calculated values, etc. + """ + preliminaryDecisionAvailable: bool + calculations: CalculationsDict + eligible: bool + comment: str + awardPercentage: float + reviewNeeded: bool + +class MeasurementType(TypedDict): + """Type of measurement. + """ + conduction: str + masking: bool + +class SettingsDict(TypedDict): + """Settings used in computing the eligibility. + """ + left: TypedDict("EarSettings", { + "measurementType": TypedDict("MeasurementType", { + "conduction": str, + "masking": bool + }), + "ptaFrequencies": List[int] + }) + right: TypedDict("EarSettings", { + "measurementType": TypedDict("MeasurementType", { + "conduction": str, + "masking": bool + }), + "ptaFrequencies": List[int] + }) + +class EligibilityDict(TypedDict): + """Eligibility information. + """ + claimantProfile: ClaimantProfileDict + settings: SettingsDict + hearingLossCriteria: HearingLossCriteriaDict + exposureCriteria: dict diff --git a/src/plot_audiogram.py b/src/plot_audiogram.py new file mode 100644 index 0000000000000000000000000000000000000000..dc0f7127a75c8d96bba2aa402540e06ae49d9aaf --- /dev/null +++ b/src/plot_audiogram.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2022, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" +from argparse import ArgumentParser +import json + +from matplotlib.offsetbox import OffsetImage, AnnotationBbox +import matplotlib.pyplot as plt +import pandas as pd + +def load_image(path, zoom=1): + return OffsetImage(plt.imread(path), zoom=zoom) + +def main(args): + + thresholds = pd.DataFrame(json.load(open(args.input))) + + # Figure + fig = plt.figure() + ax = fig.add_subplot(111) + + # x axis + axis = [250, 500, 1000, 2000, 4000, 8000, 16000] + ax.set_xscale('log') + ax.xaxis.tick_top() + ax.xaxis.set_major_formatter(plt.FuncFormatter('{:.0f}'.format)) + ax.set_xlabel('Frequency (Hz)') + ax.xaxis.set_label_position('top') + ax.set_xlim(125,16000) + plt.xticks(axis) + + # y axis + ax.set_ylim(-20, 120) + ax.invert_yaxis() + ax.set_ylabel('Threshold (dB HL)') + + plt.grid() + + for conduction in ("air", "bone"): + for masking in (True, False): + for ear in ("left", "right"): + symbol_name = f"{ear}_{conduction}_{'unmasked' if not masking else 'masked'}" + selection = thresholds[(thresholds.conduction == conduction) & (thresholds.ear == ear) & (thresholds.masking == masking)] + selection = selection.sort_values("frequency") + + # Plot the symbols + for i, threshold in selection.iterrows(): + ab = AnnotationBbox(load_image(f"src/digitizer/assets/symbols/{symbol_name}.png", zoom=0.1), (threshold.frequency, threshold.threshold), frameon=False) + ax.add_artist(ab) + + # Add joining line for air conduction thresholds + if conduction == "air": + plt.plot(selection.frequency, selection.threshold, color="red" if ear == "right" else "blue", linewidth=0.5) + + # Save audiogram plot to PNG + output = args.output + if not output.endswith(".png"): + output = output.split(".")[0] + ".png" + + plt.savefig(output) + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("-i", "--input", type=str, required=True, help="Path to the JSON file of the digitized audiogram.") + parser.add_argument("-o", "--output", type=str, required=True, help="Where the output file should be saved.") + args = parser.parse_args() + main(args) \ No newline at end of file diff --git a/src/utils/__pycache__/audiology.cpython-38.pyc b/src/utils/__pycache__/audiology.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d12c7ed83113fba09a769c4c7d4ff4efed8cf306 Binary files /dev/null and b/src/utils/__pycache__/audiology.cpython-38.pyc differ diff --git a/src/utils/__pycache__/exceptions.cpython-38.pyc b/src/utils/__pycache__/exceptions.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b0e8610534eb3949c9e695c59407c5becd21dc9 Binary files /dev/null and b/src/utils/__pycache__/exceptions.cpython-38.pyc differ diff --git a/src/utils/__pycache__/geometry.cpython-38.pyc b/src/utils/__pycache__/geometry.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2d4b77dbbac2c4fb8b0c4384c07cbe985bda8cb Binary files /dev/null and b/src/utils/__pycache__/geometry.cpython-38.pyc differ diff --git a/src/utils/audiology.py b/src/utils/audiology.py new file mode 100644 index 0000000000000000000000000000000000000000..da0af17e2ff41980bb19b39c4525f382b5ac8e7d --- /dev/null +++ b/src/utils/audiology.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import List +import numpy as np + +VALID_FREQUENCIES = [125, 250, 500, 750, 1000, 1500, 2000, 3000, 4000, 6000, 8000, 16000] +VALID_THRESHOLDS = list(range(-10, 135, 5)) +THRESHOLDS = list(range(-10, 130, 10)) +OCTAVE_FREQS_HZ = [125, 250, 500, 1000, 2000, 4000, 8000] +INTEROCTAVE_FREQS_HZ = [750, 1500, 3000, 6000] +OCTAVE_FREQS_KHZ = [0.125, 0.25, 0.5, 1, 2, 4, 8] +INTEROCTAVE_FREQS_KHZ = [0.750, 1.5, 3, 6] + +def round_threshold(threshold: float) -> int: + """Returns the nearest multiple of 5 for the threshold input. + + Parameters + ---------- + threshold : float + The threshold snapped to the nearest multiple of 5 along the y-axis. + + Returns + ------- + float + A ``snapped`` threshold value. + """ + return VALID_THRESHOLDS[np.argmin([abs(threshold - t) for t in VALID_THRESHOLDS])] + +def round_frequency(frequency: float) -> int: + """Returns the nearest audiologically meaningful frequency. + Parameters + ---------- + frequency : float + The frequency to be snapped to the nearest clinically meaningful frequency. + Returns + ------- + float + A ``snapped`` frequency value. + """ + return VALID_FREQUENCIES[np.argmin([abs(frequency - f) for f in VALID_FREQUENCIES])] + +def round_frequency_bone(frequency: float, direction: str, epsilon: float = 0.15) -> int: + """Returns the nearest audiologically meaningful frequency. + + Parameters + ---------- + frequency : float + The frequency to be snapped to the nearest clinically meaningful frequency. + + epsilon: float + Distance (in octaves) below which a frequency is considered to be + exactly on the nearest valid frequency. (default: 0.15 octaves) + + direction: str + This parameter will influence the snapping behavior as some + audiologists draw bone conduction symbols next to the target frequency, + while other draw it right on it. + + epsilon: float + The frequency will be snapped in to the nearest frequency in the + provided direction, unless the distance to the nearest + frequency is < ε (some small distance (IN OCTAVE UNITS), in which + case the frequency will be snapped to that value. + + Eg: + + 1K 2K 1K 2K + | | | | + | > | will be snapped to > | + | | | | + + but if the threshold fell directly (within a very small distance of + 1.5K, it would be snapped to that. + + 1K 2K 1K 1.5K 2K + | | | | + | > | will be snapped to | > | + | | | | + + because it is really close to 1.5 and the audiologist likely + intentionally meant to indicate 1.5K rather than 1K. + + Note: ε is a tweakable parameter that can be optimized over the + dataset. + + + Returns + ------- + float + A ``snapped`` frequency value. + """ + assert direction == "left" or direction == "right" + + distances = [abs(frequency_to_octave(frequency) - frequency_to_octave(f)) for f in VALID_FREQUENCIES] + nearest_frequency_index = np.argmin(distances) + + snapped = None + if distances[nearest_frequency_index] < epsilon: + snapped = VALID_FREQUENCIES[nearest_frequency_index] + elif direction == "left": + if VALID_FREQUENCIES[nearest_frequency_index] > frequency: + snapped = VALID_FREQUENCIES[nearest_frequency_index - 1] if nearest_frequency_index > 0 else VALID_FREQUENCIES[nearest_frequency_index] + else: + snapped = VALID_FREQUENCIES[nearest_frequency_index] + else: + if VALID_FREQUENCIES[nearest_frequency_index] > frequency: + snapped = VALID_FREQUENCIES[nearest_frequency_index] + else: + snapped = VALID_FREQUENCIES[nearest_frequency_index + 1] if nearest_frequency_index < len(VALID_FREQUENCIES) - 1 else VALID_FREQUENCIES[nearest_frequency_index] + + return snapped + +def frequency_to_octave(frequency: float) -> float: + """Converts a frequency (in Hz) to an octave value (linear units). + + By convention, the 0th octave is 125Hz. + + Parameters + ---------- + frequency : float + The frequency (a positive real) to be converted to an octave value. + + Returns + ------- + float + The octave corresponding to the input frequency. + """ + return np.log(frequency / 125) / np.log(2) + +def octave_to_frequency(octave: float) -> float: + """Converts an octave to its corresponding frequency value (in Hz). + + By convention, the 0th octave is 125Hz. + + Parameters + ---------- + octave : float + The octave to put on a frequency scale. + + Returns + ------- + float + The frequency value corresponding to the octave. + """ + return 125 * 2 ** octave + +def stringify_measurement(measurement: dict) -> str: + """Returns a string describing the measurement type that is compatible + with the NIHL portal format. + + eg. An air conduction threshold for the right ear with no masking + would yield the string `AIR_UNMASKED_RIGHT`. + + Parameters + ---------- + measurement: dict + A dictionary describing a threshold. Should have the keys `ear`, + `conduction` and `masking`. + + Returns + ------- + str + The string describing the measurement type in the NIHL portal format. + """ + masking = "masked" if measurement["masking"] else "unmasked" + return f"{measurement['conduction']}_{masking}_{measurement['ear']}".upper() + + +def measurement_string_to_dict(measurement_type: str) -> dict: + """Converts a measurement type string in the NIHL portal format into + a dictionary with the equivalent information for use with the digitizer. + + eg. `AIR_UNMASKED_RIGHT` would be equivalent to the dictionary: + {`ear`: `right`, `conduction`: `air`, `masking`: False} + + Parameters + ---------- + measurement: dict + A dictionary describing a threshold. Should have the keys `ear`, + `conduction` and `masking`. + + Returns + ------- + str + The string describing the measurement type in the NIHL portal format. + """ + return { + "ear": "left" if "LEFT" in measurement_type else "right", + "conduction": "air" if "AIR" in measurement_type else "bone", + "masking": False if "UNMASKED" in measurement_type else True + } diff --git a/src/utils/exceptions.py b/src/utils/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..7a1c9b80ebee6553bbec0f9e457c3ff530013340 --- /dev/null +++ b/src/utils/exceptions.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +class InsufficientLabelsException(Exception): + def __init__(self): + self.message = f"An insufficient number of labels was detected." + self.code = "INSUFFICIENT_LABELS" + +class InsufficientLinesException(Exception): + def __init__(self): + self.message = f"An insufficient number of lines was detected." + self.code = "INSUFFICIENT_LINES" + +class UndefinedPTAException(Exception): + def __init__(self): + self.message = f"Something prevented the calculation of a pure tone average. Verify that all the thresholds are available." + self.code = "UNDEFINED_PTA_EXCEPTION" + +class MixedEarsException(Exception): + def __init__(self, feature: str): + self.message = f"You attempted to compute the {feature} of the audiogram, but provided thresholds coming from both ears. Ensure that only thresholds from one ear are provided with the ThresholdSet." + self.code = "MIXED_EARS_EXCEPTION" diff --git a/src/utils/geometry.py b/src/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..f65b07325037601de70373a0f6ae004a66a98828 --- /dev/null +++ b/src/utils/geometry.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import List + +import numpy as np + +from digitizer.report_components.line import Line + +def compute_deviation_sum(angle: float, lines: List[Line]) -> float: + """Given a candidate angle and a list of lines, computes the sum of the + deviation of these lines from the horizontal or vertical axis. + + Parameters + ---------- + angle : float + The candidate angle in degrees. + lines : List[Line] + All the lines used in computing the sum of deviation. + + Returns + ------- + float + The sum of the deviations from all lines with the vertical or horizontal + axis. + """ + residual_angle_sum = 0 + for line in lines: + if line.get_angle() > -45 and line.get_angle() < 45: + residual = abs((line.get_angle() - angle)) + residual_angle_sum += residual + else: + residual = 90 - abs(line.get_angle() - angle) + residual_angle_sum += residual + return abs(residual_angle_sum) + +def compute_rotation_angle(perpendicular_lines: List[Line]) -> float: + """Given a list of lines, returns the angle that must be applied to + the image so that lines that all lines that intersect another line + at roughly a right angle (+/- some tolerance) as close to vertical + or horizontal as possible. + + Parameters + ---------- + perpendicular_lines : List[Line} + The lines extracted from the image. These lines are expected to come + from an isolated audiogram grid. + + tolerance : float + Two lines intersecting at 90 +/- `tolerance` degrees are considered + perpendicular. + + Returns + ------- + float + The correction angle that must be applied to the image so as to + make perpendicular lines as close as possible to horizontal or vertical. + """ + + # Find the angle that minimizes the sum of distances to the nearest axis + #angle. + angle_range = np.arange(-44, 44, step=0.5) + errors = [ + compute_deviation_sum(angle, perpendicular_lines) + for angle in angle_range + ] + correction_angle = angle_range[np.argmin(errors)] + + return correction_angle + +def apply_rotation(point: dict, rotation_angle: float) -> dict: + new_x = (np.cos(rotation_angle) * point["x"] + - np.sin(rotation_angle) * -point["y"]) + new_y = (np.sin(rotation_angle) * point["x"] + + np.cos(rotation_angle) * -point["y"]) + return { **point, "x": new_x, "y": -new_y } + + +def get_bounding_box_relative_to_original_report(bounding_box, audiogram_coordinates, correction_angle): + + absolute_corrected_coordinates = { + "x": bounding_box["x"] + audiogram_coordinates["x"], + "y": bounding_box["y"] + audiogram_coordinates["y"] + } + + raw_coordinates = apply_rotation(absolute_corrected_coordinates, -correction_angle) + correction_angle_rad = np.radians(correction_angle) + + side_length = bounding_box["width"] * np.sin(correction_angle_rad) + bounding_box["width"] * np.cos(correction_angle_rad) + if correction_angle_rad <= 0: + return { + "x": absolute_corrected_coordinates["x"] - bounding_box["width"] * np.sin(correction_angle_rad), + "y": absolute_corrected_coordinates["y"], + "width": side_length, + "height": side_length + } + else: + return { + "x": absolute_corrected_coordinates["x"], + "y": absolute_corrected_coordinates["y"] - bounding_box["height"] * np.sin(correction_angle_rad), + "width": side_length, + "height": side_length + } diff --git a/src/utils/plotting.py b/src/utils/plotting.py new file mode 100644 index 0000000000000000000000000000000000000000..72a398eb006fe01e58c3a82e715d42670304aad0 --- /dev/null +++ b/src/utils/plotting.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2020, Carleton University Biomedical Informatics Collaboratory + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. +""" +from typing import List +import os.path as path +import io + +import PIL.Image +import matplotlib.pyplot as plt +from matplotlib.offsetbox import OffsetImage, AnnotationBbox +import matplotlib + +from interfaces import Threshold + +# Path to the folder containing the symbols that get plotted +SYMBOLS_DIR = path.join(path.dirname(__file__), "..", "assets", "symbols") + +def figure_to_image(figure: matplotlib.pyplot.figure, size, dpi=300): + """Converts a matplotlib figure to a PIL Image. + + Parameters + ---------- + figure : matplotlib.pyplot.figure + The figure to convert to an image. + + size: tuple + Dimensions of the image in pixels. + + dpi: int + Resolution (default: 300) + + Returns + ------- + PIL.Image + The image of the plot in PIL format. + """ + # Save the figure to a buffer + image_buffer = io.BytesIO() + figure.set_size_inches((size[0]/dpi, size[1]/dpi)) + figure.savefig(image_buffer, dpi=dpi) + + # Open image from buffer + image_buffer.seek(0) + image = PIL.Image.open(image_buffer) + + return image + +def plot_audiogram(thresholds: List[Threshold]) -> plt.figure: + """Given a list of threshold dictionaries, plots the audiogram. + + Parameters + ---------- + thresholds : List[Threshold] + A list of dictionaries that implement the `Threshold` interface. + + Returns + ------- + matplotlib.pyplot.figure + The `Figure` object corresponding to the audiogram. + """ + # Plot setup + fig = plt.figure() + ax = plt.gca() + + # Setup axes + plt.xscale("log") + ax.set_xlim(125, 8000) # Standard audios go from 125 to 8000 Hz + ax.set_ylim(-20, 120) # Most go from -10 to 120, but let's start at -20, in case + plt.gca().invert_yaxis() + + # Add the ticks + ax.set_xticks([125, 250, 500, 1000, 2000, 4000, 8000]) # Octave frequencies + ax.set_yticks(list(range(-20, 120, 10))) # All multiples of 10 + ax.get_xaxis().set_major_formatter( + matplotlib.ticker.FormatStrFormatter("%.0f") + ) + + # Show the grid + plt.grid() + + # Iterate through the different symbols + for ear in ("left", "right"): + for masking in (True, False): + for conduction in ("air", "bone"): + # Filter out all other symbols to generate an individual curve + curve = [ + threshold + for threshold + in thresholds + if threshold["ear"] == ear + and threshold["masking"] == masking + and threshold["conduction"] == conduction + ] + + # Sort the thresholds on the curve in order of frequency + curve = sorted(curve, key=lambda t: t["frequency"]) + freq = [t["frequency"] for t in curve] + threshold = [t["threshold"] for t in curve] + + # For every threshold (freq, thresh) belonging to the curve + # considered in this loop iteration, add a symbol on the plot. + for (f, t) in zip(freq, threshold): + icon_name = f"{ear}_{conduction}_{'masked' if masking else 'unmasked'}.png" + icon_img = plt.imread(path.join(SYMBOLS_DIR, icon_name)) + icon = AnnotationBbox( + OffsetImage(icon_img, zoom=0.1), + (f, t), + frameon=False + ) + ax.add_artist(icon) + + return fig