diff --git a/.gitattributes b/.gitattributes index 944335fb8799914bc97819dcc025330f32dc942a..d5cb953a22451f8b33d2588020afe02a9170eada 100644 --- a/.gitattributes +++ b/.gitattributes @@ -85,3 +85,4 @@ result/RESULTS_FOLDER_VICCLE_1S_WITHOUT_PRETRAINED_MODEL_NOT_1000/nnUNet_ext/2d/ XCBCL/required_folder/RESULTS_FOLDER_PRETRAINED_1_SUPPORT_OFFICIAL/nnUNet_ext/2d/Task125_All/pretrained-viccle/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_CBCL_UNet/SEQ/1_support/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text pretrained-viccle/RESULTS_FOLDER_PRETRAINED_1_SUPPORT_OFFICIAL/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_CBCL_UNet/SEQ/1_support/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text pretrained-viccle/RESULTS_FOLDER_PRETRAINED_5_SUPPORT_OFFICIAL/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_CBCL_UNet/SEQ/5_support/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text +pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/debug.json b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/debug.json new file mode 100644 index 0000000000000000000000000000000000000000..9d93b0a0348fc8efad6d3380b19f82e784b1e604 --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/debug.json @@ -0,0 +1,118 @@ +{ + "LSA": "False", + "SPT": "False", + "ViT_task_specific_ln": "False", + "all_tr_losses": "[]", + "all_val_eval_metrics": "[]", + "all_val_losses": "[]", + "all_val_losses_tr_mode": "[]", + "already_trained_on": "{'0': {'finished_training_on': [], 'start_training_on': 'Task125_All', 'finished_validation_on': [], 'used_identifier': 'nnUNetPlansv2.1', 'prev_trainer': ['nnUNetTrainerSequential'], 'val_metrics_should_exist': False, 'checkpoint_should_exist': False, 'tasks_at_time_of_checkpoint': [], 'active_task_at_time_of_checkpoint': None, 'used_split': ['seg_outputs']}}", + "also_val_in_tr_mode": "False", + "amp_grad_scaler": "None", + "base_num_features": "32", + "basic_generator_patch_size": "[451 451]", + "batch_dice": "True", + "batch_size": "15", + "best_MA_tr_loss_for_patience": "None", + "best_epoch_based_on_MA_tr_loss": "None", + "best_val_eval_criterion_MA": "None", + "classes": "[1]", + "conv_per_stage": "2", + "csv": "True", + "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 200.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-3.141592653589793, 3.141592653589793), 'rotation_y': (-0.0, 0.0), 'rotation_z': (-0.0, 0.0), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([384, 384])}", + "dataset_directory": "/home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All", + "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5], [0.25, 0.25], [0.125, 0.125], [0.0625, 0.0625], [0.03125, 0.03125]]", + "del_log": "False", + "deterministic": "False", + "dl_tr": "", + "dl_val": "", + "do_dummy_2D_aug": "False", + "ds_loss_weights": "[0.51612903 0.25806452 0.12903226 0.06451613 0.03225806 0. ]", + "epoch": "0", + "eval_batch": "True", + "experiment_name": "nnUNetTrainerSequential", + "extension": "sequential", + "fold": "0", + "folder_with_preprocessed_data": "/home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/nnUNetData_plans_v2.1_2D_stage0", + "fp16": "True", + "gt_niftis_folder": "/home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/gt_segmentations", + "identifier": "nnUNetPlansv2.1", + "inference_pad_border_mode": "constant", + "inference_pad_kwargs": "{'constant_values': 0}", + "init_args": "('seg_outputs', 'Task125_All', '/home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/nnUNetPlansv2.1_plans_2D.pkl', 0, '/home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ', '/home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All', True, 0, True, False, True, 50, {'0': {'finished_training_on': [], 'start_training_on': 'Task125_All', 'finished_validation_on': [], 'used_identifier': 'nnUNetPlansv2.1', 'prev_trainer': ['nnUNetTrainerSequential'], 'val_metrics_should_exist': False, 'checkpoint_should_exist': False, 'tasks_at_time_of_checkpoint': [], 'active_task_at_time_of_checkpoint': None, 'used_split': ['seg_outputs']}}, True, 'nnUNetPlansv2.1', 'sequential', (['Task125_All'], '_'), True, True, False, False, 1, False, 'base', 1, False, True, False, False, False)", + "initial_lr": "0.01", + "log_file": "/home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/training_log_2024_7_17_10_54_28.txt", + "lr_scheduler": "None", + "lr_scheduler_eps": "0.001", + "lr_scheduler_patience": "30", + "lr_threshold": "1", + "max_num_epochs": "250", + "min_region_size_per_class": "None", + "min_size_per_class": "None", + "mixed_precision": "True", + "n_support": "1", + "net_conv_kernel_sizes": "[[3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]]", + "net_num_pool_op_kernel_sizes": "[[2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2]]", + "net_pool_per_axis": "[6, 6]", + "network_name": "2d", + "new_trainer": "True", + "normalization_schemes": "OrderedDict([(0, 'nonCT')])", + "num_batches_per_epoch": "250", + "num_classes": "2", + "num_input_channels": "1", + "num_val_batches_per_epoch": "50", + "online_eval_fn": "[]", + "online_eval_foreground_dc": "[]", + "online_eval_fp": "[]", + "online_eval_tp": "[]", + "only_keep_largest_connected_component": "None", + "optimizer": "SGD (\nParameter Group 0\n dampening: 0\n differentiable: False\n foreach: None\n lr: 0.01\n maximize: False\n momentum: 0.99\n nesterov: True\n weight_decay: 3e-05\n)", + "output_folder": "/home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0", + "output_folder_base": "/home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ", + "output_folder_orig": "/home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1", + "oversample_foreground_percent": "0.33", + "pad_all_sides": "None", + "param_split": "False", + "patch_size": "[384 384]", + "patience": "50", + "pin_memory": "True", + "plans_file": "/home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/nnUNetPlansv2.1_plans_2D.pkl", + "regions_class_order": "None", + "save_best_checkpoint": "True", + "save_every": "50", + "save_final_checkpoint": "True", + "save_intermediate_checkpoints": "True", + "save_latest_only": "True", + "split": "seg_outputs", + "split_gpu": "False", + "stage": "0", + "subject_names_raw": "[]", + "task": "Task125_All", + "tasks_joined_name": "Task125_All", + "tasks_list_with_char": "(['Task125_All'], '_')", + "threeD": "False", + "tr_gen": "", + "train_loss_MA": "None", + "train_loss_MA_alpha": "0.93", + "train_loss_MA_eps": "0.0005", + "trained_on_path": "/home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/metadata/Generic_UNet/SEQ", + "trainer_class_name": "nnUNetTrainerSequential", + "trainer_model": "None", + "trainer_path": "None", + "transfer_heads": "True", + "transpose_backward": "[0, 1, 2]", + "transpose_forward": "[0, 1, 2]", + "unpack_data": "True", + "use_cbcl": "False", + "use_mask_for_norm": "OrderedDict([(0, False)])", + "use_progress_bar": "True", + "use_vit": "False", + "val_eval_criterion_MA": "None", + "val_eval_criterion_alpha": "0.9", + "val_gen": "", + "validation_results": "{}", + "version": "V1", + "vit_type": "base", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_best.model b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_best.model new file mode 100644 index 0000000000000000000000000000000000000000..bce8c6427d68be15eca7f70b8f34a46c1e217fcf --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_best.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6d015f69d2d5d1c79944ed25dc18e9c55e63d23e91c1f06265c59989e1550ce +size 359768930 diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_best.model.pkl b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_best.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..13358aed1d30afe00bf0a2aca22f9eb302ed966e --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_best.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44dec77b3ff12a78c78cbeada530f2f3ef8997931d80b833592cf6cbcd5e24b0 +size 22662 diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_final_checkpoint.model b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_final_checkpoint.model new file mode 100644 index 0000000000000000000000000000000000000000..02add39e7b668d922c91c24a630b795676f9037b --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_final_checkpoint.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1555574d7a0d47288e2fee47318e7950889548fd212651a51eebb2ee229530bc +size 359773374 diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_final_checkpoint.model.pkl b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_final_checkpoint.model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..9460724a4ff32160a030523b7f5cd3c2fedc1ab7 --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/model_final_checkpoint.model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e725cc5e0d8403fcc1c0736060343fba1d30359560f46829922bebdd2134f46 +size 22664 diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/progress.png b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/progress.png new file mode 100644 index 0000000000000000000000000000000000000000..0df161f1e3d44d0f67af32fbda156e3dc361784c --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/progress.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b8405e10db6322f76fa08778f5978cb4c2370c1721f3486c7d592199aec979 +size 201728 diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/training_log_2024_7_17_10_54_28.txt b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/training_log_2024_7_17_10_54_28.txt new file mode 100644 index 0000000000000000000000000000000000000000..8c836bff3a62fbe1c9655b58b2cec5d3c21b5937 --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/training_log_2024_7_17_10_54_28.txt @@ -0,0 +1,3787 @@ +Starting... +2024-07-17 10:54:28.456104: /home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1 +2024-07-17 10:54:28.456518: /home/lhbac/media/XCBCL/required_folder/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All +2024-07-17 10:54:28.674161: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 10:54:28.674362: The split file contains 5 splits. +2024-07-17 10:54:28.674400: Desired fold for training: 0 +2024-07-17 10:54:28.674428: This split has 74 training and 20 validation cases. +2024-07-17 10:54:28.773744: TRAINING KEYS: + odict_keys(['121_26', '121_27', '121_28', '121_29', '121_30', '121_31', '121_33', '121_35', '121_36', '121_37', '122_00', '122_01', '122_02', '122_04', '122_05', '122_06', '122_07', '122_09', '122_10', '122_11', '122_12', '122_14', '122_16', '122_17', '122_18', '123_00', '123_01', '123_02', '123_04', '123_05', '123_06', '123_07', '123_09', '123_10', '123_11', '123_13', '123_14', '123_16', '123_17', '123_19', '123_20', '123_21', '123_22', '123_23', '123_24', '123_25', '123_27', '123_28', '123_29', '124_01', '124_02', '124_06', '124_07', '124_10', '124_13', '124_16', '124_17', '124_18', '124_21', '124_24', '124_28', '124_29', '124_32', '124_34', '124_35', '124_37', '124_38', '124_39', '124_40', '124_41', '124_43', '124_44', '124_46', '124_47']) +2024-07-17 10:54:28.773924: VALIDATION KEYS: + odict_keys(['121_01', '121_32', '121_34', '122_03', '122_08', '122_13', '122_15', '123_03', '123_08', '123_12', '123_15', '123_18', '123_26', '124_00', '124_04', '124_14', '124_20', '124_25', '124_31', '124_42']) +2024-07-17 10:54:31.128164: lr: 0.01 +2024-07-17 10:54:39.111287: Unable to plot network architecture: +2024-07-17 10:54:39.112001: module 'torch.onnx' has no attribute '_optimize_trace' +2024-07-17 10:54:39.112046: +printing the network instead: + +2024-07-17 10:54:39.112082: Generic_UNet( + (conv_blocks_context): ModuleList( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (2): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (3): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (4): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(256, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (5): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (6): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + ) + (td): ModuleList() + (tu): ModuleList( + (0): ConvTranspose2d(480, 480, kernel_size=(2, 2), stride=(2, 2), bias=False) + (1): ConvTranspose2d(480, 480, kernel_size=(2, 2), stride=(2, 2), bias=False) + (2): ConvTranspose2d(480, 256, kernel_size=(2, 2), stride=(2, 2), bias=False) + (3): ConvTranspose2d(256, 128, kernel_size=(2, 2), stride=(2, 2), bias=False) + (4): ConvTranspose2d(128, 64, kernel_size=(2, 2), stride=(2, 2), bias=False) + (5): ConvTranspose2d(64, 32, kernel_size=(2, 2), stride=(2, 2), bias=False) + ) + (conv_blocks_localization): ModuleList( + (0): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(960, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (1): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(960, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (2): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(512, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (3): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(256, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (4): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (5): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(64, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + ) + (seg_outputs): ModuleList( + (0): Conv2d(480, 2, kernel_size=(1, 1), stride=(1, 1), bias=False) + (1): Conv2d(480, 2, kernel_size=(1, 1), stride=(1, 1), bias=False) + (2): Conv2d(256, 2, kernel_size=(1, 1), stride=(1, 1), bias=False) + (3): Conv2d(128, 2, kernel_size=(1, 1), stride=(1, 1), bias=False) + (4): Conv2d(64, 2, kernel_size=(1, 1), stride=(1, 1), bias=False) + (5): Conv2d(32, 2, kernel_size=(1, 1), stride=(1, 1), bias=False) + ) +) +2024-07-17 10:54:39.115345: + +2024-07-17 10:54:39.229547: +epoch: 0 +2024-07-17 10:55:22.706467: train loss : -0.1978 +2024-07-17 10:55:26.306186: validation loss: -0.6186 +2024-07-17 10:55:26.307951: Average global foreground Dice: [0.4561] +2024-07-17 10:55:26.308027: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:55:26.938760: lr: 0.009964 +2024-07-17 10:55:26.939006: This epoch took 47.708380 s + +2024-07-17 10:55:26.939074: +epoch: 1 +2024-07-17 10:56:01.145356: train loss : -0.6150 +2024-07-17 10:56:04.734845: validation loss: -0.7430 +2024-07-17 10:56:04.735802: Average global foreground Dice: [0.7608] +2024-07-17 10:56:04.736475: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:56:05.378831: lr: 0.009928 +2024-07-17 10:56:05.516945: saving checkpoint... +2024-07-17 10:56:06.144519: done, saving took 0.76 seconds +2024-07-17 10:56:06.162321: New best epoch (train loss MA): -0.2270 +2024-07-17 10:56:06.162580: Best loss so far: -0.1978 +2024-07-17 10:56:06.162613: eps: 0.0005 +2024-07-17 10:56:06.162645: Best Epoch 0 +2024-07-17 10:56:06.162674: current epoch is 1 +2024-07-17 10:56:06.162714: This epoch took 39.223577 s + +2024-07-17 10:56:06.162750: +epoch: 2 +2024-07-17 10:56:41.014353: train loss : -0.7090 +2024-07-17 10:56:44.857882: validation loss: -0.7828 +2024-07-17 10:56:44.876981: Average global foreground Dice: [0.8119] +2024-07-17 10:56:44.877103: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:56:45.805909: lr: 0.009892 +2024-07-17 10:56:45.897837: saving checkpoint... +2024-07-17 10:56:46.494560: done, saving took 0.68 seconds +2024-07-17 10:56:47.187368: New best epoch (train loss MA): -0.2608 +2024-07-17 10:56:47.187575: Best loss so far: -0.2270 +2024-07-17 10:56:47.187608: eps: 0.0005 +2024-07-17 10:56:47.187635: Best Epoch 1 +2024-07-17 10:56:47.187658: current epoch is 2 +2024-07-17 10:56:47.187700: This epoch took 41.024914 s + +2024-07-17 10:56:47.187732: +epoch: 3 +2024-07-17 10:57:16.917812: train loss : -0.7496 +2024-07-17 10:57:20.625887: validation loss: -0.7828 +2024-07-17 10:57:20.626749: Average global foreground Dice: [0.834] +2024-07-17 10:57:20.626864: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:57:21.339396: lr: 0.009856 +2024-07-17 10:57:21.436316: saving checkpoint... +2024-07-17 10:57:22.163045: done, saving took 0.82 seconds +2024-07-17 10:57:22.704573: New best epoch (train loss MA): -0.2950 +2024-07-17 10:57:22.704736: Best loss so far: -0.2608 +2024-07-17 10:57:22.704793: eps: 0.0005 +2024-07-17 10:57:22.704842: Best Epoch 2 +2024-07-17 10:57:22.704895: current epoch is 3 +2024-07-17 10:57:22.704952: This epoch took 35.517187 s + +2024-07-17 10:57:22.705005: +epoch: 4 +2024-07-17 10:57:56.623772: train loss : -0.7708 +2024-07-17 10:58:00.185766: validation loss: -0.8084 +2024-07-17 10:58:00.186900: Average global foreground Dice: [0.8492] +2024-07-17 10:58:00.187304: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:58:00.920780: lr: 0.00982 +2024-07-17 10:58:01.011529: saving checkpoint... +2024-07-17 10:58:01.608513: done, saving took 0.69 seconds +2024-07-17 10:58:02.130819: New best epoch (train loss MA): -0.3283 +2024-07-17 10:58:02.131046: Best loss so far: -0.2950 +2024-07-17 10:58:02.131113: eps: 0.0005 +2024-07-17 10:58:02.131173: Best Epoch 3 +2024-07-17 10:58:02.131232: current epoch is 4 +2024-07-17 10:58:02.131317: This epoch took 39.426245 s + +2024-07-17 10:58:02.131412: +epoch: 5 +2024-07-17 10:58:35.151998: train loss : -0.7922 +2024-07-17 10:58:38.875069: validation loss: -0.7747 +2024-07-17 10:58:38.876152: Average global foreground Dice: [0.8528] +2024-07-17 10:58:38.876218: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:58:39.612820: lr: 0.009784 +2024-07-17 10:58:39.705933: saving checkpoint... +2024-07-17 10:58:40.637398: done, saving took 1.02 seconds +2024-07-17 10:58:40.827192: New best epoch (train loss MA): -0.3608 +2024-07-17 10:58:40.827288: Best loss so far: -0.3283 +2024-07-17 10:58:40.827370: eps: 0.0005 +2024-07-17 10:58:40.827435: Best Epoch 4 +2024-07-17 10:58:40.827494: current epoch is 5 +2024-07-17 10:58:40.827575: This epoch took 38.696063 s + +2024-07-17 10:58:40.827651: +epoch: 6 +2024-07-17 10:59:15.117521: train loss : -0.7845 +2024-07-17 10:59:18.976136: validation loss: -0.8221 +2024-07-17 10:59:18.977366: Average global foreground Dice: [0.8555] +2024-07-17 10:59:18.977479: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:59:19.533743: lr: 0.009748 +2024-07-17 10:59:19.661237: saving checkpoint... +2024-07-17 10:59:20.498938: done, saving took 0.96 seconds +2024-07-17 10:59:20.567262: New best epoch (train loss MA): -0.3904 +2024-07-17 10:59:20.567340: Best loss so far: -0.3608 +2024-07-17 10:59:20.567373: eps: 0.0005 +2024-07-17 10:59:20.567401: Best Epoch 5 +2024-07-17 10:59:20.567426: current epoch is 6 +2024-07-17 10:59:20.567461: This epoch took 39.739721 s + +2024-07-17 10:59:20.567490: +epoch: 7 +2024-07-17 10:59:54.039888: train loss : -0.8085 +2024-07-17 10:59:57.406724: validation loss: -0.8216 +2024-07-17 10:59:57.408374: Average global foreground Dice: [0.8697] +2024-07-17 10:59:57.408510: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 10:59:58.009818: lr: 0.009712 +2024-07-17 10:59:58.066754: saving checkpoint... +2024-07-17 10:59:58.878222: done, saving took 0.87 seconds +2024-07-17 10:59:59.326782: New best epoch (train loss MA): -0.4197 +2024-07-17 10:59:59.326971: Best loss so far: -0.3904 +2024-07-17 10:59:59.327011: eps: 0.0005 +2024-07-17 10:59:59.327047: Best Epoch 6 +2024-07-17 10:59:59.327081: current epoch is 7 +2024-07-17 10:59:59.327133: This epoch took 38.759608 s + +2024-07-17 10:59:59.327170: +epoch: 8 +2024-07-17 11:00:28.720250: train loss : -0.8154 +2024-07-17 11:00:32.013889: validation loss: -0.8330 +2024-07-17 11:00:32.014842: Average global foreground Dice: [0.8736] +2024-07-17 11:00:32.014903: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:00:32.816020: lr: 0.009675 +2024-07-17 11:00:32.879963: saving checkpoint... +2024-07-17 11:00:33.628409: done, saving took 0.81 seconds +2024-07-17 11:00:34.083941: New best epoch (train loss MA): -0.4474 +2024-07-17 11:00:34.084152: Best loss so far: -0.4197 +2024-07-17 11:00:34.084194: eps: 0.0005 +2024-07-17 11:00:34.084232: Best Epoch 7 +2024-07-17 11:00:34.084267: current epoch is 8 +2024-07-17 11:00:34.084328: This epoch took 34.757113 s + +2024-07-17 11:00:34.084369: +epoch: 9 +2024-07-17 11:01:08.984574: train loss : -0.8265 +2024-07-17 11:01:12.419845: validation loss: -0.8452 +2024-07-17 11:01:12.420896: Average global foreground Dice: [0.8807] +2024-07-17 11:01:12.420963: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:01:13.128764: lr: 0.009639 +2024-07-17 11:01:13.192076: saving checkpoint... +2024-07-17 11:01:14.237840: done, saving took 1.11 seconds +2024-07-17 11:01:14.240067: New best epoch (train loss MA): -0.4739 +2024-07-17 11:01:14.240152: Best loss so far: -0.4474 +2024-07-17 11:01:14.240215: eps: 0.0005 +2024-07-17 11:01:14.240272: Best Epoch 8 +2024-07-17 11:01:14.240328: current epoch is 9 +2024-07-17 11:01:14.240394: This epoch took 40.155983 s + +2024-07-17 11:01:14.240462: +epoch: 10 +2024-07-17 11:01:48.958351: train loss : -0.8296 +2024-07-17 11:01:52.677905: validation loss: -0.8460 +2024-07-17 11:01:52.679004: Average global foreground Dice: [0.8834] +2024-07-17 11:01:52.679100: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:01:53.336743: lr: 0.009603 +2024-07-17 11:01:53.395089: saving checkpoint... +2024-07-17 11:01:54.271528: done, saving took 0.93 seconds +2024-07-17 11:01:54.316203: New best epoch (train loss MA): -0.4988 +2024-07-17 11:01:54.316444: Best loss so far: -0.4739 +2024-07-17 11:01:54.316516: eps: 0.0005 +2024-07-17 11:01:54.316576: Best Epoch 9 +2024-07-17 11:01:54.316634: current epoch is 10 +2024-07-17 11:01:54.316704: This epoch took 40.076154 s + +2024-07-17 11:01:54.316774: +epoch: 11 +2024-07-17 11:02:29.196952: train loss : -0.8315 +2024-07-17 11:02:32.705087: validation loss: -0.8414 +2024-07-17 11:02:32.737898: Average global foreground Dice: [0.8841] +2024-07-17 11:02:32.739395: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:02:33.431724: lr: 0.009567 +2024-07-17 11:02:33.497751: saving checkpoint... +2024-07-17 11:02:34.955381: done, saving took 1.52 seconds +2024-07-17 11:02:34.985847: New best epoch (train loss MA): -0.5221 +2024-07-17 11:02:34.985943: Best loss so far: -0.4988 +2024-07-17 11:02:34.985990: eps: 0.0005 +2024-07-17 11:02:34.986036: Best Epoch 10 +2024-07-17 11:02:34.986078: current epoch is 11 +2024-07-17 11:02:34.986141: This epoch took 40.669259 s + +2024-07-17 11:02:34.986188: +epoch: 12 +2024-07-17 11:03:09.334155: train loss : -0.8418 +2024-07-17 11:03:12.716980: validation loss: -0.8327 +2024-07-17 11:03:12.718306: Average global foreground Dice: [0.8877] +2024-07-17 11:03:12.718406: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:03:13.671995: lr: 0.009531 +2024-07-17 11:03:13.752682: saving checkpoint... +2024-07-17 11:03:14.532569: done, saving took 0.86 seconds +2024-07-17 11:03:14.807531: New best epoch (train loss MA): -0.5445 +2024-07-17 11:03:14.807764: Best loss so far: -0.5221 +2024-07-17 11:03:14.807800: eps: 0.0005 +2024-07-17 11:03:14.807827: Best Epoch 11 +2024-07-17 11:03:14.807853: current epoch is 12 +2024-07-17 11:03:14.807905: This epoch took 39.821646 s + +2024-07-17 11:03:14.807936: +epoch: 13 +2024-07-17 11:03:44.127066: train loss : -0.8514 +2024-07-17 11:03:47.460063: validation loss: -0.8546 +2024-07-17 11:03:47.461234: Average global foreground Dice: [0.8979] +2024-07-17 11:03:47.461465: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:03:48.132903: lr: 0.009495 +2024-07-17 11:03:48.207608: saving checkpoint... +2024-07-17 11:03:48.839901: done, saving took 0.71 seconds +2024-07-17 11:03:49.436115: New best epoch (train loss MA): -0.5660 +2024-07-17 11:03:49.436287: Best loss so far: -0.5445 +2024-07-17 11:03:49.436319: eps: 0.0005 +2024-07-17 11:03:49.436348: Best Epoch 12 +2024-07-17 11:03:49.436373: current epoch is 13 +2024-07-17 11:03:49.436405: This epoch took 34.628437 s + +2024-07-17 11:03:49.436434: +epoch: 14 +2024-07-17 11:04:24.169833: train loss : -0.8515 +2024-07-17 11:04:27.600557: validation loss: -0.8413 +2024-07-17 11:04:27.601862: Average global foreground Dice: [0.8958] +2024-07-17 11:04:27.601927: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:04:28.577995: lr: 0.009458 +2024-07-17 11:04:28.651920: saving checkpoint... +2024-07-17 11:04:29.378594: done, saving took 0.80 seconds +2024-07-17 11:04:29.749434: New best epoch (train loss MA): -0.5860 +2024-07-17 11:04:29.749707: Best loss so far: -0.5660 +2024-07-17 11:04:29.749774: eps: 0.0005 +2024-07-17 11:04:29.749815: Best Epoch 13 +2024-07-17 11:04:29.749851: current epoch is 14 +2024-07-17 11:04:29.749926: This epoch took 40.313458 s + +2024-07-17 11:04:29.749969: +epoch: 15 +2024-07-17 11:05:04.388653: train loss : -0.8543 +2024-07-17 11:05:07.653921: validation loss: -0.8447 +2024-07-17 11:05:07.655103: Average global foreground Dice: [0.8965] +2024-07-17 11:05:07.655173: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:05:08.414484: lr: 0.009422 +2024-07-17 11:05:08.474927: saving checkpoint... +2024-07-17 11:05:09.362976: done, saving took 0.95 seconds +2024-07-17 11:05:09.916251: New best epoch (train loss MA): -0.6048 +2024-07-17 11:05:09.916524: Best loss so far: -0.5860 +2024-07-17 11:05:09.916595: eps: 0.0005 +2024-07-17 11:05:09.916659: Best Epoch 14 +2024-07-17 11:05:09.916722: current epoch is 15 +2024-07-17 11:05:09.916803: This epoch took 40.166791 s + +2024-07-17 11:05:09.916881: +epoch: 16 +2024-07-17 11:05:44.786275: train loss : -0.8653 +2024-07-17 11:05:48.025484: validation loss: -0.8578 +2024-07-17 11:05:48.027797: Average global foreground Dice: [0.9049] +2024-07-17 11:05:48.027882: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:05:48.819158: lr: 0.009386 +2024-07-17 11:05:48.990721: saving checkpoint... +2024-07-17 11:05:49.863653: done, saving took 1.04 seconds +2024-07-17 11:05:50.336281: New best epoch (train loss MA): -0.6230 +2024-07-17 11:05:50.336587: Best loss so far: -0.6048 +2024-07-17 11:05:50.336656: eps: 0.0005 +2024-07-17 11:05:50.336718: Best Epoch 15 +2024-07-17 11:05:50.336779: current epoch is 16 +2024-07-17 11:05:50.336870: This epoch took 40.419891 s + +2024-07-17 11:05:50.336949: +epoch: 17 +2024-07-17 11:06:24.159167: train loss : -0.8696 +2024-07-17 11:06:27.516360: validation loss: -0.8637 +2024-07-17 11:06:27.517479: Average global foreground Dice: [0.908] +2024-07-17 11:06:27.517585: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:06:28.229168: lr: 0.00935 +2024-07-17 11:06:28.300056: saving checkpoint... +2024-07-17 11:06:29.206645: done, saving took 0.97 seconds +2024-07-17 11:06:29.705178: New best epoch (train loss MA): -0.6403 +2024-07-17 11:06:29.705385: Best loss so far: -0.6230 +2024-07-17 11:06:29.706180: eps: 0.0005 +2024-07-17 11:06:29.706361: Best Epoch 16 +2024-07-17 11:06:29.706407: current epoch is 17 +2024-07-17 11:06:29.706465: This epoch took 39.369421 s + +2024-07-17 11:06:29.706511: +epoch: 18 +2024-07-17 11:06:59.849253: train loss : -0.8620 +2024-07-17 11:07:03.343248: validation loss: -0.8535 +2024-07-17 11:07:03.344426: Average global foreground Dice: [0.9028] +2024-07-17 11:07:03.344577: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:07:04.227816: lr: 0.009313 +2024-07-17 11:07:04.334992: saving checkpoint... +2024-07-17 11:07:05.292234: done, saving took 1.06 seconds +2024-07-17 11:07:05.367017: New best epoch (train loss MA): -0.6558 +2024-07-17 11:07:05.367098: Best loss so far: -0.6403 +2024-07-17 11:07:05.367146: eps: 0.0005 +2024-07-17 11:07:05.367190: Best Epoch 17 +2024-07-17 11:07:05.367230: current epoch is 18 +2024-07-17 11:07:05.367280: This epoch took 35.660722 s + +2024-07-17 11:07:05.367329: +epoch: 19 +2024-07-17 11:07:39.196161: train loss : -0.8636 +2024-07-17 11:07:42.701383: validation loss: -0.8542 +2024-07-17 11:07:42.702610: Average global foreground Dice: [0.9038] +2024-07-17 11:07:42.702724: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:07:43.503143: lr: 0.009277 +2024-07-17 11:07:43.562085: saving checkpoint... +2024-07-17 11:07:44.518152: done, saving took 1.01 seconds +2024-07-17 11:07:44.564402: New best epoch (train loss MA): -0.6703 +2024-07-17 11:07:44.564643: Best loss so far: -0.6558 +2024-07-17 11:07:44.564713: eps: 0.0005 +2024-07-17 11:07:44.564775: Best Epoch 18 +2024-07-17 11:07:44.564836: current epoch is 19 +2024-07-17 11:07:44.564904: This epoch took 39.197530 s + +2024-07-17 11:07:44.564978: +epoch: 20 +2024-07-17 11:08:18.920949: train loss : -0.8740 +2024-07-17 11:08:22.276672: validation loss: -0.8699 +2024-07-17 11:08:22.277911: Average global foreground Dice: [0.9118] +2024-07-17 11:08:22.277983: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:08:22.911122: lr: 0.009241 +2024-07-17 11:08:23.006420: saving checkpoint... +2024-07-17 11:08:23.890710: done, saving took 0.98 seconds +2024-07-17 11:08:23.893097: New best epoch (train loss MA): -0.6846 +2024-07-17 11:08:23.893186: Best loss so far: -0.6703 +2024-07-17 11:08:23.893250: eps: 0.0005 +2024-07-17 11:08:23.893306: Best Epoch 19 +2024-07-17 11:08:23.893360: current epoch is 20 +2024-07-17 11:08:23.893449: This epoch took 39.328377 s + +2024-07-17 11:08:23.893525: +epoch: 21 +2024-07-17 11:08:57.997053: train loss : -0.8802 +2024-07-17 11:09:01.392151: validation loss: -0.8703 +2024-07-17 11:09:01.401666: Average global foreground Dice: [0.9142] +2024-07-17 11:09:01.401778: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:09:02.270043: lr: 0.009204 +2024-07-17 11:09:02.338113: saving checkpoint... +2024-07-17 11:09:03.265300: done, saving took 0.99 seconds +2024-07-17 11:09:03.313600: New best epoch (train loss MA): -0.6983 +2024-07-17 11:09:03.313744: Best loss so far: -0.6846 +2024-07-17 11:09:03.313830: eps: 0.0005 +2024-07-17 11:09:03.313910: Best Epoch 20 +2024-07-17 11:09:03.313988: current epoch is 21 +2024-07-17 11:09:03.314077: This epoch took 39.420469 s + +2024-07-17 11:09:03.314172: +epoch: 22 +2024-07-17 11:09:35.995338: train loss : -0.8816 +2024-07-17 11:09:39.073385: validation loss: -0.8719 +2024-07-17 11:09:39.074885: Average global foreground Dice: [0.9155] +2024-07-17 11:09:39.074971: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:09:39.732893: lr: 0.009168 +2024-07-17 11:09:39.805129: saving checkpoint... +2024-07-17 11:09:40.699025: done, saving took 0.97 seconds +2024-07-17 11:09:40.747276: New best epoch (train loss MA): -0.7111 +2024-07-17 11:09:40.747390: Best loss so far: -0.6983 +2024-07-17 11:09:40.747444: eps: 0.0005 +2024-07-17 11:09:40.747496: Best Epoch 21 +2024-07-17 11:09:40.747544: current epoch is 22 +2024-07-17 11:09:40.747631: This epoch took 37.433345 s + +2024-07-17 11:09:40.747695: +epoch: 23 +2024-07-17 11:10:11.399928: train loss : -0.8775 +2024-07-17 11:10:14.901523: validation loss: -0.8642 +2024-07-17 11:10:14.903108: Average global foreground Dice: [0.9125] +2024-07-17 11:10:14.903178: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:10:15.529737: lr: 0.009132 +2024-07-17 11:10:15.597278: saving checkpoint... +2024-07-17 11:10:16.470735: done, saving took 0.94 seconds +2024-07-17 11:10:16.516623: New best epoch (train loss MA): -0.7228 +2024-07-17 11:10:16.516840: Best loss so far: -0.7111 +2024-07-17 11:10:16.516889: eps: 0.0005 +2024-07-17 11:10:16.516942: Best Epoch 22 +2024-07-17 11:10:16.516992: current epoch is 23 +2024-07-17 11:10:16.517050: This epoch took 35.769280 s + +2024-07-17 11:10:16.517106: +epoch: 24 +2024-07-17 11:10:49.740791: train loss : -0.8756 +2024-07-17 11:10:52.895252: validation loss: -0.8544 +2024-07-17 11:10:52.896882: Average global foreground Dice: [0.9105] +2024-07-17 11:10:52.896998: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:10:53.626002: lr: 0.009095 +2024-07-17 11:10:53.741414: saving checkpoint... +2024-07-17 11:10:54.439518: done, saving took 0.81 seconds +2024-07-17 11:10:55.057214: New best epoch (train loss MA): -0.7335 +2024-07-17 11:10:55.057531: Best loss so far: -0.7228 +2024-07-17 11:10:55.057592: eps: 0.0005 +2024-07-17 11:10:55.057641: Best Epoch 23 +2024-07-17 11:10:55.057686: current epoch is 24 +2024-07-17 11:10:55.057741: This epoch took 38.540576 s + +2024-07-17 11:10:55.057789: +epoch: 25 +2024-07-17 11:11:28.168350: train loss : -0.8776 +2024-07-17 11:11:31.487082: validation loss: -0.8661 +2024-07-17 11:11:31.488334: Average global foreground Dice: [0.9123] +2024-07-17 11:11:31.488410: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:11:32.210204: lr: 0.009059 +2024-07-17 11:11:32.277330: saving checkpoint... +2024-07-17 11:11:33.178001: done, saving took 0.96 seconds +2024-07-17 11:11:33.544656: New best epoch (train loss MA): -0.7435 +2024-07-17 11:11:33.544840: Best loss so far: -0.7335 +2024-07-17 11:11:33.544880: eps: 0.0005 +2024-07-17 11:11:33.544916: Best Epoch 24 +2024-07-17 11:11:33.544950: current epoch is 25 +2024-07-17 11:11:33.544994: This epoch took 38.487150 s + +2024-07-17 11:11:33.545030: +epoch: 26 +2024-07-17 11:12:06.982327: train loss : -0.8784 +2024-07-17 11:12:10.341575: validation loss: -0.8542 +2024-07-17 11:12:10.343049: Average global foreground Dice: [0.9116] +2024-07-17 11:12:10.343165: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:12:11.028817: lr: 0.009023 +2024-07-17 11:12:11.231471: saving checkpoint... +2024-07-17 11:12:11.991082: done, saving took 0.96 seconds +2024-07-17 11:12:12.145406: New best epoch (train loss MA): -0.7530 +2024-07-17 11:12:12.145675: Best loss so far: -0.7435 +2024-07-17 11:12:12.145778: eps: 0.0005 +2024-07-17 11:12:12.145853: Best Epoch 25 +2024-07-17 11:12:12.145925: current epoch is 26 +2024-07-17 11:12:12.146013: This epoch took 38.600937 s + +2024-07-17 11:12:12.146068: +epoch: 27 +2024-07-17 11:12:44.019980: train loss : -0.8764 +2024-07-17 11:12:47.132706: validation loss: -0.8417 +2024-07-17 11:12:47.133786: Average global foreground Dice: [0.9092] +2024-07-17 11:12:47.133879: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:12:47.734921: lr: 0.008986 +2024-07-17 11:12:47.791350: saving checkpoint... +2024-07-17 11:12:48.455476: done, saving took 0.72 seconds +2024-07-17 11:12:48.729741: New best epoch (train loss MA): -0.7616 +2024-07-17 11:12:48.730105: Best loss so far: -0.7530 +2024-07-17 11:12:48.730175: eps: 0.0005 +2024-07-17 11:12:48.730222: Best Epoch 26 +2024-07-17 11:12:48.730264: current epoch is 27 +2024-07-17 11:12:48.730326: This epoch took 36.584202 s + +2024-07-17 11:12:48.730377: +epoch: 28 +2024-07-17 11:13:19.027186: train loss : -0.8839 +2024-07-17 11:13:22.365039: validation loss: -0.8622 +2024-07-17 11:13:22.366244: Average global foreground Dice: [0.9157] +2024-07-17 11:13:22.366329: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:13:23.136051: lr: 0.00895 +2024-07-17 11:13:23.205548: saving checkpoint... +2024-07-17 11:13:24.010080: done, saving took 0.87 seconds +2024-07-17 11:13:24.423662: New best epoch (train loss MA): -0.7702 +2024-07-17 11:13:24.423906: Best loss so far: -0.7616 +2024-07-17 11:13:24.423970: eps: 0.0005 +2024-07-17 11:13:24.424070: Best Epoch 27 +2024-07-17 11:13:24.424143: current epoch is 28 +2024-07-17 11:13:24.424233: This epoch took 35.693792 s + +2024-07-17 11:13:24.424316: +epoch: 29 +2024-07-17 11:13:58.070033: train loss : -0.8871 +2024-07-17 11:14:01.310763: validation loss: -0.8740 +2024-07-17 11:14:01.311846: Average global foreground Dice: [0.9201] +2024-07-17 11:14:01.311906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:14:02.138175: lr: 0.008913 +2024-07-17 11:14:02.206461: saving checkpoint... +2024-07-17 11:14:03.064982: done, saving took 0.93 seconds +2024-07-17 11:14:03.399435: New best epoch (train loss MA): -0.7784 +2024-07-17 11:14:03.399657: Best loss so far: -0.7702 +2024-07-17 11:14:03.399697: eps: 0.0005 +2024-07-17 11:14:03.399734: Best Epoch 28 +2024-07-17 11:14:03.399769: current epoch is 29 +2024-07-17 11:14:03.399813: This epoch took 38.975402 s + +2024-07-17 11:14:03.399851: +epoch: 30 +2024-07-17 11:14:37.155807: train loss : -0.8861 +2024-07-17 11:14:40.717232: validation loss: -0.8679 +2024-07-17 11:14:40.718270: Average global foreground Dice: [0.9176] +2024-07-17 11:14:40.718340: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:14:41.273952: lr: 0.008877 +2024-07-17 11:14:41.338399: saving checkpoint... +2024-07-17 11:14:42.275244: done, saving took 1.00 seconds +2024-07-17 11:14:42.321340: New best epoch (train loss MA): -0.7859 +2024-07-17 11:14:42.321580: Best loss so far: -0.7784 +2024-07-17 11:14:42.321621: eps: 0.0005 +2024-07-17 11:14:42.321661: Best Epoch 29 +2024-07-17 11:14:42.321697: current epoch is 30 +2024-07-17 11:14:42.321742: This epoch took 38.921848 s + +2024-07-17 11:14:42.321780: +epoch: 31 +2024-07-17 11:15:16.735700: train loss : -0.8909 +2024-07-17 11:15:19.987261: validation loss: -0.8662 +2024-07-17 11:15:19.988349: Average global foreground Dice: [0.9201] +2024-07-17 11:15:19.988420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:15:20.597048: lr: 0.00884 +2024-07-17 11:15:20.684768: saving checkpoint... +2024-07-17 11:15:21.686108: done, saving took 1.09 seconds +2024-07-17 11:15:22.026046: New best epoch (train loss MA): -0.7933 +2024-07-17 11:15:22.026279: Best loss so far: -0.7859 +2024-07-17 11:15:22.026340: eps: 0.0005 +2024-07-17 11:15:22.026398: Best Epoch 30 +2024-07-17 11:15:22.026455: current epoch is 31 +2024-07-17 11:15:22.026517: This epoch took 39.704656 s + +2024-07-17 11:15:22.026577: +epoch: 32 +2024-07-17 11:15:54.915153: train loss : -0.8938 +2024-07-17 11:15:58.088018: validation loss: -0.8696 +2024-07-17 11:15:58.089122: Average global foreground Dice: [0.9232] +2024-07-17 11:15:58.089213: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:15:58.792108: lr: 0.008804 +2024-07-17 11:15:58.847592: saving checkpoint... +2024-07-17 11:15:59.699316: done, saving took 0.91 seconds +2024-07-17 11:15:59.701746: New best epoch (train loss MA): -0.8003 +2024-07-17 11:15:59.701879: Best loss so far: -0.7933 +2024-07-17 11:15:59.701945: eps: 0.0005 +2024-07-17 11:15:59.702001: Best Epoch 31 +2024-07-17 11:15:59.702054: current epoch is 32 +2024-07-17 11:15:59.702130: This epoch took 37.675393 s + +2024-07-17 11:15:59.702198: +epoch: 33 +2024-07-17 11:16:29.641786: train loss : -0.8954 +2024-07-17 11:16:33.226310: validation loss: -0.8735 +2024-07-17 11:16:33.227690: Average global foreground Dice: [0.9234] +2024-07-17 11:16:33.227770: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:16:33.835711: lr: 0.008767 +2024-07-17 11:16:33.892655: saving checkpoint... +2024-07-17 11:16:34.898470: done, saving took 1.06 seconds +2024-07-17 11:16:35.159099: New best epoch (train loss MA): -0.8070 +2024-07-17 11:16:35.159323: Best loss so far: -0.8003 +2024-07-17 11:16:35.159424: eps: 0.0005 +2024-07-17 11:16:35.159488: Best Epoch 32 +2024-07-17 11:16:35.159549: current epoch is 33 +2024-07-17 11:16:35.159620: This epoch took 35.457338 s + +2024-07-17 11:16:35.159693: +epoch: 34 +2024-07-17 11:17:09.792018: train loss : -0.8975 +2024-07-17 11:17:13.311177: validation loss: -0.8665 +2024-07-17 11:17:13.312538: Average global foreground Dice: [0.9241] +2024-07-17 11:17:13.312640: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:17:14.025239: lr: 0.008731 +2024-07-17 11:17:14.108505: saving checkpoint... +2024-07-17 11:17:15.100931: done, saving took 1.08 seconds +2024-07-17 11:17:15.318336: New best epoch (train loss MA): -0.8133 +2024-07-17 11:17:15.318548: Best loss so far: -0.8070 +2024-07-17 11:17:15.318591: eps: 0.0005 +2024-07-17 11:17:15.318629: Best Epoch 33 +2024-07-17 11:17:15.318662: current epoch is 34 +2024-07-17 11:17:15.318703: This epoch took 40.158918 s + +2024-07-17 11:17:15.318738: +epoch: 35 +2024-07-17 11:17:48.724032: train loss : -0.9046 +2024-07-17 11:17:52.207897: validation loss: -0.8714 +2024-07-17 11:17:52.208881: Average global foreground Dice: [0.9292] +2024-07-17 11:17:52.208945: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:17:53.071569: lr: 0.008694 +2024-07-17 11:17:53.126145: saving checkpoint... +2024-07-17 11:17:54.104726: done, saving took 1.03 seconds +2024-07-17 11:17:54.146195: New best epoch (train loss MA): -0.8197 +2024-07-17 11:17:54.146312: Best loss so far: -0.8133 +2024-07-17 11:17:54.146358: eps: 0.0005 +2024-07-17 11:17:54.146396: Best Epoch 34 +2024-07-17 11:17:54.146430: current epoch is 35 +2024-07-17 11:17:54.146475: This epoch took 38.827697 s + +2024-07-17 11:17:54.146512: +epoch: 36 +2024-07-17 11:18:27.926505: train loss : -0.9023 +2024-07-17 11:18:31.110019: validation loss: -0.8710 +2024-07-17 11:18:31.111073: Average global foreground Dice: [0.9268] +2024-07-17 11:18:31.111136: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:18:31.699357: lr: 0.008658 +2024-07-17 11:18:31.757716: saving checkpoint... +2024-07-17 11:18:32.919935: done, saving took 1.22 seconds +2024-07-17 11:18:32.935610: New best epoch (train loss MA): -0.8255 +2024-07-17 11:18:32.935702: Best loss so far: -0.8197 +2024-07-17 11:18:32.935765: eps: 0.0005 +2024-07-17 11:18:32.935824: Best Epoch 35 +2024-07-17 11:18:32.935881: current epoch is 36 +2024-07-17 11:18:32.935946: This epoch took 38.789394 s + +2024-07-17 11:18:32.936013: +epoch: 37 +2024-07-17 11:19:04.736282: train loss : -0.8905 +2024-07-17 11:19:07.877410: validation loss: -0.8593 +2024-07-17 11:19:07.878438: Average global foreground Dice: [0.9197] +2024-07-17 11:19:07.879493: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:19:08.598289: lr: 0.008621 +2024-07-17 11:19:08.661168: saving checkpoint... +2024-07-17 11:19:09.946692: done, saving took 1.35 seconds +2024-07-17 11:19:09.957232: New best epoch (train loss MA): -0.8300 +2024-07-17 11:19:09.957308: Best loss so far: -0.8255 +2024-07-17 11:19:09.957349: eps: 0.0005 +2024-07-17 11:19:09.957408: Best Epoch 36 +2024-07-17 11:19:09.957443: current epoch is 37 +2024-07-17 11:19:09.957486: This epoch took 37.021389 s + +2024-07-17 11:19:09.957523: +epoch: 38 +2024-07-17 11:19:38.581076: train loss : -0.8988 +2024-07-17 11:19:41.882330: validation loss: -0.8749 +2024-07-17 11:19:41.883440: Average global foreground Dice: [0.9259] +2024-07-17 11:19:41.883509: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:19:42.565817: lr: 0.008584 +2024-07-17 11:19:42.620677: saving checkpoint... +2024-07-17 11:19:43.683142: done, saving took 1.12 seconds +2024-07-17 11:19:43.687516: New best epoch (train loss MA): -0.8348 +2024-07-17 11:19:43.687619: Best loss so far: -0.8300 +2024-07-17 11:19:43.687681: eps: 0.0005 +2024-07-17 11:19:43.687738: Best Epoch 37 +2024-07-17 11:19:43.687793: current epoch is 38 +2024-07-17 11:19:43.687856: This epoch took 33.730290 s + +2024-07-17 11:19:43.687923: +epoch: 39 +2024-07-17 11:20:17.186377: train loss : -0.9045 +2024-07-17 11:20:20.401832: validation loss: -0.8738 +2024-07-17 11:20:20.403014: Average global foreground Dice: [0.9288] +2024-07-17 11:20:20.403115: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:20:21.120514: lr: 0.008548 +2024-07-17 11:20:21.175131: saving checkpoint... +2024-07-17 11:20:22.250646: done, saving took 1.13 seconds +2024-07-17 11:20:22.260323: New best epoch (train loss MA): -0.8397 +2024-07-17 11:20:22.260377: Best loss so far: -0.8348 +2024-07-17 11:20:22.260409: eps: 0.0005 +2024-07-17 11:20:22.260435: Best Epoch 38 +2024-07-17 11:20:22.260460: current epoch is 39 +2024-07-17 11:20:22.260491: This epoch took 38.572484 s + +2024-07-17 11:20:22.260518: +epoch: 40 +2024-07-17 11:20:55.848912: train loss : -0.8988 +2024-07-17 11:20:59.623227: validation loss: -0.8644 +2024-07-17 11:20:59.624281: Average global foreground Dice: [0.9248] +2024-07-17 11:20:59.624347: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:21:00.369073: lr: 0.008511 +2024-07-17 11:21:00.424670: saving checkpoint... +2024-07-17 11:21:01.140089: done, saving took 0.77 seconds +2024-07-17 11:21:01.141783: New best epoch (train loss MA): -0.8438 +2024-07-17 11:21:01.141843: Best loss so far: -0.8397 +2024-07-17 11:21:01.141879: eps: 0.0005 +2024-07-17 11:21:01.141907: Best Epoch 39 +2024-07-17 11:21:01.141932: current epoch is 40 +2024-07-17 11:21:01.141963: This epoch took 38.881417 s + +2024-07-17 11:21:01.141989: +epoch: 41 +2024-07-17 11:21:35.214489: train loss : -0.9026 +2024-07-17 11:21:38.747038: validation loss: -0.8716 +2024-07-17 11:21:38.748506: Average global foreground Dice: [0.9279] +2024-07-17 11:21:38.748675: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:21:39.325718: lr: 0.008474 +2024-07-17 11:21:39.385750: saving checkpoint... +2024-07-17 11:21:40.373200: done, saving took 1.05 seconds +2024-07-17 11:21:40.408957: New best epoch (train loss MA): -0.8480 +2024-07-17 11:21:40.409023: Best loss so far: -0.8438 +2024-07-17 11:21:40.409076: eps: 0.0005 +2024-07-17 11:21:40.409161: Best Epoch 40 +2024-07-17 11:21:40.409221: current epoch is 41 +2024-07-17 11:21:40.409289: This epoch took 39.267271 s + +2024-07-17 11:21:40.409360: +epoch: 42 +2024-07-17 11:22:12.146564: train loss : -0.8970 +2024-07-17 11:22:15.095912: validation loss: -0.8414 +2024-07-17 11:22:15.098244: Average global foreground Dice: [0.9215] +2024-07-17 11:22:15.099410: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:22:15.866857: lr: 0.008438 +2024-07-17 11:22:15.925323: saving checkpoint... +2024-07-17 11:22:16.850917: done, saving took 0.98 seconds +2024-07-17 11:22:16.965241: New best epoch (train loss MA): -0.8514 +2024-07-17 11:22:16.965430: Best loss so far: -0.8480 +2024-07-17 11:22:16.965466: eps: 0.0005 +2024-07-17 11:22:16.965496: Best Epoch 41 +2024-07-17 11:22:16.965522: current epoch is 42 +2024-07-17 11:22:16.965555: This epoch took 36.556082 s + +2024-07-17 11:22:16.965585: +epoch: 43 +2024-07-17 11:22:45.218285: train loss : -0.8883 +2024-07-17 11:22:48.776613: validation loss: -0.8622 +2024-07-17 11:22:48.783837: Average global foreground Dice: [0.9179] +2024-07-17 11:22:48.784043: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:22:49.437358: lr: 0.008401 +2024-07-17 11:22:49.494738: saving checkpoint... +2024-07-17 11:22:50.720128: done, saving took 1.28 seconds +2024-07-17 11:22:50.736046: New best epoch (train loss MA): -0.8540 +2024-07-17 11:22:50.736103: Best loss so far: -0.8514 +2024-07-17 11:22:50.736135: eps: 0.0005 +2024-07-17 11:22:50.736163: Best Epoch 42 +2024-07-17 11:22:50.736189: current epoch is 43 +2024-07-17 11:22:50.736222: This epoch took 33.770591 s + +2024-07-17 11:22:50.736251: +epoch: 44 +2024-07-17 11:23:23.687113: train loss : -0.9015 +2024-07-17 11:23:27.084106: validation loss: -0.8684 +2024-07-17 11:23:27.085119: Average global foreground Dice: [0.9274] +2024-07-17 11:23:27.085185: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:23:27.590192: lr: 0.008364 +2024-07-17 11:23:27.649756: saving checkpoint... +2024-07-17 11:23:28.310350: done, saving took 0.72 seconds +2024-07-17 11:23:28.844424: New best epoch (train loss MA): -0.8573 +2024-07-17 11:23:28.844633: Best loss so far: -0.8540 +2024-07-17 11:23:28.844673: eps: 0.0005 +2024-07-17 11:23:28.844709: Best Epoch 43 +2024-07-17 11:23:28.844741: current epoch is 44 +2024-07-17 11:23:28.844781: This epoch took 38.108499 s + +2024-07-17 11:23:28.844826: +epoch: 45 +2024-07-17 11:24:01.289350: train loss : -0.9035 +2024-07-17 11:24:04.599535: validation loss: -0.8674 +2024-07-17 11:24:04.600708: Average global foreground Dice: [0.927] +2024-07-17 11:24:04.600781: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:24:05.149904: lr: 0.008328 +2024-07-17 11:24:05.201273: saving checkpoint... +2024-07-17 11:24:05.912910: done, saving took 0.76 seconds +2024-07-17 11:24:06.404372: New best epoch (train loss MA): -0.8605 +2024-07-17 11:24:06.404611: Best loss so far: -0.8573 +2024-07-17 11:24:06.404646: eps: 0.0005 +2024-07-17 11:24:06.404676: Best Epoch 44 +2024-07-17 11:24:06.404727: current epoch is 45 +2024-07-17 11:24:06.404778: This epoch took 37.559910 s + +2024-07-17 11:24:06.404818: +epoch: 46 +2024-07-17 11:24:39.373985: train loss : -0.9047 +2024-07-17 11:24:42.785113: validation loss: -0.8253 +2024-07-17 11:24:42.796247: Average global foreground Dice: [0.9237] +2024-07-17 11:24:42.796560: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:24:43.474594: lr: 0.008291 +2024-07-17 11:24:43.552893: saving checkpoint... +2024-07-17 11:24:44.638496: done, saving took 1.16 seconds +2024-07-17 11:24:44.708682: New best epoch (train loss MA): -0.8636 +2024-07-17 11:24:44.708905: Best loss so far: -0.8605 +2024-07-17 11:24:44.708947: eps: 0.0005 +2024-07-17 11:24:44.708986: Best Epoch 45 +2024-07-17 11:24:44.709022: current epoch is 46 +2024-07-17 11:24:44.709067: This epoch took 38.304205 s + +2024-07-17 11:24:44.709105: +epoch: 47 +2024-07-17 11:25:15.201247: train loss : -0.9012 +2024-07-17 11:25:17.941933: validation loss: -0.8773 +2024-07-17 11:25:17.950497: Average global foreground Dice: [0.9284] +2024-07-17 11:25:17.953995: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:25:18.729040: lr: 0.008254 +2024-07-17 11:25:18.833169: saving checkpoint... +2024-07-17 11:25:19.980528: done, saving took 1.20 seconds +2024-07-17 11:25:20.059338: New best epoch (train loss MA): -0.8663 +2024-07-17 11:25:20.059427: Best loss so far: -0.8636 +2024-07-17 11:25:20.059460: eps: 0.0005 +2024-07-17 11:25:20.059497: Best Epoch 46 +2024-07-17 11:25:20.059523: current epoch is 47 +2024-07-17 11:25:20.059560: This epoch took 35.350413 s + +2024-07-17 11:25:20.059596: +epoch: 48 +2024-07-17 11:25:49.468490: train loss : -0.9067 +2024-07-17 11:25:52.553417: validation loss: -0.8661 +2024-07-17 11:25:52.584394: Average global foreground Dice: [0.9306] +2024-07-17 11:25:52.589288: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:25:53.122793: lr: 0.008217 +2024-07-17 11:25:53.173950: saving checkpoint... +2024-07-17 11:25:55.075216: done, saving took 1.95 seconds +2024-07-17 11:25:55.168423: New best epoch (train loss MA): -0.8691 +2024-07-17 11:25:55.168674: Best loss so far: -0.8663 +2024-07-17 11:25:55.168712: eps: 0.0005 +2024-07-17 11:25:55.168817: Best Epoch 47 +2024-07-17 11:25:55.168870: current epoch is 48 +2024-07-17 11:25:55.168939: This epoch took 35.109308 s + +2024-07-17 11:25:55.168971: +epoch: 49 +2024-07-17 11:26:28.440787: train loss : -0.9021 +2024-07-17 11:26:32.337913: validation loss: -0.8603 +2024-07-17 11:26:32.339154: Average global foreground Dice: [0.9262] +2024-07-17 11:26:32.339230: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:26:33.066300: lr: 0.008181 +2024-07-17 11:26:33.066526: saving scheduled checkpoint file... +2024-07-17 11:26:33.137298: saving checkpoint... +2024-07-17 11:26:33.711674: done, saving took 0.64 seconds +2024-07-17 11:26:33.712943: done +2024-07-17 11:26:33.822931: saving checkpoint... +2024-07-17 11:26:34.830903: done, saving took 1.06 seconds +2024-07-17 11:26:34.834514: New best epoch (train loss MA): -0.8714 +2024-07-17 11:26:34.834702: Best loss so far: -0.8691 +2024-07-17 11:26:34.834771: eps: 0.0005 +2024-07-17 11:26:34.834836: Best Epoch 48 +2024-07-17 11:26:34.834899: current epoch is 49 +2024-07-17 11:26:34.853492: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 11:26:34.853772: The split file contains 5 splits. +2024-07-17 11:26:34.853848: Desired fold for training: 0 +2024-07-17 11:26:34.853926: This split has 74 training and 20 validation cases. +2024-07-17 11:26:36.298952: Performing validation with validation data from task Task125_All. +2024-07-17 11:26:58.801184: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 11:26:58.801801: The split file contains 5 splits. +2024-07-17 11:26:58.801898: Desired fold for training: 0 +2024-07-17 11:26:58.801978: This split has 74 training and 20 validation cases. +2024-07-17 11:26:59.152141: This epoch took 63.983129 s + +2024-07-17 11:26:59.152253: +epoch: 50 +2024-07-17 11:27:35.101245: train loss : -0.9101 +2024-07-17 11:27:39.770152: validation loss: -0.8697 +2024-07-17 11:27:39.773642: Average global foreground Dice: [0.9318] +2024-07-17 11:27:39.781355: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:27:40.599991: lr: 0.008144 +2024-07-17 11:27:40.759769: saving checkpoint... +2024-07-17 11:27:41.462271: done, saving took 0.86 seconds +2024-07-17 11:27:41.464604: New best epoch (train loss MA): -0.8741 +2024-07-17 11:27:41.464658: Best loss so far: -0.8714 +2024-07-17 11:27:41.464688: eps: 0.0005 +2024-07-17 11:27:41.464721: Best Epoch 49 +2024-07-17 11:27:41.464747: current epoch is 50 +2024-07-17 11:27:41.464789: This epoch took 42.312494 s + +2024-07-17 11:27:41.464826: +epoch: 51 +2024-07-17 11:28:15.221724: train loss : -0.9104 +2024-07-17 11:28:18.391265: validation loss: -0.8693 +2024-07-17 11:28:18.392449: Average global foreground Dice: [0.9315] +2024-07-17 11:28:18.392535: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:28:19.218527: lr: 0.008107 +2024-07-17 11:28:19.467228: saving checkpoint... +2024-07-17 11:28:21.228869: done, saving took 1.99 seconds +2024-07-17 11:28:21.363268: New best epoch (train loss MA): -0.8766 +2024-07-17 11:28:21.363455: Best loss so far: -0.8741 +2024-07-17 11:28:21.363489: eps: 0.0005 +2024-07-17 11:28:21.363518: Best Epoch 50 +2024-07-17 11:28:21.363544: current epoch is 51 +2024-07-17 11:28:21.363600: This epoch took 39.898731 s + +2024-07-17 11:28:21.363635: +epoch: 52 +2024-07-17 11:28:53.737920: train loss : -0.9111 +2024-07-17 11:28:57.281719: validation loss: -0.8670 +2024-07-17 11:28:57.314635: Average global foreground Dice: [0.9326] +2024-07-17 11:28:57.314827: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:28:58.028342: lr: 0.00807 +2024-07-17 11:28:58.095026: saving checkpoint... +2024-07-17 11:28:59.188231: done, saving took 1.16 seconds +2024-07-17 11:28:59.211942: New best epoch (train loss MA): -0.8791 +2024-07-17 11:28:59.212001: Best loss so far: -0.8766 +2024-07-17 11:28:59.212033: eps: 0.0005 +2024-07-17 11:28:59.212063: Best Epoch 51 +2024-07-17 11:28:59.212090: current epoch is 52 +2024-07-17 11:28:59.212134: This epoch took 37.848457 s + +2024-07-17 11:28:59.212168: +epoch: 53 +2024-07-17 11:29:34.481611: train loss : -0.9094 +2024-07-17 11:29:38.699718: validation loss: -0.8671 +2024-07-17 11:29:38.701126: Average global foreground Dice: [0.9305] +2024-07-17 11:29:38.701228: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:29:39.255538: lr: 0.008033 +2024-07-17 11:29:39.315621: saving checkpoint... +2024-07-17 11:29:40.133475: done, saving took 0.88 seconds +2024-07-17 11:29:40.207926: New best epoch (train loss MA): -0.8812 +2024-07-17 11:29:40.208161: Best loss so far: -0.8791 +2024-07-17 11:29:40.208223: eps: 0.0005 +2024-07-17 11:29:40.208274: Best Epoch 52 +2024-07-17 11:29:40.208321: current epoch is 53 +2024-07-17 11:29:40.208394: This epoch took 40.996193 s + +2024-07-17 11:29:40.208448: +epoch: 54 +2024-07-17 11:30:15.661291: train loss : -0.9082 +2024-07-17 11:30:20.539344: validation loss: -0.8783 +2024-07-17 11:30:20.542475: Average global foreground Dice: [0.9319] +2024-07-17 11:30:20.542818: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:30:21.030743: lr: 0.007996 +2024-07-17 11:30:21.090202: saving checkpoint... +2024-07-17 11:30:22.209405: done, saving took 1.18 seconds +2024-07-17 11:30:22.279023: New best epoch (train loss MA): -0.8831 +2024-07-17 11:30:22.279212: Best loss so far: -0.8812 +2024-07-17 11:30:22.279278: eps: 0.0005 +2024-07-17 11:30:22.279340: Best Epoch 53 +2024-07-17 11:30:22.279412: current epoch is 54 +2024-07-17 11:30:22.279483: This epoch took 42.070976 s + +2024-07-17 11:30:22.279556: +epoch: 55 +2024-07-17 11:30:58.790746: train loss : -0.9045 +2024-07-17 11:31:03.357339: validation loss: -0.8658 +2024-07-17 11:31:03.358742: Average global foreground Dice: [0.9286] +2024-07-17 11:31:03.358832: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:31:03.998214: lr: 0.007959 +2024-07-17 11:31:04.069809: saving checkpoint... +2024-07-17 11:31:04.859189: done, saving took 0.86 seconds +2024-07-17 11:31:05.234563: New best epoch (train loss MA): -0.8846 +2024-07-17 11:31:05.234782: Best loss so far: -0.8831 +2024-07-17 11:31:05.234818: eps: 0.0005 +2024-07-17 11:31:05.234848: Best Epoch 54 +2024-07-17 11:31:05.234876: current epoch is 55 +2024-07-17 11:31:05.234927: This epoch took 42.955278 s + +2024-07-17 11:31:05.234962: +epoch: 56 +2024-07-17 11:31:38.144970: train loss : -0.9036 +2024-07-17 11:31:42.523347: validation loss: -0.8673 +2024-07-17 11:31:42.524651: Average global foreground Dice: [0.9279] +2024-07-17 11:31:42.524786: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:31:43.297663: lr: 0.007922 +2024-07-17 11:31:43.463841: saving checkpoint... +2024-07-17 11:31:44.768149: done, saving took 1.47 seconds +2024-07-17 11:31:44.911735: New best epoch (train loss MA): -0.8859 +2024-07-17 11:31:44.912161: Best loss so far: -0.8846 +2024-07-17 11:31:44.912242: eps: 0.0005 +2024-07-17 11:31:44.912349: Best Epoch 55 +2024-07-17 11:31:44.912418: current epoch is 56 +2024-07-17 11:31:44.912547: This epoch took 39.677548 s + +2024-07-17 11:31:44.912632: +epoch: 57 +2024-07-17 11:32:19.742799: train loss : -0.9092 +2024-07-17 11:32:24.786241: validation loss: -0.8709 +2024-07-17 11:32:24.787971: Average global foreground Dice: [0.9312] +2024-07-17 11:32:24.788047: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:32:25.580244: lr: 0.007885 +2024-07-17 11:32:25.643930: saving checkpoint... +2024-07-17 11:32:26.718332: done, saving took 1.14 seconds +2024-07-17 11:32:26.809276: New best epoch (train loss MA): -0.8875 +2024-07-17 11:32:26.809538: Best loss so far: -0.8859 +2024-07-17 11:32:26.809595: eps: 0.0005 +2024-07-17 11:32:26.809644: Best Epoch 56 +2024-07-17 11:32:26.809688: current epoch is 57 +2024-07-17 11:32:26.809741: This epoch took 41.897007 s + +2024-07-17 11:32:26.809773: +epoch: 58 +2024-07-17 11:33:01.934694: train loss : -0.9116 +2024-07-17 11:33:06.425969: validation loss: -0.8688 +2024-07-17 11:33:06.427274: Average global foreground Dice: [0.9327] +2024-07-17 11:33:06.427359: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:33:07.263836: lr: 0.007848 +2024-07-17 11:33:07.424661: saving checkpoint... +2024-07-17 11:33:08.516307: done, saving took 1.25 seconds +2024-07-17 11:33:08.574132: New best epoch (train loss MA): -0.8892 +2024-07-17 11:33:08.574398: Best loss so far: -0.8875 +2024-07-17 11:33:08.574456: eps: 0.0005 +2024-07-17 11:33:08.574508: Best Epoch 57 +2024-07-17 11:33:08.574556: current epoch is 58 +2024-07-17 11:33:08.574653: This epoch took 41.764848 s + +2024-07-17 11:33:08.574715: +epoch: 59 +2024-07-17 11:33:44.726649: train loss : -0.9094 +2024-07-17 11:33:49.689181: validation loss: -0.8692 +2024-07-17 11:33:49.690620: Average global foreground Dice: [0.9317] +2024-07-17 11:33:49.691021: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:33:50.641503: lr: 0.007811 +2024-07-17 11:33:50.695561: saving checkpoint... +2024-07-17 11:33:51.815904: done, saving took 1.17 seconds +2024-07-17 11:33:51.925854: New best epoch (train loss MA): -0.8906 +2024-07-17 11:33:51.925968: Best loss so far: -0.8892 +2024-07-17 11:33:51.926009: eps: 0.0005 +2024-07-17 11:33:51.926048: Best Epoch 58 +2024-07-17 11:33:51.926083: current epoch is 59 +2024-07-17 11:33:51.926127: This epoch took 43.351336 s + +2024-07-17 11:33:51.926183: +epoch: 60 +2024-07-17 11:34:27.426116: train loss : -0.9144 +2024-07-17 11:34:32.152564: validation loss: -0.8746 +2024-07-17 11:34:32.153472: Average global foreground Dice: [0.9357] +2024-07-17 11:34:32.153537: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:34:32.607694: lr: 0.007774 +2024-07-17 11:34:32.660143: saving checkpoint... +2024-07-17 11:34:33.808707: done, saving took 1.20 seconds +2024-07-17 11:34:33.853770: New best epoch (train loss MA): -0.8923 +2024-07-17 11:34:33.853940: Best loss so far: -0.8906 +2024-07-17 11:34:33.853989: eps: 0.0005 +2024-07-17 11:34:33.854034: Best Epoch 59 +2024-07-17 11:34:33.854076: current epoch is 60 +2024-07-17 11:34:33.854132: This epoch took 41.927902 s + +2024-07-17 11:34:33.854189: +epoch: 61 +2024-07-17 11:35:04.756284: train loss : -0.9210 +2024-07-17 11:35:08.883407: validation loss: -0.8718 +2024-07-17 11:35:08.884567: Average global foreground Dice: [0.9381] +2024-07-17 11:35:08.884668: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:35:09.979629: lr: 0.007737 +2024-07-17 11:35:10.137818: saving checkpoint... +2024-07-17 11:35:11.318887: done, saving took 1.34 seconds +2024-07-17 11:35:11.370031: New best epoch (train loss MA): -0.8943 +2024-07-17 11:35:11.370255: Best loss so far: -0.8923 +2024-07-17 11:35:11.370303: eps: 0.0005 +2024-07-17 11:35:11.370350: Best Epoch 60 +2024-07-17 11:35:11.370402: current epoch is 61 +2024-07-17 11:35:11.370465: This epoch took 37.516215 s + +2024-07-17 11:35:11.370506: +epoch: 62 +2024-07-17 11:35:46.398038: train loss : -0.9204 +2024-07-17 11:35:50.990658: validation loss: -0.8729 +2024-07-17 11:35:50.991710: Average global foreground Dice: [0.9382] +2024-07-17 11:35:50.991776: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:35:51.814777: lr: 0.0077 +2024-07-17 11:35:51.880925: saving checkpoint... +2024-07-17 11:35:53.303936: done, saving took 1.49 seconds +2024-07-17 11:35:53.339573: New best epoch (train loss MA): -0.8961 +2024-07-17 11:35:53.339638: Best loss so far: -0.8943 +2024-07-17 11:35:53.339680: eps: 0.0005 +2024-07-17 11:35:53.339719: Best Epoch 61 +2024-07-17 11:35:53.339755: current epoch is 62 +2024-07-17 11:35:53.339803: This epoch took 41.969231 s + +2024-07-17 11:35:53.339842: +epoch: 63 +2024-07-17 11:36:29.475675: train loss : -0.9174 +2024-07-17 11:36:34.512102: validation loss: -0.8617 +2024-07-17 11:36:34.513744: Average global foreground Dice: [0.9353] +2024-07-17 11:36:34.513874: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:36:35.056511: lr: 0.007663 +2024-07-17 11:36:35.126876: saving checkpoint... +2024-07-17 11:36:36.298023: done, saving took 1.24 seconds +2024-07-17 11:36:36.323006: New best epoch (train loss MA): -0.8976 +2024-07-17 11:36:36.323080: Best loss so far: -0.8961 +2024-07-17 11:36:36.323119: eps: 0.0005 +2024-07-17 11:36:36.323159: Best Epoch 62 +2024-07-17 11:36:36.323194: current epoch is 63 +2024-07-17 11:36:36.323236: This epoch took 42.983348 s + +2024-07-17 11:36:36.323273: +epoch: 64 +2024-07-17 11:37:13.346875: train loss : -0.9179 +2024-07-17 11:37:17.839779: validation loss: -0.8688 +2024-07-17 11:37:17.840969: Average global foreground Dice: [0.9363] +2024-07-17 11:37:17.841043: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:37:18.858446: lr: 0.007626 +2024-07-17 11:37:18.934365: saving checkpoint... +2024-07-17 11:37:20.152729: done, saving took 1.29 seconds +2024-07-17 11:37:20.160743: New best epoch (train loss MA): -0.8990 +2024-07-17 11:37:20.160830: Best loss so far: -0.8976 +2024-07-17 11:37:20.160890: eps: 0.0005 +2024-07-17 11:37:20.160949: Best Epoch 63 +2024-07-17 11:37:20.161002: current epoch is 64 +2024-07-17 11:37:20.161067: This epoch took 43.837751 s + +2024-07-17 11:37:20.161146: +epoch: 65 +2024-07-17 11:37:57.036775: train loss : -0.9159 +2024-07-17 11:38:01.354147: validation loss: -0.8767 +2024-07-17 11:38:01.355518: Average global foreground Dice: [0.9361] +2024-07-17 11:38:01.355618: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:38:02.070710: lr: 0.007589 +2024-07-17 11:38:02.144441: saving checkpoint... +2024-07-17 11:38:03.338275: done, saving took 1.27 seconds +2024-07-17 11:38:03.340600: New best epoch (train loss MA): -0.9002 +2024-07-17 11:38:03.340655: Best loss so far: -0.8990 +2024-07-17 11:38:03.340693: eps: 0.0005 +2024-07-17 11:38:03.340729: Best Epoch 64 +2024-07-17 11:38:03.340764: current epoch is 65 +2024-07-17 11:38:03.340804: This epoch took 43.179565 s + +2024-07-17 11:38:03.340840: +epoch: 66 +2024-07-17 11:38:37.404382: train loss : -0.9208 +2024-07-17 11:38:42.323860: validation loss: -0.8760 +2024-07-17 11:38:42.324966: Average global foreground Dice: [0.9387] +2024-07-17 11:38:42.325040: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:38:42.810138: lr: 0.007552 +2024-07-17 11:38:42.882925: saving checkpoint... +2024-07-17 11:38:43.987909: done, saving took 1.18 seconds +2024-07-17 11:38:44.074474: New best epoch (train loss MA): -0.9017 +2024-07-17 11:38:44.074629: Best loss so far: -0.9002 +2024-07-17 11:38:44.074678: eps: 0.0005 +2024-07-17 11:38:44.074732: Best Epoch 65 +2024-07-17 11:38:44.074785: current epoch is 66 +2024-07-17 11:38:44.074875: This epoch took 40.733993 s + +2024-07-17 11:38:44.075024: +epoch: 67 +2024-07-17 11:39:21.587747: train loss : -0.9175 +2024-07-17 11:39:26.932661: validation loss: -0.8856 +2024-07-17 11:39:26.933735: Average global foreground Dice: [0.9381] +2024-07-17 11:39:26.933810: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:39:27.368132: lr: 0.007515 +2024-07-17 11:39:27.480882: saving checkpoint... +2024-07-17 11:39:28.882334: done, saving took 1.51 seconds +2024-07-17 11:39:28.933769: New best epoch (train loss MA): -0.9028 +2024-07-17 11:39:28.933976: Best loss so far: -0.9017 +2024-07-17 11:39:28.934027: eps: 0.0005 +2024-07-17 11:39:28.934073: Best Epoch 66 +2024-07-17 11:39:28.934103: current epoch is 67 +2024-07-17 11:39:28.934155: This epoch took 44.859060 s + +2024-07-17 11:39:28.934194: +epoch: 68 +2024-07-17 11:40:05.268027: train loss : -0.9207 +2024-07-17 11:40:10.447181: validation loss: -0.8688 +2024-07-17 11:40:10.448229: Average global foreground Dice: [0.9371] +2024-07-17 11:40:10.448298: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:40:11.044674: lr: 0.007478 +2024-07-17 11:40:11.148526: saving checkpoint... +2024-07-17 11:40:12.410209: done, saving took 1.36 seconds +2024-07-17 11:40:12.452618: New best epoch (train loss MA): -0.9040 +2024-07-17 11:40:12.452830: Best loss so far: -0.9028 +2024-07-17 11:40:12.452863: eps: 0.0005 +2024-07-17 11:40:12.452891: Best Epoch 67 +2024-07-17 11:40:12.452928: current epoch is 68 +2024-07-17 11:40:12.452996: This epoch took 43.518744 s + +2024-07-17 11:40:12.453047: +epoch: 69 +2024-07-17 11:40:48.454697: train loss : -0.9215 +2024-07-17 11:40:53.859165: validation loss: -0.8739 +2024-07-17 11:40:53.867267: Average global foreground Dice: [0.9389] +2024-07-17 11:40:53.867372: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:40:54.725151: lr: 0.00744 +2024-07-17 11:40:54.776518: saving checkpoint... +2024-07-17 11:40:55.905149: done, saving took 1.18 seconds +2024-07-17 11:40:55.957770: New best epoch (train loss MA): -0.9052 +2024-07-17 11:40:55.957985: Best loss so far: -0.9040 +2024-07-17 11:40:55.958019: eps: 0.0005 +2024-07-17 11:40:55.958046: Best Epoch 68 +2024-07-17 11:40:55.958071: current epoch is 69 +2024-07-17 11:40:55.958107: This epoch took 43.505016 s + +2024-07-17 11:40:55.958136: +epoch: 70 +2024-07-17 11:41:28.022095: train loss : -0.9202 +2024-07-17 11:41:32.270914: validation loss: -0.8769 +2024-07-17 11:41:32.273027: Average global foreground Dice: [0.938] +2024-07-17 11:41:32.273171: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:41:33.190208: lr: 0.007403 +2024-07-17 11:41:33.249006: saving checkpoint... +2024-07-17 11:41:35.166068: done, saving took 1.97 seconds +2024-07-17 11:41:35.216110: New best epoch (train loss MA): -0.9063 +2024-07-17 11:41:35.216236: Best loss so far: -0.9052 +2024-07-17 11:41:35.216295: eps: 0.0005 +2024-07-17 11:41:35.216334: Best Epoch 69 +2024-07-17 11:41:35.216363: current epoch is 70 +2024-07-17 11:41:35.216404: This epoch took 39.258238 s + +2024-07-17 11:41:35.216440: +epoch: 71 +2024-07-17 11:42:12.914903: train loss : -0.9200 +2024-07-17 11:42:17.798394: validation loss: -0.8791 +2024-07-17 11:42:17.800122: Average global foreground Dice: [0.9386] +2024-07-17 11:42:17.800196: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:42:18.477665: lr: 0.007366 +2024-07-17 11:42:18.532477: saving checkpoint... +2024-07-17 11:42:19.636661: done, saving took 1.16 seconds +2024-07-17 11:42:19.683038: New best epoch (train loss MA): -0.9073 +2024-07-17 11:42:19.723934: Best loss so far: -0.9063 +2024-07-17 11:42:19.724179: eps: 0.0005 +2024-07-17 11:42:19.724215: Best Epoch 70 +2024-07-17 11:42:19.724243: current epoch is 71 +2024-07-17 11:42:19.724281: This epoch took 44.507800 s + +2024-07-17 11:42:19.724311: +epoch: 72 +2024-07-17 11:42:56.024655: train loss : -0.9227 +2024-07-17 11:43:00.813805: validation loss: -0.8743 +2024-07-17 11:43:00.818286: Average global foreground Dice: [0.9398] +2024-07-17 11:43:00.818494: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:43:01.481871: lr: 0.007329 +2024-07-17 11:43:01.643256: saving checkpoint... +2024-07-17 11:43:02.930539: done, saving took 1.44 seconds +2024-07-17 11:43:02.965176: New best epoch (train loss MA): -0.9083 +2024-07-17 11:43:02.965235: Best loss so far: -0.9073 +2024-07-17 11:43:02.965268: eps: 0.0005 +2024-07-17 11:43:02.965296: Best Epoch 71 +2024-07-17 11:43:02.965321: current epoch is 72 +2024-07-17 11:43:02.965365: This epoch took 43.241022 s + +2024-07-17 11:43:02.965411: +epoch: 73 +2024-07-17 11:43:39.593652: train loss : -0.9226 +2024-07-17 11:43:44.042561: validation loss: -0.8716 +2024-07-17 11:43:44.044734: Average global foreground Dice: [0.9385] +2024-07-17 11:43:44.045223: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:43:45.334093: lr: 0.007291 +2024-07-17 11:43:45.402442: saving checkpoint... +2024-07-17 11:43:46.666173: done, saving took 1.33 seconds +2024-07-17 11:43:46.700388: New best epoch (train loss MA): -0.9093 +2024-07-17 11:43:46.700460: Best loss so far: -0.9083 +2024-07-17 11:43:46.700498: eps: 0.0005 +2024-07-17 11:43:46.700535: Best Epoch 72 +2024-07-17 11:43:46.700569: current epoch is 73 +2024-07-17 11:43:46.700615: This epoch took 43.735172 s + +2024-07-17 11:43:46.700654: +epoch: 74 +2024-07-17 11:44:21.576976: train loss : -0.9247 +2024-07-17 11:44:26.256129: validation loss: -0.8723 +2024-07-17 11:44:26.258036: Average global foreground Dice: [0.941] +2024-07-17 11:44:26.258138: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:44:26.841504: lr: 0.007254 +2024-07-17 11:44:26.899897: saving checkpoint... +2024-07-17 11:44:28.136319: done, saving took 1.29 seconds +2024-07-17 11:44:28.138701: New best epoch (train loss MA): -0.9104 +2024-07-17 11:44:28.138756: Best loss so far: -0.9093 +2024-07-17 11:44:28.138786: eps: 0.0005 +2024-07-17 11:44:28.138813: Best Epoch 73 +2024-07-17 11:44:28.138838: current epoch is 74 +2024-07-17 11:44:28.138869: This epoch took 41.438173 s + +2024-07-17 11:44:28.138896: +epoch: 75 +2024-07-17 11:45:02.196969: train loss : -0.9224 +2024-07-17 11:45:06.798087: validation loss: -0.8659 +2024-07-17 11:45:06.814436: Average global foreground Dice: [0.9388] +2024-07-17 11:45:06.814553: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:45:07.568923: lr: 0.007217 +2024-07-17 11:45:07.634693: saving checkpoint... +2024-07-17 11:45:09.119272: done, saving took 1.55 seconds +2024-07-17 11:45:09.129774: New best epoch (train loss MA): -0.9112 +2024-07-17 11:45:09.129837: Best loss so far: -0.9104 +2024-07-17 11:45:09.129868: eps: 0.0005 +2024-07-17 11:45:09.129894: Best Epoch 74 +2024-07-17 11:45:09.129919: current epoch is 75 +2024-07-17 11:45:09.129949: This epoch took 40.991024 s + +2024-07-17 11:45:09.129976: +epoch: 76 +2024-07-17 11:45:46.075962: train loss : -0.9200 +2024-07-17 11:45:50.991267: validation loss: -0.8740 +2024-07-17 11:45:50.992578: Average global foreground Dice: [0.9383] +2024-07-17 11:45:50.992668: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:45:51.642371: lr: 0.00718 +2024-07-17 11:45:51.703331: saving checkpoint... +2024-07-17 11:45:52.777501: done, saving took 1.13 seconds +2024-07-17 11:45:52.827985: New best epoch (train loss MA): -0.9119 +2024-07-17 11:45:52.828158: Best loss so far: -0.9112 +2024-07-17 11:45:52.828195: eps: 0.0005 +2024-07-17 11:45:52.828228: Best Epoch 75 +2024-07-17 11:45:52.828261: current epoch is 76 +2024-07-17 11:45:52.828309: This epoch took 43.698301 s + +2024-07-17 11:45:52.828344: +epoch: 77 +2024-07-17 11:46:29.076797: train loss : -0.9292 +2024-07-17 11:46:34.280545: validation loss: -0.8696 +2024-07-17 11:46:34.281644: Average global foreground Dice: [0.943] +2024-07-17 11:46:34.281712: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:46:34.785598: lr: 0.007142 +2024-07-17 11:46:34.880639: saving checkpoint... +2024-07-17 11:46:36.004315: done, saving took 1.22 seconds +2024-07-17 11:46:36.047730: New best epoch (train loss MA): -0.9131 +2024-07-17 11:46:36.047829: Best loss so far: -0.9119 +2024-07-17 11:46:36.047868: eps: 0.0005 +2024-07-17 11:46:36.047901: Best Epoch 76 +2024-07-17 11:46:36.047926: current epoch is 77 +2024-07-17 11:46:36.047958: This epoch took 43.219573 s + +2024-07-17 11:46:36.047984: +epoch: 78 +2024-07-17 11:47:10.589348: train loss : -0.9253 +2024-07-17 11:47:14.846886: validation loss: -0.8683 +2024-07-17 11:47:14.848363: Average global foreground Dice: [0.9403] +2024-07-17 11:47:14.848458: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:47:15.557904: lr: 0.007105 +2024-07-17 11:47:15.614604: saving checkpoint... +2024-07-17 11:47:16.790237: done, saving took 1.23 seconds +2024-07-17 11:47:16.834528: New best epoch (train loss MA): -0.9139 +2024-07-17 11:47:16.834620: Best loss so far: -0.9131 +2024-07-17 11:47:16.834658: eps: 0.0005 +2024-07-17 11:47:16.834694: Best Epoch 77 +2024-07-17 11:47:16.834728: current epoch is 78 +2024-07-17 11:47:16.834769: This epoch took 40.786754 s + +2024-07-17 11:47:16.834805: +epoch: 79 +2024-07-17 11:47:49.018957: train loss : -0.9266 +2024-07-17 11:47:54.076807: validation loss: -0.8736 +2024-07-17 11:47:54.077802: Average global foreground Dice: [0.9426] +2024-07-17 11:47:54.077882: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:47:54.715008: lr: 0.007067 +2024-07-17 11:47:54.772091: saving checkpoint... +2024-07-17 11:47:56.029123: done, saving took 1.31 seconds +2024-07-17 11:47:56.065263: New best epoch (train loss MA): -0.9148 +2024-07-17 11:47:56.065344: Best loss so far: -0.9139 +2024-07-17 11:47:56.065386: eps: 0.0005 +2024-07-17 11:47:56.065414: Best Epoch 78 +2024-07-17 11:47:56.065439: current epoch is 79 +2024-07-17 11:47:56.065471: This epoch took 39.230627 s + +2024-07-17 11:47:56.065499: +epoch: 80 +2024-07-17 11:48:32.921030: train loss : -0.9215 +2024-07-17 11:48:37.660498: validation loss: -0.8742 +2024-07-17 11:48:37.662075: Average global foreground Dice: [0.939] +2024-07-17 11:48:37.662149: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:48:38.234569: lr: 0.00703 +2024-07-17 11:48:38.786787: saving checkpoint... +2024-07-17 11:48:40.052989: done, saving took 1.32 seconds +2024-07-17 11:48:40.055256: This epoch took 43.989726 s + +2024-07-17 11:48:40.055325: +epoch: 81 +2024-07-17 11:49:16.466785: train loss : -0.9265 +2024-07-17 11:49:20.789167: validation loss: -0.8739 +2024-07-17 11:49:20.791806: Average global foreground Dice: [0.9419] +2024-07-17 11:49:20.792004: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:49:21.690461: lr: 0.006992 +2024-07-17 11:49:21.829785: saving checkpoint... +2024-07-17 11:49:23.073527: done, saving took 1.38 seconds +2024-07-17 11:49:23.076202: New best epoch (train loss MA): -0.9161 +2024-07-17 11:49:23.076271: Best loss so far: -0.9148 +2024-07-17 11:49:23.076305: eps: 0.0005 +2024-07-17 11:49:23.076337: Best Epoch 79 +2024-07-17 11:49:23.076368: current epoch is 81 +2024-07-17 11:49:23.076407: This epoch took 43.021025 s + +2024-07-17 11:49:23.076440: +epoch: 82 +2024-07-17 11:49:58.933326: train loss : -0.9267 +2024-07-17 11:50:03.335654: validation loss: -0.8835 +2024-07-17 11:50:03.337144: Average global foreground Dice: [0.9435] +2024-07-17 11:50:03.337243: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:50:03.882775: lr: 0.006955 +2024-07-17 11:50:03.937393: saving checkpoint... +2024-07-17 11:50:05.061666: done, saving took 1.18 seconds +2024-07-17 11:50:05.076469: New best epoch (train loss MA): -0.9168 +2024-07-17 11:50:05.076683: Best loss so far: -0.9161 +2024-07-17 11:50:05.076727: eps: 0.0005 +2024-07-17 11:50:05.076770: Best Epoch 81 +2024-07-17 11:50:05.076808: current epoch is 82 +2024-07-17 11:50:05.076857: This epoch took 42.000381 s + +2024-07-17 11:50:05.076898: +epoch: 83 +2024-07-17 11:50:36.797394: train loss : -0.9253 +2024-07-17 11:50:41.631327: validation loss: -0.8612 +2024-07-17 11:50:41.633255: Average global foreground Dice: [0.9394] +2024-07-17 11:50:41.633344: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:50:42.243179: lr: 0.006918 +2024-07-17 11:50:42.341421: saving checkpoint... +2024-07-17 11:50:43.586005: done, saving took 1.34 seconds +2024-07-17 11:50:43.635163: New best epoch (train loss MA): -0.9174 +2024-07-17 11:50:43.635329: Best loss so far: -0.9168 +2024-07-17 11:50:43.635392: eps: 0.0005 +2024-07-17 11:50:43.635451: Best Epoch 82 +2024-07-17 11:50:43.635508: current epoch is 83 +2024-07-17 11:50:43.635573: This epoch took 38.558629 s + +2024-07-17 11:50:43.635642: +epoch: 84 +2024-07-17 11:51:19.940627: train loss : -0.9275 +2024-07-17 11:51:25.049906: validation loss: -0.8745 +2024-07-17 11:51:25.050974: Average global foreground Dice: [0.9428] +2024-07-17 11:51:25.051095: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:51:25.561752: lr: 0.00688 +2024-07-17 11:51:25.619904: saving checkpoint... +2024-07-17 11:51:26.684243: done, saving took 1.12 seconds +2024-07-17 11:51:26.775514: New best epoch (train loss MA): -0.9181 +2024-07-17 11:51:26.775654: Best loss so far: -0.9174 +2024-07-17 11:51:26.775719: eps: 0.0005 +2024-07-17 11:51:26.775779: Best Epoch 83 +2024-07-17 11:51:26.775838: current epoch is 84 +2024-07-17 11:51:26.775909: This epoch took 43.140180 s + +2024-07-17 11:51:26.775979: +epoch: 85 +2024-07-17 11:52:02.512779: train loss : -0.9271 +2024-07-17 11:52:07.623515: validation loss: -0.8791 +2024-07-17 11:52:07.624460: Average global foreground Dice: [0.943] +2024-07-17 11:52:07.624526: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:52:08.201720: lr: 0.006842 +2024-07-17 11:52:08.259954: saving checkpoint... +2024-07-17 11:52:08.822241: done, saving took 0.62 seconds +2024-07-17 11:52:09.402515: New best epoch (train loss MA): -0.9187 +2024-07-17 11:52:09.402730: Best loss so far: -0.9181 +2024-07-17 11:52:09.402769: eps: 0.0005 +2024-07-17 11:52:09.402838: Best Epoch 84 +2024-07-17 11:52:09.402914: current epoch is 85 +2024-07-17 11:52:09.402983: This epoch took 42.626916 s + +2024-07-17 11:52:09.403059: +epoch: 86 +2024-07-17 11:52:44.400036: train loss : -0.9284 +2024-07-17 11:52:48.781276: validation loss: -0.8712 +2024-07-17 11:52:48.782590: Average global foreground Dice: [0.9428] +2024-07-17 11:52:48.782660: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:52:49.500034: lr: 0.006805 +2024-07-17 11:52:49.557358: saving checkpoint... +2024-07-17 11:52:51.307970: done, saving took 1.81 seconds +2024-07-17 11:52:51.352719: New best epoch (train loss MA): -0.9194 +2024-07-17 11:52:51.352843: Best loss so far: -0.9187 +2024-07-17 11:52:51.352908: eps: 0.0005 +2024-07-17 11:52:51.352967: Best Epoch 85 +2024-07-17 11:52:51.353036: current epoch is 86 +2024-07-17 11:52:51.353106: This epoch took 41.949957 s + +2024-07-17 11:52:51.353179: +epoch: 87 +2024-07-17 11:53:18.254333: train loss : -0.9270 +2024-07-17 11:53:22.458819: validation loss: -0.8638 +2024-07-17 11:53:22.460141: Average global foreground Dice: [0.942] +2024-07-17 11:53:22.460237: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:53:23.414994: lr: 0.006767 +2024-07-17 11:53:23.473330: saving checkpoint... +2024-07-17 11:53:24.481232: done, saving took 1.07 seconds +2024-07-17 11:53:24.532161: New best epoch (train loss MA): -0.9199 +2024-07-17 11:53:24.532266: Best loss so far: -0.9194 +2024-07-17 11:53:24.532343: eps: 0.0005 +2024-07-17 11:53:24.532405: Best Epoch 86 +2024-07-17 11:53:24.532462: current epoch is 87 +2024-07-17 11:53:24.532529: This epoch took 33.179262 s + +2024-07-17 11:53:24.532596: +epoch: 88 +2024-07-17 11:53:51.011400: train loss : -0.9306 +2024-07-17 11:53:55.199756: validation loss: -0.8698 +2024-07-17 11:53:55.201110: Average global foreground Dice: [0.9438] +2024-07-17 11:53:55.201181: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:53:56.050108: lr: 0.00673 +2024-07-17 11:53:56.101045: saving checkpoint... +2024-07-17 11:53:57.112052: done, saving took 1.06 seconds +2024-07-17 11:53:57.328912: New best epoch (train loss MA): -0.9207 +2024-07-17 11:53:57.329035: Best loss so far: -0.9199 +2024-07-17 11:53:57.329098: eps: 0.0005 +2024-07-17 11:53:57.329154: Best Epoch 87 +2024-07-17 11:53:57.329206: current epoch is 88 +2024-07-17 11:53:57.329276: This epoch took 32.796602 s + +2024-07-17 11:53:57.329363: +epoch: 89 +2024-07-17 11:54:25.094213: train loss : -0.9336 +2024-07-17 11:54:29.076911: validation loss: -0.8686 +2024-07-17 11:54:29.090474: Average global foreground Dice: [0.9455] +2024-07-17 11:54:29.127178: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:54:29.974417: lr: 0.006692 +2024-07-17 11:54:30.036086: saving checkpoint... +2024-07-17 11:54:30.923530: done, saving took 0.94 seconds +2024-07-17 11:54:31.119860: New best epoch (train loss MA): -0.9216 +2024-07-17 11:54:31.119950: Best loss so far: -0.9207 +2024-07-17 11:54:31.119987: eps: 0.0005 +2024-07-17 11:54:31.120022: Best Epoch 88 +2024-07-17 11:54:31.120053: current epoch is 89 +2024-07-17 11:54:31.120100: This epoch took 33.790616 s + +2024-07-17 11:54:31.120140: +epoch: 90 +2024-07-17 11:54:58.780368: train loss : -0.9283 +2024-07-17 11:55:02.283102: validation loss: -0.8734 +2024-07-17 11:55:02.284446: Average global foreground Dice: [0.9432] +2024-07-17 11:55:02.284541: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:55:03.326742: lr: 0.006654 +2024-07-17 11:55:03.380110: saving checkpoint... +2024-07-17 11:55:04.298990: done, saving took 0.97 seconds +2024-07-17 11:55:04.534942: This epoch took 33.414751 s + +2024-07-17 11:55:04.535175: +epoch: 91 +2024-07-17 11:55:31.141285: train loss : -0.9316 +2024-07-17 11:55:34.958687: validation loss: -0.8759 +2024-07-17 11:55:34.959988: Average global foreground Dice: [0.9455] +2024-07-17 11:55:34.960085: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:55:35.800790: lr: 0.006617 +2024-07-17 11:55:35.870230: saving checkpoint... +2024-07-17 11:55:36.822102: done, saving took 1.02 seconds +2024-07-17 11:55:36.867418: New best epoch (train loss MA): -0.9227 +2024-07-17 11:55:36.867686: Best loss so far: -0.9216 +2024-07-17 11:55:36.867752: eps: 0.0005 +2024-07-17 11:55:36.867811: Best Epoch 89 +2024-07-17 11:55:36.867867: current epoch is 91 +2024-07-17 11:55:36.867935: This epoch took 32.332715 s + +2024-07-17 11:55:36.868014: +epoch: 92 +2024-07-17 11:56:04.649536: train loss : -0.9333 +2024-07-17 11:56:08.400380: validation loss: -0.8780 +2024-07-17 11:56:08.401417: Average global foreground Dice: [0.9464] +2024-07-17 11:56:08.401516: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:56:09.123630: lr: 0.006579 +2024-07-17 11:56:09.176757: saving checkpoint... +2024-07-17 11:56:10.322953: done, saving took 1.20 seconds +2024-07-17 11:56:10.341495: New best epoch (train loss MA): -0.9235 +2024-07-17 11:56:10.341594: Best loss so far: -0.9227 +2024-07-17 11:56:10.341626: eps: 0.0005 +2024-07-17 11:56:10.341655: Best Epoch 91 +2024-07-17 11:56:10.341687: current epoch is 92 +2024-07-17 11:56:10.341749: This epoch took 33.473646 s + +2024-07-17 11:56:10.341793: +epoch: 93 +2024-07-17 11:56:37.580839: train loss : -0.9304 +2024-07-17 11:56:41.741599: validation loss: -0.8703 +2024-07-17 11:56:41.742796: Average global foreground Dice: [0.9437] +2024-07-17 11:56:41.742889: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:56:42.343123: lr: 0.006541 +2024-07-17 11:56:42.399421: saving checkpoint... +2024-07-17 11:56:43.279490: done, saving took 0.94 seconds +2024-07-17 11:56:43.281902: This epoch took 32.940068 s + +2024-07-17 11:56:43.281955: +epoch: 94 +2024-07-17 11:57:10.778031: train loss : -0.9325 +2024-07-17 11:57:14.812403: validation loss: -0.8727 +2024-07-17 11:57:14.813740: Average global foreground Dice: [0.9447] +2024-07-17 11:57:14.813863: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:57:15.512114: lr: 0.006504 +2024-07-17 11:57:15.601766: saving checkpoint... +2024-07-17 11:57:17.120856: done, saving took 1.61 seconds +2024-07-17 11:57:17.123169: New best epoch (train loss MA): -0.9246 +2024-07-17 11:57:17.123223: Best loss so far: -0.9235 +2024-07-17 11:57:17.123258: eps: 0.0005 +2024-07-17 11:57:17.123291: Best Epoch 92 +2024-07-17 11:57:17.123321: current epoch is 94 +2024-07-17 11:57:17.123365: This epoch took 33.841370 s + +2024-07-17 11:57:17.123412: +epoch: 95 +2024-07-17 11:57:54.981258: train loss : -0.9335 +2024-07-17 11:57:59.940334: validation loss: -0.8717 +2024-07-17 11:57:59.942322: Average global foreground Dice: [0.946] +2024-07-17 11:57:59.942420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:58:00.446992: lr: 0.006466 +2024-07-17 11:58:00.511740: saving checkpoint... +2024-07-17 11:58:01.686384: done, saving took 1.24 seconds +2024-07-17 11:58:01.725002: New best epoch (train loss MA): -0.9252 +2024-07-17 11:58:01.725103: Best loss so far: -0.9246 +2024-07-17 11:58:01.725165: eps: 0.0005 +2024-07-17 11:58:01.725221: Best Epoch 94 +2024-07-17 11:58:01.725276: current epoch is 95 +2024-07-17 11:58:01.725343: This epoch took 44.601869 s + +2024-07-17 11:58:01.725448: +epoch: 96 +2024-07-17 11:58:38.270401: train loss : -0.9313 +2024-07-17 11:58:42.182146: validation loss: -0.8732 +2024-07-17 11:58:42.183404: Average global foreground Dice: [0.9443] +2024-07-17 11:58:42.183475: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:58:43.113859: lr: 0.006428 +2024-07-17 11:58:43.173918: saving checkpoint... +2024-07-17 11:58:44.113051: done, saving took 1.00 seconds +2024-07-17 11:58:44.130326: This epoch took 42.404787 s + +2024-07-17 11:58:44.130398: +epoch: 97 +2024-07-17 11:59:20.784351: train loss : -0.9330 +2024-07-17 11:59:24.946575: validation loss: -0.8803 +2024-07-17 11:59:24.948253: Average global foreground Dice: [0.9466] +2024-07-17 11:59:24.948352: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 11:59:25.710717: lr: 0.00639 +2024-07-17 11:59:25.769082: saving checkpoint... +2024-07-17 11:59:26.977828: done, saving took 1.27 seconds +2024-07-17 11:59:26.980052: New best epoch (train loss MA): -0.9261 +2024-07-17 11:59:26.980110: Best loss so far: -0.9252 +2024-07-17 11:59:26.980148: eps: 0.0005 +2024-07-17 11:59:26.980240: Best Epoch 95 +2024-07-17 11:59:26.980295: current epoch is 97 +2024-07-17 11:59:26.980361: This epoch took 42.849905 s + +2024-07-17 11:59:26.980430: +epoch: 98 +2024-07-17 12:00:03.257841: train loss : -0.9352 +2024-07-17 12:00:06.532360: validation loss: -0.8789 +2024-07-17 12:00:06.533559: Average global foreground Dice: [0.9477] +2024-07-17 12:00:06.533656: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:00:07.125101: lr: 0.006352 +2024-07-17 12:00:07.183494: saving checkpoint... +2024-07-17 12:00:08.190892: done, saving took 1.07 seconds +2024-07-17 12:00:08.240697: New best epoch (train loss MA): -0.9268 +2024-07-17 12:00:08.240950: Best loss so far: -0.9261 +2024-07-17 12:00:08.241004: eps: 0.0005 +2024-07-17 12:00:08.241047: Best Epoch 97 +2024-07-17 12:00:08.241089: current epoch is 98 +2024-07-17 12:00:08.241138: This epoch took 41.260625 s + +2024-07-17 12:00:08.241301: +epoch: 99 +2024-07-17 12:00:42.091795: train loss : -0.9323 +2024-07-17 12:00:45.693049: validation loss: -0.8779 +2024-07-17 12:00:45.693959: Average global foreground Dice: [0.9462] +2024-07-17 12:00:45.694036: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:00:46.129410: lr: 0.006314 +2024-07-17 12:00:46.129748: saving scheduled checkpoint file... +2024-07-17 12:00:46.197979: saving checkpoint... +2024-07-17 12:00:47.275091: done, saving took 1.14 seconds +2024-07-17 12:00:47.279454: done +2024-07-17 12:00:47.308064: saving checkpoint... +2024-07-17 12:00:48.332857: done, saving took 1.05 seconds +2024-07-17 12:00:48.422725: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 12:00:48.422925: The split file contains 5 splits. +2024-07-17 12:00:48.422965: Desired fold for training: 0 +2024-07-17 12:00:48.422998: This split has 74 training and 20 validation cases. +2024-07-17 12:00:49.561822: Performing validation with validation data from task Task125_All. +2024-07-17 12:01:04.786605: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 12:01:04.786990: The split file contains 5 splits. +2024-07-17 12:01:04.787041: Desired fold for training: 0 +2024-07-17 12:01:04.787091: This split has 74 training and 20 validation cases. +2024-07-17 12:01:05.063936: This epoch took 56.822577 s + +2024-07-17 12:01:05.064065: +epoch: 100 +2024-07-17 12:01:40.432062: train loss : -0.9351 +2024-07-17 12:01:44.306398: validation loss: -0.8658 +2024-07-17 12:01:44.307851: Average global foreground Dice: [0.9457] +2024-07-17 12:01:44.307937: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:01:45.151490: lr: 0.006277 +2024-07-17 12:01:45.290164: saving checkpoint... +2024-07-17 12:01:46.237816: done, saving took 1.09 seconds +2024-07-17 12:01:46.389800: New best epoch (train loss MA): -0.9277 +2024-07-17 12:01:46.390030: Best loss so far: -0.9268 +2024-07-17 12:01:46.390065: eps: 0.0005 +2024-07-17 12:01:46.390096: Best Epoch 98 +2024-07-17 12:01:46.390122: current epoch is 100 +2024-07-17 12:01:46.390201: This epoch took 41.326091 s + +2024-07-17 12:01:46.390253: +epoch: 101 +2024-07-17 12:02:20.754317: train loss : -0.9314 +2024-07-17 12:02:23.458198: validation loss: -0.8730 +2024-07-17 12:02:23.459511: Average global foreground Dice: [0.9454] +2024-07-17 12:02:23.459623: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:02:24.312595: lr: 0.006239 +2024-07-17 12:02:24.382483: saving checkpoint... +2024-07-17 12:02:25.576700: done, saving took 1.26 seconds +2024-07-17 12:02:25.581428: This epoch took 39.191137 s + +2024-07-17 12:02:25.581497: +epoch: 102 +2024-07-17 12:03:02.827525: train loss : -0.9197 +2024-07-17 12:03:06.064348: validation loss: -0.8744 +2024-07-17 12:03:06.065503: Average global foreground Dice: [0.9385] +2024-07-17 12:03:06.065629: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:03:06.585158: lr: 0.006201 +2024-07-17 12:03:06.585351: This epoch took 41.003808 s + +2024-07-17 12:03:06.585405: +epoch: 103 +2024-07-17 12:03:40.342093: train loss : -0.9234 +2024-07-17 12:03:43.119284: validation loss: -0.8657 +2024-07-17 12:03:43.137820: Average global foreground Dice: [0.9388] +2024-07-17 12:03:43.138103: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:03:43.860742: lr: 0.006163 +2024-07-17 12:03:43.860932: This epoch took 37.275491 s + +2024-07-17 12:03:43.860968: +epoch: 104 +2024-07-17 12:04:16.979376: train loss : -0.9316 +2024-07-17 12:04:20.337150: validation loss: -0.8771 +2024-07-17 12:04:20.338231: Average global foreground Dice: [0.9457] +2024-07-17 12:04:20.338296: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:04:21.093794: lr: 0.006125 +2024-07-17 12:04:21.094077: This epoch took 37.233074 s + +2024-07-17 12:04:21.094118: +epoch: 105 +2024-07-17 12:04:56.976389: train loss : -0.9338 +2024-07-17 12:05:00.559450: validation loss: -0.8726 +2024-07-17 12:05:00.560727: Average global foreground Dice: [0.9467] +2024-07-17 12:05:00.560796: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:05:01.247398: lr: 0.006087 +2024-07-17 12:05:01.247673: This epoch took 40.153517 s + +2024-07-17 12:05:01.247718: +epoch: 106 +2024-07-17 12:05:34.639814: train loss : -0.9329 +2024-07-17 12:05:38.099325: validation loss: -0.8801 +2024-07-17 12:05:38.100510: Average global foreground Dice: [0.9468] +2024-07-17 12:05:38.100609: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:05:38.790821: lr: 0.006049 +2024-07-17 12:05:38.791036: New best epoch (train loss MA): -0.9282 +2024-07-17 12:05:38.791082: Best loss so far: -0.9277 +2024-07-17 12:05:38.791119: eps: 0.0005 +2024-07-17 12:05:38.791155: Best Epoch 100 +2024-07-17 12:05:38.791237: current epoch is 106 +2024-07-17 12:05:38.791310: This epoch took 37.543552 s + +2024-07-17 12:05:38.791386: +epoch: 107 +2024-07-17 12:06:12.910785: train loss : -0.9322 +2024-07-17 12:06:16.128478: validation loss: -0.8530 +2024-07-17 12:06:16.129422: Average global foreground Dice: [0.9431] +2024-07-17 12:06:16.129484: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:06:16.847069: lr: 0.006011 +2024-07-17 12:06:16.847322: This epoch took 38.055839 s + +2024-07-17 12:06:16.847376: +epoch: 108 +2024-07-17 12:06:53.689120: train loss : -0.9265 +2024-07-17 12:06:57.363422: validation loss: -0.8645 +2024-07-17 12:06:57.364791: Average global foreground Dice: [0.9416] +2024-07-17 12:06:57.364859: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:06:57.862194: lr: 0.005972 +2024-07-17 12:06:57.862446: This epoch took 41.015019 s + +2024-07-17 12:06:57.862494: +epoch: 109 +2024-07-17 12:07:33.183552: train loss : -0.9297 +2024-07-17 12:07:36.984379: validation loss: -0.8789 +2024-07-17 12:07:36.985483: Average global foreground Dice: [0.9455] +2024-07-17 12:07:36.985577: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:07:37.507507: lr: 0.005934 +2024-07-17 12:07:37.507708: This epoch took 39.645178 s + +2024-07-17 12:07:37.507755: +epoch: 110 +2024-07-17 12:08:12.088963: train loss : -0.9328 +2024-07-17 12:08:15.715803: validation loss: -0.8824 +2024-07-17 12:08:15.716799: Average global foreground Dice: [0.9463] +2024-07-17 12:08:15.718370: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:08:16.455925: lr: 0.005896 +2024-07-17 12:08:16.456106: New best epoch (train loss MA): -0.9288 +2024-07-17 12:08:16.456143: Best loss so far: -0.9282 +2024-07-17 12:08:16.456169: eps: 0.0005 +2024-07-17 12:08:16.456194: Best Epoch 106 +2024-07-17 12:08:16.456217: current epoch is 110 +2024-07-17 12:08:16.456253: This epoch took 38.948462 s + +2024-07-17 12:08:16.456283: +epoch: 111 +2024-07-17 12:08:49.846171: train loss : -0.9364 +2024-07-17 12:08:53.299995: validation loss: -0.8800 +2024-07-17 12:08:53.301021: Average global foreground Dice: [0.9481] +2024-07-17 12:08:53.301114: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:08:53.844463: lr: 0.005858 +2024-07-17 12:08:53.918740: saving checkpoint... +2024-07-17 12:08:54.879859: done, saving took 1.03 seconds +2024-07-17 12:08:55.102576: New best epoch (train loss MA): -0.9293 +2024-07-17 12:08:55.102842: Best loss so far: -0.9288 +2024-07-17 12:08:55.102906: eps: 0.0005 +2024-07-17 12:08:55.102960: Best Epoch 110 +2024-07-17 12:08:55.103078: current epoch is 111 +2024-07-17 12:08:55.103145: This epoch took 38.646830 s + +2024-07-17 12:08:55.103202: +epoch: 112 +2024-07-17 12:09:30.054923: train loss : -0.9367 +2024-07-17 12:09:33.914610: validation loss: -0.8722 +2024-07-17 12:09:33.915671: Average global foreground Dice: [0.9475] +2024-07-17 12:09:33.915736: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:09:34.612452: lr: 0.00582 +2024-07-17 12:09:34.684208: saving checkpoint... +2024-07-17 12:09:35.560692: done, saving took 0.95 seconds +2024-07-17 12:09:35.736714: New best epoch (train loss MA): -0.9298 +2024-07-17 12:09:35.736899: Best loss so far: -0.9293 +2024-07-17 12:09:35.736947: eps: 0.0005 +2024-07-17 12:09:35.736991: Best Epoch 111 +2024-07-17 12:09:35.737030: current epoch is 112 +2024-07-17 12:09:35.737078: This epoch took 40.633818 s + +2024-07-17 12:09:35.737113: +epoch: 113 +2024-07-17 12:10:11.058050: train loss : -0.9384 +2024-07-17 12:10:14.583179: validation loss: -0.8735 +2024-07-17 12:10:14.598205: Average global foreground Dice: [0.9489] +2024-07-17 12:10:14.599876: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:10:15.277122: lr: 0.005781 +2024-07-17 12:10:15.336648: saving checkpoint... +2024-07-17 12:10:16.147275: done, saving took 0.87 seconds +2024-07-17 12:10:16.224010: New best epoch (train loss MA): -0.9304 +2024-07-17 12:10:16.224128: Best loss so far: -0.9298 +2024-07-17 12:10:16.224158: eps: 0.0005 +2024-07-17 12:10:16.224183: Best Epoch 112 +2024-07-17 12:10:16.224205: current epoch is 113 +2024-07-17 12:10:16.224236: This epoch took 40.487077 s + +2024-07-17 12:10:16.224262: +epoch: 114 +2024-07-17 12:10:47.667575: train loss : -0.9375 +2024-07-17 12:10:51.038062: validation loss: -0.8781 +2024-07-17 12:10:51.039305: Average global foreground Dice: [0.9491] +2024-07-17 12:10:51.039404: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:10:51.560506: lr: 0.005743 +2024-07-17 12:10:51.702458: saving checkpoint... +2024-07-17 12:10:52.597507: done, saving took 1.02 seconds +2024-07-17 12:10:52.665790: This epoch took 36.441484 s + +2024-07-17 12:10:52.666062: +epoch: 115 +2024-07-17 12:11:28.395589: train loss : -0.9362 +2024-07-17 12:11:31.995203: validation loss: -0.8772 +2024-07-17 12:11:32.002282: Average global foreground Dice: [0.9484] +2024-07-17 12:11:32.007901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:11:32.716011: lr: 0.005705 +2024-07-17 12:11:32.775055: saving checkpoint... +2024-07-17 12:11:33.661651: done, saving took 0.95 seconds +2024-07-17 12:11:33.743656: New best epoch (train loss MA): -0.9313 +2024-07-17 12:11:33.743893: Best loss so far: -0.9304 +2024-07-17 12:11:33.743938: eps: 0.0005 +2024-07-17 12:11:33.743981: Best Epoch 113 +2024-07-17 12:11:33.744020: current epoch is 115 +2024-07-17 12:11:33.744068: This epoch took 41.077951 s + +2024-07-17 12:11:33.744117: +epoch: 116 +2024-07-17 12:12:10.047082: train loss : -0.9326 +2024-07-17 12:12:14.285211: validation loss: -0.8759 +2024-07-17 12:12:14.287246: Average global foreground Dice: [0.9459] +2024-07-17 12:12:14.287340: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:12:14.758493: lr: 0.005667 +2024-07-17 12:12:14.758770: This epoch took 41.014603 s + +2024-07-17 12:12:14.758814: +epoch: 117 +2024-07-17 12:12:50.373228: train loss : -0.9384 +2024-07-17 12:12:54.033986: validation loss: -0.8724 +2024-07-17 12:12:54.035694: Average global foreground Dice: [0.9489] +2024-07-17 12:12:54.035780: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:12:54.689292: lr: 0.005628 +2024-07-17 12:12:54.747394: saving checkpoint... +2024-07-17 12:12:55.665743: done, saving took 0.98 seconds +2024-07-17 12:12:55.854135: New best epoch (train loss MA): -0.9319 +2024-07-17 12:12:55.854326: Best loss so far: -0.9313 +2024-07-17 12:12:55.854367: eps: 0.0005 +2024-07-17 12:12:55.854402: Best Epoch 115 +2024-07-17 12:12:55.854434: current epoch is 117 +2024-07-17 12:12:55.854475: This epoch took 41.095432 s + +2024-07-17 12:12:55.854519: +epoch: 118 +2024-07-17 12:13:32.617827: train loss : -0.9395 +2024-07-17 12:13:36.568372: validation loss: -0.8632 +2024-07-17 12:13:36.569670: Average global foreground Dice: [0.9486] +2024-07-17 12:13:36.569821: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:13:37.039272: lr: 0.00559 +2024-07-17 12:13:37.123438: saving checkpoint... +2024-07-17 12:13:38.131634: done, saving took 1.09 seconds +2024-07-17 12:13:38.226733: New best epoch (train loss MA): -0.9324 +2024-07-17 12:13:38.226833: Best loss so far: -0.9319 +2024-07-17 12:13:38.226863: eps: 0.0005 +2024-07-17 12:13:38.226892: Best Epoch 117 +2024-07-17 12:13:38.226918: current epoch is 118 +2024-07-17 12:13:38.226993: This epoch took 42.372434 s + +2024-07-17 12:13:38.227029: +epoch: 119 +2024-07-17 12:14:15.568730: train loss : -0.9394 +2024-07-17 12:14:19.836549: validation loss: -0.8780 +2024-07-17 12:14:19.837897: Average global foreground Dice: [0.9508] +2024-07-17 12:14:19.837990: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:14:20.249022: lr: 0.005551 +2024-07-17 12:14:20.303055: saving checkpoint... +2024-07-17 12:14:21.008500: done, saving took 0.76 seconds +2024-07-17 12:14:21.010430: This epoch took 42.783368 s + +2024-07-17 12:14:21.010491: +epoch: 120 +2024-07-17 12:14:57.007456: train loss : -0.9353 +2024-07-17 12:15:00.825442: validation loss: -0.8746 +2024-07-17 12:15:00.826780: Average global foreground Dice: [0.9469] +2024-07-17 12:15:00.826877: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:15:01.372941: lr: 0.005513 +2024-07-17 12:15:01.426438: saving checkpoint... +2024-07-17 12:15:02.564418: done, saving took 1.19 seconds +2024-07-17 12:15:02.582295: New best epoch (train loss MA): -0.9331 +2024-07-17 12:15:02.582377: Best loss so far: -0.9324 +2024-07-17 12:15:02.582410: eps: 0.0005 +2024-07-17 12:15:02.582437: Best Epoch 118 +2024-07-17 12:15:02.582464: current epoch is 120 +2024-07-17 12:15:02.582506: This epoch took 41.571974 s + +2024-07-17 12:15:02.582537: +epoch: 121 +2024-07-17 12:15:41.071610: train loss : -0.9370 +2024-07-17 12:15:44.899788: validation loss: -0.8751 +2024-07-17 12:15:44.901230: Average global foreground Dice: [0.9483] +2024-07-17 12:15:44.901297: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:15:45.607075: lr: 0.005474 +2024-07-17 12:15:45.666065: saving checkpoint... +2024-07-17 12:15:46.589695: done, saving took 0.98 seconds +2024-07-17 12:15:46.819516: This epoch took 44.236938 s + +2024-07-17 12:15:46.819727: +epoch: 122 +2024-07-17 12:16:25.109258: train loss : -0.9398 +2024-07-17 12:16:28.789095: validation loss: -0.8674 +2024-07-17 12:16:28.790291: Average global foreground Dice: [0.9495] +2024-07-17 12:16:28.790366: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:16:29.427714: lr: 0.005436 +2024-07-17 12:16:29.482826: saving checkpoint... +2024-07-17 12:16:30.383532: done, saving took 0.95 seconds +2024-07-17 12:16:30.587008: New best epoch (train loss MA): -0.9338 +2024-07-17 12:16:30.587226: Best loss so far: -0.9331 +2024-07-17 12:16:30.587259: eps: 0.0005 +2024-07-17 12:16:30.587287: Best Epoch 120 +2024-07-17 12:16:30.587321: current epoch is 122 +2024-07-17 12:16:30.587360: This epoch took 43.767584 s + +2024-07-17 12:16:30.587387: +epoch: 123 +2024-07-17 12:17:07.118184: train loss : -0.9402 +2024-07-17 12:17:10.823998: validation loss: -0.8767 +2024-07-17 12:17:10.825712: Average global foreground Dice: [0.9505] +2024-07-17 12:17:10.825823: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:17:11.506499: lr: 0.005397 +2024-07-17 12:17:11.691192: saving checkpoint... +2024-07-17 12:17:12.701123: done, saving took 1.19 seconds +2024-07-17 12:17:12.729555: This epoch took 42.142136 s + +2024-07-17 12:17:12.729613: +epoch: 124 +2024-07-17 12:17:50.636647: train loss : -0.9392 +2024-07-17 12:17:54.425357: validation loss: -0.8748 +2024-07-17 12:17:54.426868: Average global foreground Dice: [0.9499] +2024-07-17 12:17:54.427044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:17:54.930122: lr: 0.005359 +2024-07-17 12:17:54.995250: saving checkpoint... +2024-07-17 12:17:56.064586: done, saving took 1.13 seconds +2024-07-17 12:17:56.074951: New best epoch (train loss MA): -0.9346 +2024-07-17 12:17:56.075021: Best loss so far: -0.9338 +2024-07-17 12:17:56.075051: eps: 0.0005 +2024-07-17 12:17:56.075078: Best Epoch 122 +2024-07-17 12:17:56.075103: current epoch is 124 +2024-07-17 12:17:56.075133: This epoch took 43.345483 s + +2024-07-17 12:17:56.075159: +epoch: 125 +2024-07-17 12:18:30.418661: train loss : -0.9393 +2024-07-17 12:18:33.971528: validation loss: -0.8775 +2024-07-17 12:18:33.972678: Average global foreground Dice: [0.9506] +2024-07-17 12:18:33.978747: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:18:34.500260: lr: 0.00532 +2024-07-17 12:18:34.640781: saving checkpoint... +2024-07-17 12:18:35.823905: done, saving took 1.32 seconds +2024-07-17 12:18:35.854065: This epoch took 39.778872 s + +2024-07-17 12:18:35.854189: +epoch: 126 +2024-07-17 12:19:10.544358: train loss : -0.9395 +2024-07-17 12:19:14.943659: validation loss: -0.8806 +2024-07-17 12:19:14.944748: Average global foreground Dice: [0.9506] +2024-07-17 12:19:14.944841: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:19:15.899214: lr: 0.005282 +2024-07-17 12:19:16.044085: saving checkpoint... +2024-07-17 12:19:17.094269: done, saving took 1.19 seconds +2024-07-17 12:19:17.323131: New best epoch (train loss MA): -0.9352 +2024-07-17 12:19:17.323349: Best loss so far: -0.9346 +2024-07-17 12:19:17.323400: eps: 0.0005 +2024-07-17 12:19:17.323450: Best Epoch 124 +2024-07-17 12:19:17.323497: current epoch is 126 +2024-07-17 12:19:17.323551: This epoch took 41.469287 s + +2024-07-17 12:19:17.323606: +epoch: 127 +2024-07-17 12:19:49.045001: train loss : -0.9425 +2024-07-17 12:19:53.116505: validation loss: -0.8792 +2024-07-17 12:19:53.117669: Average global foreground Dice: [0.9525] +2024-07-17 12:19:53.117739: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:19:53.808164: lr: 0.005243 +2024-07-17 12:19:53.965497: saving checkpoint... +2024-07-17 12:19:54.915941: done, saving took 1.11 seconds +2024-07-17 12:19:54.956374: New best epoch (train loss MA): -0.9358 +2024-07-17 12:19:54.956455: Best loss so far: -0.9352 +2024-07-17 12:19:54.956488: eps: 0.0005 +2024-07-17 12:19:54.956519: Best Epoch 126 +2024-07-17 12:19:54.956546: current epoch is 127 +2024-07-17 12:19:54.956580: This epoch took 37.632911 s + +2024-07-17 12:19:54.956609: +epoch: 128 +2024-07-17 12:20:25.970686: train loss : -0.9446 +2024-07-17 12:20:29.896994: validation loss: -0.8847 +2024-07-17 12:20:29.898153: Average global foreground Dice: [0.954] +2024-07-17 12:20:29.898221: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:20:30.695549: lr: 0.005204 +2024-07-17 12:20:30.782388: saving checkpoint... +2024-07-17 12:20:31.434579: done, saving took 0.74 seconds +2024-07-17 12:20:31.630915: New best epoch (train loss MA): -0.9364 +2024-07-17 12:20:31.631113: Best loss so far: -0.9358 +2024-07-17 12:20:31.631146: eps: 0.0005 +2024-07-17 12:20:31.631174: Best Epoch 127 +2024-07-17 12:20:31.631198: current epoch is 128 +2024-07-17 12:20:31.631248: This epoch took 36.674607 s + +2024-07-17 12:20:31.631292: +epoch: 129 +2024-07-17 12:20:59.610287: train loss : -0.9419 +2024-07-17 12:21:03.555855: validation loss: -0.8693 +2024-07-17 12:21:03.557342: Average global foreground Dice: [0.9512] +2024-07-17 12:21:03.557437: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:21:04.318800: lr: 0.005166 +2024-07-17 12:21:04.411579: saving checkpoint... +2024-07-17 12:21:05.200856: done, saving took 0.88 seconds +2024-07-17 12:21:05.208068: This epoch took 33.576742 s + +2024-07-17 12:21:05.208163: +epoch: 130 +2024-07-17 12:21:38.315051: train loss : -0.9450 +2024-07-17 12:21:42.587484: validation loss: -0.8764 +2024-07-17 12:21:42.594764: Average global foreground Dice: [0.9545] +2024-07-17 12:21:42.594978: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:21:43.353287: lr: 0.005127 +2024-07-17 12:21:43.408718: saving checkpoint... +2024-07-17 12:21:44.584793: done, saving took 1.23 seconds +2024-07-17 12:21:44.637071: New best epoch (train loss MA): -0.9373 +2024-07-17 12:21:44.637189: Best loss so far: -0.9364 +2024-07-17 12:21:44.637243: eps: 0.0005 +2024-07-17 12:21:44.637291: Best Epoch 128 +2024-07-17 12:21:44.637338: current epoch is 130 +2024-07-17 12:21:44.637411: This epoch took 39.429154 s + +2024-07-17 12:21:44.637464: +epoch: 131 +2024-07-17 12:22:15.563654: train loss : -0.9470 +2024-07-17 12:22:19.258612: validation loss: -0.8770 +2024-07-17 12:22:19.259851: Average global foreground Dice: [0.955] +2024-07-17 12:22:19.259929: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:22:19.995221: lr: 0.005088 +2024-07-17 12:22:20.052377: saving checkpoint... +2024-07-17 12:22:21.085130: done, saving took 1.09 seconds +2024-07-17 12:22:21.198483: New best epoch (train loss MA): -0.9380 +2024-07-17 12:22:21.198615: Best loss so far: -0.9373 +2024-07-17 12:22:21.198654: eps: 0.0005 +2024-07-17 12:22:21.198690: Best Epoch 130 +2024-07-17 12:22:21.198721: current epoch is 131 +2024-07-17 12:22:21.198763: This epoch took 36.561246 s + +2024-07-17 12:22:21.198796: +epoch: 132 +2024-07-17 12:22:51.752939: train loss : -0.9421 +2024-07-17 12:22:55.479841: validation loss: -0.8682 +2024-07-17 12:22:55.480878: Average global foreground Dice: [0.9501] +2024-07-17 12:22:55.480974: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:22:56.096004: lr: 0.005049 +2024-07-17 12:22:56.099521: This epoch took 34.900681 s + +2024-07-17 12:22:56.099606: +epoch: 133 +2024-07-17 12:23:24.320487: train loss : -0.9428 +2024-07-17 12:23:28.496884: validation loss: -0.8785 +2024-07-17 12:23:28.497955: Average global foreground Dice: [0.9523] +2024-07-17 12:23:28.498053: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:23:28.902359: lr: 0.00501 +2024-07-17 12:23:28.960058: saving checkpoint... +2024-07-17 12:23:29.524236: done, saving took 0.62 seconds +2024-07-17 12:23:29.663763: New best epoch (train loss MA): -0.9386 +2024-07-17 12:23:29.663948: Best loss so far: -0.9380 +2024-07-17 12:23:29.663980: eps: 0.0005 +2024-07-17 12:23:29.664008: Best Epoch 131 +2024-07-17 12:23:29.664031: current epoch is 133 +2024-07-17 12:23:29.664062: This epoch took 33.564377 s + +2024-07-17 12:23:29.664089: +epoch: 134 +2024-07-17 12:24:01.146421: train loss : -0.9433 +2024-07-17 12:24:05.938334: validation loss: -0.8773 +2024-07-17 12:24:05.939342: Average global foreground Dice: [0.9531] +2024-07-17 12:24:05.939414: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:24:06.415695: lr: 0.004971 +2024-07-17 12:24:06.472350: saving checkpoint... +2024-07-17 12:24:07.431985: done, saving took 1.02 seconds +2024-07-17 12:24:07.653701: This epoch took 37.989572 s + +2024-07-17 12:24:07.653895: +epoch: 135 +2024-07-17 12:24:38.574898: train loss : -0.9404 +2024-07-17 12:24:42.723495: validation loss: -0.8679 +2024-07-17 12:24:42.724947: Average global foreground Dice: [0.9497] +2024-07-17 12:24:42.725020: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:24:43.193455: lr: 0.004933 +2024-07-17 12:24:43.193681: This epoch took 35.539744 s + +2024-07-17 12:24:43.193722: +epoch: 136 +2024-07-17 12:25:13.617968: train loss : -0.9420 +2024-07-17 12:25:17.285067: validation loss: -0.8776 +2024-07-17 12:25:17.286091: Average global foreground Dice: [0.9525] +2024-07-17 12:25:17.286154: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:25:17.932633: lr: 0.004894 +2024-07-17 12:25:18.023897: saving checkpoint... +2024-07-17 12:25:19.094013: done, saving took 1.16 seconds +2024-07-17 12:25:19.155172: New best epoch (train loss MA): -0.9393 +2024-07-17 12:25:19.155942: Best loss so far: -0.9386 +2024-07-17 12:25:19.155977: eps: 0.0005 +2024-07-17 12:25:19.156007: Best Epoch 133 +2024-07-17 12:25:19.156033: current epoch is 136 +2024-07-17 12:25:19.156066: This epoch took 35.962310 s + +2024-07-17 12:25:19.156095: +epoch: 137 +2024-07-17 12:25:49.274047: train loss : -0.9429 +2024-07-17 12:25:53.132357: validation loss: -0.8707 +2024-07-17 12:25:53.133564: Average global foreground Dice: [0.9519] +2024-07-17 12:25:53.133653: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:25:53.894999: lr: 0.004855 +2024-07-17 12:25:53.950468: saving checkpoint... +2024-07-17 12:25:54.792964: done, saving took 0.90 seconds +2024-07-17 12:25:54.799811: This epoch took 35.643677 s + +2024-07-17 12:25:54.799888: +epoch: 138 +2024-07-17 12:26:22.297972: train loss : -0.9440 +2024-07-17 12:26:26.026349: validation loss: -0.8665 +2024-07-17 12:26:26.027529: Average global foreground Dice: [0.9519] +2024-07-17 12:26:26.027625: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:26:26.596875: lr: 0.004816 +2024-07-17 12:26:26.664036: saving checkpoint... +2024-07-17 12:26:27.756668: done, saving took 1.16 seconds +2024-07-17 12:26:27.771175: New best epoch (train loss MA): -0.9398 +2024-07-17 12:26:27.771236: Best loss so far: -0.9393 +2024-07-17 12:26:27.771268: eps: 0.0005 +2024-07-17 12:26:27.771295: Best Epoch 136 +2024-07-17 12:26:27.771321: current epoch is 138 +2024-07-17 12:26:27.771355: This epoch took 32.971431 s + +2024-07-17 12:26:27.771391: +epoch: 139 +2024-07-17 12:26:51.566027: train loss : -0.9459 +2024-07-17 12:26:54.922515: validation loss: -0.8787 +2024-07-17 12:26:54.923599: Average global foreground Dice: [0.9539] +2024-07-17 12:26:54.923728: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:26:55.635434: lr: 0.004776 +2024-07-17 12:26:55.702342: saving checkpoint... +2024-07-17 12:26:56.825490: done, saving took 1.18 seconds +2024-07-17 12:26:56.834618: This epoch took 29.063186 s + +2024-07-17 12:26:56.834674: +epoch: 140 +2024-07-17 12:27:21.042000: train loss : -0.9384 +2024-07-17 12:27:24.507598: validation loss: -0.8695 +2024-07-17 12:27:24.508811: Average global foreground Dice: [0.9487] +2024-07-17 12:27:24.508911: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:27:25.191887: lr: 0.004737 +2024-07-17 12:27:25.192089: This epoch took 28.357370 s + +2024-07-17 12:27:25.192125: +epoch: 141 +2024-07-17 12:27:49.257985: train loss : -0.9400 +2024-07-17 12:27:52.790576: validation loss: -0.8752 +2024-07-17 12:27:52.792208: Average global foreground Dice: [0.9505] +2024-07-17 12:27:52.792287: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:27:53.437420: lr: 0.004698 +2024-07-17 12:27:53.437630: This epoch took 28.245464 s + +2024-07-17 12:27:53.437667: +epoch: 142 +2024-07-17 12:28:17.627059: train loss : -0.9428 +2024-07-17 12:28:20.621388: validation loss: -0.8688 +2024-07-17 12:28:20.622752: Average global foreground Dice: [0.9517] +2024-07-17 12:28:20.622822: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:28:21.206693: lr: 0.004659 +2024-07-17 12:28:21.206935: This epoch took 27.769232 s + +2024-07-17 12:28:21.206979: +epoch: 143 +2024-07-17 12:28:45.326473: train loss : -0.9424 +2024-07-17 12:28:48.538209: validation loss: -0.8635 +2024-07-17 12:28:48.548270: Average global foreground Dice: [0.9514] +2024-07-17 12:28:48.548597: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:28:49.054081: lr: 0.00462 +2024-07-17 12:28:49.054284: New best epoch (train loss MA): -0.9404 +2024-07-17 12:28:49.054331: Best loss so far: -0.9398 +2024-07-17 12:28:49.054368: eps: 0.0005 +2024-07-17 12:28:49.054409: Best Epoch 138 +2024-07-17 12:28:49.054444: current epoch is 143 +2024-07-17 12:28:49.054531: This epoch took 27.847512 s + +2024-07-17 12:28:49.054607: +epoch: 144 +2024-07-17 12:29:14.001752: train loss : -0.9462 +2024-07-17 12:29:17.374669: validation loss: -0.8776 +2024-07-17 12:29:17.375695: Average global foreground Dice: [0.9541] +2024-07-17 12:29:17.375754: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:29:18.105904: lr: 0.004581 +2024-07-17 12:29:18.282772: saving checkpoint... +2024-07-17 12:29:19.362851: done, saving took 1.26 seconds +2024-07-17 12:29:19.396488: This epoch took 30.341798 s + +2024-07-17 12:29:19.396547: +epoch: 145 +2024-07-17 12:29:43.435515: train loss : -0.9463 +2024-07-17 12:29:46.956504: validation loss: -0.8650 +2024-07-17 12:29:46.959066: Average global foreground Dice: [0.9528] +2024-07-17 12:29:46.959135: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:29:47.497539: lr: 0.004541 +2024-07-17 12:29:47.550926: saving checkpoint... +2024-07-17 12:29:48.597114: done, saving took 1.10 seconds +2024-07-17 12:29:48.646847: New best epoch (train loss MA): -0.9412 +2024-07-17 12:29:48.646932: Best loss so far: -0.9404 +2024-07-17 12:29:48.646974: eps: 0.0005 +2024-07-17 12:29:48.647015: Best Epoch 143 +2024-07-17 12:29:48.647051: current epoch is 145 +2024-07-17 12:29:48.647095: This epoch took 29.250496 s + +2024-07-17 12:29:48.647134: +epoch: 146 +2024-07-17 12:30:12.594385: train loss : -0.9469 +2024-07-17 12:30:15.942829: validation loss: -0.8626 +2024-07-17 12:30:15.943866: Average global foreground Dice: [0.9533] +2024-07-17 12:30:15.943954: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:30:16.516883: lr: 0.004502 +2024-07-17 12:30:16.579195: saving checkpoint... +2024-07-17 12:30:17.673859: done, saving took 1.16 seconds +2024-07-17 12:30:17.752534: This epoch took 29.105361 s + +2024-07-17 12:30:17.752594: +epoch: 147 +2024-07-17 12:30:41.905647: train loss : -0.9473 +2024-07-17 12:30:45.198453: validation loss: -0.8684 +2024-07-17 12:30:45.199618: Average global foreground Dice: [0.9537] +2024-07-17 12:30:45.199711: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:30:45.881778: lr: 0.004463 +2024-07-17 12:30:45.948251: saving checkpoint... +2024-07-17 12:30:47.063084: done, saving took 1.18 seconds +2024-07-17 12:30:47.098790: New best epoch (train loss MA): -0.9420 +2024-07-17 12:30:47.098846: Best loss so far: -0.9412 +2024-07-17 12:30:47.098875: eps: 0.0005 +2024-07-17 12:30:47.098902: Best Epoch 145 +2024-07-17 12:30:47.098927: current epoch is 147 +2024-07-17 12:30:47.098957: This epoch took 29.346326 s + +2024-07-17 12:30:47.098983: +epoch: 148 +2024-07-17 12:31:11.792238: train loss : -0.9465 +2024-07-17 12:31:15.022162: validation loss: -0.8669 +2024-07-17 12:31:15.023244: Average global foreground Dice: [0.9535] +2024-07-17 12:31:15.023304: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:31:15.854941: lr: 0.004423 +2024-07-17 12:31:15.931600: saving checkpoint... +2024-07-17 12:31:17.013623: done, saving took 1.16 seconds +2024-07-17 12:31:17.015530: This epoch took 29.916516 s + +2024-07-17 12:31:17.015579: +epoch: 149 +2024-07-17 12:31:40.663210: train loss : -0.9416 +2024-07-17 12:31:44.180967: validation loss: -0.8758 +2024-07-17 12:31:44.182104: Average global foreground Dice: [0.9518] +2024-07-17 12:31:44.182165: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:31:44.795817: lr: 0.004384 +2024-07-17 12:31:44.795993: saving scheduled checkpoint file... +2024-07-17 12:31:44.861239: saving checkpoint... +2024-07-17 12:31:45.876634: done, saving took 1.08 seconds +2024-07-17 12:31:45.911899: done +2024-07-17 12:31:45.922798: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 12:31:45.922996: The split file contains 5 splits. +2024-07-17 12:31:45.923036: Desired fold for training: 0 +2024-07-17 12:31:45.923070: This split has 74 training and 20 validation cases. +2024-07-17 12:31:47.058392: Performing validation with validation data from task Task125_All. +2024-07-17 12:32:03.556936: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 12:32:03.557447: The split file contains 5 splits. +2024-07-17 12:32:03.557499: Desired fold for training: 0 +2024-07-17 12:32:03.557553: This split has 74 training and 20 validation cases. +2024-07-17 12:32:03.885041: This epoch took 46.869404 s + +2024-07-17 12:32:03.885142: +epoch: 150 +2024-07-17 12:32:27.878015: train loss : -0.9447 +2024-07-17 12:32:31.347003: validation loss: -0.8783 +2024-07-17 12:32:31.348130: Average global foreground Dice: [0.9541] +2024-07-17 12:32:31.348392: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:32:31.893398: lr: 0.004344 +2024-07-17 12:32:32.055389: saving checkpoint... +2024-07-17 12:32:33.244975: done, saving took 1.35 seconds +2024-07-17 12:32:33.274198: This epoch took 29.389015 s + +2024-07-17 12:32:33.274268: +epoch: 151 +2024-07-17 12:32:55.669750: train loss : -0.9424 +2024-07-17 12:32:57.956421: validation loss: -0.8649 +2024-07-17 12:32:57.957565: Average global foreground Dice: [0.9505] +2024-07-17 12:32:57.957624: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:32:58.493926: lr: 0.004305 +2024-07-17 12:32:58.494145: This epoch took 25.219822 s + +2024-07-17 12:32:58.494286: +epoch: 152 +2024-07-17 12:33:21.126186: train loss : -0.9464 +2024-07-17 12:33:23.528902: validation loss: -0.8621 +2024-07-17 12:33:23.529874: Average global foreground Dice: [0.9527] +2024-07-17 12:33:23.529998: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:33:24.133767: lr: 0.004265 +2024-07-17 12:33:24.133975: New best epoch (train loss MA): -0.9427 +2024-07-17 12:33:24.134021: Best loss so far: -0.9420 +2024-07-17 12:33:24.134054: eps: 0.0005 +2024-07-17 12:33:24.134086: Best Epoch 147 +2024-07-17 12:33:24.134115: current epoch is 152 +2024-07-17 12:33:24.134150: This epoch took 25.639799 s + +2024-07-17 12:33:24.134182: +epoch: 153 +2024-07-17 12:33:46.904557: train loss : -0.9460 +2024-07-17 12:33:49.522852: validation loss: -0.8719 +2024-07-17 12:33:49.524105: Average global foreground Dice: [0.9536] +2024-07-17 12:33:49.524173: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:33:50.127995: lr: 0.004226 +2024-07-17 12:33:50.214809: saving checkpoint... +2024-07-17 12:33:51.035844: done, saving took 0.91 seconds +2024-07-17 12:33:51.260480: This epoch took 27.126258 s + +2024-07-17 12:33:51.260593: +epoch: 154 +2024-07-17 12:34:14.645924: train loss : -0.9503 +2024-07-17 12:34:17.497692: validation loss: -0.8680 +2024-07-17 12:34:17.499830: Average global foreground Dice: [0.956] +2024-07-17 12:34:17.506052: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:34:18.133899: lr: 0.004186 +2024-07-17 12:34:18.360286: saving checkpoint... +2024-07-17 12:34:19.317944: done, saving took 1.18 seconds +2024-07-17 12:34:19.320189: New best epoch (train loss MA): -0.9435 +2024-07-17 12:34:19.320239: Best loss so far: -0.9427 +2024-07-17 12:34:19.320269: eps: 0.0005 +2024-07-17 12:34:19.320295: Best Epoch 152 +2024-07-17 12:34:19.320319: current epoch is 154 +2024-07-17 12:34:19.320364: This epoch took 28.059735 s + +2024-07-17 12:34:19.320394: +epoch: 155 +2024-07-17 12:34:42.182590: train loss : -0.9522 +2024-07-17 12:34:44.898784: validation loss: -0.8684 +2024-07-17 12:34:44.899943: Average global foreground Dice: [0.9574] +2024-07-17 12:34:44.900064: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:34:45.519917: lr: 0.004146 +2024-07-17 12:34:45.609925: saving checkpoint... +2024-07-17 12:34:46.502891: done, saving took 0.98 seconds +2024-07-17 12:34:46.712918: New best epoch (train loss MA): -0.9441 +2024-07-17 12:34:46.712992: Best loss so far: -0.9435 +2024-07-17 12:34:46.713026: eps: 0.0005 +2024-07-17 12:34:46.713054: Best Epoch 154 +2024-07-17 12:34:46.713079: current epoch is 155 +2024-07-17 12:34:46.713110: This epoch took 27.392684 s + +2024-07-17 12:34:46.713137: +epoch: 156 +2024-07-17 12:35:10.207339: train loss : -0.9459 +2024-07-17 12:35:13.066891: validation loss: -0.8775 +2024-07-17 12:35:13.068146: Average global foreground Dice: [0.9541] +2024-07-17 12:35:13.068215: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:35:13.636241: lr: 0.004107 +2024-07-17 12:35:13.702690: saving checkpoint... +2024-07-17 12:35:14.496103: done, saving took 0.86 seconds +2024-07-17 12:35:14.681773: This epoch took 27.968597 s + +2024-07-17 12:35:14.681958: +epoch: 157 +2024-07-17 12:35:38.250701: train loss : -0.9480 +2024-07-17 12:35:41.114221: validation loss: -0.8724 +2024-07-17 12:35:41.115673: Average global foreground Dice: [0.9539] +2024-07-17 12:35:41.115798: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:35:41.690081: lr: 0.004067 +2024-07-17 12:35:41.752471: saving checkpoint... +2024-07-17 12:35:42.671010: done, saving took 0.98 seconds +2024-07-17 12:35:42.672669: This epoch took 27.990654 s + +2024-07-17 12:35:42.672718: +epoch: 158 +2024-07-17 12:36:06.314191: train loss : -0.9450 +2024-07-17 12:36:09.200444: validation loss: -0.8744 +2024-07-17 12:36:09.201920: Average global foreground Dice: [0.9544] +2024-07-17 12:36:09.201991: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:36:09.876703: lr: 0.004027 +2024-07-17 12:36:09.970375: saving checkpoint... +2024-07-17 12:36:10.848651: done, saving took 0.96 seconds +2024-07-17 12:36:11.084006: This epoch took 28.411246 s + +2024-07-17 12:36:11.084206: +epoch: 159 +2024-07-17 12:36:34.628628: train loss : -0.9476 +2024-07-17 12:36:37.493354: validation loss: -0.8711 +2024-07-17 12:36:37.494555: Average global foreground Dice: [0.955] +2024-07-17 12:36:37.494623: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:36:38.105258: lr: 0.003987 +2024-07-17 12:36:38.166690: saving checkpoint... +2024-07-17 12:36:39.054827: done, saving took 0.95 seconds +2024-07-17 12:36:39.284840: New best epoch (train loss MA): -0.9447 +2024-07-17 12:36:39.284954: Best loss so far: -0.9441 +2024-07-17 12:36:39.284984: eps: 0.0005 +2024-07-17 12:36:39.285010: Best Epoch 155 +2024-07-17 12:36:39.285034: current epoch is 159 +2024-07-17 12:36:39.285067: This epoch took 28.200807 s + +2024-07-17 12:36:39.285093: +epoch: 160 +2024-07-17 12:37:02.808395: train loss : -0.9455 +2024-07-17 12:37:05.734771: validation loss: -0.8711 +2024-07-17 12:37:05.736338: Average global foreground Dice: [0.953] +2024-07-17 12:37:05.736422: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:37:06.380088: lr: 0.003947 +2024-07-17 12:37:06.380475: This epoch took 27.095347 s + +2024-07-17 12:37:06.380517: +epoch: 161 +2024-07-17 12:37:29.884171: train loss : -0.9488 +2024-07-17 12:37:32.909427: validation loss: -0.8642 +2024-07-17 12:37:32.910639: Average global foreground Dice: [0.9548] +2024-07-17 12:37:32.910700: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:37:33.337211: lr: 0.003907 +2024-07-17 12:37:33.393818: saving checkpoint... +2024-07-17 12:37:34.300534: done, saving took 0.96 seconds +2024-07-17 12:37:34.302403: This epoch took 27.921851 s + +2024-07-17 12:37:34.302459: +epoch: 162 +2024-07-17 12:37:58.071659: train loss : -0.9502 +2024-07-17 12:38:01.212286: validation loss: -0.8625 +2024-07-17 12:38:01.213287: Average global foreground Dice: [0.9555] +2024-07-17 12:38:01.213355: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:38:01.746064: lr: 0.003867 +2024-07-17 12:38:01.802943: saving checkpoint... +2024-07-17 12:38:02.703145: done, saving took 0.96 seconds +2024-07-17 12:38:02.705331: New best epoch (train loss MA): -0.9454 +2024-07-17 12:38:02.705483: Best loss so far: -0.9447 +2024-07-17 12:38:02.705547: eps: 0.0005 +2024-07-17 12:38:02.705604: Best Epoch 159 +2024-07-17 12:38:02.705657: current epoch is 162 +2024-07-17 12:38:02.705720: This epoch took 28.403224 s + +2024-07-17 12:38:02.705786: +epoch: 163 +2024-07-17 12:38:26.361366: train loss : -0.9497 +2024-07-17 12:38:29.128838: validation loss: -0.8659 +2024-07-17 12:38:29.130016: Average global foreground Dice: [0.9553] +2024-07-17 12:38:29.130110: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:38:29.843158: lr: 0.003827 +2024-07-17 12:38:29.906716: saving checkpoint... +2024-07-17 12:38:30.763952: done, saving took 0.91 seconds +2024-07-17 12:38:30.964723: This epoch took 28.258844 s + +2024-07-17 12:38:30.964983: +epoch: 164 +2024-07-17 12:38:55.092144: train loss : -0.9502 +2024-07-17 12:38:57.971497: validation loss: -0.8703 +2024-07-17 12:38:57.972599: Average global foreground Dice: [0.9566] +2024-07-17 12:38:57.972674: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:38:58.715744: lr: 0.003787 +2024-07-17 12:38:58.790149: saving checkpoint... +2024-07-17 12:38:59.655231: done, saving took 0.94 seconds +2024-07-17 12:38:59.888261: New best epoch (train loss MA): -0.9460 +2024-07-17 12:38:59.888413: Best loss so far: -0.9454 +2024-07-17 12:38:59.888449: eps: 0.0005 +2024-07-17 12:38:59.888482: Best Epoch 162 +2024-07-17 12:38:59.888507: current epoch is 164 +2024-07-17 12:38:59.888540: This epoch took 28.923518 s + +2024-07-17 12:38:59.888566: +epoch: 165 +2024-07-17 12:39:23.631762: train loss : -0.9500 +2024-07-17 12:39:26.409896: validation loss: -0.8687 +2024-07-17 12:39:26.417146: Average global foreground Dice: [0.9558] +2024-07-17 12:39:26.417324: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:39:27.096537: lr: 0.003747 +2024-07-17 12:39:27.152174: saving checkpoint... +2024-07-17 12:39:28.001482: done, saving took 0.90 seconds +2024-07-17 12:39:28.003693: This epoch took 28.115096 s + +2024-07-17 12:39:28.003750: +epoch: 166 +2024-07-17 12:39:52.240754: train loss : -0.9520 +2024-07-17 12:39:55.235838: validation loss: -0.8717 +2024-07-17 12:39:55.236947: Average global foreground Dice: [0.9573] +2024-07-17 12:39:55.237077: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:39:55.810415: lr: 0.003707 +2024-07-17 12:39:55.866153: saving checkpoint... +2024-07-17 12:39:56.715422: done, saving took 0.90 seconds +2024-07-17 12:39:56.717222: New best epoch (train loss MA): -0.9467 +2024-07-17 12:39:56.717272: Best loss so far: -0.9460 +2024-07-17 12:39:56.717302: eps: 0.0005 +2024-07-17 12:39:56.717329: Best Epoch 164 +2024-07-17 12:39:56.717353: current epoch is 166 +2024-07-17 12:39:56.717396: This epoch took 28.713612 s + +2024-07-17 12:39:56.717424: +epoch: 167 +2024-07-17 12:40:20.677600: train loss : -0.9522 +2024-07-17 12:40:23.492445: validation loss: -0.8690 +2024-07-17 12:40:23.493967: Average global foreground Dice: [0.957] +2024-07-17 12:40:23.494066: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:40:24.071979: lr: 0.003667 +2024-07-17 12:40:24.132280: saving checkpoint... +2024-07-17 12:40:25.018828: done, saving took 0.95 seconds +2024-07-17 12:40:25.239211: This epoch took 28.521747 s + +2024-07-17 12:40:25.239414: +epoch: 168 +2024-07-17 12:40:48.980368: train loss : -0.9529 +2024-07-17 12:40:52.073810: validation loss: -0.8798 +2024-07-17 12:40:52.075102: Average global foreground Dice: [0.9589] +2024-07-17 12:40:52.075171: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:40:52.628038: lr: 0.003627 +2024-07-17 12:40:52.812352: saving checkpoint... +2024-07-17 12:40:53.753185: done, saving took 1.12 seconds +2024-07-17 12:40:54.008230: New best epoch (train loss MA): -0.9475 +2024-07-17 12:40:54.008414: Best loss so far: -0.9467 +2024-07-17 12:40:54.008444: eps: 0.0005 +2024-07-17 12:40:54.008473: Best Epoch 166 +2024-07-17 12:40:54.008497: current epoch is 168 +2024-07-17 12:40:54.008530: This epoch took 28.769070 s + +2024-07-17 12:40:54.008557: +epoch: 169 +2024-07-17 12:41:19.601207: train loss : -0.9542 +2024-07-17 12:41:22.851330: validation loss: -0.8780 +2024-07-17 12:41:22.865025: Average global foreground Dice: [0.9591] +2024-07-17 12:41:22.865142: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:41:23.423050: lr: 0.003586 +2024-07-17 12:41:23.483301: saving checkpoint... +2024-07-17 12:41:24.386866: done, saving took 0.96 seconds +2024-07-17 12:41:24.642475: This epoch took 30.633883 s + +2024-07-17 12:41:24.642630: +epoch: 170 +2024-07-17 12:41:50.139425: train loss : -0.9533 +2024-07-17 12:41:53.317909: validation loss: -0.8748 +2024-07-17 12:41:53.319197: Average global foreground Dice: [0.9583] +2024-07-17 12:41:53.325322: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:41:54.053508: lr: 0.003546 +2024-07-17 12:41:54.173163: saving checkpoint... +2024-07-17 12:41:55.056644: done, saving took 1.00 seconds +2024-07-17 12:41:55.058539: New best epoch (train loss MA): -0.9483 +2024-07-17 12:41:55.058591: Best loss so far: -0.9475 +2024-07-17 12:41:55.058621: eps: 0.0005 +2024-07-17 12:41:55.058646: Best Epoch 168 +2024-07-17 12:41:55.058671: current epoch is 170 +2024-07-17 12:41:55.058702: This epoch took 30.416035 s + +2024-07-17 12:41:55.058728: +epoch: 171 +2024-07-17 12:42:20.684477: train loss : -0.9520 +2024-07-17 12:42:24.247949: validation loss: -0.8641 +2024-07-17 12:42:24.249059: Average global foreground Dice: [0.9567] +2024-07-17 12:42:24.249140: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:42:24.718734: lr: 0.003505 +2024-07-17 12:42:24.779769: saving checkpoint... +2024-07-17 12:42:25.766856: done, saving took 1.05 seconds +2024-07-17 12:42:25.968921: This epoch took 30.910156 s + +2024-07-17 12:42:25.969088: +epoch: 172 +2024-07-17 12:42:51.978480: train loss : -0.9528 +2024-07-17 12:42:55.224002: validation loss: -0.8662 +2024-07-17 12:42:55.225134: Average global foreground Dice: [0.957] +2024-07-17 12:42:55.225210: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:42:55.890076: lr: 0.003465 +2024-07-17 12:42:55.958352: saving checkpoint... +2024-07-17 12:42:56.833426: done, saving took 0.94 seconds +2024-07-17 12:42:56.877346: New best epoch (train loss MA): -0.9489 +2024-07-17 12:42:56.877470: Best loss so far: -0.9483 +2024-07-17 12:42:56.877525: eps: 0.0005 +2024-07-17 12:42:56.877575: Best Epoch 170 +2024-07-17 12:42:56.877621: current epoch is 172 +2024-07-17 12:42:56.877675: This epoch took 30.908543 s + +2024-07-17 12:42:56.877724: +epoch: 173 +2024-07-17 12:43:22.752156: train loss : -0.9533 +2024-07-17 12:43:25.991422: validation loss: -0.8675 +2024-07-17 12:43:25.998679: Average global foreground Dice: [0.9574] +2024-07-17 12:43:25.998783: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:43:26.546755: lr: 0.003424 +2024-07-17 12:43:26.604095: saving checkpoint... +2024-07-17 12:43:27.519920: done, saving took 0.97 seconds +2024-07-17 12:43:27.752812: This epoch took 30.875033 s + +2024-07-17 12:43:27.752876: +epoch: 174 +2024-07-17 12:43:53.615238: train loss : -0.9539 +2024-07-17 12:43:56.771078: validation loss: -0.8612 +2024-07-17 12:43:56.778996: Average global foreground Dice: [0.958] +2024-07-17 12:43:56.779104: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:43:57.462544: lr: 0.003384 +2024-07-17 12:43:57.565065: saving checkpoint... +2024-07-17 12:43:58.440900: done, saving took 0.98 seconds +2024-07-17 12:43:58.663104: New best epoch (train loss MA): -0.9495 +2024-07-17 12:43:58.663287: Best loss so far: -0.9489 +2024-07-17 12:43:58.663337: eps: 0.0005 +2024-07-17 12:43:58.663380: Best Epoch 172 +2024-07-17 12:43:58.663419: current epoch is 174 +2024-07-17 12:43:58.663465: This epoch took 30.910551 s + +2024-07-17 12:43:58.663506: +epoch: 175 +2024-07-17 12:44:24.572619: train loss : -0.9539 +2024-07-17 12:44:27.746741: validation loss: -0.8710 +2024-07-17 12:44:27.748338: Average global foreground Dice: [0.9586] +2024-07-17 12:44:27.748438: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:44:28.391306: lr: 0.003343 +2024-07-17 12:44:28.447809: saving checkpoint... +2024-07-17 12:44:29.352466: done, saving took 0.96 seconds +2024-07-17 12:44:29.354398: This epoch took 30.690845 s + +2024-07-17 12:44:29.354447: +epoch: 176 +2024-07-17 12:44:54.962681: train loss : -0.9556 +2024-07-17 12:44:58.163441: validation loss: -0.8706 +2024-07-17 12:44:58.166158: Average global foreground Dice: [0.9598] +2024-07-17 12:44:58.166235: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:44:58.907560: lr: 0.003303 +2024-07-17 12:44:59.060036: saving checkpoint... +2024-07-17 12:44:59.997981: done, saving took 1.09 seconds +2024-07-17 12:45:00.227588: New best epoch (train loss MA): -0.9502 +2024-07-17 12:45:00.227663: Best loss so far: -0.9495 +2024-07-17 12:45:00.227693: eps: 0.0005 +2024-07-17 12:45:00.227719: Best Epoch 174 +2024-07-17 12:45:00.227741: current epoch is 176 +2024-07-17 12:45:00.227812: This epoch took 30.873314 s + +2024-07-17 12:45:00.227842: +epoch: 177 +2024-07-17 12:45:26.107338: train loss : -0.9531 +2024-07-17 12:45:29.481061: validation loss: -0.8633 +2024-07-17 12:45:29.501199: Average global foreground Dice: [0.9578] +2024-07-17 12:45:29.501394: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:45:30.193216: lr: 0.003262 +2024-07-17 12:45:30.252617: saving checkpoint... +2024-07-17 12:45:31.136317: done, saving took 0.94 seconds +2024-07-17 12:45:31.350277: This epoch took 31.122395 s + +2024-07-17 12:45:31.350489: +epoch: 178 +2024-07-17 12:45:57.291199: train loss : -0.9535 +2024-07-17 12:46:00.776169: validation loss: -0.8660 +2024-07-17 12:46:00.777354: Average global foreground Dice: [0.9574] +2024-07-17 12:46:00.777460: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:46:01.327104: lr: 0.003221 +2024-07-17 12:46:01.519524: saving checkpoint... +2024-07-17 12:46:02.378444: done, saving took 1.05 seconds +2024-07-17 12:46:02.564558: This epoch took 31.214002 s + +2024-07-17 12:46:02.564759: +epoch: 179 +2024-07-17 12:46:28.462811: train loss : -0.9519 +2024-07-17 12:46:31.819963: validation loss: -0.8630 +2024-07-17 12:46:31.821758: Average global foreground Dice: [0.9569] +2024-07-17 12:46:31.821905: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:46:32.439832: lr: 0.00318 +2024-07-17 12:46:32.440048: New best epoch (train loss MA): -0.9507 +2024-07-17 12:46:32.440092: Best loss so far: -0.9502 +2024-07-17 12:46:32.440122: eps: 0.0005 +2024-07-17 12:46:32.440152: Best Epoch 176 +2024-07-17 12:46:32.440177: current epoch is 179 +2024-07-17 12:46:32.440212: This epoch took 29.875396 s + +2024-07-17 12:46:32.440243: +epoch: 180 +2024-07-17 12:46:58.300640: train loss : -0.9531 +2024-07-17 12:47:01.882508: validation loss: -0.8666 +2024-07-17 12:47:01.883703: Average global foreground Dice: [0.9577] +2024-07-17 12:47:01.883795: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:47:02.384158: lr: 0.003139 +2024-07-17 12:47:02.442571: saving checkpoint... +2024-07-17 12:47:03.285321: done, saving took 0.90 seconds +2024-07-17 12:47:03.478962: This epoch took 31.038676 s + +2024-07-17 12:47:03.479250: +epoch: 181 +2024-07-17 12:47:29.553641: train loss : -0.9548 +2024-07-17 12:47:32.742127: validation loss: -0.8675 +2024-07-17 12:47:32.756837: Average global foreground Dice: [0.9587] +2024-07-17 12:47:32.756930: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:47:33.376443: lr: 0.003098 +2024-07-17 12:47:33.445839: saving checkpoint... +2024-07-17 12:47:34.307759: done, saving took 0.93 seconds +2024-07-17 12:47:34.547783: This epoch took 31.068429 s + +2024-07-17 12:47:34.547907: +epoch: 182 +2024-07-17 12:47:59.438714: train loss : -0.9550 +2024-07-17 12:48:02.726893: validation loss: -0.8833 +2024-07-17 12:48:02.727974: Average global foreground Dice: [0.9602] +2024-07-17 12:48:02.728035: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:48:03.372403: lr: 0.003057 +2024-07-17 12:48:03.442543: saving checkpoint... +2024-07-17 12:48:04.363870: done, saving took 0.99 seconds +2024-07-17 12:48:04.365585: New best epoch (train loss MA): -0.9514 +2024-07-17 12:48:04.365632: Best loss so far: -0.9507 +2024-07-17 12:48:04.365660: eps: 0.0005 +2024-07-17 12:48:04.365686: Best Epoch 179 +2024-07-17 12:48:04.365710: current epoch is 182 +2024-07-17 12:48:04.365740: This epoch took 29.817798 s + +2024-07-17 12:48:04.365765: +epoch: 183 +2024-07-17 12:48:29.508299: train loss : -0.9562 +2024-07-17 12:48:32.831759: validation loss: -0.8751 +2024-07-17 12:48:32.833527: Average global foreground Dice: [0.9602] +2024-07-17 12:48:32.833622: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:48:33.434312: lr: 0.003016 +2024-07-17 12:48:33.503513: saving checkpoint... +2024-07-17 12:48:34.369164: done, saving took 0.93 seconds +2024-07-17 12:48:34.570472: This epoch took 30.204670 s + +2024-07-17 12:48:34.570605: +epoch: 184 +2024-07-17 12:48:59.687358: train loss : -0.9542 +2024-07-17 12:49:02.744142: validation loss: -0.8645 +2024-07-17 12:49:02.745331: Average global foreground Dice: [0.958] +2024-07-17 12:49:02.745413: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:49:03.435129: lr: 0.002975 +2024-07-17 12:49:03.496781: saving checkpoint... +2024-07-17 12:49:04.387410: done, saving took 0.95 seconds +2024-07-17 12:49:04.560706: This epoch took 29.990062 s + +2024-07-17 12:49:04.560771: +epoch: 185 +2024-07-17 12:49:29.948864: train loss : -0.9566 +2024-07-17 12:49:33.283575: validation loss: -0.8635 +2024-07-17 12:49:33.285463: Average global foreground Dice: [0.9597] +2024-07-17 12:49:33.285548: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:49:34.076706: lr: 0.002934 +2024-07-17 12:49:34.159181: saving checkpoint... +2024-07-17 12:49:35.073677: done, saving took 1.00 seconds +2024-07-17 12:49:35.291441: New best epoch (train loss MA): -0.9523 +2024-07-17 12:49:35.291517: Best loss so far: -0.9514 +2024-07-17 12:49:35.291549: eps: 0.0005 +2024-07-17 12:49:35.291574: Best Epoch 182 +2024-07-17 12:49:35.291598: current epoch is 185 +2024-07-17 12:49:35.291635: This epoch took 30.730826 s + +2024-07-17 12:49:35.291666: +epoch: 186 +2024-07-17 12:50:00.777635: train loss : -0.9561 +2024-07-17 12:50:03.994315: validation loss: -0.8846 +2024-07-17 12:50:03.995764: Average global foreground Dice: [0.9608] +2024-07-17 12:50:03.995835: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:50:04.706535: lr: 0.002892 +2024-07-17 12:50:04.764050: saving checkpoint... +2024-07-17 12:50:05.655301: done, saving took 0.95 seconds +2024-07-17 12:50:05.657088: This epoch took 30.365392 s + +2024-07-17 12:50:05.657138: +epoch: 187 +2024-07-17 12:50:30.883062: train loss : -0.9566 +2024-07-17 12:50:34.309874: validation loss: -0.8687 +2024-07-17 12:50:34.311697: Average global foreground Dice: [0.9603] +2024-07-17 12:50:34.311764: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:50:34.860327: lr: 0.002851 +2024-07-17 12:50:35.057428: saving checkpoint... +2024-07-17 12:50:35.729485: done, saving took 0.87 seconds +2024-07-17 12:50:36.052534: New best epoch (train loss MA): -0.9528 +2024-07-17 12:50:36.052684: Best loss so far: -0.9523 +2024-07-17 12:50:36.052727: eps: 0.0005 +2024-07-17 12:50:36.052767: Best Epoch 185 +2024-07-17 12:50:36.052803: current epoch is 187 +2024-07-17 12:50:36.052917: This epoch took 30.395742 s + +2024-07-17 12:50:36.052960: +epoch: 188 +2024-07-17 12:51:01.640494: train loss : -0.9562 +2024-07-17 12:51:04.770104: validation loss: -0.8801 +2024-07-17 12:51:04.777757: Average global foreground Dice: [0.9601] +2024-07-17 12:51:04.777853: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:51:05.557357: lr: 0.00281 +2024-07-17 12:51:05.635632: saving checkpoint... +2024-07-17 12:51:06.484387: done, saving took 0.93 seconds +2024-07-17 12:51:06.671463: This epoch took 30.618452 s + +2024-07-17 12:51:06.671637: +epoch: 189 +2024-07-17 12:51:31.977261: train loss : -0.9577 +2024-07-17 12:51:35.401602: validation loss: -0.8703 +2024-07-17 12:51:35.404333: Average global foreground Dice: [0.9608] +2024-07-17 12:51:35.404500: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:51:36.006909: lr: 0.002768 +2024-07-17 12:51:36.128139: saving checkpoint... +2024-07-17 12:51:36.805076: done, saving took 0.80 seconds +2024-07-17 12:51:37.151479: New best epoch (train loss MA): -0.9534 +2024-07-17 12:51:37.151649: Best loss so far: -0.9528 +2024-07-17 12:51:37.151706: eps: 0.0005 +2024-07-17 12:51:37.151756: Best Epoch 187 +2024-07-17 12:51:37.151800: current epoch is 189 +2024-07-17 12:51:37.151861: This epoch took 30.480173 s + +2024-07-17 12:51:37.151912: +epoch: 190 +2024-07-17 12:52:02.373581: train loss : -0.9580 +2024-07-17 12:52:05.622361: validation loss: -0.8743 +2024-07-17 12:52:05.623636: Average global foreground Dice: [0.9612] +2024-07-17 12:52:05.623716: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:52:06.357718: lr: 0.002727 +2024-07-17 12:52:06.481344: saving checkpoint... +2024-07-17 12:52:07.327401: done, saving took 0.97 seconds +2024-07-17 12:52:07.524573: This epoch took 30.372604 s + +2024-07-17 12:52:07.524639: +epoch: 191 +2024-07-17 12:52:32.642732: train loss : -0.9593 +2024-07-17 12:52:35.794833: validation loss: -0.8609 +2024-07-17 12:52:35.796007: Average global foreground Dice: [0.9606] +2024-07-17 12:52:35.796078: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:52:36.408486: lr: 0.002685 +2024-07-17 12:52:36.526025: saving checkpoint... +2024-07-17 12:52:37.426833: done, saving took 1.02 seconds +2024-07-17 12:52:37.663657: New best epoch (train loss MA): -0.9541 +2024-07-17 12:52:37.663799: Best loss so far: -0.9534 +2024-07-17 12:52:37.663839: eps: 0.0005 +2024-07-17 12:52:37.663880: Best Epoch 189 +2024-07-17 12:52:37.663916: current epoch is 191 +2024-07-17 12:52:37.663959: This epoch took 30.139283 s + +2024-07-17 12:52:37.663994: +epoch: 192 +2024-07-17 12:53:02.949369: train loss : -0.9585 +2024-07-17 12:53:06.224200: validation loss: -0.8803 +2024-07-17 12:53:06.225687: Average global foreground Dice: [0.9624] +2024-07-17 12:53:06.225830: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:53:07.070180: lr: 0.002643 +2024-07-17 12:53:07.128246: saving checkpoint... +2024-07-17 12:53:08.006269: done, saving took 0.94 seconds +2024-07-17 12:53:08.228382: This epoch took 30.564340 s + +2024-07-17 12:53:08.228497: +epoch: 193 +2024-07-17 12:53:33.499715: train loss : -0.9574 +2024-07-17 12:53:36.967813: validation loss: -0.8727 +2024-07-17 12:53:36.969768: Average global foreground Dice: [0.9608] +2024-07-17 12:53:36.969852: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:53:37.569247: lr: 0.002601 +2024-07-17 12:53:37.651539: saving checkpoint... +2024-07-17 12:53:38.531695: done, saving took 0.96 seconds +2024-07-17 12:53:38.715127: New best epoch (train loss MA): -0.9546 +2024-07-17 12:53:38.715208: Best loss so far: -0.9541 +2024-07-17 12:53:38.715238: eps: 0.0005 +2024-07-17 12:53:38.715266: Best Epoch 191 +2024-07-17 12:53:38.715290: current epoch is 193 +2024-07-17 12:53:38.715325: This epoch took 30.486788 s + +2024-07-17 12:53:38.715353: +epoch: 194 +2024-07-17 12:54:03.885664: train loss : -0.9583 +2024-07-17 12:54:07.160476: validation loss: -0.8733 +2024-07-17 12:54:07.161714: Average global foreground Dice: [0.9617] +2024-07-17 12:54:07.161786: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:54:07.829911: lr: 0.00256 +2024-07-17 12:54:07.887165: saving checkpoint... +2024-07-17 12:54:08.982422: done, saving took 1.15 seconds +2024-07-17 12:54:08.986840: This epoch took 30.271457 s + +2024-07-17 12:54:08.986891: +epoch: 195 +2024-07-17 12:54:33.914908: train loss : -0.9579 +2024-07-17 12:54:37.315370: validation loss: -0.8700 +2024-07-17 12:54:37.316732: Average global foreground Dice: [0.961] +2024-07-17 12:54:37.316881: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:54:38.009241: lr: 0.002518 +2024-07-17 12:54:38.066289: saving checkpoint... +2024-07-17 12:54:38.961550: done, saving took 0.95 seconds +2024-07-17 12:54:39.183181: This epoch took 30.196249 s + +2024-07-17 12:54:39.183306: +epoch: 196 +2024-07-17 12:55:04.118172: train loss : -0.9598 +2024-07-17 12:55:07.406805: validation loss: -0.8672 +2024-07-17 12:55:07.414377: Average global foreground Dice: [0.9623] +2024-07-17 12:55:07.414580: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:55:08.013108: lr: 0.002476 +2024-07-17 12:55:08.069985: saving checkpoint... +2024-07-17 12:55:08.927848: done, saving took 0.91 seconds +2024-07-17 12:55:09.135876: New best epoch (train loss MA): -0.9554 +2024-07-17 12:55:09.135947: Best loss so far: -0.9546 +2024-07-17 12:55:09.135978: eps: 0.0005 +2024-07-17 12:55:09.136003: Best Epoch 193 +2024-07-17 12:55:09.136026: current epoch is 196 +2024-07-17 12:55:09.136061: This epoch took 29.952710 s + +2024-07-17 12:55:09.136087: +epoch: 197 +2024-07-17 12:55:34.085497: train loss : -0.9581 +2024-07-17 12:55:37.526352: validation loss: -0.8680 +2024-07-17 12:55:37.527551: Average global foreground Dice: [0.9605] +2024-07-17 12:55:37.527618: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:55:38.175421: lr: 0.002434 +2024-07-17 12:55:38.232556: saving checkpoint... +2024-07-17 12:55:39.089619: done, saving took 0.91 seconds +2024-07-17 12:55:39.091382: This epoch took 29.955265 s + +2024-07-17 12:55:39.091429: +epoch: 198 +2024-07-17 12:56:03.721668: train loss : -0.9590 +2024-07-17 12:56:07.153574: validation loss: -0.8729 +2024-07-17 12:56:07.155534: Average global foreground Dice: [0.9616] +2024-07-17 12:56:07.155619: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:56:07.831667: lr: 0.002391 +2024-07-17 12:56:07.888559: saving checkpoint... +2024-07-17 12:56:08.792274: done, saving took 0.96 seconds +2024-07-17 12:56:09.022824: This epoch took 29.931359 s + +2024-07-17 12:56:09.022887: +epoch: 199 +2024-07-17 12:56:33.882186: train loss : -0.9606 +2024-07-17 12:56:37.177322: validation loss: -0.8721 +2024-07-17 12:56:37.178572: Average global foreground Dice: [0.9626] +2024-07-17 12:56:37.178635: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:56:38.018594: lr: 0.002349 +2024-07-17 12:56:38.018799: saving scheduled checkpoint file... +2024-07-17 12:56:38.077355: saving checkpoint... +2024-07-17 12:56:38.974446: done, saving took 0.96 seconds +2024-07-17 12:56:39.198790: done +2024-07-17 12:56:39.224153: saving checkpoint... +2024-07-17 12:56:40.106265: done, saving took 0.91 seconds +2024-07-17 12:56:40.314203: New best epoch (train loss MA): -0.9562 +2024-07-17 12:56:40.314284: Best loss so far: -0.9554 +2024-07-17 12:56:40.314348: eps: 0.0005 +2024-07-17 12:56:40.314409: Best Epoch 196 +2024-07-17 12:56:40.314456: current epoch is 199 +2024-07-17 12:56:40.326804: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 12:56:40.327056: The split file contains 5 splits. +2024-07-17 12:56:40.327108: Desired fold for training: 0 +2024-07-17 12:56:40.327162: This split has 74 training and 20 validation cases. +2024-07-17 12:56:41.563822: Performing validation with validation data from task Task125_All. +2024-07-17 12:56:58.342312: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 12:56:58.342815: The split file contains 5 splits. +2024-07-17 12:56:58.342867: Desired fold for training: 0 +2024-07-17 12:56:58.342919: This split has 74 training and 20 validation cases. +2024-07-17 12:56:58.683980: This epoch took 49.661030 s + +2024-07-17 12:56:58.684140: +epoch: 200 +2024-07-17 12:57:24.830095: train loss : -0.9590 +2024-07-17 12:57:29.056690: validation loss: -0.8750 +2024-07-17 12:57:29.058466: Average global foreground Dice: [0.9616] +2024-07-17 12:57:29.058578: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:57:29.844067: lr: 0.002307 +2024-07-17 12:57:29.965386: saving checkpoint... +2024-07-17 12:57:30.784787: done, saving took 0.94 seconds +2024-07-17 12:57:30.786707: This epoch took 32.102499 s + +2024-07-17 12:57:30.786765: +epoch: 201 +2024-07-17 12:57:55.552631: train loss : -0.9577 +2024-07-17 12:57:58.415153: validation loss: -0.8665 +2024-07-17 12:57:58.416371: Average global foreground Dice: [0.9603] +2024-07-17 12:57:58.416445: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:57:58.994704: lr: 0.002264 +2024-07-17 12:57:58.994913: This epoch took 28.208079 s + +2024-07-17 12:57:58.994962: +epoch: 202 +2024-07-17 12:58:24.310155: train loss : -0.9586 +2024-07-17 12:58:27.208571: validation loss: -0.8800 +2024-07-17 12:58:27.209963: Average global foreground Dice: [0.9627] +2024-07-17 12:58:27.210061: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:58:27.857716: lr: 0.002222 +2024-07-17 12:58:27.961900: saving checkpoint... +2024-07-17 12:58:28.779166: done, saving took 0.92 seconds +2024-07-17 12:58:28.930704: This epoch took 29.935670 s + +2024-07-17 12:58:28.930773: +epoch: 203 +2024-07-17 12:58:53.941892: train loss : -0.9592 +2024-07-17 12:58:57.041421: validation loss: -0.8714 +2024-07-17 12:58:57.043259: Average global foreground Dice: [0.962] +2024-07-17 12:58:57.043340: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:58:57.743812: lr: 0.002179 +2024-07-17 12:58:57.812645: saving checkpoint... +2024-07-17 12:58:58.433606: done, saving took 0.69 seconds +2024-07-17 12:58:58.502422: New best epoch (train loss MA): -0.9568 +2024-07-17 12:58:58.502484: Best loss so far: -0.9562 +2024-07-17 12:58:58.502531: eps: 0.0005 +2024-07-17 12:58:58.502571: Best Epoch 199 +2024-07-17 12:58:58.502608: current epoch is 203 +2024-07-17 12:58:58.502657: This epoch took 29.571846 s + +2024-07-17 12:58:58.502698: +epoch: 204 +2024-07-17 12:59:23.612567: train loss : -0.9607 +2024-07-17 12:59:27.090622: validation loss: -0.8796 +2024-07-17 12:59:27.092043: Average global foreground Dice: [0.9637] +2024-07-17 12:59:27.092145: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:59:27.828364: lr: 0.002137 +2024-07-17 12:59:27.889643: saving checkpoint... +2024-07-17 12:59:28.796463: done, saving took 0.97 seconds +2024-07-17 12:59:28.912740: This epoch took 30.409989 s + +2024-07-17 12:59:28.912852: +epoch: 205 +2024-07-17 12:59:53.368761: train loss : -0.9591 +2024-07-17 12:59:56.738536: validation loss: -0.8671 +2024-07-17 12:59:56.739585: Average global foreground Dice: [0.9618] +2024-07-17 12:59:56.739648: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 12:59:57.652416: lr: 0.002094 +2024-07-17 12:59:57.704695: saving checkpoint... +2024-07-17 12:59:58.522049: done, saving took 0.87 seconds +2024-07-17 12:59:58.720055: This epoch took 29.807158 s + +2024-07-17 12:59:58.720355: +epoch: 206 +2024-07-17 13:00:23.669854: train loss : -0.9613 +2024-07-17 13:00:27.465366: validation loss: -0.8703 +2024-07-17 13:00:27.466606: Average global foreground Dice: [0.9629] +2024-07-17 13:00:27.466677: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:00:27.939154: lr: 0.002051 +2024-07-17 13:00:27.993833: saving checkpoint... +2024-07-17 13:00:28.870536: done, saving took 0.93 seconds +2024-07-17 13:00:29.094138: New best epoch (train loss MA): -0.9575 +2024-07-17 13:00:29.094270: Best loss so far: -0.9568 +2024-07-17 13:00:29.094302: eps: 0.0005 +2024-07-17 13:00:29.094330: Best Epoch 203 +2024-07-17 13:00:29.094354: current epoch is 206 +2024-07-17 13:00:29.094387: This epoch took 30.373975 s + +2024-07-17 13:00:29.094414: +epoch: 207 +2024-07-17 13:00:53.871596: train loss : -0.9613 +2024-07-17 13:00:57.477206: validation loss: -0.8751 +2024-07-17 13:00:57.485610: Average global foreground Dice: [0.964] +2024-07-17 13:00:57.485785: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:00:58.135601: lr: 0.002008 +2024-07-17 13:00:58.193221: saving checkpoint... +2024-07-17 13:00:59.079250: done, saving took 0.94 seconds +2024-07-17 13:00:59.299100: This epoch took 30.204652 s + +2024-07-17 13:00:59.299163: +epoch: 208 +2024-07-17 13:01:23.538662: train loss : -0.9590 +2024-07-17 13:01:26.995661: validation loss: -0.8745 +2024-07-17 13:01:26.997077: Average global foreground Dice: [0.9624] +2024-07-17 13:01:26.997163: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:01:27.774838: lr: 0.001965 +2024-07-17 13:01:27.829789: saving checkpoint... +2024-07-17 13:01:28.726918: done, saving took 0.95 seconds +2024-07-17 13:01:28.944509: This epoch took 29.645273 s + +2024-07-17 13:01:28.944716: +epoch: 209 +2024-07-17 13:01:53.475068: train loss : -0.9594 +2024-07-17 13:01:57.040236: validation loss: -0.8664 +2024-07-17 13:01:57.041612: Average global foreground Dice: [0.9622] +2024-07-17 13:01:57.041711: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:01:57.773324: lr: 0.001922 +2024-07-17 13:01:57.833366: saving checkpoint... +2024-07-17 13:01:58.701753: done, saving took 0.93 seconds +2024-07-17 13:01:58.763805: This epoch took 29.819046 s + +2024-07-17 13:01:58.763884: +epoch: 210 +2024-07-17 13:02:23.251719: train loss : -0.9586 +2024-07-17 13:02:26.793976: validation loss: -0.8743 +2024-07-17 13:02:26.795187: Average global foreground Dice: [0.9625] +2024-07-17 13:02:26.795274: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:02:27.468975: lr: 0.001879 +2024-07-17 13:02:27.537847: saving checkpoint... +2024-07-17 13:02:28.338351: done, saving took 0.86 seconds +2024-07-17 13:02:28.540104: New best epoch (train loss MA): -0.9580 +2024-07-17 13:02:28.540298: Best loss so far: -0.9575 +2024-07-17 13:02:28.540341: eps: 0.0005 +2024-07-17 13:02:28.540379: Best Epoch 206 +2024-07-17 13:02:28.540411: current epoch is 210 +2024-07-17 13:02:28.540444: This epoch took 29.776521 s + +2024-07-17 13:02:28.540472: +epoch: 211 +2024-07-17 13:02:53.230085: train loss : -0.9604 +2024-07-17 13:02:56.414216: validation loss: -0.8678 +2024-07-17 13:02:56.415355: Average global foreground Dice: [0.9622] +2024-07-17 13:02:56.415534: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:02:57.102801: lr: 0.001835 +2024-07-17 13:02:57.162304: saving checkpoint... +2024-07-17 13:02:58.152280: done, saving took 1.05 seconds +2024-07-17 13:02:58.358956: This epoch took 29.818442 s + +2024-07-17 13:02:58.359233: +epoch: 212 +2024-07-17 13:03:22.828302: train loss : -0.9623 +2024-07-17 13:03:26.448983: validation loss: -0.8689 +2024-07-17 13:03:26.463145: Average global foreground Dice: [0.9638] +2024-07-17 13:03:26.463273: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:03:27.119151: lr: 0.001792 +2024-07-17 13:03:27.171416: saving checkpoint... +2024-07-17 13:03:28.029182: done, saving took 0.91 seconds +2024-07-17 13:03:28.208427: This epoch took 29.849116 s + +2024-07-17 13:03:28.208635: +epoch: 213 +2024-07-17 13:03:52.981318: train loss : -0.9630 +2024-07-17 13:03:56.452374: validation loss: -0.8768 +2024-07-17 13:03:56.453665: Average global foreground Dice: [0.9642] +2024-07-17 13:03:56.453766: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:03:57.115582: lr: 0.001748 +2024-07-17 13:03:57.376561: saving checkpoint... +2024-07-17 13:03:58.240680: done, saving took 1.12 seconds +2024-07-17 13:03:58.243968: New best epoch (train loss MA): -0.9588 +2024-07-17 13:03:58.244177: Best loss so far: -0.9580 +2024-07-17 13:03:58.244241: eps: 0.0005 +2024-07-17 13:03:58.244304: Best Epoch 210 +2024-07-17 13:03:58.244362: current epoch is 213 +2024-07-17 13:03:58.244433: This epoch took 30.035760 s + +2024-07-17 13:03:58.244502: +epoch: 214 +2024-07-17 13:04:22.796847: train loss : -0.9618 +2024-07-17 13:04:26.232353: validation loss: -0.8705 +2024-07-17 13:04:26.237063: Average global foreground Dice: [0.9634] +2024-07-17 13:04:26.237150: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:04:26.870985: lr: 0.001704 +2024-07-17 13:04:26.944559: saving checkpoint... +2024-07-17 13:04:27.815986: done, saving took 0.94 seconds +2024-07-17 13:04:28.047271: This epoch took 29.802630 s + +2024-07-17 13:04:28.047463: +epoch: 215 +2024-07-17 13:04:52.090516: train loss : -0.9621 +2024-07-17 13:04:55.555277: validation loss: -0.8667 +2024-07-17 13:04:55.556831: Average global foreground Dice: [0.9627] +2024-07-17 13:04:55.556942: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:04:56.231886: lr: 0.00166 +2024-07-17 13:04:56.290170: saving checkpoint... +2024-07-17 13:04:57.160110: done, saving took 0.93 seconds +2024-07-17 13:04:57.266831: This epoch took 29.219318 s + +2024-07-17 13:04:57.267037: +epoch: 216 +2024-07-17 13:05:21.456751: train loss : -0.9628 +2024-07-17 13:05:24.773981: validation loss: -0.8654 +2024-07-17 13:05:24.775462: Average global foreground Dice: [0.9636] +2024-07-17 13:05:24.775554: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:05:25.436323: lr: 0.001616 +2024-07-17 13:05:25.624361: saving checkpoint... +2024-07-17 13:05:26.306741: done, saving took 0.86 seconds +2024-07-17 13:05:26.490918: New best epoch (train loss MA): -0.9595 +2024-07-17 13:05:26.491058: Best loss so far: -0.9588 +2024-07-17 13:05:26.491092: eps: 0.0005 +2024-07-17 13:05:26.491121: Best Epoch 213 +2024-07-17 13:05:26.491147: current epoch is 216 +2024-07-17 13:05:26.491179: This epoch took 29.224102 s + +2024-07-17 13:05:26.491205: +epoch: 217 +2024-07-17 13:05:50.759767: train loss : -0.9637 +2024-07-17 13:05:54.120864: validation loss: -0.8769 +2024-07-17 13:05:54.122393: Average global foreground Dice: [0.9654] +2024-07-17 13:05:54.122491: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:05:54.721526: lr: 0.001572 +2024-07-17 13:05:54.776139: saving checkpoint... +2024-07-17 13:05:55.630926: done, saving took 0.91 seconds +2024-07-17 13:05:55.872899: This epoch took 29.381652 s + +2024-07-17 13:05:55.873087: +epoch: 218 +2024-07-17 13:06:19.822181: train loss : -0.9619 +2024-07-17 13:06:23.215137: validation loss: -0.8666 +2024-07-17 13:06:23.216394: Average global foreground Dice: [0.9633] +2024-07-17 13:06:23.216461: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:06:24.254063: lr: 0.001528 +2024-07-17 13:06:24.339027: saving checkpoint... +2024-07-17 13:06:25.259081: done, saving took 1.00 seconds +2024-07-17 13:06:25.512194: This epoch took 29.639064 s + +2024-07-17 13:06:25.512269: +epoch: 219 +2024-07-17 13:06:49.867385: train loss : -0.9628 +2024-07-17 13:06:53.484342: validation loss: -0.8675 +2024-07-17 13:06:53.485474: Average global foreground Dice: [0.9632] +2024-07-17 13:06:53.528521: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:06:54.378915: lr: 0.001483 +2024-07-17 13:06:54.436758: saving checkpoint... +2024-07-17 13:06:55.316681: done, saving took 0.94 seconds +2024-07-17 13:06:55.553401: New best epoch (train loss MA): -0.9601 +2024-07-17 13:06:55.553635: Best loss so far: -0.9595 +2024-07-17 13:06:55.553668: eps: 0.0005 +2024-07-17 13:06:55.553697: Best Epoch 216 +2024-07-17 13:06:55.553722: current epoch is 219 +2024-07-17 13:06:55.553771: This epoch took 30.041465 s + +2024-07-17 13:06:55.553806: +epoch: 220 +2024-07-17 13:07:19.701164: train loss : -0.9625 +2024-07-17 13:07:23.345114: validation loss: -0.8791 +2024-07-17 13:07:23.346575: Average global foreground Dice: [0.9651] +2024-07-17 13:07:23.346663: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:07:24.111917: lr: 0.001439 +2024-07-17 13:07:24.170038: saving checkpoint... +2024-07-17 13:07:25.042144: done, saving took 0.93 seconds +2024-07-17 13:07:25.229428: This epoch took 29.675583 s + +2024-07-17 13:07:25.229609: +epoch: 221 +2024-07-17 13:07:49.100552: train loss : -0.9641 +2024-07-17 13:07:52.682525: validation loss: -0.8760 +2024-07-17 13:07:52.684828: Average global foreground Dice: [0.9656] +2024-07-17 13:07:52.684953: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:07:53.359602: lr: 0.001394 +2024-07-17 13:07:53.418592: saving checkpoint... +2024-07-17 13:07:54.260588: done, saving took 0.90 seconds +2024-07-17 13:07:54.447204: This epoch took 29.217524 s + +2024-07-17 13:07:54.447367: +epoch: 222 +2024-07-17 13:08:18.442071: train loss : -0.9617 +2024-07-17 13:08:22.076007: validation loss: -0.8690 +2024-07-17 13:08:22.077724: Average global foreground Dice: [0.9622] +2024-07-17 13:08:22.077808: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:08:22.830962: lr: 0.001349 +2024-07-17 13:08:22.831169: New best epoch (train loss MA): -0.9606 +2024-07-17 13:08:22.831204: Best loss so far: -0.9601 +2024-07-17 13:08:22.831230: eps: 0.0005 +2024-07-17 13:08:22.831257: Best Epoch 219 +2024-07-17 13:08:22.831281: current epoch is 222 +2024-07-17 13:08:22.831314: This epoch took 28.383910 s + +2024-07-17 13:08:22.831342: +epoch: 223 +2024-07-17 13:08:47.000993: train loss : -0.9631 +2024-07-17 13:08:50.600270: validation loss: -0.8827 +2024-07-17 13:08:50.601803: Average global foreground Dice: [0.9648] +2024-07-17 13:08:50.601883: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:08:51.282153: lr: 0.001304 +2024-07-17 13:08:51.414496: saving checkpoint... +2024-07-17 13:08:52.399093: done, saving took 1.11 seconds +2024-07-17 13:08:52.492088: This epoch took 29.660707 s + +2024-07-17 13:08:52.492206: +epoch: 224 +2024-07-17 13:09:16.420945: train loss : -0.9625 +2024-07-17 13:09:20.223696: validation loss: -0.8664 +2024-07-17 13:09:20.225400: Average global foreground Dice: [0.9629] +2024-07-17 13:09:20.225502: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:09:20.902217: lr: 0.001259 +2024-07-17 13:09:20.902459: This epoch took 28.410213 s + +2024-07-17 13:09:20.902497: +epoch: 225 +2024-07-17 13:09:45.108207: train loss : -0.9631 +2024-07-17 13:09:48.651189: validation loss: -0.8699 +2024-07-17 13:09:48.652573: Average global foreground Dice: [0.9641] +2024-07-17 13:09:48.652703: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:09:49.373320: lr: 0.001214 +2024-07-17 13:09:49.483916: saving checkpoint... +2024-07-17 13:09:50.477790: done, saving took 1.10 seconds +2024-07-17 13:09:50.696354: This epoch took 29.793816 s + +2024-07-17 13:09:50.696541: +epoch: 226 +2024-07-17 13:10:15.017575: train loss : -0.9644 +2024-07-17 13:10:18.647689: validation loss: -0.8752 +2024-07-17 13:10:18.649537: Average global foreground Dice: [0.9655] +2024-07-17 13:10:18.649671: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:10:19.364459: lr: 0.001168 +2024-07-17 13:10:19.423655: saving checkpoint... +2024-07-17 13:10:20.325421: done, saving took 0.96 seconds +2024-07-17 13:10:20.327481: New best epoch (train loss MA): -0.9613 +2024-07-17 13:10:20.327530: Best loss so far: -0.9606 +2024-07-17 13:10:20.327560: eps: 0.0005 +2024-07-17 13:10:20.327587: Best Epoch 222 +2024-07-17 13:10:20.327610: current epoch is 226 +2024-07-17 13:10:20.327644: This epoch took 29.631059 s + +2024-07-17 13:10:20.327672: +epoch: 227 +2024-07-17 13:10:44.478794: train loss : -0.9621 +2024-07-17 13:10:47.679209: validation loss: -0.8735 +2024-07-17 13:10:47.683063: Average global foreground Dice: [0.964] +2024-07-17 13:10:47.683226: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:10:48.390601: lr: 0.001122 +2024-07-17 13:10:48.461251: saving checkpoint... +2024-07-17 13:10:49.328331: done, saving took 0.94 seconds +2024-07-17 13:10:49.405579: This epoch took 29.077872 s + +2024-07-17 13:10:49.405922: +epoch: 228 +2024-07-17 13:11:13.684715: train loss : -0.9646 +2024-07-17 13:11:16.884085: validation loss: -0.8725 +2024-07-17 13:11:16.885358: Average global foreground Dice: [0.9655] +2024-07-17 13:11:16.885467: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:11:17.513021: lr: 0.001076 +2024-07-17 13:11:17.583844: saving checkpoint... +2024-07-17 13:11:18.232528: done, saving took 0.72 seconds +2024-07-17 13:11:18.307451: This epoch took 28.901463 s + +2024-07-17 13:11:18.307550: +epoch: 229 +2024-07-17 13:11:42.619389: train loss : -0.9658 +2024-07-17 13:11:46.030287: validation loss: -0.8715 +2024-07-17 13:11:46.031656: Average global foreground Dice: [0.9658] +2024-07-17 13:11:46.031891: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:11:46.646449: lr: 0.00103 +2024-07-17 13:11:46.723419: saving checkpoint... +2024-07-17 13:11:47.534362: done, saving took 0.89 seconds +2024-07-17 13:11:47.771608: New best epoch (train loss MA): -0.9619 +2024-07-17 13:11:47.771685: Best loss so far: -0.9613 +2024-07-17 13:11:47.771716: eps: 0.0005 +2024-07-17 13:11:47.771741: Best Epoch 226 +2024-07-17 13:11:47.771765: current epoch is 229 +2024-07-17 13:11:47.771798: This epoch took 29.464212 s + +2024-07-17 13:11:47.771823: +epoch: 230 +2024-07-17 13:12:11.883837: train loss : -0.9647 +2024-07-17 13:12:14.960873: validation loss: -0.8783 +2024-07-17 13:12:14.962259: Average global foreground Dice: [0.9657] +2024-07-17 13:12:14.962326: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:12:15.670153: lr: 0.000983 +2024-07-17 13:12:15.743016: saving checkpoint... +2024-07-17 13:12:16.684921: done, saving took 1.01 seconds +2024-07-17 13:12:16.922102: This epoch took 29.150235 s + +2024-07-17 13:12:16.922608: +epoch: 231 +2024-07-17 13:12:40.973528: train loss : -0.9639 +2024-07-17 13:12:44.351669: validation loss: -0.8828 +2024-07-17 13:12:44.353072: Average global foreground Dice: [0.9661] +2024-07-17 13:12:44.353174: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:12:45.123929: lr: 0.000937 +2024-07-17 13:12:45.203735: saving checkpoint... +2024-07-17 13:12:46.097664: done, saving took 0.97 seconds +2024-07-17 13:12:46.099388: This epoch took 29.176726 s + +2024-07-17 13:12:46.099437: +epoch: 232 +2024-07-17 13:13:10.258464: train loss : -0.9656 +2024-07-17 13:13:13.460984: validation loss: -0.8651 +2024-07-17 13:13:13.463097: Average global foreground Dice: [0.9656] +2024-07-17 13:13:13.463202: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:13:14.102369: lr: 0.00089 +2024-07-17 13:13:14.229728: saving checkpoint... +2024-07-17 13:13:15.180903: done, saving took 1.08 seconds +2024-07-17 13:13:15.418936: New best epoch (train loss MA): -0.9624 +2024-07-17 13:13:15.419010: Best loss so far: -0.9619 +2024-07-17 13:13:15.419041: eps: 0.0005 +2024-07-17 13:13:15.419072: Best Epoch 229 +2024-07-17 13:13:15.419098: current epoch is 232 +2024-07-17 13:13:15.419128: This epoch took 29.319656 s + +2024-07-17 13:13:15.419155: +epoch: 233 +2024-07-17 13:13:39.673208: train loss : -0.9647 +2024-07-17 13:13:42.773969: validation loss: -0.8773 +2024-07-17 13:13:42.775100: Average global foreground Dice: [0.9655] +2024-07-17 13:13:42.775184: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:13:43.458196: lr: 0.000842 +2024-07-17 13:13:43.518416: saving checkpoint... +2024-07-17 13:13:44.419115: done, saving took 0.96 seconds +2024-07-17 13:13:44.647810: This epoch took 29.228623 s + +2024-07-17 13:13:44.647881: +epoch: 234 +2024-07-17 13:14:08.692613: train loss : -0.9639 +2024-07-17 13:14:11.855108: validation loss: -0.8654 +2024-07-17 13:14:11.856221: Average global foreground Dice: [0.9648] +2024-07-17 13:14:11.856386: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:14:12.578516: lr: 0.000795 +2024-07-17 13:14:12.631080: saving checkpoint... +2024-07-17 13:14:13.525244: done, saving took 0.95 seconds +2024-07-17 13:14:13.765665: This epoch took 29.117742 s + +2024-07-17 13:14:13.765787: +epoch: 235 +2024-07-17 13:14:37.724305: train loss : -0.9641 +2024-07-17 13:14:41.130575: validation loss: -0.8640 +2024-07-17 13:14:41.131681: Average global foreground Dice: [0.9642] +2024-07-17 13:14:41.131803: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:14:41.639619: lr: 0.000747 +2024-07-17 13:14:41.639822: This epoch took 27.873986 s + +2024-07-17 13:14:41.639859: +epoch: 236 +2024-07-17 13:15:05.926485: train loss : -0.9652 +2024-07-17 13:15:09.177574: validation loss: -0.8677 +2024-07-17 13:15:09.178786: Average global foreground Dice: [0.9656] +2024-07-17 13:15:09.178914: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:15:09.892004: lr: 0.000699 +2024-07-17 13:15:09.946821: saving checkpoint... +2024-07-17 13:15:10.856068: done, saving took 0.96 seconds +2024-07-17 13:15:10.858072: New best epoch (train loss MA): -0.9630 +2024-07-17 13:15:10.858131: Best loss so far: -0.9624 +2024-07-17 13:15:10.858162: eps: 0.0005 +2024-07-17 13:15:10.858187: Best Epoch 232 +2024-07-17 13:15:10.858220: current epoch is 236 +2024-07-17 13:15:10.858253: This epoch took 29.218360 s + +2024-07-17 13:15:10.858280: +epoch: 237 +2024-07-17 13:15:34.893868: train loss : -0.9653 +2024-07-17 13:15:38.054230: validation loss: -0.8855 +2024-07-17 13:15:38.056435: Average global foreground Dice: [0.9671] +2024-07-17 13:15:38.056578: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:15:38.552079: lr: 0.00065 +2024-07-17 13:15:38.653358: saving checkpoint... +2024-07-17 13:15:39.694891: done, saving took 1.14 seconds +2024-07-17 13:15:39.712874: This epoch took 28.854558 s + +2024-07-17 13:15:39.712934: +epoch: 238 +2024-07-17 13:16:03.964320: train loss : -0.9659 +2024-07-17 13:16:07.373415: validation loss: -0.8695 +2024-07-17 13:16:07.380971: Average global foreground Dice: [0.966] +2024-07-17 13:16:07.381208: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:16:08.021546: lr: 0.000601 +2024-07-17 13:16:08.077763: saving checkpoint... +2024-07-17 13:16:08.722117: done, saving took 0.70 seconds +2024-07-17 13:16:08.796084: This epoch took 29.083113 s + +2024-07-17 13:16:08.796148: +epoch: 239 +2024-07-17 13:16:33.050072: train loss : -0.9661 +2024-07-17 13:16:36.486912: validation loss: -0.8663 +2024-07-17 13:16:36.488265: Average global foreground Dice: [0.9654] +2024-07-17 13:16:36.491443: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:16:36.932721: lr: 0.000552 +2024-07-17 13:16:36.990399: saving checkpoint... +2024-07-17 13:16:37.689169: done, saving took 0.76 seconds +2024-07-17 13:16:37.705686: New best epoch (train loss MA): -0.9635 +2024-07-17 13:16:37.705787: Best loss so far: -0.9630 +2024-07-17 13:16:37.705818: eps: 0.0005 +2024-07-17 13:16:37.705844: Best Epoch 236 +2024-07-17 13:16:37.705869: current epoch is 239 +2024-07-17 13:16:37.705901: This epoch took 28.909696 s + +2024-07-17 13:16:37.705929: +epoch: 240 +2024-07-17 13:17:01.729270: train loss : -0.9660 +2024-07-17 13:17:04.903843: validation loss: -0.8632 +2024-07-17 13:17:04.904927: Average global foreground Dice: [0.9656] +2024-07-17 13:17:04.904989: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:17:05.579218: lr: 0.000502 +2024-07-17 13:17:05.636581: saving checkpoint... +2024-07-17 13:17:06.271587: done, saving took 0.69 seconds +2024-07-17 13:17:06.273506: This epoch took 28.567546 s + +2024-07-17 13:17:06.273562: +epoch: 241 +2024-07-17 13:17:30.653594: train loss : -0.9652 +2024-07-17 13:17:33.587247: validation loss: -0.8666 +2024-07-17 13:17:33.588319: Average global foreground Dice: [0.9652] +2024-07-17 13:17:33.588426: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:17:34.467436: lr: 0.000451 +2024-07-17 13:17:34.539788: saving checkpoint... +2024-07-17 13:17:35.213916: done, saving took 0.75 seconds +2024-07-17 13:17:35.386038: This epoch took 29.112437 s + +2024-07-17 13:17:35.386104: +epoch: 242 +2024-07-17 13:17:59.710544: train loss : -0.9662 +2024-07-17 13:18:02.803250: validation loss: -0.8673 +2024-07-17 13:18:02.804356: Average global foreground Dice: [0.9668] +2024-07-17 13:18:02.804431: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:18:03.463862: lr: 0.0004 +2024-07-17 13:18:03.520770: saving checkpoint... +2024-07-17 13:18:04.443427: done, saving took 0.98 seconds +2024-07-17 13:18:04.658933: This epoch took 29.272791 s + +2024-07-17 13:18:04.659013: +epoch: 243 +2024-07-17 13:18:28.807385: train loss : -0.9665 +2024-07-17 13:18:32.217483: validation loss: -0.8732 +2024-07-17 13:18:32.243656: Average global foreground Dice: [0.9666] +2024-07-17 13:18:32.244217: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:18:32.738265: lr: 0.000348 +2024-07-17 13:18:32.797287: saving checkpoint... +2024-07-17 13:18:33.709401: done, saving took 0.97 seconds +2024-07-17 13:18:33.929279: New best epoch (train loss MA): -0.9641 +2024-07-17 13:18:33.929457: Best loss so far: -0.9635 +2024-07-17 13:18:33.929490: eps: 0.0005 +2024-07-17 13:18:33.929518: Best Epoch 239 +2024-07-17 13:18:33.929541: current epoch is 243 +2024-07-17 13:18:33.929575: This epoch took 29.270497 s + +2024-07-17 13:18:33.929602: +epoch: 244 +2024-07-17 13:18:58.558378: train loss : -0.9665 +2024-07-17 13:19:01.664099: validation loss: -0.8684 +2024-07-17 13:19:01.665981: Average global foreground Dice: [0.9665] +2024-07-17 13:19:01.666053: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:19:02.330830: lr: 0.000296 +2024-07-17 13:19:02.385809: saving checkpoint... +2024-07-17 13:19:03.262642: done, saving took 0.93 seconds +2024-07-17 13:19:03.264343: This epoch took 29.334709 s + +2024-07-17 13:19:03.264390: +epoch: 245 +2024-07-17 13:19:27.653112: train loss : -0.9657 +2024-07-17 13:19:30.844435: validation loss: -0.8639 +2024-07-17 13:19:30.851633: Average global foreground Dice: [0.9652] +2024-07-17 13:19:30.860659: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:19:31.463532: lr: 0.000242 +2024-07-17 13:19:31.463736: This epoch took 28.199307 s + +2024-07-17 13:19:31.463772: +epoch: 246 +2024-07-17 13:19:55.522173: train loss : -0.9670 +2024-07-17 13:19:58.601352: validation loss: -0.8725 +2024-07-17 13:19:58.603231: Average global foreground Dice: [0.9671] +2024-07-17 13:19:58.603319: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:19:59.235984: lr: 0.000187 +2024-07-17 13:19:59.290626: saving checkpoint... +2024-07-17 13:19:59.918605: done, saving took 0.68 seconds +2024-07-17 13:19:59.920353: This epoch took 28.456547 s + +2024-07-17 13:19:59.920403: +epoch: 247 +2024-07-17 13:20:24.000534: train loss : -0.9657 +2024-07-17 13:20:27.085530: validation loss: -0.8711 +2024-07-17 13:20:27.086905: Average global foreground Dice: [0.9669] +2024-07-17 13:20:27.087001: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:20:27.649837: lr: 0.00013 +2024-07-17 13:20:27.724777: saving checkpoint... +2024-07-17 13:20:28.629731: done, saving took 0.98 seconds +2024-07-17 13:20:28.874115: New best epoch (train loss MA): -0.9647 +2024-07-17 13:20:28.874213: Best loss so far: -0.9641 +2024-07-17 13:20:28.874244: eps: 0.0005 +2024-07-17 13:20:28.874272: Best Epoch 243 +2024-07-17 13:20:28.874296: current epoch is 247 +2024-07-17 13:20:28.874328: This epoch took 28.953889 s + +2024-07-17 13:20:28.874355: +epoch: 248 +2024-07-17 13:20:53.101962: train loss : -0.9667 +2024-07-17 13:20:56.538952: validation loss: -0.8776 +2024-07-17 13:20:56.548133: Average global foreground Dice: [0.9669] +2024-07-17 13:20:56.548244: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:20:57.155656: lr: 6.9e-05 +2024-07-17 13:20:57.210757: saving checkpoint... +2024-07-17 13:20:58.126694: done, saving took 0.97 seconds +2024-07-17 13:20:58.257300: This epoch took 29.382905 s + +2024-07-17 13:20:58.257514: +epoch: 249 +2024-07-17 13:21:22.615805: train loss : -0.9649 +2024-07-17 13:21:25.781318: validation loss: -0.8702 +2024-07-17 13:21:25.789499: Average global foreground Dice: [0.9653] +2024-07-17 13:21:25.789846: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2024-07-17 13:21:26.501009: lr: 0.0 +2024-07-17 13:21:26.501180: saving scheduled checkpoint file... +2024-07-17 13:21:26.557520: saving checkpoint... +2024-07-17 13:21:27.452110: done, saving took 0.95 seconds +2024-07-17 13:21:27.453771: done +2024-07-17 13:21:27.465080: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 13:21:27.465358: The split file contains 5 splits. +2024-07-17 13:21:27.465423: Desired fold for training: 0 +2024-07-17 13:21:27.465467: This split has 74 training and 20 validation cases. +2024-07-17 13:21:28.650417: Performing validation with validation data from task Task125_All. +2024-07-17 13:21:42.732409: Using splits from existing split file: /home/lhbac/media/XCBCL/required_folder/nnUNet_preprocessed/Task125_All/splits_final.pkl +2024-07-17 13:21:42.732786: The split file contains 5 splits. +2024-07-17 13:21:42.732834: Desired fold for training: 0 +2024-07-17 13:21:42.732872: This split has 74 training and 20 validation cases. +2024-07-17 13:21:43.093801: This epoch took 44.836237 s + +2024-07-17 13:21:43.159327: saving checkpoint... +2024-07-17 13:21:43.542540: done, saving took 0.45 seconds diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/val_metrics.csv b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/val_metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..ae12ec2cbc1fa7943dd7751053550b32954ab9cc --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/val_metrics.csv @@ -0,0 +1,201 @@ +Epoch Task subject_id seg_mask metric value +epoch_49 Task125_All 121_01 mask_1 IoU 0.8190414905548096 +epoch_49 Task125_All 121_01 mask_1 Dice 0.900519847869873 +epoch_49 Task125_All 121_32 mask_1 IoU 0.8433037996292114 +epoch_49 Task125_All 121_32 mask_1 Dice 0.9149916768074036 +epoch_49 Task125_All 121_34 mask_1 IoU 0.8565046191215515 +epoch_49 Task125_All 121_34 mask_1 Dice 0.9227067232131958 +epoch_49 Task125_All 122_03 mask_1 IoU 0.8295673131942749 +epoch_49 Task125_All 122_03 mask_1 Dice 0.9068453907966614 +epoch_49 Task125_All 122_08 mask_1 IoU 0.8541257977485657 +epoch_49 Task125_All 122_08 mask_1 Dice 0.9213245511054993 +epoch_49 Task125_All 122_13 mask_1 IoU 0.8204582333564758 +epoch_49 Task125_All 122_13 mask_1 Dice 0.9013755321502686 +epoch_49 Task125_All 122_15 mask_1 IoU 0.8348348736763 +epoch_49 Task125_All 122_15 mask_1 Dice 0.9099836349487305 +epoch_49 Task125_All 123_03 mask_1 IoU 0.8123469948768616 +epoch_49 Task125_All 123_03 mask_1 Dice 0.8964585661888123 +epoch_49 Task125_All 123_08 mask_1 IoU 0.8557426333427429 +epoch_49 Task125_All 123_08 mask_1 Dice 0.9222643375396729 +epoch_49 Task125_All 123_12 mask_1 IoU 0.8032535910606384 +epoch_49 Task125_All 123_12 mask_1 Dice 0.8908936977386475 +epoch_49 Task125_All 123_15 mask_1 IoU 0.8570336103439331 +epoch_49 Task125_All 123_15 mask_1 Dice 0.9230135679244995 +epoch_49 Task125_All 123_18 mask_1 IoU 0.8444092869758606 +epoch_49 Task125_All 123_18 mask_1 Dice 0.9156419634819031 +epoch_49 Task125_All 123_26 mask_1 IoU 0.8524324297904968 +epoch_49 Task125_All 123_26 mask_1 Dice 0.9203384518623352 +epoch_49 Task125_All 124_00 mask_1 IoU 0.8273141980171204 +epoch_49 Task125_All 124_00 mask_1 Dice 0.9054974913597107 +epoch_49 Task125_All 124_04 mask_1 IoU 0.8348763585090637 +epoch_49 Task125_All 124_04 mask_1 Dice 0.9100083112716675 +epoch_49 Task125_All 124_14 mask_1 IoU 0.8251117467880249 +epoch_49 Task125_All 124_14 mask_1 Dice 0.904176652431488 +epoch_49 Task125_All 124_20 mask_1 IoU 0.8404054641723633 +epoch_49 Task125_All 124_20 mask_1 Dice 0.9132829308509827 +epoch_49 Task125_All 124_25 mask_1 IoU 0.8591132164001465 +epoch_49 Task125_All 124_25 mask_1 Dice 0.9242182970046997 +epoch_49 Task125_All 124_31 mask_1 IoU 0.8663323521614075 +epoch_49 Task125_All 124_31 mask_1 Dice 0.928379476070404 +epoch_49 Task125_All 124_42 mask_1 IoU 0.8374714255332947 +epoch_49 Task125_All 124_42 mask_1 Dice 0.9115477204322815 +epoch_99 Task125_All 121_01 mask_1 IoU 0.8566195964813232 +epoch_99 Task125_All 121_01 mask_1 Dice 0.9227734208106995 +epoch_99 Task125_All 121_32 mask_1 IoU 0.8633441925048828 +epoch_99 Task125_All 121_32 mask_1 Dice 0.9266610145568848 +epoch_99 Task125_All 121_34 mask_1 IoU 0.830015242099762 +epoch_99 Task125_All 121_34 mask_1 Dice 0.9071128964424133 +epoch_99 Task125_All 122_03 mask_1 IoU 0.8172138929367065 +epoch_99 Task125_All 122_03 mask_1 Dice 0.8994141221046448 +epoch_99 Task125_All 122_08 mask_1 IoU 0.8082484602928162 +epoch_99 Task125_All 122_08 mask_1 Dice 0.8939573168754578 +epoch_99 Task125_All 122_13 mask_1 IoU 0.8305820226669312 +epoch_99 Task125_All 122_13 mask_1 Dice 0.9074512720108032 +epoch_99 Task125_All 122_15 mask_1 IoU 0.7707474827766418 +epoch_99 Task125_All 122_15 mask_1 Dice 0.8705334663391113 +epoch_99 Task125_All 123_03 mask_1 IoU 0.8588523864746094 +epoch_99 Task125_All 123_03 mask_1 Dice 0.9240673184394836 +epoch_99 Task125_All 123_08 mask_1 IoU 0.8445738554000854 +epoch_99 Task125_All 123_08 mask_1 Dice 0.9157387018203735 +epoch_99 Task125_All 123_12 mask_1 IoU 0.8351500034332275 +epoch_99 Task125_All 123_12 mask_1 Dice 0.91017085313797 +epoch_99 Task125_All 123_15 mask_1 IoU 0.8173587322235107 +epoch_99 Task125_All 123_15 mask_1 Dice 0.8995018005371094 +epoch_99 Task125_All 123_18 mask_1 IoU 0.8322774767875671 +epoch_99 Task125_All 123_18 mask_1 Dice 0.9084622859954834 +epoch_99 Task125_All 123_26 mask_1 IoU 0.8488346934318542 +epoch_99 Task125_All 123_26 mask_1 Dice 0.9182375073432922 +epoch_99 Task125_All 124_00 mask_1 IoU 0.840190052986145 +epoch_99 Task125_All 124_00 mask_1 Dice 0.913155734539032 +epoch_99 Task125_All 124_04 mask_1 IoU 0.8453475832939148 +epoch_99 Task125_All 124_04 mask_1 Dice 0.9161933660507202 +epoch_99 Task125_All 124_14 mask_1 IoU 0.8615167737007141 +epoch_99 Task125_All 124_14 mask_1 Dice 0.9256073236465454 +epoch_99 Task125_All 124_20 mask_1 IoU 0.8461165428161621 +epoch_99 Task125_All 124_20 mask_1 Dice 0.9166447520256042 +epoch_99 Task125_All 124_25 mask_1 IoU 0.8607038259506226 +epoch_99 Task125_All 124_25 mask_1 Dice 0.9251379370689392 +epoch_99 Task125_All 124_31 mask_1 IoU 0.8529249429702759 +epoch_99 Task125_All 124_31 mask_1 Dice 0.9206254482269287 +epoch_99 Task125_All 124_42 mask_1 IoU 0.7938142418861389 +epoch_99 Task125_All 124_42 mask_1 Dice 0.8850573301315308 +epoch_149 Task125_All 121_01 mask_1 IoU 0.8418939113616943 +epoch_149 Task125_All 121_01 mask_1 Dice 0.9141611456871033 +epoch_149 Task125_All 121_32 mask_1 IoU 0.8663614392280579 +epoch_149 Task125_All 121_32 mask_1 Dice 0.9283962249755859 +epoch_149 Task125_All 121_34 mask_1 IoU 0.8479877710342407 +epoch_149 Task125_All 121_34 mask_1 Dice 0.9177417755126953 +epoch_149 Task125_All 122_03 mask_1 IoU 0.8712738156318665 +epoch_149 Task125_All 122_03 mask_1 Dice 0.9312093257904053 +epoch_149 Task125_All 122_08 mask_1 IoU 0.8334442973136902 +epoch_149 Task125_All 122_08 mask_1 Dice 0.9091569185256958 +epoch_149 Task125_All 122_13 mask_1 IoU 0.8369640707969666 +epoch_149 Task125_All 122_13 mask_1 Dice 0.9112470746040344 +epoch_149 Task125_All 122_15 mask_1 IoU 0.825735867023468 +epoch_149 Task125_All 122_15 mask_1 Dice 0.9045512676239014 +epoch_149 Task125_All 123_03 mask_1 IoU 0.7752001285552979 +epoch_149 Task125_All 123_03 mask_1 Dice 0.8733664751052856 +epoch_149 Task125_All 123_08 mask_1 IoU 0.8314370512962341 +epoch_149 Task125_All 123_08 mask_1 Dice 0.907961368560791 +epoch_149 Task125_All 123_12 mask_1 IoU 0.7932608723640442 +epoch_149 Task125_All 123_12 mask_1 Dice 0.8847132921218872 +epoch_149 Task125_All 123_15 mask_1 IoU 0.8462263345718384 +epoch_149 Task125_All 123_15 mask_1 Dice 0.9167091846466064 +epoch_149 Task125_All 123_18 mask_1 IoU 0.8563746809959412 +epoch_149 Task125_All 123_18 mask_1 Dice 0.9226313233375549 +epoch_149 Task125_All 123_26 mask_1 IoU 0.8624430298805237 +epoch_149 Task125_All 123_26 mask_1 Dice 0.9261416792869568 +epoch_149 Task125_All 124_00 mask_1 IoU 0.8616271018981934 +epoch_149 Task125_All 124_00 mask_1 Dice 0.9256709814071655 +epoch_149 Task125_All 124_04 mask_1 IoU 0.831540584564209 +epoch_149 Task125_All 124_04 mask_1 Dice 0.9080231189727783 +epoch_149 Task125_All 124_14 mask_1 IoU 0.8469969630241394 +epoch_149 Task125_All 124_14 mask_1 Dice 0.9171611666679382 +epoch_149 Task125_All 124_20 mask_1 IoU 0.8115898370742798 +epoch_149 Task125_All 124_20 mask_1 Dice 0.8959973454475403 +epoch_149 Task125_All 124_25 mask_1 IoU 0.8792508244514465 +epoch_149 Task125_All 124_25 mask_1 Dice 0.9357461333274841 +epoch_149 Task125_All 124_31 mask_1 IoU 0.794054388999939 +epoch_149 Task125_All 124_31 mask_1 Dice 0.8852065801620483 +epoch_149 Task125_All 124_42 mask_1 IoU 0.8389893174171448 +epoch_149 Task125_All 124_42 mask_1 Dice 0.912446141242981 +epoch_199 Task125_All 121_01 mask_1 IoU 0.8784856200218201 +epoch_199 Task125_All 121_01 mask_1 Dice 0.9353125691413879 +epoch_199 Task125_All 121_32 mask_1 IoU 0.8476223945617676 +epoch_199 Task125_All 121_32 mask_1 Dice 0.9175277352333069 +epoch_199 Task125_All 121_34 mask_1 IoU 0.8607978224754333 +epoch_199 Task125_All 121_34 mask_1 Dice 0.9251921772956848 +epoch_199 Task125_All 122_03 mask_1 IoU 0.8442391753196716 +epoch_199 Task125_All 122_03 mask_1 Dice 0.91554194688797 +epoch_199 Task125_All 122_08 mask_1 IoU 0.849332332611084 +epoch_199 Task125_All 122_08 mask_1 Dice 0.9185286164283752 +epoch_199 Task125_All 122_13 mask_1 IoU 0.8142664432525635 +epoch_199 Task125_All 122_13 mask_1 Dice 0.8976260423660278 +epoch_199 Task125_All 122_15 mask_1 IoU 0.8593831062316895 +epoch_199 Task125_All 122_15 mask_1 Dice 0.9243744611740112 +epoch_199 Task125_All 123_03 mask_1 IoU 0.8357996344566345 +epoch_199 Task125_All 123_03 mask_1 Dice 0.9105564951896667 +epoch_199 Task125_All 123_08 mask_1 IoU 0.8279212713241577 +epoch_199 Task125_All 123_08 mask_1 Dice 0.905860960483551 +epoch_199 Task125_All 123_12 mask_1 IoU 0.8407734036445618 +epoch_199 Task125_All 123_12 mask_1 Dice 0.913500189781189 +epoch_199 Task125_All 123_15 mask_1 IoU 0.8218513131141663 +epoch_199 Task125_All 123_15 mask_1 Dice 0.9022156000137329 +epoch_199 Task125_All 123_18 mask_1 IoU 0.8808693885803223 +epoch_199 Task125_All 123_18 mask_1 Dice 0.936661958694458 +epoch_199 Task125_All 123_26 mask_1 IoU 0.838599681854248 +epoch_199 Task125_All 123_26 mask_1 Dice 0.9122155904769897 +epoch_199 Task125_All 124_00 mask_1 IoU 0.804621696472168 +epoch_199 Task125_All 124_00 mask_1 Dice 0.8917344808578491 +epoch_199 Task125_All 124_04 mask_1 IoU 0.8054866194725037 +epoch_199 Task125_All 124_04 mask_1 Dice 0.8922653794288635 +epoch_199 Task125_All 124_14 mask_1 IoU 0.8518298268318176 +epoch_199 Task125_All 124_14 mask_1 Dice 0.919987142086029 +epoch_199 Task125_All 124_20 mask_1 IoU 0.8598655462265015 +epoch_199 Task125_All 124_20 mask_1 Dice 0.9246534705162048 +epoch_199 Task125_All 124_25 mask_1 IoU 0.8394500017166138 +epoch_199 Task125_All 124_25 mask_1 Dice 0.9127184748649597 +epoch_199 Task125_All 124_31 mask_1 IoU 0.8838265538215637 +epoch_199 Task125_All 124_31 mask_1 Dice 0.938331127166748 +epoch_199 Task125_All 124_42 mask_1 IoU 0.8574174046516418 +epoch_199 Task125_All 124_42 mask_1 Dice 0.923236072063446 +epoch_249 Task125_All 121_01 mask_1 IoU 0.8424583077430725 +epoch_249 Task125_All 121_01 mask_1 Dice 0.9144937992095947 +epoch_249 Task125_All 121_32 mask_1 IoU 0.8430827260017395 +epoch_249 Task125_All 121_32 mask_1 Dice 0.9148615002632141 +epoch_249 Task125_All 121_34 mask_1 IoU 0.8621417284011841 +epoch_249 Task125_All 121_34 mask_1 Dice 0.9259678721427917 +epoch_249 Task125_All 122_03 mask_1 IoU 0.8131737112998962 +epoch_249 Task125_All 122_03 mask_1 Dice 0.8969617486000061 +epoch_249 Task125_All 122_08 mask_1 IoU 0.839440643787384 +epoch_249 Task125_All 122_08 mask_1 Dice 0.9127129316329956 +epoch_249 Task125_All 122_13 mask_1 IoU 0.885624349117279 +epoch_249 Task125_All 122_13 mask_1 Dice 0.9393433332443237 +epoch_249 Task125_All 122_15 mask_1 IoU 0.8455602526664734 +epoch_249 Task125_All 122_15 mask_1 Dice 0.9163182377815247 +epoch_249 Task125_All 123_03 mask_1 IoU 0.8490548729896545 +epoch_249 Task125_All 123_03 mask_1 Dice 0.9183663725852966 +epoch_249 Task125_All 123_08 mask_1 IoU 0.8475263714790344 +epoch_249 Task125_All 123_08 mask_1 Dice 0.9174714684486389 +epoch_249 Task125_All 123_12 mask_1 IoU 0.8411496877670288 +epoch_249 Task125_All 123_12 mask_1 Dice 0.9137222170829773 +epoch_249 Task125_All 123_15 mask_1 IoU 0.8418524265289307 +epoch_249 Task125_All 123_15 mask_1 Dice 0.9141366481781006 +epoch_249 Task125_All 123_18 mask_1 IoU 0.8673768639564514 +epoch_249 Task125_All 123_18 mask_1 Dice 0.9289789199829102 +epoch_249 Task125_All 123_26 mask_1 IoU 0.7453387975692749 +epoch_249 Task125_All 123_26 mask_1 Dice 0.854090690612793 +epoch_249 Task125_All 124_00 mask_1 IoU 0.8261633515357971 +epoch_249 Task125_All 124_00 mask_1 Dice 0.9048077464103699 +epoch_249 Task125_All 124_04 mask_1 IoU 0.8356912732124329 +epoch_249 Task125_All 124_04 mask_1 Dice 0.9104921817779541 +epoch_249 Task125_All 124_14 mask_1 IoU 0.8398071527481079 +epoch_249 Task125_All 124_14 mask_1 Dice 0.9129295349121094 +epoch_249 Task125_All 124_20 mask_1 IoU 0.8595260381698608 +epoch_249 Task125_All 124_20 mask_1 Dice 0.9244571328163147 +epoch_249 Task125_All 124_25 mask_1 IoU 0.8202120661735535 +epoch_249 Task125_All 124_25 mask_1 Dice 0.9012269377708435 +epoch_249 Task125_All 124_31 mask_1 IoU 0.8788493871688843 +epoch_249 Task125_All 124_31 mask_1 Dice 0.935518741607666 +epoch_249 Task125_All 124_42 mask_1 IoU 0.8354772329330444 +epoch_249 Task125_All 124_42 mask_1 Dice 0.9103651642799377 diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/val_metrics.json b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/val_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fa6b30853bfdd779736154b01544a5712643dc78 --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/fold_0/val_metrics.json @@ -0,0 +1,622 @@ +{ + "epoch_49": { + "Task125_All": { + "121_01": { + "mask_1": { + "IoU": 0.8190414905548096, + "Dice": 0.900519847869873 + } + }, + "121_32": { + "mask_1": { + "IoU": 0.8433037996292114, + "Dice": 0.9149916768074036 + } + }, + "121_34": { + "mask_1": { + "IoU": 0.8565046191215515, + "Dice": 0.9227067232131958 + } + }, + "122_03": { + "mask_1": { + "IoU": 0.8295673131942749, + "Dice": 0.9068453907966614 + } + }, + "122_08": { + "mask_1": { + "IoU": 0.8541257977485657, + "Dice": 0.9213245511054993 + } + }, + "122_13": { + "mask_1": { + "IoU": 0.8204582333564758, + "Dice": 0.9013755321502686 + } + }, + "122_15": { + "mask_1": { + "IoU": 0.8348348736763, + "Dice": 0.9099836349487305 + } + }, + "123_03": { + "mask_1": { + "IoU": 0.8123469948768616, + "Dice": 0.8964585661888123 + } + }, + "123_08": { + "mask_1": { + "IoU": 0.8557426333427429, + "Dice": 0.9222643375396729 + } + }, + "123_12": { + "mask_1": { + "IoU": 0.8032535910606384, + "Dice": 0.8908936977386475 + } + }, + "123_15": { + "mask_1": { + "IoU": 0.8570336103439331, + "Dice": 0.9230135679244995 + } + }, + "123_18": { + "mask_1": { + "IoU": 0.8444092869758606, + "Dice": 0.9156419634819031 + } + }, + "123_26": { + "mask_1": { + "IoU": 0.8524324297904968, + "Dice": 0.9203384518623352 + } + }, + "124_00": { + "mask_1": { + "IoU": 0.8273141980171204, + "Dice": 0.9054974913597107 + } + }, + "124_04": { + "mask_1": { + "IoU": 0.8348763585090637, + "Dice": 0.9100083112716675 + } + }, + "124_14": { + "mask_1": { + "IoU": 0.8251117467880249, + "Dice": 0.904176652431488 + } + }, + "124_20": { + "mask_1": { + "IoU": 0.8404054641723633, + "Dice": 0.9132829308509827 + } + }, + "124_25": { + "mask_1": { + "IoU": 0.8591132164001465, + "Dice": 0.9242182970046997 + } + }, + "124_31": { + "mask_1": { + "IoU": 0.8663323521614075, + "Dice": 0.928379476070404 + } + }, + "124_42": { + "mask_1": { + "IoU": 0.8374714255332947, + "Dice": 0.9115477204322815 + } + } + } + }, + "epoch_99": { + "Task125_All": { + "121_01": { + "mask_1": { + "IoU": 0.8566195964813232, + "Dice": 0.9227734208106995 + } + }, + "121_32": { + "mask_1": { + "IoU": 0.8633441925048828, + "Dice": 0.9266610145568848 + } + }, + "121_34": { + "mask_1": { + "IoU": 0.830015242099762, + "Dice": 0.9071128964424133 + } + }, + "122_03": { + "mask_1": { + "IoU": 0.8172138929367065, + "Dice": 0.8994141221046448 + } + }, + "122_08": { + "mask_1": { + "IoU": 0.8082484602928162, + "Dice": 0.8939573168754578 + } + }, + "122_13": { + "mask_1": { + "IoU": 0.8305820226669312, + "Dice": 0.9074512720108032 + } + }, + "122_15": { + "mask_1": { + "IoU": 0.7707474827766418, + "Dice": 0.8705334663391113 + } + }, + "123_03": { + "mask_1": { + "IoU": 0.8588523864746094, + "Dice": 0.9240673184394836 + } + }, + "123_08": { + "mask_1": { + "IoU": 0.8445738554000854, + "Dice": 0.9157387018203735 + } + }, + "123_12": { + "mask_1": { + "IoU": 0.8351500034332275, + "Dice": 0.91017085313797 + } + }, + "123_15": { + "mask_1": { + "IoU": 0.8173587322235107, + "Dice": 0.8995018005371094 + } + }, + "123_18": { + "mask_1": { + "IoU": 0.8322774767875671, + "Dice": 0.9084622859954834 + } + }, + "123_26": { + "mask_1": { + "IoU": 0.8488346934318542, + "Dice": 0.9182375073432922 + } + }, + "124_00": { + "mask_1": { + "IoU": 0.840190052986145, + "Dice": 0.913155734539032 + } + }, + "124_04": { + "mask_1": { + "IoU": 0.8453475832939148, + "Dice": 0.9161933660507202 + } + }, + "124_14": { + "mask_1": { + "IoU": 0.8615167737007141, + "Dice": 0.9256073236465454 + } + }, + "124_20": { + "mask_1": { + "IoU": 0.8461165428161621, + "Dice": 0.9166447520256042 + } + }, + "124_25": { + "mask_1": { + "IoU": 0.8607038259506226, + "Dice": 0.9251379370689392 + } + }, + "124_31": { + "mask_1": { + "IoU": 0.8529249429702759, + "Dice": 0.9206254482269287 + } + }, + "124_42": { + "mask_1": { + "IoU": 0.7938142418861389, + "Dice": 0.8850573301315308 + } + } + } + }, + "epoch_149": { + "Task125_All": { + "121_01": { + "mask_1": { + "IoU": 0.8418939113616943, + "Dice": 0.9141611456871033 + } + }, + "121_32": { + "mask_1": { + "IoU": 0.8663614392280579, + "Dice": 0.9283962249755859 + } + }, + "121_34": { + "mask_1": { + "IoU": 0.8479877710342407, + "Dice": 0.9177417755126953 + } + }, + "122_03": { + "mask_1": { + "IoU": 0.8712738156318665, + "Dice": 0.9312093257904053 + } + }, + "122_08": { + "mask_1": { + "IoU": 0.8334442973136902, + "Dice": 0.9091569185256958 + } + }, + "122_13": { + "mask_1": { + "IoU": 0.8369640707969666, + "Dice": 0.9112470746040344 + } + }, + "122_15": { + "mask_1": { + "IoU": 0.825735867023468, + "Dice": 0.9045512676239014 + } + }, + "123_03": { + "mask_1": { + "IoU": 0.7752001285552979, + "Dice": 0.8733664751052856 + } + }, + "123_08": { + "mask_1": { + "IoU": 0.8314370512962341, + "Dice": 0.907961368560791 + } + }, + "123_12": { + "mask_1": { + "IoU": 0.7932608723640442, + "Dice": 0.8847132921218872 + } + }, + "123_15": { + "mask_1": { + "IoU": 0.8462263345718384, + "Dice": 0.9167091846466064 + } + }, + "123_18": { + "mask_1": { + "IoU": 0.8563746809959412, + "Dice": 0.9226313233375549 + } + }, + "123_26": { + "mask_1": { + "IoU": 0.8624430298805237, + "Dice": 0.9261416792869568 + } + }, + "124_00": { + "mask_1": { + "IoU": 0.8616271018981934, + "Dice": 0.9256709814071655 + } + }, + "124_04": { + "mask_1": { + "IoU": 0.831540584564209, + "Dice": 0.9080231189727783 + } + }, + "124_14": { + "mask_1": { + "IoU": 0.8469969630241394, + "Dice": 0.9171611666679382 + } + }, + "124_20": { + "mask_1": { + "IoU": 0.8115898370742798, + "Dice": 0.8959973454475403 + } + }, + "124_25": { + "mask_1": { + "IoU": 0.8792508244514465, + "Dice": 0.9357461333274841 + } + }, + "124_31": { + "mask_1": { + "IoU": 0.794054388999939, + "Dice": 0.8852065801620483 + } + }, + "124_42": { + "mask_1": { + "IoU": 0.8389893174171448, + "Dice": 0.912446141242981 + } + } + } + }, + "epoch_199": { + "Task125_All": { + "121_01": { + "mask_1": { + "IoU": 0.8784856200218201, + "Dice": 0.9353125691413879 + } + }, + "121_32": { + "mask_1": { + "IoU": 0.8476223945617676, + "Dice": 0.9175277352333069 + } + }, + "121_34": { + "mask_1": { + "IoU": 0.8607978224754333, + "Dice": 0.9251921772956848 + } + }, + "122_03": { + "mask_1": { + "IoU": 0.8442391753196716, + "Dice": 0.91554194688797 + } + }, + "122_08": { + "mask_1": { + "IoU": 0.849332332611084, + "Dice": 0.9185286164283752 + } + }, + "122_13": { + "mask_1": { + "IoU": 0.8142664432525635, + "Dice": 0.8976260423660278 + } + }, + "122_15": { + "mask_1": { + "IoU": 0.8593831062316895, + "Dice": 0.9243744611740112 + } + }, + "123_03": { + "mask_1": { + "IoU": 0.8357996344566345, + "Dice": 0.9105564951896667 + } + }, + "123_08": { + "mask_1": { + "IoU": 0.8279212713241577, + "Dice": 0.905860960483551 + } + }, + "123_12": { + "mask_1": { + "IoU": 0.8407734036445618, + "Dice": 0.913500189781189 + } + }, + "123_15": { + "mask_1": { + "IoU": 0.8218513131141663, + "Dice": 0.9022156000137329 + } + }, + "123_18": { + "mask_1": { + "IoU": 0.8808693885803223, + "Dice": 0.936661958694458 + } + }, + "123_26": { + "mask_1": { + "IoU": 0.838599681854248, + "Dice": 0.9122155904769897 + } + }, + "124_00": { + "mask_1": { + "IoU": 0.804621696472168, + "Dice": 0.8917344808578491 + } + }, + "124_04": { + "mask_1": { + "IoU": 0.8054866194725037, + "Dice": 0.8922653794288635 + } + }, + "124_14": { + "mask_1": { + "IoU": 0.8518298268318176, + "Dice": 0.919987142086029 + } + }, + "124_20": { + "mask_1": { + "IoU": 0.8598655462265015, + "Dice": 0.9246534705162048 + } + }, + "124_25": { + "mask_1": { + "IoU": 0.8394500017166138, + "Dice": 0.9127184748649597 + } + }, + "124_31": { + "mask_1": { + "IoU": 0.8838265538215637, + "Dice": 0.938331127166748 + } + }, + "124_42": { + "mask_1": { + "IoU": 0.8574174046516418, + "Dice": 0.923236072063446 + } + } + } + }, + "epoch_249": { + "Task125_All": { + "121_01": { + "mask_1": { + "IoU": 0.8424583077430725, + "Dice": 0.9144937992095947 + } + }, + "121_32": { + "mask_1": { + "IoU": 0.8430827260017395, + "Dice": 0.9148615002632141 + } + }, + "121_34": { + "mask_1": { + "IoU": 0.8621417284011841, + "Dice": 0.9259678721427917 + } + }, + "122_03": { + "mask_1": { + "IoU": 0.8131737112998962, + "Dice": 0.8969617486000061 + } + }, + "122_08": { + "mask_1": { + "IoU": 0.839440643787384, + "Dice": 0.9127129316329956 + } + }, + "122_13": { + "mask_1": { + "IoU": 0.885624349117279, + "Dice": 0.9393433332443237 + } + }, + "122_15": { + "mask_1": { + "IoU": 0.8455602526664734, + "Dice": 0.9163182377815247 + } + }, + "123_03": { + "mask_1": { + "IoU": 0.8490548729896545, + "Dice": 0.9183663725852966 + } + }, + "123_08": { + "mask_1": { + "IoU": 0.8475263714790344, + "Dice": 0.9174714684486389 + } + }, + "123_12": { + "mask_1": { + "IoU": 0.8411496877670288, + "Dice": 0.9137222170829773 + } + }, + "123_15": { + "mask_1": { + "IoU": 0.8418524265289307, + "Dice": 0.9141366481781006 + } + }, + "123_18": { + "mask_1": { + "IoU": 0.8673768639564514, + "Dice": 0.9289789199829102 + } + }, + "123_26": { + "mask_1": { + "IoU": 0.7453387975692749, + "Dice": 0.854090690612793 + } + }, + "124_00": { + "mask_1": { + "IoU": 0.8261633515357971, + "Dice": 0.9048077464103699 + } + }, + "124_04": { + "mask_1": { + "IoU": 0.8356912732124329, + "Dice": 0.9104921817779541 + } + }, + "124_14": { + "mask_1": { + "IoU": 0.8398071527481079, + "Dice": 0.9129295349121094 + } + }, + "124_20": { + "mask_1": { + "IoU": 0.8595260381698608, + "Dice": 0.9244571328163147 + } + }, + "124_25": { + "mask_1": { + "IoU": 0.8202120661735535, + "Dice": 0.9012269377708435 + } + }, + "124_31": { + "mask_1": { + "IoU": 0.8788493871688843, + "Dice": 0.935518741607666 + } + }, + "124_42": { + "mask_1": { + "IoU": 0.8354772329330444, + "Dice": 0.9103651642799377 + } + } + } + } +} \ No newline at end of file diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/plans.pkl b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/plans.pkl new file mode 100644 index 0000000000000000000000000000000000000000..580844a943384ca822b1bee238311f5fa39eb9eb --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/Task125_All/nnUNetTrainerSequential__nnUNetPlansv2.1/Generic_UNet/SEQ/plans.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ae403940a83943c233fce36e18a8f48d52ea13719fd99142a929590169ed750 +size 21833 diff --git a/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/metadata/Generic_UNet/SEQ/sequential_trained_on.pkl b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/metadata/Generic_UNet/SEQ/sequential_trained_on.pkl new file mode 100644 index 0000000000000000000000000000000000000000..73e7fa7244f972f76785e8f3b28ef4cfd236194e --- /dev/null +++ b/pretrained-lifelong/RESULTS_FOLDER_LIFELONG/nnUNet_ext/2d/Task125_All/metadata/Generic_UNet/SEQ/sequential_trained_on.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdc1241e4e5d7a67629b5acb51a9283afbae770b80124f92203149ac54c64274 +size 552