diff --git a/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/config.yaml b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9863c7a017626f3f3675ad789ed7a0e65d494b3c --- /dev/null +++ b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/config.yaml @@ -0,0 +1,40 @@ +general_cfg: + algo_name: DQN + device: cuda + env_name: Acrobot-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_Acrobot-v1_DQN_20221122-120436 + max_steps: 100000 + mode: test + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.002 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/logs/log.txt b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..ae3f0a459b29c4458b498054ac15df23a03556b5 --- /dev/null +++ b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/logs/log.txt @@ -0,0 +1,14 @@ +2022-11-22 12:06:10 - r - INFO: - n_states: 6, n_actions: 3 +2022-11-22 12:06:13 - r - INFO: - Start testing! +2022-11-22 12:06:13 - r - INFO: - Env: Acrobot-v1, Algorithm: DQN, Device: cuda +2022-11-22 12:06:13 - r - INFO: - Episode: 1/10, Reward: -90.000, Step: 91 +2022-11-22 12:06:13 - r - INFO: - Episode: 2/10, Reward: -116.000, Step: 117 +2022-11-22 12:06:13 - r - INFO: - Episode: 3/10, Reward: -120.000, Step: 121 +2022-11-22 12:06:13 - r - INFO: - Episode: 4/10, Reward: -110.000, Step: 111 +2022-11-22 12:06:13 - r - INFO: - Episode: 5/10, Reward: -71.000, Step: 72 +2022-11-22 12:06:13 - r - INFO: - Episode: 6/10, Reward: -72.000, Step: 73 +2022-11-22 12:06:13 - r - INFO: - Episode: 7/10, Reward: -96.000, Step: 97 +2022-11-22 12:06:13 - r - INFO: - Episode: 8/10, Reward: -106.000, Step: 107 +2022-11-22 12:06:13 - r - INFO: - Episode: 9/10, Reward: -100.000, Step: 101 +2022-11-22 12:06:13 - r - INFO: - Episode: 10/10, Reward: -87.000, Step: 88 +2022-11-22 12:06:13 - r - INFO: - Finish testing! diff --git a/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/models/checkpoint.pt b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2c09dad2e8c3e8479647d8caf8bdc6b89f4af36 --- /dev/null +++ b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca574bc347296936e61d7a259d139825d1809a3332b5f9725e5ca72c87efbbfa +size 275543 diff --git a/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/results/learning_curve.png b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..b79b6bc944b82c33d74b5aa16f89ab53ccf4d4f4 Binary files /dev/null and b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/results/learning_curve.png differ diff --git a/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/results/res.csv b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..369c7af3f325583071a2e2d4f877d465328b7035 --- /dev/null +++ b/Acrobot-v1/Test_Acrobot-v1_DQN_20221122-120610/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,-90.0,91 +1,-116.0,117 +2,-120.0,121 +3,-110.0,111 +4,-71.0,72 +5,-72.0,73 +6,-96.0,97 +7,-106.0,107 +8,-100.0,101 +9,-87.0,88 diff --git a/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/config.yaml b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39ff176dc74f44d8095f952977766667895fdaf1 --- /dev/null +++ b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/config.yaml @@ -0,0 +1,40 @@ +general_cfg: + algo_name: DQN + device: cuda + env_name: Acrobot-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/logs/log.txt b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..93399e2d12145ccf931c286c9e5133cccd57a307 --- /dev/null +++ b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/logs/log.txt @@ -0,0 +1,110 @@ +2022-11-22 12:04:36 - r - INFO: - n_states: 6, n_actions: 3 +2022-11-22 12:04:39 - r - INFO: - Start training! +2022-11-22 12:04:39 - r - INFO: - Env: Acrobot-v1, Algorithm: DQN, Device: cuda +2022-11-22 12:04:39 - r - INFO: - Episode: 1/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:40 - r - INFO: - Episode: 2/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:40 - r - INFO: - Episode: 3/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:40 - r - INFO: - Episode: 4/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:41 - r - INFO: - Episode: 5/100, Reward: -189.000, Step: 190 +2022-11-22 12:04:41 - r - INFO: - Current episode 5 has the best eval reward: -200.000 +2022-11-22 12:04:42 - r - INFO: - Episode: 6/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:42 - r - INFO: - Episode: 7/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:42 - r - INFO: - Episode: 8/100, Reward: -132.000, Step: 133 +2022-11-22 12:04:43 - r - INFO: - Episode: 9/100, Reward: -101.000, Step: 102 +2022-11-22 12:04:43 - r - INFO: - Episode: 10/100, Reward: -148.000, Step: 149 +2022-11-22 12:04:44 - r - INFO: - Current episode 10 has the best eval reward: -200.000 +2022-11-22 12:04:44 - r - INFO: - Episode: 11/100, Reward: -188.000, Step: 189 +2022-11-22 12:04:44 - r - INFO: - Episode: 12/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:45 - r - INFO: - Episode: 13/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:45 - r - INFO: - Episode: 14/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:45 - r - INFO: - Episode: 15/100, Reward: -181.000, Step: 182 +2022-11-22 12:04:46 - r - INFO: - Current episode 15 has the best eval reward: -193.900 +2022-11-22 12:04:46 - r - INFO: - Episode: 16/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:47 - r - INFO: - Episode: 17/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:47 - r - INFO: - Episode: 18/100, Reward: -184.000, Step: 185 +2022-11-22 12:04:47 - r - INFO: - Episode: 19/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:48 - r - INFO: - Episode: 20/100, Reward: -180.000, Step: 181 +2022-11-22 12:04:48 - r - INFO: - Current episode 20 has the best eval reward: -151.000 +2022-11-22 12:04:49 - r - INFO: - Episode: 21/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:49 - r - INFO: - Episode: 22/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:49 - r - INFO: - Episode: 23/100, Reward: -139.000, Step: 140 +2022-11-22 12:04:50 - r - INFO: - Episode: 24/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:50 - r - INFO: - Episode: 25/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:51 - r - INFO: - Episode: 26/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:51 - r - INFO: - Episode: 27/100, Reward: -159.000, Step: 160 +2022-11-22 12:04:52 - r - INFO: - Episode: 28/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:52 - r - INFO: - Episode: 29/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:52 - r - INFO: - Episode: 30/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:53 - r - INFO: - Episode: 31/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:54 - r - INFO: - Episode: 32/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:54 - r - INFO: - Episode: 33/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:55 - r - INFO: - Episode: 34/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:55 - r - INFO: - Episode: 35/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:56 - r - INFO: - Episode: 36/100, Reward: -165.000, Step: 166 +2022-11-22 12:04:56 - r - INFO: - Episode: 37/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:57 - r - INFO: - Episode: 38/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:57 - r - INFO: - Episode: 39/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:57 - r - INFO: - Episode: 40/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:58 - r - INFO: - Current episode 40 has the best eval reward: -141.800 +2022-11-22 12:04:58 - r - INFO: - Episode: 41/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:58 - r - INFO: - Episode: 42/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:59 - r - INFO: - Episode: 43/100, Reward: -200.000, Step: 200 +2022-11-22 12:04:59 - r - INFO: - Episode: 44/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:00 - r - INFO: - Episode: 45/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:01 - r - INFO: - Episode: 46/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:01 - r - INFO: - Episode: 47/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:01 - r - INFO: - Episode: 48/100, Reward: -186.000, Step: 187 +2022-11-22 12:05:02 - r - INFO: - Episode: 49/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:02 - r - INFO: - Episode: 50/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:03 - r - INFO: - Episode: 51/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:03 - r - INFO: - Episode: 52/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:04 - r - INFO: - Episode: 53/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:04 - r - INFO: - Episode: 54/100, Reward: -156.000, Step: 157 +2022-11-22 12:05:04 - r - INFO: - Episode: 55/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:05 - r - INFO: - Episode: 56/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:06 - r - INFO: - Episode: 57/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:06 - r - INFO: - Episode: 58/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:06 - r - INFO: - Episode: 59/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:07 - r - INFO: - Episode: 60/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:08 - r - INFO: - Episode: 61/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:08 - r - INFO: - Episode: 62/100, Reward: -191.000, Step: 192 +2022-11-22 12:05:08 - r - INFO: - Episode: 63/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:09 - r - INFO: - Episode: 64/100, Reward: -187.000, Step: 188 +2022-11-22 12:05:09 - r - INFO: - Episode: 65/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:10 - r - INFO: - Episode: 66/100, Reward: -158.000, Step: 159 +2022-11-22 12:05:10 - r - INFO: - Episode: 67/100, Reward: -164.000, Step: 165 +2022-11-22 12:05:11 - r - INFO: - Episode: 68/100, Reward: -134.000, Step: 135 +2022-11-22 12:05:11 - r - INFO: - Episode: 69/100, Reward: -200.000, Step: 200 +2022-11-22 12:05:11 - r - INFO: - Episode: 70/100, Reward: -144.000, Step: 145 +2022-11-22 12:05:12 - r - INFO: - Episode: 71/100, Reward: -193.000, Step: 194 +2022-11-22 12:05:13 - r - INFO: - Episode: 72/100, Reward: -137.000, Step: 138 +2022-11-22 12:05:13 - r - INFO: - Episode: 73/100, Reward: -183.000, Step: 184 +2022-11-22 12:05:13 - r - INFO: - Episode: 74/100, Reward: -141.000, Step: 142 +2022-11-22 12:05:14 - r - INFO: - Episode: 75/100, Reward: -163.000, Step: 164 +2022-11-22 12:05:14 - r - INFO: - Current episode 75 has the best eval reward: -93.500 +2022-11-22 12:05:14 - r - INFO: - Episode: 76/100, Reward: -71.000, Step: 72 +2022-11-22 12:05:14 - r - INFO: - Episode: 77/100, Reward: -71.000, Step: 72 +2022-11-22 12:05:14 - r - INFO: - Episode: 78/100, Reward: -154.000, Step: 155 +2022-11-22 12:05:15 - r - INFO: - Episode: 79/100, Reward: -117.000, Step: 118 +2022-11-22 12:05:15 - r - INFO: - Episode: 80/100, Reward: -175.000, Step: 176 +2022-11-22 12:05:16 - r - INFO: - Episode: 81/100, Reward: -187.000, Step: 188 +2022-11-22 12:05:16 - r - INFO: - Episode: 82/100, Reward: -111.000, Step: 112 +2022-11-22 12:05:16 - r - INFO: - Episode: 83/100, Reward: -86.000, Step: 87 +2022-11-22 12:05:16 - r - INFO: - Episode: 84/100, Reward: -85.000, Step: 86 +2022-11-22 12:05:17 - r - INFO: - Episode: 85/100, Reward: -93.000, Step: 94 +2022-11-22 12:05:17 - r - INFO: - Episode: 86/100, Reward: -131.000, Step: 132 +2022-11-22 12:05:18 - r - INFO: - Episode: 87/100, Reward: -128.000, Step: 129 +2022-11-22 12:05:18 - r - INFO: - Episode: 88/100, Reward: -194.000, Step: 195 +2022-11-22 12:05:18 - r - INFO: - Episode: 89/100, Reward: -93.000, Step: 94 +2022-11-22 12:05:18 - r - INFO: - Episode: 90/100, Reward: -91.000, Step: 92 +2022-11-22 12:05:19 - r - INFO: - Episode: 91/100, Reward: -161.000, Step: 162 +2022-11-22 12:05:19 - r - INFO: - Episode: 92/100, Reward: -103.000, Step: 104 +2022-11-22 12:05:19 - r - INFO: - Episode: 93/100, Reward: -110.000, Step: 111 +2022-11-22 12:05:20 - r - INFO: - Episode: 94/100, Reward: -191.000, Step: 192 +2022-11-22 12:05:20 - r - INFO: - Episode: 95/100, Reward: -70.000, Step: 71 +2022-11-22 12:05:20 - r - INFO: - Episode: 96/100, Reward: -71.000, Step: 72 +2022-11-22 12:05:21 - r - INFO: - Episode: 97/100, Reward: -125.000, Step: 126 +2022-11-22 12:05:21 - r - INFO: - Episode: 98/100, Reward: -109.000, Step: 110 +2022-11-22 12:05:21 - r - INFO: - Episode: 99/100, Reward: -104.000, Step: 105 +2022-11-22 12:05:21 - r - INFO: - Episode: 100/100, Reward: -93.000, Step: 94 +2022-11-22 12:05:22 - r - INFO: - Finish training! diff --git a/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/models/checkpoint.pt b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2c09dad2e8c3e8479647d8caf8bdc6b89f4af36 --- /dev/null +++ b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca574bc347296936e61d7a259d139825d1809a3332b5f9725e5ca72c87efbbfa +size 275543 diff --git a/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/results/learning_curve.png b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..024a9b03eccb5bf0f0fa22faabaa3e1932ecf2b4 Binary files /dev/null and b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/results/learning_curve.png differ diff --git a/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/results/res.csv b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab4cac3b8c64286aaa77774578658c6ea7b721a9 --- /dev/null +++ b/Acrobot-v1/Train_Acrobot-v1_DQN_20221122-120436/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,-200.0,200 +1,-200.0,200 +2,-200.0,200 +3,-200.0,200 +4,-189.0,190 +5,-200.0,200 +6,-200.0,200 +7,-132.0,133 +8,-101.0,102 +9,-148.0,149 +10,-188.0,189 +11,-200.0,200 +12,-200.0,200 +13,-200.0,200 +14,-181.0,182 +15,-200.0,200 +16,-200.0,200 +17,-184.0,185 +18,-200.0,200 +19,-180.0,181 +20,-200.0,200 +21,-200.0,200 +22,-139.0,140 +23,-200.0,200 +24,-200.0,200 +25,-200.0,200 +26,-159.0,160 +27,-200.0,200 +28,-200.0,200 +29,-200.0,200 +30,-200.0,200 +31,-200.0,200 +32,-200.0,200 +33,-200.0,200 +34,-200.0,200 +35,-165.0,166 +36,-200.0,200 +37,-200.0,200 +38,-200.0,200 +39,-200.0,200 +40,-200.0,200 +41,-200.0,200 +42,-200.0,200 +43,-200.0,200 +44,-200.0,200 +45,-200.0,200 +46,-200.0,200 +47,-186.0,187 +48,-200.0,200 +49,-200.0,200 +50,-200.0,200 +51,-200.0,200 +52,-200.0,200 +53,-156.0,157 +54,-200.0,200 +55,-200.0,200 +56,-200.0,200 +57,-200.0,200 +58,-200.0,200 +59,-200.0,200 +60,-200.0,200 +61,-191.0,192 +62,-200.0,200 +63,-187.0,188 +64,-200.0,200 +65,-158.0,159 +66,-164.0,165 +67,-134.0,135 +68,-200.0,200 +69,-144.0,145 +70,-193.0,194 +71,-137.0,138 +72,-183.0,184 +73,-141.0,142 +74,-163.0,164 +75,-71.0,72 +76,-71.0,72 +77,-154.0,155 +78,-117.0,118 +79,-175.0,176 +80,-187.0,188 +81,-111.0,112 +82,-86.0,87 +83,-85.0,86 +84,-93.0,94 +85,-131.0,132 +86,-128.0,129 +87,-194.0,195 +88,-93.0,94 +89,-91.0,92 +90,-161.0,162 +91,-103.0,104 +92,-110.0,111 +93,-191.0,192 +94,-70.0,71 +95,-71.0,72 +96,-125.0,126 +97,-109.0,110 +98,-104.0,105 +99,-93.0,94 diff --git a/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/config.yaml b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16c73e4d16a7643e965cc5aa952312af830cc877 --- /dev/null +++ b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/config.yaml @@ -0,0 +1,30 @@ +general_cfg: + algo_name: DDPG + device: cpu + env_name: BipedalWalker-v3 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_BipedalWalker-v3_DDPG_20230414-214211 + max_steps: 900 + mode: test + mp_backend: mp + n_workers: 1 + new_step_api: true + render: true + render_mode: rgb_array + save_fig: true + seed: 2023 + show_fig: false + test_eps: 3 + train_eps: 2000 + wrapper: null +algo_cfg: + actor_hidden_dim: 128 + actor_lr: 5.0e-05 + batch_size: 256 + buffer_size: 20480 + critic_hidden_dim: 128 + critic_lr: 0.001 + gamma: 0.99 + tau: 0.01 diff --git a/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/logs/log.txt b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..9a26cadaaf1aeaa5ac71e1c417d4fadde4829be7 --- /dev/null +++ b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/logs/log.txt @@ -0,0 +1,45 @@ +2023-04-15 18:44:48 - r - INFO: - Hyperparameters: +2023-04-15 18:44:48 - r - INFO: - ================================================================================ +2023-04-15 18:44:48 - r - INFO: - Name Value Type +2023-04-15 18:44:48 - r - INFO: - env_name BipedalWalker-v3 +2023-04-15 18:44:48 - r - INFO: - new_step_api 1 +2023-04-15 18:44:48 - r - INFO: - wrapper None +2023-04-15 18:44:48 - r - INFO: - render 1 +2023-04-15 18:44:48 - r - INFO: - render_mode rgb_array +2023-04-15 18:44:48 - r - INFO: - algo_name DDPG +2023-04-15 18:44:48 - r - INFO: - mode test +2023-04-15 18:44:48 - r - INFO: - mp_backend mp +2023-04-15 18:44:48 - r - INFO: - seed 2023 +2023-04-15 18:44:48 - r - INFO: - device cpu +2023-04-15 18:44:48 - r - INFO: - train_eps 2000 +2023-04-15 18:44:48 - r - INFO: - test_eps 3 +2023-04-15 18:44:48 - r - INFO: - eval_eps 10 +2023-04-15 18:44:48 - r - INFO: - eval_per_episode 5 +2023-04-15 18:44:48 - r - INFO: - max_steps 900 +2023-04-15 18:44:48 - r - INFO: - load_checkpoint 1 +2023-04-15 18:44:48 - r - INFO: - load_path Train_BipedalWalker-v3_DDPG_20230414-214211 +2023-04-15 18:44:48 - r - INFO: - show_fig 0 +2023-04-15 18:44:48 - r - INFO: - save_fig 1 +2023-04-15 18:44:48 - r - INFO: - n_workers 1 +2023-04-15 18:44:48 - r - INFO: - gamma 0.99 +2023-04-15 18:44:48 - r - INFO: - critic_lr 0.001 +2023-04-15 18:44:48 - r - INFO: - actor_lr 5e-05 +2023-04-15 18:44:48 - r - INFO: - buffer_size 20480 +2023-04-15 18:44:48 - r - INFO: - batch_size 256 +2023-04-15 18:44:48 - r - INFO: - tau 0.01 +2023-04-15 18:44:48 - r - INFO: - critic_hidden_dim 128 +2023-04-15 18:44:48 - r - INFO: - actor_hidden_dim 128 +2023-04-15 18:44:48 - r - INFO: - task_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Test_BipedalWalker-v3_DDPG_20230415-184448 +2023-04-15 18:44:48 - r - INFO: - res_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Test_BipedalWalker-v3_DDPG_20230415-184448/results +2023-04-15 18:44:48 - r - INFO: - log_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Test_BipedalWalker-v3_DDPG_20230415-184448/logs +2023-04-15 18:44:48 - r - INFO: - traj_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Test_BipedalWalker-v3_DDPG_20230415-184448/traj +2023-04-15 18:44:48 - r - INFO: - video_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Test_BipedalWalker-v3_DDPG_20230415-184448/videos +2023-04-15 18:44:48 - r - INFO: - ================================================================================ +2023-04-15 18:44:49 - r - INFO: - action_bound: 1.0 +2023-04-15 18:44:49 - r - INFO: - n_states: 24, n_actions: 4 +2023-04-15 18:44:49 - r - INFO: - Start testing! +2023-04-15 18:44:49 - r - INFO: - Env: BipedalWalker-v3, Algorithm: DDPG, Device: cpu +2023-04-15 18:45:15 - r - INFO: - Episode: 1/3, Reward: 189.378, Step: 900 +2023-04-15 18:47:08 - r - INFO: - Episode: 2/3, Reward: 189.378, Step: 900 +2023-04-15 18:47:36 - r - INFO: - Episode: 3/3, Reward: 193.824, Step: 900 +2023-04-15 18:47:36 - r - INFO: - Finish testing! diff --git a/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/models/actor_checkpoint.pt b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..694d1ed28c84f81d1ed05ea93f0c141b7f68dda6 --- /dev/null +++ b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e21820818072463e981ee39f73418521857414fbb150ee3e9de8a70b4a29cc5c +size 83167 diff --git a/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/results/learning_curve.png b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..782ed580729b0e14bada2602a629c87ccabf4b05 Binary files /dev/null and b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/results/learning_curve.png differ diff --git a/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/results/res.csv b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..ee3d9f7d6ff85613105a26485784e8087f557b23 --- /dev/null +++ b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/results/res.csv @@ -0,0 +1,4 @@ +episodes,rewards,steps +0,189.37752135212605,900 +1,189.37752135212605,900 +2,193.82441135536328,900 diff --git a/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/videos/video.gif b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/videos/video.gif new file mode 100644 index 0000000000000000000000000000000000000000..9b399a9473a1ce5a30dcb061f42046470906bee3 --- /dev/null +++ b/BipedalWalker-v3/Test_BipedalWalker-v3_DDPG_20230415-184448/videos/video.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a45fb2090b422dec04384f8f6f12a5bd1d7f764d5d14b091dc741455427054 +size 3479690 diff --git a/BipedalWalker-v3/Test_gym_SAC_20230416-022652/config.yaml b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94e21b37dd0e30b84bc742a74b4a136bff61c22d --- /dev/null +++ b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/config.yaml @@ -0,0 +1,38 @@ +general_cfg: + algo_name: SAC + continous: true + device: cuda + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_gym_SAC_20230415-140928 + max_steps: 800 + mode: test + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 666 + show_fig: false + test_eps: 20 + train_eps: 2000 + wrapper: null +algo_cfg: + alpha: 0.01 + automatic_entropy_tuning: false + batch_size: 64 + buffer_size: 1000000 + gamma: 0.98 + hidden_dim: 256 + lr: 0.0004 + n_epochs: 1 + policy_type: Gaussian + start_steps: 10000 + target_update_fre: 1 + tau: 0.01 +env_cfg: + id: BipedalWalker-v3 + new_step_api: true + render_mode: null diff --git a/BipedalWalker-v3/Test_gym_SAC_20230416-022652/logs/log.txt b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a672910e78380fadf73c7186e12d2159d2080e27 --- /dev/null +++ b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/logs/log.txt @@ -0,0 +1,67 @@ +2023-04-16 02:26:52 - r - INFO: - Hyperparameters: +2023-04-16 02:26:52 - r - INFO: - ================================================================================ +2023-04-16 02:26:52 - r - INFO: - Name Value Type +2023-04-16 02:26:52 - r - INFO: - env_name gym +2023-04-16 02:26:52 - r - INFO: - new_step_api 1 +2023-04-16 02:26:52 - r - INFO: - wrapper None +2023-04-16 02:26:52 - r - INFO: - render 0 +2023-04-16 02:26:52 - r - INFO: - render_mode None +2023-04-16 02:26:52 - r - INFO: - algo_name SAC +2023-04-16 02:26:52 - r - INFO: - mode test +2023-04-16 02:26:52 - r - INFO: - mp_backend mp +2023-04-16 02:26:52 - r - INFO: - seed 666 +2023-04-16 02:26:52 - r - INFO: - device cuda +2023-04-16 02:26:52 - r - INFO: - train_eps 2000 +2023-04-16 02:26:52 - r - INFO: - test_eps 20 +2023-04-16 02:26:52 - r - INFO: - eval_eps 10 +2023-04-16 02:26:52 - r - INFO: - eval_per_episode 5 +2023-04-16 02:26:52 - r - INFO: - max_steps 800 +2023-04-16 02:26:52 - r - INFO: - load_checkpoint 1 +2023-04-16 02:26:52 - r - INFO: - load_path Train_gym_SAC_20230415-140928 +2023-04-16 02:26:52 - r - INFO: - show_fig 0 +2023-04-16 02:26:52 - r - INFO: - save_fig 1 +2023-04-16 02:26:52 - r - INFO: - continous 1 +2023-04-16 02:26:52 - r - INFO: - policy_type Gaussian +2023-04-16 02:26:52 - r - INFO: - lr 0.0004 +2023-04-16 02:26:52 - r - INFO: - gamma 0.98 +2023-04-16 02:26:52 - r - INFO: - tau 0.01 +2023-04-16 02:26:52 - r - INFO: - alpha 0.01 +2023-04-16 02:26:52 - r - INFO: - automatic_entropy_tuning 0 +2023-04-16 02:26:52 - r - INFO: - batch_size 64 +2023-04-16 02:26:52 - r - INFO: - hidden_dim 256 +2023-04-16 02:26:52 - r - INFO: - n_epochs 1 +2023-04-16 02:26:52 - r - INFO: - start_steps 10000 +2023-04-16 02:26:52 - r - INFO: - target_update_fre 1 +2023-04-16 02:26:52 - r - INFO: - buffer_size 1000000 +2023-04-16 02:26:52 - r - INFO: - id BipedalWalker-v3 +2023-04-16 02:26:52 - r - INFO: - task_dir C:\Users\86153\Desktop\lab\joyrl-offline/tasks/Test_gym_SAC_20230416-022652 +2023-04-16 02:26:52 - r - INFO: - res_dir C:\Users\86153\Desktop\lab\joyrl-offline/tasks/Test_gym_SAC_20230416-022652/results +2023-04-16 02:26:52 - r - INFO: - log_dir C:\Users\86153\Desktop\lab\joyrl-offline/tasks/Test_gym_SAC_20230416-022652/logs +2023-04-16 02:26:52 - r - INFO: - traj_dir C:\Users\86153\Desktop\lab\joyrl-offline/tasks/Test_gym_SAC_20230416-022652/traj +2023-04-16 02:26:52 - r - INFO: - video_dir C:\Users\86153\Desktop\lab\joyrl-offline/tasks/Test_gym_SAC_20230416-022652/videos +2023-04-16 02:26:52 - r - INFO: - ================================================================================ +2023-04-16 02:26:52 - r - INFO: - action_bound: 1.0 +2023-04-16 02:26:52 - r - INFO: - n_states: 24, n_actions: 4 +2023-04-16 02:26:54 - r - INFO: - Start testing! +2023-04-16 02:26:54 - r - INFO: - Env: gym, Algorithm: SAC, Device: cuda +2023-04-16 02:26:56 - r - INFO: - Episode: 1/20, Reward: 317.463, Step: 749 +2023-04-16 02:26:57 - r - INFO: - Episode: 2/20, Reward: 317.753, Step: 755 +2023-04-16 02:26:58 - r - INFO: - Episode: 3/20, Reward: 317.815, Step: 736 +2023-04-16 02:26:59 - r - INFO: - Episode: 4/20, Reward: 317.523, Step: 757 +2023-04-16 02:27:00 - r - INFO: - Episode: 5/20, Reward: 317.422, Step: 755 +2023-04-16 02:27:01 - r - INFO: - Episode: 6/20, Reward: 317.201, Step: 753 +2023-04-16 02:27:02 - r - INFO: - Episode: 7/20, Reward: 317.512, Step: 753 +2023-04-16 02:27:03 - r - INFO: - Episode: 8/20, Reward: 316.893, Step: 757 +2023-04-16 02:27:04 - r - INFO: - Episode: 9/20, Reward: 316.954, Step: 758 +2023-04-16 02:27:06 - r - INFO: - Episode: 10/20, Reward: 317.564, Step: 746 +2023-04-16 02:27:07 - r - INFO: - Episode: 11/20, Reward: 317.620, Step: 754 +2023-04-16 02:27:09 - r - INFO: - Episode: 12/20, Reward: 317.115, Step: 753 +2023-04-16 02:27:10 - r - INFO: - Episode: 13/20, Reward: 317.953, Step: 750 +2023-04-16 02:27:11 - r - INFO: - Episode: 14/20, Reward: 317.991, Step: 747 +2023-04-16 02:27:13 - r - INFO: - Episode: 15/20, Reward: 317.402, Step: 761 +2023-04-16 02:27:14 - r - INFO: - Episode: 16/20, Reward: 318.334, Step: 741 +2023-04-16 02:27:16 - r - INFO: - Episode: 17/20, Reward: 318.368, Step: 743 +2023-04-16 02:27:17 - r - INFO: - Episode: 18/20, Reward: 316.972, Step: 759 +2023-04-16 02:27:19 - r - INFO: - Episode: 19/20, Reward: 316.778, Step: 757 +2023-04-16 02:27:20 - r - INFO: - Episode: 20/20, Reward: 317.033, Step: 752 +2023-04-16 02:27:20 - r - INFO: - Finish testing! diff --git a/BipedalWalker-v3/Test_gym_SAC_20230416-022652/models/checkpoint.pt b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..868e3ef211f6ada934b8b423dfd664f3102c8ffe --- /dev/null +++ b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e57f7720e0d181816c759181e068984e6533e701e78fe8355947e539090488d2 +size 3263885 diff --git a/BipedalWalker-v3/Test_gym_SAC_20230416-022652/results/learning_curve.png b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..ff3eaba797f844036ca000587b4d6d90368d0e8c Binary files /dev/null and b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/results/learning_curve.png differ diff --git a/BipedalWalker-v3/Test_gym_SAC_20230416-022652/results/res.csv b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..c363949bc462e84af392a6aefc32140d73c71b73 --- /dev/null +++ b/BipedalWalker-v3/Test_gym_SAC_20230416-022652/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,317.4629323907178,749 +1,317.75301182115135,755 +2,317.81452193879414,736 +3,317.52279608642925,757 +4,317.42152056943814,755 +5,317.2009972314697,753 +6,317.511599273757,753 +7,316.8930851386963,757 +8,316.9540603024222,758 +9,317.5642415119148,746 +10,317.6201045948496,754 +11,317.1149633966414,753 +12,317.9526517411536,750 +13,317.99103767269946,747 +14,317.4024140685487,761 +15,318.3339004381753,741 +16,318.36781152977653,743 +17,316.9721525340675,759 +18,316.7777139291063,757 +19,317.0330598132471,752 diff --git a/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/config.yaml b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3657e7d5788e8607b645f979368f91287182fc9 --- /dev/null +++ b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/config.yaml @@ -0,0 +1,30 @@ +general_cfg: + algo_name: DDPG + device: cpu + env_name: BipedalWalker-v3 + eval_eps: 10 + eval_per_episode: 20 + load_checkpoint: false + load_path: Train_BipedalWalker-v3_DDPG_20230414-201956 + max_steps: 300 + mode: train + mp_backend: mp + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 2023 + show_fig: false + test_eps: 3 + train_eps: 2000 + wrapper: null +algo_cfg: + actor_hidden_dim: 128 + actor_lr: 5.0e-05 + batch_size: 256 + buffer_size: 20480 + critic_hidden_dim: 128 + critic_lr: 0.001 + gamma: 0.99 + tau: 0.01 diff --git a/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/logs/log.txt b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..e736f318e6a8133e6b93c37e0356371f0d87907f --- /dev/null +++ b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/logs/log.txt @@ -0,0 +1,2051 @@ +2023-04-14 21:42:11 - r - INFO: - Hyperparameters: +2023-04-14 21:42:11 - r - INFO: - ================================================================================ +2023-04-14 21:42:11 - r - INFO: - Name Value Type +2023-04-14 21:42:11 - r - INFO: - env_name BipedalWalker-v3 +2023-04-14 21:42:11 - r - INFO: - new_step_api 1 +2023-04-14 21:42:11 - r - INFO: - wrapper None +2023-04-14 21:42:11 - r - INFO: - render 0 +2023-04-14 21:42:11 - r - INFO: - render_mode human +2023-04-14 21:42:11 - r - INFO: - algo_name DDPG +2023-04-14 21:42:11 - r - INFO: - mode train +2023-04-14 21:42:11 - r - INFO: - mp_backend mp +2023-04-14 21:42:11 - r - INFO: - seed 2023 +2023-04-14 21:42:11 - r - INFO: - device cpu +2023-04-14 21:42:11 - r - INFO: - train_eps 2000 +2023-04-14 21:42:11 - r - INFO: - test_eps 3 +2023-04-14 21:42:11 - r - INFO: - eval_eps 10 +2023-04-14 21:42:11 - r - INFO: - eval_per_episode 20 +2023-04-14 21:42:11 - r - INFO: - max_steps 300 +2023-04-14 21:42:11 - r - INFO: - load_checkpoint 0 +2023-04-14 21:42:11 - r - INFO: - load_path Train_BipedalWalker-v3_DDPG_20230414-201956 +2023-04-14 21:42:11 - r - INFO: - show_fig 0 +2023-04-14 21:42:11 - r - INFO: - save_fig 1 +2023-04-14 21:42:11 - r - INFO: - n_workers 1 +2023-04-14 21:42:11 - r - INFO: - gamma 0.99 +2023-04-14 21:42:11 - r - INFO: - critic_lr 0.001 +2023-04-14 21:42:11 - r - INFO: - actor_lr 5e-05 +2023-04-14 21:42:11 - r - INFO: - buffer_size 20480 +2023-04-14 21:42:11 - r - INFO: - batch_size 256 +2023-04-14 21:42:11 - r - INFO: - tau 0.01 +2023-04-14 21:42:11 - r - INFO: - critic_hidden_dim 128 +2023-04-14 21:42:11 - r - INFO: - actor_hidden_dim 128 +2023-04-14 21:42:11 - r - INFO: - task_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Train_BipedalWalker-v3_DDPG_20230414-214211 +2023-04-14 21:42:11 - r - INFO: - res_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Train_BipedalWalker-v3_DDPG_20230414-214211/results +2023-04-14 21:42:11 - r - INFO: - log_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Train_BipedalWalker-v3_DDPG_20230414-214211/logs +2023-04-14 21:42:11 - r - INFO: - traj_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Train_BipedalWalker-v3_DDPG_20230414-214211/traj +2023-04-14 21:42:11 - r - INFO: - video_dir D:\Python_data\gihub_project\joyrl-offline/tasks/Train_BipedalWalker-v3_DDPG_20230414-214211/videos +2023-04-14 21:42:11 - r - INFO: - ================================================================================ +2023-04-14 21:42:11 - r - INFO: - action_bound: 1.0 +2023-04-14 21:42:11 - r - INFO: - n_states: 24, n_actions: 4 +2023-04-14 21:42:11 - r - INFO: - Start training! +2023-04-14 21:42:11 - r - INFO: - Env: BipedalWalker-v3, Algorithm: DDPG, Device: cpu +2023-04-14 21:42:11 - r - INFO: - Episode: 1/2000, Reward: -109.112, Step: 185 +2023-04-14 21:42:12 - r - INFO: - Episode: 2/2000, Reward: -100.860, Step: 78 +2023-04-14 21:42:13 - r - INFO: - Episode: 3/2000, Reward: -106.094, Step: 57 +2023-04-14 21:42:15 - r - INFO: - Episode: 4/2000, Reward: -122.972, Step: 105 +2023-04-14 21:42:17 - r - INFO: - Episode: 5/2000, Reward: -115.262, Step: 44 +2023-04-14 21:42:22 - r - INFO: - Episode: 6/2000, Reward: -13.242, Step: 300 +2023-04-14 21:42:23 - r - INFO: - Episode: 7/2000, Reward: -121.542, Step: 42 +2023-04-14 21:42:25 - r - INFO: - Episode: 8/2000, Reward: -110.970, Step: 105 +2023-04-14 21:42:28 - r - INFO: - Episode: 9/2000, Reward: -115.359, Step: 127 +2023-04-14 21:42:32 - r - INFO: - Episode: 10/2000, Reward: -130.550, Step: 236 +2023-04-14 21:42:38 - r - INFO: - Episode: 11/2000, Reward: -33.805, Step: 300 +2023-04-14 21:42:41 - r - INFO: - Episode: 12/2000, Reward: -118.993, Step: 153 +2023-04-14 21:42:46 - r - INFO: - Episode: 13/2000, Reward: -17.596, Step: 300 +2023-04-14 21:42:48 - r - INFO: - Episode: 14/2000, Reward: -117.660, Step: 117 +2023-04-14 21:42:54 - r - INFO: - Episode: 15/2000, Reward: -16.641, Step: 300 +2023-04-14 21:43:00 - r - INFO: - Episode: 16/2000, Reward: -23.547, Step: 300 +2023-04-14 21:43:05 - r - INFO: - Episode: 17/2000, Reward: -21.802, Step: 300 +2023-04-14 21:43:10 - r - INFO: - Episode: 18/2000, Reward: -22.084, Step: 300 +2023-04-14 21:43:16 - r - INFO: - Episode: 19/2000, Reward: -26.890, Step: 300 +2023-04-14 21:43:23 - r - INFO: - Episode: 20/2000, Reward: -19.628, Step: 300 +2023-04-14 21:43:29 - r - INFO: - Current episode 20 has the best eval reward: -23.388 +2023-04-14 21:43:36 - r - INFO: - Episode: 21/2000, Reward: -113.434, Step: 282 +2023-04-14 21:43:42 - r - INFO: - Episode: 22/2000, Reward: -16.413, Step: 300 +2023-04-14 21:43:48 - r - INFO: - Episode: 23/2000, Reward: -24.658, Step: 300 +2023-04-14 21:43:53 - r - INFO: - Episode: 24/2000, Reward: -19.759, Step: 300 +2023-04-14 21:43:58 - r - INFO: - Episode: 25/2000, Reward: -18.339, Step: 300 +2023-04-14 21:44:03 - r - INFO: - Episode: 26/2000, Reward: -20.764, Step: 300 +2023-04-14 21:44:08 - r - INFO: - Episode: 27/2000, Reward: -25.039, Step: 300 +2023-04-14 21:44:13 - r - INFO: - Episode: 28/2000, Reward: -21.535, Step: 300 +2023-04-14 21:44:17 - r - INFO: - Episode: 29/2000, Reward: -118.795, Step: 224 +2023-04-14 21:44:18 - r - INFO: - Episode: 30/2000, Reward: -115.984, Step: 67 +2023-04-14 21:44:20 - r - INFO: - Episode: 31/2000, Reward: -115.901, Step: 67 +2023-04-14 21:44:20 - r - INFO: - Episode: 32/2000, Reward: -112.128, Step: 50 +2023-04-14 21:44:21 - r - INFO: - Episode: 33/2000, Reward: -111.155, Step: 44 +2023-04-14 21:44:23 - r - INFO: - Episode: 34/2000, Reward: -104.336, Step: 93 +2023-04-14 21:44:24 - r - INFO: - Episode: 35/2000, Reward: -114.617, Step: 62 +2023-04-14 21:44:25 - r - INFO: - Episode: 36/2000, Reward: -113.158, Step: 52 +2023-04-14 21:44:25 - r - INFO: - Episode: 37/2000, Reward: -111.612, Step: 41 +2023-04-14 21:44:26 - r - INFO: - Episode: 38/2000, Reward: -110.226, Step: 46 +2023-04-14 21:44:27 - r - INFO: - Episode: 39/2000, Reward: -114.134, Step: 64 +2023-04-14 21:44:32 - r - INFO: - Episode: 40/2000, Reward: -29.079, Step: 300 +2023-04-14 21:44:43 - r - INFO: - Episode: 41/2000, Reward: -34.994, Step: 300 +2023-04-14 21:44:48 - r - INFO: - Episode: 42/2000, Reward: -28.210, Step: 300 +2023-04-14 21:44:53 - r - INFO: - Episode: 43/2000, Reward: -26.793, Step: 300 +2023-04-14 21:44:58 - r - INFO: - Episode: 44/2000, Reward: -29.962, Step: 300 +2023-04-14 21:45:04 - r - INFO: - Episode: 45/2000, Reward: -28.751, Step: 300 +2023-04-14 21:45:09 - r - INFO: - Episode: 46/2000, Reward: -28.505, Step: 300 +2023-04-14 21:45:10 - r - INFO: - Episode: 47/2000, Reward: -114.122, Step: 36 +2023-04-14 21:45:10 - r - INFO: - Episode: 48/2000, Reward: -114.088, Step: 41 +2023-04-14 21:45:19 - r - INFO: - Episode: 49/2000, Reward: -25.577, Step: 300 +2023-04-14 21:45:26 - r - INFO: - Episode: 50/2000, Reward: -23.633, Step: 300 +2023-04-14 21:45:27 - r - INFO: - Episode: 51/2000, Reward: -113.772, Step: 42 +2023-04-14 21:45:30 - r - INFO: - Episode: 52/2000, Reward: -125.206, Step: 116 +2023-04-14 21:45:35 - r - INFO: - Episode: 53/2000, Reward: -21.013, Step: 300 +2023-04-14 21:45:36 - r - INFO: - Episode: 54/2000, Reward: -117.215, Step: 48 +2023-04-14 21:45:37 - r - INFO: - Episode: 55/2000, Reward: -117.234, Step: 52 +2023-04-14 21:45:42 - r - INFO: - Episode: 56/2000, Reward: -29.327, Step: 300 +2023-04-14 21:45:43 - r - INFO: - Episode: 57/2000, Reward: -117.063, Step: 47 +2023-04-14 21:45:44 - r - INFO: - Episode: 58/2000, Reward: -113.742, Step: 39 +2023-04-14 21:45:45 - r - INFO: - Episode: 59/2000, Reward: -123.294, Step: 89 +2023-04-14 21:45:51 - r - INFO: - Episode: 60/2000, Reward: -23.202, Step: 300 +2023-04-14 21:45:58 - r - INFO: - Episode: 61/2000, Reward: -33.771, Step: 300 +2023-04-14 21:46:00 - r - INFO: - Episode: 62/2000, Reward: -108.161, Step: 64 +2023-04-14 21:46:06 - r - INFO: - Episode: 63/2000, Reward: -29.419, Step: 300 +2023-04-14 21:46:11 - r - INFO: - Episode: 64/2000, Reward: -101.811, Step: 122 +2023-04-14 21:46:14 - r - INFO: - Episode: 65/2000, Reward: -108.055, Step: 124 +2023-04-14 21:46:21 - r - INFO: - Episode: 66/2000, Reward: -26.285, Step: 300 +2023-04-14 21:46:22 - r - INFO: - Episode: 67/2000, Reward: -107.422, Step: 75 +2023-04-14 21:46:24 - r - INFO: - Episode: 68/2000, Reward: -111.807, Step: 129 +2023-04-14 21:46:27 - r - INFO: - Episode: 69/2000, Reward: -101.434, Step: 131 +2023-04-14 21:46:29 - r - INFO: - Episode: 70/2000, Reward: -112.775, Step: 128 +2023-04-14 21:46:31 - r - INFO: - Episode: 71/2000, Reward: -110.311, Step: 71 +2023-04-14 21:46:33 - r - INFO: - Episode: 72/2000, Reward: -99.563, Step: 114 +2023-04-14 21:46:36 - r - INFO: - Episode: 73/2000, Reward: -109.423, Step: 126 +2023-04-14 21:46:42 - r - INFO: - Episode: 74/2000, Reward: -31.711, Step: 300 +2023-04-14 21:46:48 - r - INFO: - Episode: 75/2000, Reward: -17.547, Step: 300 +2023-04-14 21:46:50 - r - INFO: - Episode: 76/2000, Reward: -119.460, Step: 64 +2023-04-14 21:46:55 - r - INFO: - Episode: 77/2000, Reward: -22.546, Step: 300 +2023-04-14 21:46:58 - r - INFO: - Episode: 78/2000, Reward: -106.006, Step: 163 +2023-04-14 21:47:05 - r - INFO: - Episode: 79/2000, Reward: -30.913, Step: 300 +2023-04-14 21:47:13 - r - INFO: - Episode: 80/2000, Reward: -20.373, Step: 300 +2023-04-14 21:47:23 - r - INFO: - Episode: 81/2000, Reward: -25.565, Step: 300 +2023-04-14 21:47:30 - r - INFO: - Episode: 82/2000, Reward: -28.357, Step: 300 +2023-04-14 21:47:37 - r - INFO: - Episode: 83/2000, Reward: -26.506, Step: 300 +2023-04-14 21:47:47 - r - INFO: - Episode: 84/2000, Reward: -27.160, Step: 300 +2023-04-14 21:47:54 - r - INFO: - Episode: 85/2000, Reward: -27.436, Step: 300 +2023-04-14 21:48:01 - r - INFO: - Episode: 86/2000, Reward: -28.297, Step: 300 +2023-04-14 21:48:08 - r - INFO: - Episode: 87/2000, Reward: -25.462, Step: 300 +2023-04-14 21:48:13 - r - INFO: - Episode: 88/2000, Reward: -28.484, Step: 300 +2023-04-14 21:48:20 - r - INFO: - Episode: 89/2000, Reward: -28.667, Step: 300 +2023-04-14 21:48:28 - r - INFO: - Episode: 90/2000, Reward: -27.735, Step: 300 +2023-04-14 21:48:37 - r - INFO: - Episode: 91/2000, Reward: -26.747, Step: 300 +2023-04-14 21:48:46 - r - INFO: - Episode: 92/2000, Reward: -18.252, Step: 300 +2023-04-14 21:48:48 - r - INFO: - Episode: 93/2000, Reward: -106.641, Step: 124 +2023-04-14 21:48:53 - r - INFO: - Episode: 94/2000, Reward: -22.984, Step: 300 +2023-04-14 21:48:54 - r - INFO: - Episode: 95/2000, Reward: -110.767, Step: 55 +2023-04-14 21:48:55 - r - INFO: - Episode: 96/2000, Reward: -122.389, Step: 58 +2023-04-14 21:48:58 - r - INFO: - Episode: 97/2000, Reward: -113.438, Step: 144 +2023-04-14 21:49:00 - r - INFO: - Episode: 98/2000, Reward: -117.842, Step: 118 +2023-04-14 21:49:02 - r - INFO: - Episode: 99/2000, Reward: -104.387, Step: 96 +2023-04-14 21:49:04 - r - INFO: - Episode: 100/2000, Reward: -129.942, Step: 119 +2023-04-14 21:49:08 - r - INFO: - Current episode 100 has the best eval reward: -23.255 +2023-04-14 21:49:09 - r - INFO: - Episode: 101/2000, Reward: -121.974, Step: 69 +2023-04-14 21:49:10 - r - INFO: - Episode: 102/2000, Reward: -119.786, Step: 78 +2023-04-14 21:49:16 - r - INFO: - Episode: 103/2000, Reward: -23.827, Step: 300 +2023-04-14 21:49:18 - r - INFO: - Episode: 104/2000, Reward: -127.492, Step: 99 +2023-04-14 21:49:23 - r - INFO: - Episode: 105/2000, Reward: -21.964, Step: 300 +2023-04-14 21:49:29 - r - INFO: - Episode: 106/2000, Reward: -25.955, Step: 300 +2023-04-14 21:49:35 - r - INFO: - Episode: 107/2000, Reward: -18.989, Step: 300 +2023-04-14 21:49:42 - r - INFO: - Episode: 108/2000, Reward: -24.117, Step: 300 +2023-04-14 21:49:43 - r - INFO: - Episode: 109/2000, Reward: -118.645, Step: 53 +2023-04-14 21:49:45 - r - INFO: - Episode: 110/2000, Reward: -121.282, Step: 72 +2023-04-14 21:49:51 - r - INFO: - Episode: 111/2000, Reward: -15.122, Step: 300 +2023-04-14 21:49:53 - r - INFO: - Episode: 112/2000, Reward: -128.649, Step: 101 +2023-04-14 21:49:59 - r - INFO: - Episode: 113/2000, Reward: -20.965, Step: 300 +2023-04-14 21:50:05 - r - INFO: - Episode: 114/2000, Reward: -19.886, Step: 300 +2023-04-14 21:50:06 - r - INFO: - Episode: 115/2000, Reward: -114.630, Step: 73 +2023-04-14 21:50:13 - r - INFO: - Episode: 116/2000, Reward: -28.694, Step: 300 +2023-04-14 21:50:19 - r - INFO: - Episode: 117/2000, Reward: -22.584, Step: 300 +2023-04-14 21:50:26 - r - INFO: - Episode: 118/2000, Reward: -25.956, Step: 300 +2023-04-14 21:50:32 - r - INFO: - Episode: 119/2000, Reward: -26.005, Step: 300 +2023-04-14 21:50:38 - r - INFO: - Episode: 120/2000, Reward: -20.102, Step: 300 +2023-04-14 21:50:43 - r - INFO: - Current episode 120 has the best eval reward: -20.415 +2023-04-14 21:50:49 - r - INFO: - Episode: 121/2000, Reward: -15.634, Step: 300 +2023-04-14 21:50:55 - r - INFO: - Episode: 122/2000, Reward: -18.242, Step: 300 +2023-04-14 21:51:01 - r - INFO: - Episode: 123/2000, Reward: -12.302, Step: 300 +2023-04-14 21:51:06 - r - INFO: - Episode: 124/2000, Reward: -115.354, Step: 281 +2023-04-14 21:51:12 - r - INFO: - Episode: 125/2000, Reward: -16.132, Step: 300 +2023-04-14 21:51:18 - r - INFO: - Episode: 126/2000, Reward: -12.939, Step: 300 +2023-04-14 21:51:26 - r - INFO: - Episode: 127/2000, Reward: -17.105, Step: 300 +2023-04-14 21:51:34 - r - INFO: - Episode: 128/2000, Reward: -22.028, Step: 300 +2023-04-14 21:51:35 - r - INFO: - Episode: 129/2000, Reward: -111.802, Step: 53 +2023-04-14 21:51:40 - r - INFO: - Episode: 130/2000, Reward: -12.520, Step: 300 +2023-04-14 21:51:48 - r - INFO: - Episode: 131/2000, Reward: -13.983, Step: 300 +2023-04-14 21:51:49 - r - INFO: - Episode: 132/2000, Reward: -112.494, Step: 48 +2023-04-14 21:52:03 - r - INFO: - Episode: 133/2000, Reward: -16.831, Step: 300 +2023-04-14 21:52:20 - r - INFO: - Episode: 134/2000, Reward: -17.937, Step: 300 +2023-04-14 21:52:33 - r - INFO: - Episode: 135/2000, Reward: -13.942, Step: 300 +2023-04-14 21:52:34 - r - INFO: - Episode: 136/2000, Reward: -109.476, Step: 40 +2023-04-14 21:52:44 - r - INFO: - Episode: 137/2000, Reward: -20.929, Step: 300 +2023-04-14 21:52:51 - r - INFO: - Episode: 138/2000, Reward: -104.726, Step: 126 +2023-04-14 21:52:58 - r - INFO: - Episode: 139/2000, Reward: -102.115, Step: 138 +2023-04-14 21:53:01 - r - INFO: - Episode: 140/2000, Reward: -107.004, Step: 67 +2023-04-14 21:53:13 - r - INFO: - Current episode 140 has the best eval reward: -12.785 +2023-04-14 21:53:24 - r - INFO: - Episode: 141/2000, Reward: -17.928, Step: 300 +2023-04-14 21:53:29 - r - INFO: - Episode: 142/2000, Reward: -111.173, Step: 147 +2023-04-14 21:53:30 - r - INFO: - Episode: 143/2000, Reward: -106.466, Step: 48 +2023-04-14 21:53:32 - r - INFO: - Episode: 144/2000, Reward: -116.868, Step: 51 +2023-04-14 21:53:45 - r - INFO: - Episode: 145/2000, Reward: -12.773, Step: 300 +2023-04-14 21:53:50 - r - INFO: - Episode: 146/2000, Reward: -106.853, Step: 162 +2023-04-14 21:53:59 - r - INFO: - Episode: 147/2000, Reward: -131.277, Step: 175 +2023-04-14 21:54:03 - r - INFO: - Episode: 148/2000, Reward: -106.481, Step: 80 +2023-04-14 21:54:15 - r - INFO: - Episode: 149/2000, Reward: -20.066, Step: 300 +2023-04-14 21:54:18 - r - INFO: - Episode: 150/2000, Reward: -104.687, Step: 151 +2023-04-14 21:54:20 - r - INFO: - Episode: 151/2000, Reward: -107.593, Step: 62 +2023-04-14 21:54:27 - r - INFO: - Episode: 152/2000, Reward: -19.555, Step: 300 +2023-04-14 21:54:34 - r - INFO: - Episode: 153/2000, Reward: -23.645, Step: 300 +2023-04-14 21:54:43 - r - INFO: - Episode: 154/2000, Reward: -12.603, Step: 300 +2023-04-14 21:54:52 - r - INFO: - Episode: 155/2000, Reward: -19.999, Step: 300 +2023-04-14 21:55:00 - r - INFO: - Episode: 156/2000, Reward: -13.257, Step: 300 +2023-04-14 21:55:07 - r - INFO: - Episode: 157/2000, Reward: -110.936, Step: 249 +2023-04-14 21:55:14 - r - INFO: - Episode: 158/2000, Reward: -16.879, Step: 300 +2023-04-14 21:55:22 - r - INFO: - Episode: 159/2000, Reward: -15.017, Step: 300 +2023-04-14 21:55:30 - r - INFO: - Episode: 160/2000, Reward: -25.627, Step: 300 +2023-04-14 21:55:46 - r - INFO: - Episode: 161/2000, Reward: -16.128, Step: 300 +2023-04-14 21:55:54 - r - INFO: - Episode: 162/2000, Reward: -14.815, Step: 300 +2023-04-14 21:55:59 - r - INFO: - Episode: 163/2000, Reward: -112.886, Step: 221 +2023-04-14 21:56:07 - r - INFO: - Episode: 164/2000, Reward: -19.762, Step: 300 +2023-04-14 21:56:15 - r - INFO: - Episode: 165/2000, Reward: -24.051, Step: 300 +2023-04-14 21:56:23 - r - INFO: - Episode: 166/2000, Reward: -17.533, Step: 300 +2023-04-14 21:56:30 - r - INFO: - Episode: 167/2000, Reward: -14.292, Step: 300 +2023-04-14 21:56:38 - r - INFO: - Episode: 168/2000, Reward: -13.066, Step: 300 +2023-04-14 21:56:46 - r - INFO: - Episode: 169/2000, Reward: -8.492, Step: 300 +2023-04-14 21:56:55 - r - INFO: - Episode: 170/2000, Reward: -11.667, Step: 300 +2023-04-14 21:57:03 - r - INFO: - Episode: 171/2000, Reward: -17.485, Step: 300 +2023-04-14 21:57:12 - r - INFO: - Episode: 172/2000, Reward: -18.128, Step: 300 +2023-04-14 21:57:21 - r - INFO: - Episode: 173/2000, Reward: -13.669, Step: 300 +2023-04-14 21:57:31 - r - INFO: - Episode: 174/2000, Reward: -8.722, Step: 300 +2023-04-14 21:57:38 - r - INFO: - Episode: 175/2000, Reward: -16.476, Step: 300 +2023-04-14 21:57:46 - r - INFO: - Episode: 176/2000, Reward: -6.403, Step: 300 +2023-04-14 21:57:52 - r - INFO: - Episode: 177/2000, Reward: -100.833, Step: 200 +2023-04-14 21:57:55 - r - INFO: - Episode: 178/2000, Reward: -132.506, Step: 104 +2023-04-14 21:57:58 - r - INFO: - Episode: 179/2000, Reward: -98.940, Step: 84 +2023-04-14 21:58:00 - r - INFO: - Episode: 180/2000, Reward: -101.245, Step: 69 +2023-04-14 21:58:03 - r - INFO: - Episode: 181/2000, Reward: -100.234, Step: 80 +2023-04-14 21:58:06 - r - INFO: - Episode: 182/2000, Reward: -107.108, Step: 117 +2023-04-14 21:58:08 - r - INFO: - Episode: 183/2000, Reward: -99.627, Step: 59 +2023-04-14 21:58:10 - r - INFO: - Episode: 184/2000, Reward: -99.011, Step: 80 +2023-04-14 21:58:13 - r - INFO: - Episode: 185/2000, Reward: -118.452, Step: 53 +2023-04-14 21:58:16 - r - INFO: - Episode: 186/2000, Reward: -136.912, Step: 123 +2023-04-14 21:58:20 - r - INFO: - Episode: 187/2000, Reward: -140.037, Step: 175 +2023-04-14 21:58:22 - r - INFO: - Episode: 188/2000, Reward: -101.002, Step: 94 +2023-04-14 21:58:24 - r - INFO: - Episode: 189/2000, Reward: -123.934, Step: 81 +2023-04-14 21:58:26 - r - INFO: - Episode: 190/2000, Reward: -108.270, Step: 72 +2023-04-14 21:58:28 - r - INFO: - Episode: 191/2000, Reward: -103.413, Step: 79 +2023-04-14 21:58:29 - r - INFO: - Episode: 192/2000, Reward: -102.722, Step: 61 +2023-04-14 21:58:33 - r - INFO: - Episode: 193/2000, Reward: -105.449, Step: 133 +2023-04-14 21:58:34 - r - INFO: - Episode: 194/2000, Reward: -103.778, Step: 52 +2023-04-14 21:58:38 - r - INFO: - Episode: 195/2000, Reward: -141.932, Step: 162 +2023-04-14 21:58:40 - r - INFO: - Episode: 196/2000, Reward: -101.548, Step: 86 +2023-04-14 21:58:42 - r - INFO: - Episode: 197/2000, Reward: -103.067, Step: 64 +2023-04-14 21:58:44 - r - INFO: - Episode: 198/2000, Reward: -101.696, Step: 84 +2023-04-14 21:58:47 - r - INFO: - Episode: 199/2000, Reward: -103.446, Step: 113 +2023-04-14 21:58:50 - r - INFO: - Episode: 200/2000, Reward: -124.965, Step: 106 +2023-04-14 21:58:53 - r - INFO: - Episode: 201/2000, Reward: -108.299, Step: 88 +2023-04-14 21:58:55 - r - INFO: - Episode: 202/2000, Reward: -126.026, Step: 90 +2023-04-14 21:58:57 - r - INFO: - Episode: 203/2000, Reward: -103.314, Step: 51 +2023-04-14 21:58:58 - r - INFO: - Episode: 204/2000, Reward: -124.904, Step: 76 +2023-04-14 21:59:01 - r - INFO: - Episode: 205/2000, Reward: -124.513, Step: 99 +2023-04-14 21:59:02 - r - INFO: - Episode: 206/2000, Reward: -125.793, Step: 59 +2023-04-14 21:59:04 - r - INFO: - Episode: 207/2000, Reward: -124.365, Step: 56 +2023-04-14 21:59:05 - r - INFO: - Episode: 208/2000, Reward: -120.568, Step: 74 +2023-04-14 21:59:07 - r - INFO: - Episode: 209/2000, Reward: -126.892, Step: 61 +2023-04-14 21:59:10 - r - INFO: - Episode: 210/2000, Reward: -135.588, Step: 128 +2023-04-14 21:59:12 - r - INFO: - Episode: 211/2000, Reward: -128.087, Step: 71 +2023-04-14 21:59:14 - r - INFO: - Episode: 212/2000, Reward: -130.150, Step: 105 +2023-04-14 21:59:16 - r - INFO: - Episode: 213/2000, Reward: -124.440, Step: 70 +2023-04-14 21:59:23 - r - INFO: - Episode: 214/2000, Reward: -27.892, Step: 300 +2023-04-14 21:59:30 - r - INFO: - Episode: 215/2000, Reward: -26.541, Step: 300 +2023-04-14 21:59:32 - r - INFO: - Episode: 216/2000, Reward: -123.667, Step: 86 +2023-04-14 21:59:39 - r - INFO: - Episode: 217/2000, Reward: -37.209, Step: 300 +2023-04-14 21:59:47 - r - INFO: - Episode: 218/2000, Reward: -14.759, Step: 300 +2023-04-14 21:59:55 - r - INFO: - Episode: 219/2000, Reward: -20.737, Step: 300 +2023-04-14 22:00:02 - r - INFO: - Episode: 220/2000, Reward: -29.673, Step: 300 +2023-04-14 22:00:16 - r - INFO: - Episode: 221/2000, Reward: -33.928, Step: 300 +2023-04-14 22:00:28 - r - INFO: - Episode: 222/2000, Reward: -23.773, Step: 300 +2023-04-14 22:00:35 - r - INFO: - Episode: 223/2000, Reward: -26.535, Step: 300 +2023-04-14 22:00:42 - r - INFO: - Episode: 224/2000, Reward: -26.489, Step: 300 +2023-04-14 22:00:44 - r - INFO: - Episode: 225/2000, Reward: -113.939, Step: 118 +2023-04-14 22:00:50 - r - INFO: - Episode: 226/2000, Reward: -114.985, Step: 261 +2023-04-14 22:00:55 - r - INFO: - Episode: 227/2000, Reward: -109.480, Step: 203 +2023-04-14 22:01:00 - r - INFO: - Episode: 228/2000, Reward: -104.623, Step: 160 +2023-04-14 22:01:02 - r - INFO: - Episode: 229/2000, Reward: -114.187, Step: 102 +2023-04-14 22:01:04 - r - INFO: - Episode: 230/2000, Reward: -104.979, Step: 123 +2023-04-14 22:01:08 - r - INFO: - Episode: 231/2000, Reward: -107.174, Step: 151 +2023-04-14 22:01:12 - r - INFO: - Episode: 232/2000, Reward: -110.844, Step: 115 +2023-04-14 22:01:15 - r - INFO: - Episode: 233/2000, Reward: -108.186, Step: 150 +2023-04-14 22:01:20 - r - INFO: - Episode: 234/2000, Reward: -114.269, Step: 128 +2023-04-14 22:01:22 - r - INFO: - Episode: 235/2000, Reward: -122.510, Step: 98 +2023-04-14 22:01:25 - r - INFO: - Episode: 236/2000, Reward: -112.232, Step: 114 +2023-04-14 22:01:30 - r - INFO: - Episode: 237/2000, Reward: -104.013, Step: 113 +2023-04-14 22:01:35 - r - INFO: - Episode: 238/2000, Reward: -113.432, Step: 123 +2023-04-14 22:01:40 - r - INFO: - Episode: 239/2000, Reward: -104.170, Step: 134 +2023-04-14 22:01:47 - r - INFO: - Episode: 240/2000, Reward: -108.710, Step: 147 +2023-04-14 22:01:59 - r - INFO: - Episode: 241/2000, Reward: -108.712, Step: 157 +2023-04-14 22:02:06 - r - INFO: - Episode: 242/2000, Reward: -105.903, Step: 176 +2023-04-14 22:02:16 - r - INFO: - Episode: 243/2000, Reward: -5.803, Step: 300 +2023-04-14 22:02:23 - r - INFO: - Episode: 244/2000, Reward: -118.068, Step: 288 +2023-04-14 22:02:25 - r - INFO: - Episode: 245/2000, Reward: -106.936, Step: 115 +2023-04-14 22:02:34 - r - INFO: - Episode: 246/2000, Reward: -108.424, Step: 296 +2023-04-14 22:02:44 - r - INFO: - Episode: 247/2000, Reward: -16.543, Step: 300 +2023-04-14 22:02:52 - r - INFO: - Episode: 248/2000, Reward: -16.007, Step: 300 +2023-04-14 22:03:03 - r - INFO: - Episode: 249/2000, Reward: -25.237, Step: 300 +2023-04-14 22:03:12 - r - INFO: - Episode: 250/2000, Reward: -26.478, Step: 300 +2023-04-14 22:03:23 - r - INFO: - Episode: 251/2000, Reward: -26.722, Step: 300 +2023-04-14 22:03:30 - r - INFO: - Episode: 252/2000, Reward: -19.996, Step: 300 +2023-04-14 22:03:38 - r - INFO: - Episode: 253/2000, Reward: -38.016, Step: 300 +2023-04-14 22:03:45 - r - INFO: - Episode: 254/2000, Reward: -141.589, Step: 273 +2023-04-14 22:03:50 - r - INFO: - Episode: 255/2000, Reward: -106.863, Step: 195 +2023-04-14 22:03:52 - r - INFO: - Episode: 256/2000, Reward: -100.874, Step: 77 +2023-04-14 22:03:54 - r - INFO: - Episode: 257/2000, Reward: -99.885, Step: 75 +2023-04-14 22:03:57 - r - INFO: - Episode: 258/2000, Reward: -100.532, Step: 102 +2023-04-14 22:04:06 - r - INFO: - Episode: 259/2000, Reward: -28.615, Step: 300 +2023-04-14 22:04:14 - r - INFO: - Episode: 260/2000, Reward: -9.074, Step: 300 +2023-04-14 22:04:20 - r - INFO: - Episode: 261/2000, Reward: -110.718, Step: 61 +2023-04-14 22:04:24 - r - INFO: - Episode: 262/2000, Reward: -101.965, Step: 144 +2023-04-14 22:04:27 - r - INFO: - Episode: 263/2000, Reward: -107.552, Step: 75 +2023-04-14 22:04:28 - r - INFO: - Episode: 264/2000, Reward: -99.243, Step: 64 +2023-04-14 22:04:30 - r - INFO: - Episode: 265/2000, Reward: -111.676, Step: 60 +2023-04-14 22:04:34 - r - INFO: - Episode: 266/2000, Reward: -95.963, Step: 183 +2023-04-14 22:04:42 - r - INFO: - Episode: 267/2000, Reward: -7.738, Step: 300 +2023-04-14 22:04:43 - r - INFO: - Episode: 268/2000, Reward: -111.072, Step: 58 +2023-04-14 22:04:45 - r - INFO: - Episode: 269/2000, Reward: -122.073, Step: 78 +2023-04-14 22:04:48 - r - INFO: - Episode: 270/2000, Reward: -128.796, Step: 118 +2023-04-14 22:04:56 - r - INFO: - Episode: 271/2000, Reward: -10.651, Step: 300 +2023-04-14 22:05:05 - r - INFO: - Episode: 272/2000, Reward: -18.478, Step: 300 +2023-04-14 22:05:07 - r - INFO: - Episode: 273/2000, Reward: -107.653, Step: 75 +2023-04-14 22:05:16 - r - INFO: - Episode: 274/2000, Reward: -13.718, Step: 300 +2023-04-14 22:05:17 - r - INFO: - Episode: 275/2000, Reward: -102.534, Step: 62 +2023-04-14 22:05:20 - r - INFO: - Episode: 276/2000, Reward: -104.850, Step: 94 +2023-04-14 22:05:22 - r - INFO: - Episode: 277/2000, Reward: -104.585, Step: 74 +2023-04-14 22:05:30 - r - INFO: - Episode: 278/2000, Reward: -20.520, Step: 300 +2023-04-14 22:05:39 - r - INFO: - Episode: 279/2000, Reward: -25.098, Step: 300 +2023-04-14 22:05:47 - r - INFO: - Episode: 280/2000, Reward: -31.482, Step: 300 +2023-04-14 22:06:02 - r - INFO: - Episode: 281/2000, Reward: -25.983, Step: 300 +2023-04-14 22:06:11 - r - INFO: - Episode: 282/2000, Reward: -18.615, Step: 300 +2023-04-14 22:06:19 - r - INFO: - Episode: 283/2000, Reward: -21.484, Step: 300 +2023-04-14 22:06:28 - r - INFO: - Episode: 284/2000, Reward: -18.972, Step: 300 +2023-04-14 22:06:31 - r - INFO: - Episode: 285/2000, Reward: -100.927, Step: 92 +2023-04-14 22:06:40 - r - INFO: - Episode: 286/2000, Reward: -26.809, Step: 300 +2023-04-14 22:06:49 - r - INFO: - Episode: 287/2000, Reward: -25.908, Step: 300 +2023-04-14 22:06:58 - r - INFO: - Episode: 288/2000, Reward: -24.729, Step: 300 +2023-04-14 22:07:07 - r - INFO: - Episode: 289/2000, Reward: -25.593, Step: 300 +2023-04-14 22:07:15 - r - INFO: - Episode: 290/2000, Reward: -21.192, Step: 300 +2023-04-14 22:07:17 - r - INFO: - Episode: 291/2000, Reward: -100.526, Step: 76 +2023-04-14 22:07:26 - r - INFO: - Episode: 292/2000, Reward: -29.540, Step: 300 +2023-04-14 22:07:34 - r - INFO: - Episode: 293/2000, Reward: -23.734, Step: 300 +2023-04-14 22:07:42 - r - INFO: - Episode: 294/2000, Reward: -38.095, Step: 300 +2023-04-14 22:07:43 - r - INFO: - Episode: 295/2000, Reward: -115.290, Step: 46 +2023-04-14 22:07:50 - r - INFO: - Episode: 296/2000, Reward: -23.177, Step: 300 +2023-04-14 22:07:53 - r - INFO: - Episode: 297/2000, Reward: -113.365, Step: 101 +2023-04-14 22:07:58 - r - INFO: - Episode: 298/2000, Reward: -118.093, Step: 196 +2023-04-14 22:08:03 - r - INFO: - Episode: 299/2000, Reward: -142.048, Step: 190 +2023-04-14 22:08:06 - r - INFO: - Episode: 300/2000, Reward: -112.208, Step: 122 +2023-04-14 22:08:15 - r - INFO: - Episode: 301/2000, Reward: -114.126, Step: 109 +2023-04-14 22:08:17 - r - INFO: - Episode: 302/2000, Reward: -112.334, Step: 111 +2023-04-14 22:08:25 - r - INFO: - Episode: 303/2000, Reward: -26.616, Step: 300 +2023-04-14 22:08:33 - r - INFO: - Episode: 304/2000, Reward: -6.406, Step: 300 +2023-04-14 22:08:37 - r - INFO: - Episode: 305/2000, Reward: -115.125, Step: 172 +2023-04-14 22:08:40 - r - INFO: - Episode: 306/2000, Reward: -133.177, Step: 130 +2023-04-14 22:08:48 - r - INFO: - Episode: 307/2000, Reward: -21.432, Step: 300 +2023-04-14 22:08:52 - r - INFO: - Episode: 308/2000, Reward: -106.487, Step: 161 +2023-04-14 22:08:57 - r - INFO: - Episode: 309/2000, Reward: -108.145, Step: 184 +2023-04-14 22:09:04 - r - INFO: - Episode: 310/2000, Reward: -13.788, Step: 300 +2023-04-14 22:09:08 - r - INFO: - Episode: 311/2000, Reward: -109.357, Step: 165 +2023-04-14 22:09:16 - r - INFO: - Episode: 312/2000, Reward: -14.325, Step: 300 +2023-04-14 22:09:20 - r - INFO: - Episode: 313/2000, Reward: -107.740, Step: 141 +2023-04-14 22:09:28 - r - INFO: - Episode: 314/2000, Reward: -19.355, Step: 300 +2023-04-14 22:09:35 - r - INFO: - Episode: 315/2000, Reward: -11.451, Step: 300 +2023-04-14 22:09:42 - r - INFO: - Episode: 316/2000, Reward: -17.650, Step: 300 +2023-04-14 22:09:47 - r - INFO: - Episode: 317/2000, Reward: -118.744, Step: 142 +2023-04-14 22:09:54 - r - INFO: - Episode: 318/2000, Reward: -41.644, Step: 300 +2023-04-14 22:10:01 - r - INFO: - Episode: 319/2000, Reward: -126.153, Step: 250 +2023-04-14 22:10:08 - r - INFO: - Episode: 320/2000, Reward: -105.399, Step: 242 +2023-04-14 22:10:19 - r - INFO: - Episode: 321/2000, Reward: -100.893, Step: 257 +2023-04-14 22:10:26 - r - INFO: - Episode: 322/2000, Reward: -111.042, Step: 252 +2023-04-14 22:10:30 - r - INFO: - Episode: 323/2000, Reward: -106.812, Step: 159 +2023-04-14 22:10:37 - r - INFO: - Episode: 324/2000, Reward: -7.826, Step: 300 +2023-04-14 22:10:45 - r - INFO: - Episode: 325/2000, Reward: -37.343, Step: 300 +2023-04-14 22:10:46 - r - INFO: - Episode: 326/2000, Reward: -121.399, Step: 67 +2023-04-14 22:10:50 - r - INFO: - Episode: 327/2000, Reward: -107.256, Step: 148 +2023-04-14 22:10:53 - r - INFO: - Episode: 328/2000, Reward: -108.018, Step: 135 +2023-04-14 22:10:56 - r - INFO: - Episode: 329/2000, Reward: -126.524, Step: 121 +2023-04-14 22:10:58 - r - INFO: - Episode: 330/2000, Reward: -117.286, Step: 84 +2023-04-14 22:11:01 - r - INFO: - Episode: 331/2000, Reward: -99.335, Step: 114 +2023-04-14 22:11:06 - r - INFO: - Episode: 332/2000, Reward: -128.469, Step: 167 +2023-04-14 22:11:09 - r - INFO: - Episode: 333/2000, Reward: -125.342, Step: 117 +2023-04-14 22:11:10 - r - INFO: - Episode: 334/2000, Reward: -101.527, Step: 63 +2023-04-14 22:11:17 - r - INFO: - Episode: 335/2000, Reward: -102.121, Step: 183 +2023-04-14 22:11:21 - r - INFO: - Episode: 336/2000, Reward: -106.517, Step: 130 +2023-04-14 22:11:24 - r - INFO: - Episode: 337/2000, Reward: -104.048, Step: 105 +2023-04-14 22:11:27 - r - INFO: - Episode: 338/2000, Reward: -105.707, Step: 130 +2023-04-14 22:11:30 - r - INFO: - Episode: 339/2000, Reward: -103.940, Step: 94 +2023-04-14 22:11:36 - r - INFO: - Episode: 340/2000, Reward: -100.597, Step: 235 +2023-04-14 22:11:44 - r - INFO: - Episode: 341/2000, Reward: -111.749, Step: 185 +2023-04-14 22:11:47 - r - INFO: - Episode: 342/2000, Reward: -102.824, Step: 139 +2023-04-14 22:11:59 - r - INFO: - Episode: 343/2000, Reward: 4.364, Step: 300 +2023-04-14 22:12:01 - r - INFO: - Episode: 344/2000, Reward: -105.986, Step: 92 +2023-04-14 22:12:11 - r - INFO: - Episode: 345/2000, Reward: -0.487, Step: 300 +2023-04-14 22:12:23 - r - INFO: - Episode: 346/2000, Reward: -23.065, Step: 300 +2023-04-14 22:12:31 - r - INFO: - Episode: 347/2000, Reward: -28.013, Step: 300 +2023-04-14 22:12:35 - r - INFO: - Episode: 348/2000, Reward: -103.631, Step: 148 +2023-04-14 22:12:39 - r - INFO: - Episode: 349/2000, Reward: -93.968, Step: 137 +2023-04-14 22:12:45 - r - INFO: - Episode: 350/2000, Reward: -1.300, Step: 300 +2023-04-14 22:12:48 - r - INFO: - Episode: 351/2000, Reward: -105.389, Step: 139 +2023-04-14 22:12:51 - r - INFO: - Episode: 352/2000, Reward: -108.773, Step: 147 +2023-04-14 22:12:58 - r - INFO: - Episode: 353/2000, Reward: -0.373, Step: 300 +2023-04-14 22:13:06 - r - INFO: - Episode: 354/2000, Reward: 8.801, Step: 300 +2023-04-14 22:13:12 - r - INFO: - Episode: 355/2000, Reward: -95.032, Step: 241 +2023-04-14 22:13:19 - r - INFO: - Episode: 356/2000, Reward: -2.565, Step: 300 +2023-04-14 22:13:25 - r - INFO: - Episode: 357/2000, Reward: 8.099, Step: 300 +2023-04-14 22:15:23 - r - INFO: - Episode: 358/2000, Reward: -107.432, Step: 120 +2023-04-14 22:16:29 - r - INFO: - Episode: 359/2000, Reward: -101.845, Step: 149 +2023-04-14 22:16:35 - r - INFO: - Episode: 360/2000, Reward: 18.248, Step: 300 +2023-04-14 22:16:41 - r - INFO: - Current episode 360 has the best eval reward: 9.623 +2023-04-14 22:17:10 - r - INFO: - Episode: 361/2000, Reward: -98.748, Step: 99 +2023-04-14 22:17:15 - r - INFO: - Episode: 362/2000, Reward: -95.939, Step: 196 +2023-04-14 22:17:17 - r - INFO: - Episode: 363/2000, Reward: -106.797, Step: 91 +2023-04-14 22:17:24 - r - INFO: - Episode: 364/2000, Reward: -89.971, Step: 300 +2023-04-14 22:17:27 - r - INFO: - Episode: 365/2000, Reward: -100.816, Step: 118 +2023-04-14 22:17:34 - r - INFO: - Episode: 366/2000, Reward: -24.075, Step: 300 +2023-04-14 22:17:40 - r - INFO: - Episode: 367/2000, Reward: -106.835, Step: 83 +2023-04-14 22:18:18 - r - INFO: - Episode: 368/2000, Reward: 8.414, Step: 300 +2023-04-14 22:18:21 - r - INFO: - Episode: 369/2000, Reward: -103.143, Step: 101 +2023-04-14 22:18:27 - r - INFO: - Episode: 370/2000, Reward: -87.694, Step: 139 +2023-04-14 22:18:52 - r - INFO: - Episode: 371/2000, Reward: -96.281, Step: 190 +2023-04-14 22:19:03 - r - INFO: - Episode: 372/2000, Reward: -6.461, Step: 300 +2023-04-14 22:19:06 - r - INFO: - Episode: 373/2000, Reward: -112.144, Step: 115 +2023-04-14 22:19:11 - r - INFO: - Episode: 374/2000, Reward: -91.016, Step: 182 +2023-04-14 22:19:13 - r - INFO: - Episode: 375/2000, Reward: -107.397, Step: 60 +2023-04-14 22:19:17 - r - INFO: - Episode: 376/2000, Reward: -95.471, Step: 189 +2023-04-14 22:19:19 - r - INFO: - Episode: 377/2000, Reward: -103.090, Step: 67 +2023-04-14 22:19:21 - r - INFO: - Episode: 378/2000, Reward: -107.553, Step: 119 +2023-04-14 22:19:26 - r - INFO: - Episode: 379/2000, Reward: -88.947, Step: 196 +2023-04-14 22:19:30 - r - INFO: - Episode: 380/2000, Reward: -99.564, Step: 193 +2023-04-14 22:19:37 - r - INFO: - Episode: 381/2000, Reward: -107.201, Step: 99 +2023-04-14 22:19:44 - r - INFO: - Episode: 382/2000, Reward: -11.114, Step: 300 +2023-04-14 22:19:49 - r - INFO: - Episode: 383/2000, Reward: -102.629, Step: 218 +2023-04-14 22:19:56 - r - INFO: - Episode: 384/2000, Reward: -15.805, Step: 300 +2023-04-14 22:19:59 - r - INFO: - Episode: 385/2000, Reward: -122.989, Step: 72 +2023-04-14 22:20:05 - r - INFO: - Episode: 386/2000, Reward: -99.103, Step: 146 +2023-04-14 22:20:09 - r - INFO: - Episode: 387/2000, Reward: -97.800, Step: 99 +2023-04-14 22:20:12 - r - INFO: - Episode: 388/2000, Reward: -122.384, Step: 78 +2023-04-14 22:20:14 - r - INFO: - Episode: 389/2000, Reward: -127.850, Step: 83 +2023-04-14 22:20:16 - r - INFO: - Episode: 390/2000, Reward: -123.968, Step: 73 +2023-04-14 22:20:20 - r - INFO: - Episode: 391/2000, Reward: -96.547, Step: 166 +2023-04-14 22:20:23 - r - INFO: - Episode: 392/2000, Reward: -100.428, Step: 106 +2023-04-14 22:20:39 - r - INFO: - Episode: 393/2000, Reward: -88.349, Step: 224 +2023-04-14 22:20:42 - r - INFO: - Episode: 394/2000, Reward: -96.914, Step: 104 +2023-04-14 22:20:47 - r - INFO: - Episode: 395/2000, Reward: -97.018, Step: 184 +2023-04-14 22:20:54 - r - INFO: - Episode: 396/2000, Reward: -22.148, Step: 300 +2023-04-14 22:20:55 - r - INFO: - Episode: 397/2000, Reward: -101.655, Step: 51 +2023-04-14 22:21:03 - r - INFO: - Episode: 398/2000, Reward: -30.572, Step: 300 +2023-04-14 22:21:09 - r - INFO: - Episode: 399/2000, Reward: -89.325, Step: 230 +2023-04-14 22:21:17 - r - INFO: - Episode: 400/2000, Reward: -24.982, Step: 300 +2023-04-14 22:21:26 - r - INFO: - Episode: 401/2000, Reward: -35.876, Step: 300 +2023-04-14 22:21:27 - r - INFO: - Episode: 402/2000, Reward: -103.959, Step: 56 +2023-04-14 22:21:35 - r - INFO: - Episode: 403/2000, Reward: -20.680, Step: 300 +2023-04-14 22:21:45 - r - INFO: - Episode: 404/2000, Reward: -29.679, Step: 300 +2023-04-14 22:21:53 - r - INFO: - Episode: 405/2000, Reward: -14.902, Step: 300 +2023-04-14 22:22:01 - r - INFO: - Episode: 406/2000, Reward: -16.149, Step: 300 +2023-04-14 22:22:08 - r - INFO: - Episode: 407/2000, Reward: -29.861, Step: 300 +2023-04-14 22:22:15 - r - INFO: - Episode: 408/2000, Reward: 2.079, Step: 300 +2023-04-14 22:22:21 - r - INFO: - Episode: 409/2000, Reward: -90.663, Step: 233 +2023-04-14 22:22:28 - r - INFO: - Episode: 410/2000, Reward: -31.267, Step: 300 +2023-04-14 22:22:35 - r - INFO: - Episode: 411/2000, Reward: 7.243, Step: 300 +2023-04-14 22:22:38 - r - INFO: - Episode: 412/2000, Reward: -113.023, Step: 114 +2023-04-14 22:22:46 - r - INFO: - Episode: 413/2000, Reward: -33.594, Step: 300 +2023-04-14 22:22:55 - r - INFO: - Episode: 414/2000, Reward: -32.560, Step: 300 +2023-04-14 22:22:57 - r - INFO: - Episode: 415/2000, Reward: -116.987, Step: 71 +2023-04-14 22:23:06 - r - INFO: - Episode: 416/2000, Reward: -27.541, Step: 300 +2023-04-14 22:23:13 - r - INFO: - Episode: 417/2000, Reward: 8.136, Step: 300 +2023-04-14 22:23:19 - r - INFO: - Episode: 418/2000, Reward: -100.190, Step: 239 +2023-04-14 22:23:31 - r - INFO: - Episode: 419/2000, Reward: 5.091, Step: 300 +2023-04-14 22:23:42 - r - INFO: - Episode: 420/2000, Reward: -24.965, Step: 300 +2023-04-14 22:23:56 - r - INFO: - Episode: 421/2000, Reward: -30.805, Step: 300 +2023-04-14 22:24:00 - r - INFO: - Episode: 422/2000, Reward: -129.883, Step: 131 +2023-04-14 22:24:08 - r - INFO: - Episode: 423/2000, Reward: -17.671, Step: 300 +2023-04-14 22:24:16 - r - INFO: - Episode: 424/2000, Reward: -32.110, Step: 300 +2023-04-14 22:24:24 - r - INFO: - Episode: 425/2000, Reward: 9.918, Step: 300 +2023-04-14 22:24:31 - r - INFO: - Episode: 426/2000, Reward: -2.897, Step: 300 +2023-04-14 22:24:38 - r - INFO: - Episode: 427/2000, Reward: -38.354, Step: 300 +2023-04-14 22:24:45 - r - INFO: - Episode: 428/2000, Reward: -88.929, Step: 245 +2023-04-14 22:24:52 - r - INFO: - Episode: 429/2000, Reward: -84.612, Step: 254 +2023-04-14 22:24:59 - r - INFO: - Episode: 430/2000, Reward: 21.768, Step: 300 +2023-04-14 22:25:06 - r - INFO: - Episode: 431/2000, Reward: -104.165, Step: 259 +2023-04-14 22:25:17 - r - INFO: - Episode: 432/2000, Reward: 20.208, Step: 300 +2023-04-14 22:25:27 - r - INFO: - Episode: 433/2000, Reward: -28.827, Step: 300 +2023-04-14 22:25:31 - r - INFO: - Episode: 434/2000, Reward: -113.642, Step: 126 +2023-04-14 22:25:32 - r - INFO: - Episode: 435/2000, Reward: -107.973, Step: 47 +2023-04-14 22:25:35 - r - INFO: - Episode: 436/2000, Reward: -95.109, Step: 123 +2023-04-14 22:25:39 - r - INFO: - Episode: 437/2000, Reward: -119.975, Step: 137 +2023-04-14 22:25:45 - r - INFO: - Episode: 438/2000, Reward: -77.773, Step: 188 +2023-04-14 22:25:54 - r - INFO: - Episode: 439/2000, Reward: -22.616, Step: 300 +2023-04-14 22:25:59 - r - INFO: - Episode: 440/2000, Reward: -123.255, Step: 203 +2023-04-14 22:26:04 - r - INFO: - Current episode 440 has the best eval reward: 22.200 +2023-04-14 22:26:12 - r - INFO: - Episode: 441/2000, Reward: 9.308, Step: 300 +2023-04-14 22:26:15 - r - INFO: - Episode: 442/2000, Reward: -105.124, Step: 130 +2023-04-14 22:26:19 - r - INFO: - Episode: 443/2000, Reward: -116.231, Step: 152 +2023-04-14 22:26:22 - r - INFO: - Episode: 444/2000, Reward: -83.744, Step: 159 +2023-04-14 22:26:30 - r - INFO: - Episode: 445/2000, Reward: -15.606, Step: 300 +2023-04-14 22:26:33 - r - INFO: - Episode: 446/2000, Reward: -99.108, Step: 144 +2023-04-14 22:26:42 - r - INFO: - Episode: 447/2000, Reward: 2.313, Step: 300 +2023-04-14 22:26:49 - r - INFO: - Episode: 448/2000, Reward: 27.423, Step: 300 +2023-04-14 22:26:55 - r - INFO: - Episode: 449/2000, Reward: -76.635, Step: 201 +2023-04-14 22:27:04 - r - INFO: - Episode: 450/2000, Reward: -23.147, Step: 300 +2023-04-14 22:27:14 - r - INFO: - Episode: 451/2000, Reward: 15.336, Step: 300 +2023-04-14 22:27:23 - r - INFO: - Episode: 452/2000, Reward: 16.753, Step: 300 +2023-04-14 22:27:31 - r - INFO: - Episode: 453/2000, Reward: 13.257, Step: 300 +2023-04-14 22:27:38 - r - INFO: - Episode: 454/2000, Reward: -96.749, Step: 181 +2023-04-14 22:27:40 - r - INFO: - Episode: 455/2000, Reward: -105.633, Step: 64 +2023-04-14 22:27:42 - r - INFO: - Episode: 456/2000, Reward: -102.030, Step: 82 +2023-04-14 22:27:49 - r - INFO: - Episode: 457/2000, Reward: 4.277, Step: 300 +2023-04-14 22:27:57 - r - INFO: - Episode: 458/2000, Reward: 1.738, Step: 300 +2023-04-14 22:28:06 - r - INFO: - Episode: 459/2000, Reward: -5.712, Step: 300 +2023-04-14 22:28:14 - r - INFO: - Episode: 460/2000, Reward: -18.370, Step: 300 +2023-04-14 22:28:27 - r - INFO: - Episode: 461/2000, Reward: -22.801, Step: 300 +2023-04-14 22:28:34 - r - INFO: - Episode: 462/2000, Reward: 33.394, Step: 300 +2023-04-14 22:28:43 - r - INFO: - Episode: 463/2000, Reward: -13.769, Step: 300 +2023-04-14 22:28:47 - r - INFO: - Episode: 464/2000, Reward: -100.123, Step: 133 +2023-04-14 22:28:50 - r - INFO: - Episode: 465/2000, Reward: -89.480, Step: 116 +2023-04-14 22:28:53 - r - INFO: - Episode: 466/2000, Reward: -89.064, Step: 127 +2023-04-14 22:28:55 - r - INFO: - Episode: 467/2000, Reward: -95.629, Step: 108 +2023-04-14 22:28:59 - r - INFO: - Episode: 468/2000, Reward: -99.953, Step: 173 +2023-04-14 22:29:02 - r - INFO: - Episode: 469/2000, Reward: -95.822, Step: 120 +2023-04-14 22:29:05 - r - INFO: - Episode: 470/2000, Reward: -113.192, Step: 167 +2023-04-14 22:29:09 - r - INFO: - Episode: 471/2000, Reward: -98.207, Step: 141 +2023-04-14 22:29:16 - r - INFO: - Episode: 472/2000, Reward: -113.958, Step: 295 +2023-04-14 22:29:20 - r - INFO: - Episode: 473/2000, Reward: -88.162, Step: 151 +2023-04-14 22:29:27 - r - INFO: - Episode: 474/2000, Reward: -16.505, Step: 300 +2023-04-14 22:29:30 - r - INFO: - Episode: 475/2000, Reward: -93.540, Step: 142 +2023-04-14 22:29:32 - r - INFO: - Episode: 476/2000, Reward: -111.651, Step: 85 +2023-04-14 22:29:35 - r - INFO: - Episode: 477/2000, Reward: -93.043, Step: 120 +2023-04-14 22:29:38 - r - INFO: - Episode: 478/2000, Reward: -96.347, Step: 109 +2023-04-14 22:29:42 - r - INFO: - Episode: 479/2000, Reward: -78.850, Step: 187 +2023-04-14 22:29:43 - r - INFO: - Episode: 480/2000, Reward: -109.342, Step: 42 +2023-04-14 22:29:53 - r - INFO: - Episode: 481/2000, Reward: -63.813, Step: 285 +2023-04-14 22:29:55 - r - INFO: - Episode: 482/2000, Reward: -124.407, Step: 68 +2023-04-14 22:29:58 - r - INFO: - Episode: 483/2000, Reward: -127.752, Step: 114 +2023-04-14 22:30:01 - r - INFO: - Episode: 484/2000, Reward: -87.256, Step: 144 +2023-04-14 22:30:04 - r - INFO: - Episode: 485/2000, Reward: -76.895, Step: 144 +2023-04-14 22:30:07 - r - INFO: - Episode: 486/2000, Reward: -130.569, Step: 110 +2023-04-14 22:30:10 - r - INFO: - Episode: 487/2000, Reward: -128.633, Step: 113 +2023-04-14 22:30:11 - r - INFO: - Episode: 488/2000, Reward: -105.620, Step: 52 +2023-04-14 22:30:14 - r - INFO: - Episode: 489/2000, Reward: -95.280, Step: 141 +2023-04-14 22:30:17 - r - INFO: - Episode: 490/2000, Reward: -122.461, Step: 138 +2023-04-14 22:30:19 - r - INFO: - Episode: 491/2000, Reward: -123.629, Step: 77 +2023-04-14 22:30:22 - r - INFO: - Episode: 492/2000, Reward: -126.030, Step: 99 +2023-04-14 22:30:25 - r - INFO: - Episode: 493/2000, Reward: -116.169, Step: 124 +2023-04-14 22:30:31 - r - INFO: - Episode: 494/2000, Reward: -138.274, Step: 265 +2023-04-14 22:30:38 - r - INFO: - Episode: 495/2000, Reward: -7.869, Step: 300 +2023-04-14 22:30:45 - r - INFO: - Episode: 496/2000, Reward: 4.876, Step: 300 +2023-04-14 22:30:52 - r - INFO: - Episode: 497/2000, Reward: -7.278, Step: 300 +2023-04-14 22:30:53 - r - INFO: - Episode: 498/2000, Reward: -117.146, Step: 62 +2023-04-14 22:31:01 - r - INFO: - Episode: 499/2000, Reward: 15.530, Step: 300 +2023-04-14 22:31:08 - r - INFO: - Episode: 500/2000, Reward: 23.718, Step: 300 +2023-04-14 22:31:13 - r - INFO: - Current episode 500 has the best eval reward: 29.794 +2023-04-14 22:31:20 - r - INFO: - Episode: 501/2000, Reward: 22.247, Step: 300 +2023-04-14 22:31:27 - r - INFO: - Episode: 502/2000, Reward: 31.020, Step: 300 +2023-04-14 22:31:33 - r - INFO: - Episode: 503/2000, Reward: -84.095, Step: 254 +2023-04-14 22:31:36 - r - INFO: - Episode: 504/2000, Reward: -103.550, Step: 80 +2023-04-14 22:31:46 - r - INFO: - Episode: 505/2000, Reward: 13.073, Step: 300 +2023-04-14 22:31:54 - r - INFO: - Episode: 506/2000, Reward: 44.738, Step: 300 +2023-04-14 22:32:01 - r - INFO: - Episode: 507/2000, Reward: -72.733, Step: 253 +2023-04-14 22:32:04 - r - INFO: - Episode: 508/2000, Reward: -124.510, Step: 85 +2023-04-14 22:32:12 - r - INFO: - Episode: 509/2000, Reward: -69.958, Step: 253 +2023-04-14 22:32:20 - r - INFO: - Episode: 510/2000, Reward: 7.528, Step: 300 +2023-04-14 22:32:28 - r - INFO: - Episode: 511/2000, Reward: -18.036, Step: 300 +2023-04-14 22:32:35 - r - INFO: - Episode: 512/2000, Reward: -0.229, Step: 300 +2023-04-14 22:32:38 - r - INFO: - Episode: 513/2000, Reward: -128.102, Step: 125 +2023-04-14 22:32:47 - r - INFO: - Episode: 514/2000, Reward: 27.667, Step: 300 +2023-04-14 22:32:49 - r - INFO: - Episode: 515/2000, Reward: -117.388, Step: 85 +2023-04-14 22:32:59 - r - INFO: - Episode: 516/2000, Reward: 24.092, Step: 300 +2023-04-14 22:33:04 - r - INFO: - Episode: 517/2000, Reward: -101.990, Step: 190 +2023-04-14 22:33:11 - r - INFO: - Episode: 518/2000, Reward: 1.162, Step: 300 +2023-04-14 22:33:19 - r - INFO: - Episode: 519/2000, Reward: -13.124, Step: 300 +2023-04-14 22:33:28 - r - INFO: - Episode: 520/2000, Reward: -131.202, Step: 299 +2023-04-14 22:33:39 - r - INFO: - Episode: 521/2000, Reward: -83.270, Step: 197 +2023-04-14 22:33:43 - r - INFO: - Episode: 522/2000, Reward: -91.839, Step: 127 +2023-04-14 22:33:51 - r - INFO: - Episode: 523/2000, Reward: -6.724, Step: 300 +2023-04-14 22:33:59 - r - INFO: - Episode: 524/2000, Reward: 7.276, Step: 300 +2023-04-14 22:34:02 - r - INFO: - Episode: 525/2000, Reward: -99.513, Step: 132 +2023-04-14 22:34:05 - r - INFO: - Episode: 526/2000, Reward: -100.401, Step: 111 +2023-04-14 22:34:07 - r - INFO: - Episode: 527/2000, Reward: -103.588, Step: 69 +2023-04-14 22:34:09 - r - INFO: - Episode: 528/2000, Reward: -103.973, Step: 84 +2023-04-14 22:34:16 - r - INFO: - Episode: 529/2000, Reward: -80.158, Step: 213 +2023-04-14 22:34:18 - r - INFO: - Episode: 530/2000, Reward: -107.177, Step: 70 +2023-04-14 22:34:19 - r - INFO: - Episode: 531/2000, Reward: -107.575, Step: 46 +2023-04-14 22:34:25 - r - INFO: - Episode: 532/2000, Reward: -97.890, Step: 239 +2023-04-14 22:34:30 - r - INFO: - Episode: 533/2000, Reward: -81.038, Step: 177 +2023-04-14 22:34:32 - r - INFO: - Episode: 534/2000, Reward: -99.902, Step: 61 +2023-04-14 22:34:35 - r - INFO: - Episode: 535/2000, Reward: -93.585, Step: 127 +2023-04-14 22:34:44 - r - INFO: - Episode: 536/2000, Reward: 21.766, Step: 300 +2023-04-14 22:34:52 - r - INFO: - Episode: 537/2000, Reward: 30.344, Step: 300 +2023-04-14 22:35:00 - r - INFO: - Episode: 538/2000, Reward: 30.996, Step: 300 +2023-04-14 22:35:06 - r - INFO: - Episode: 539/2000, Reward: -85.714, Step: 222 +2023-04-14 22:35:07 - r - INFO: - Episode: 540/2000, Reward: -110.874, Step: 43 +2023-04-14 22:35:20 - r - INFO: - Episode: 541/2000, Reward: 39.435, Step: 300 +2023-04-14 22:35:24 - r - INFO: - Episode: 542/2000, Reward: -95.660, Step: 169 +2023-04-14 22:35:27 - r - INFO: - Episode: 543/2000, Reward: -108.559, Step: 99 +2023-04-14 22:35:34 - r - INFO: - Episode: 544/2000, Reward: 15.237, Step: 300 +2023-04-14 22:35:39 - r - INFO: - Episode: 545/2000, Reward: -107.618, Step: 194 +2023-04-14 22:35:43 - r - INFO: - Episode: 546/2000, Reward: -78.937, Step: 176 +2023-04-14 22:35:50 - r - INFO: - Episode: 547/2000, Reward: 36.827, Step: 300 +2023-04-14 22:35:53 - r - INFO: - Episode: 548/2000, Reward: -82.974, Step: 133 +2023-04-14 22:35:59 - r - INFO: - Episode: 549/2000, Reward: -89.010, Step: 232 +2023-04-14 22:36:08 - r - INFO: - Episode: 550/2000, Reward: 7.724, Step: 300 +2023-04-14 22:36:15 - r - INFO: - Episode: 551/2000, Reward: 8.609, Step: 300 +2023-04-14 22:36:22 - r - INFO: - Episode: 552/2000, Reward: 8.295, Step: 300 +2023-04-14 22:36:29 - r - INFO: - Episode: 553/2000, Reward: 31.499, Step: 300 +2023-04-14 22:36:36 - r - INFO: - Episode: 554/2000, Reward: 1.121, Step: 300 +2023-04-14 22:36:44 - r - INFO: - Episode: 555/2000, Reward: 33.334, Step: 300 +2023-04-14 22:36:51 - r - INFO: - Episode: 556/2000, Reward: -82.817, Step: 296 +2023-04-14 22:36:57 - r - INFO: - Episode: 557/2000, Reward: 29.726, Step: 300 +2023-04-14 22:37:01 - r - INFO: - Episode: 558/2000, Reward: -86.928, Step: 183 +2023-04-14 22:37:06 - r - INFO: - Episode: 559/2000, Reward: -64.629, Step: 221 +2023-04-14 22:37:10 - r - INFO: - Episode: 560/2000, Reward: -83.511, Step: 194 +2023-04-14 22:37:22 - r - INFO: - Episode: 561/2000, Reward: 19.644, Step: 300 +2023-04-14 22:37:24 - r - INFO: - Episode: 562/2000, Reward: -104.485, Step: 92 +2023-04-14 22:37:28 - r - INFO: - Episode: 563/2000, Reward: -108.059, Step: 177 +2023-04-14 22:37:33 - r - INFO: - Episode: 564/2000, Reward: -64.374, Step: 265 +2023-04-14 22:37:37 - r - INFO: - Episode: 565/2000, Reward: -84.667, Step: 174 +2023-04-14 22:37:41 - r - INFO: - Episode: 566/2000, Reward: -130.736, Step: 149 +2023-04-14 22:37:45 - r - INFO: - Episode: 567/2000, Reward: -90.957, Step: 161 +2023-04-14 22:37:52 - r - INFO: - Episode: 568/2000, Reward: -3.764, Step: 300 +2023-04-14 22:37:59 - r - INFO: - Episode: 569/2000, Reward: 16.058, Step: 300 +2023-04-14 22:38:02 - r - INFO: - Episode: 570/2000, Reward: -96.846, Step: 120 +2023-04-14 22:38:08 - r - INFO: - Episode: 571/2000, Reward: -81.247, Step: 250 +2023-04-14 22:38:12 - r - INFO: - Episode: 572/2000, Reward: -97.397, Step: 174 +2023-04-14 22:38:14 - r - INFO: - Episode: 573/2000, Reward: -124.635, Step: 110 +2023-04-14 22:38:16 - r - INFO: - Episode: 574/2000, Reward: -124.741, Step: 95 +2023-04-14 22:38:21 - r - INFO: - Episode: 575/2000, Reward: -114.464, Step: 167 +2023-04-14 22:38:24 - r - INFO: - Episode: 576/2000, Reward: -120.341, Step: 80 +2023-04-14 22:38:27 - r - INFO: - Episode: 577/2000, Reward: -120.213, Step: 96 +2023-04-14 22:38:31 - r - INFO: - Episode: 578/2000, Reward: -88.383, Step: 128 +2023-04-14 22:38:33 - r - INFO: - Episode: 579/2000, Reward: -83.491, Step: 123 +2023-04-14 22:38:35 - r - INFO: - Episode: 580/2000, Reward: -104.120, Step: 73 +2023-04-14 22:38:43 - r - INFO: - Episode: 581/2000, Reward: 12.368, Step: 300 +2023-04-14 22:38:47 - r - INFO: - Episode: 582/2000, Reward: -87.133, Step: 157 +2023-04-14 22:38:48 - r - INFO: - Episode: 583/2000, Reward: -101.335, Step: 70 +2023-04-14 22:38:51 - r - INFO: - Episode: 584/2000, Reward: -96.509, Step: 74 +2023-04-14 22:38:52 - r - INFO: - Episode: 585/2000, Reward: -98.575, Step: 68 +2023-04-14 22:38:54 - r - INFO: - Episode: 586/2000, Reward: -98.553, Step: 74 +2023-04-14 22:38:55 - r - INFO: - Episode: 587/2000, Reward: -97.977, Step: 64 +2023-04-14 22:38:57 - r - INFO: - Episode: 588/2000, Reward: -123.441, Step: 75 +2023-04-14 22:38:59 - r - INFO: - Episode: 589/2000, Reward: -100.699, Step: 62 +2023-04-14 22:39:00 - r - INFO: - Episode: 590/2000, Reward: -108.131, Step: 82 +2023-04-14 22:39:02 - r - INFO: - Episode: 591/2000, Reward: -96.457, Step: 74 +2023-04-14 22:39:03 - r - INFO: - Episode: 592/2000, Reward: -116.816, Step: 61 +2023-04-14 22:39:05 - r - INFO: - Episode: 593/2000, Reward: -112.311, Step: 87 +2023-04-14 22:39:07 - r - INFO: - Episode: 594/2000, Reward: -119.607, Step: 69 +2023-04-14 22:39:14 - r - INFO: - Episode: 595/2000, Reward: 3.532, Step: 300 +2023-04-14 22:39:16 - r - INFO: - Episode: 596/2000, Reward: -96.115, Step: 116 +2023-04-14 22:39:18 - r - INFO: - Episode: 597/2000, Reward: -114.402, Step: 85 +2023-04-14 22:39:21 - r - INFO: - Episode: 598/2000, Reward: -107.910, Step: 97 +2023-04-14 22:39:28 - r - INFO: - Episode: 599/2000, Reward: -8.219, Step: 300 +2023-04-14 22:39:34 - r - INFO: - Episode: 600/2000, Reward: -115.683, Step: 293 +2023-04-14 22:39:44 - r - INFO: - Episode: 601/2000, Reward: -19.558, Step: 300 +2023-04-14 22:39:47 - r - INFO: - Episode: 602/2000, Reward: -100.150, Step: 105 +2023-04-14 22:39:54 - r - INFO: - Episode: 603/2000, Reward: 33.392, Step: 300 +2023-04-14 22:39:56 - r - INFO: - Episode: 604/2000, Reward: -107.879, Step: 102 +2023-04-14 22:40:00 - r - INFO: - Episode: 605/2000, Reward: -133.540, Step: 174 +2023-04-14 22:40:02 - r - INFO: - Episode: 606/2000, Reward: -121.886, Step: 98 +2023-04-14 22:40:09 - r - INFO: - Episode: 607/2000, Reward: -99.980, Step: 293 +2023-04-14 22:40:11 - r - INFO: - Episode: 608/2000, Reward: -101.122, Step: 98 +2023-04-14 22:40:16 - r - INFO: - Episode: 609/2000, Reward: -125.611, Step: 209 +2023-04-14 22:40:23 - r - INFO: - Episode: 610/2000, Reward: -3.215, Step: 300 +2023-04-14 22:40:27 - r - INFO: - Episode: 611/2000, Reward: -113.461, Step: 178 +2023-04-14 22:40:34 - r - INFO: - Episode: 612/2000, Reward: -14.623, Step: 300 +2023-04-14 22:40:41 - r - INFO: - Episode: 613/2000, Reward: 17.713, Step: 300 +2023-04-14 22:40:48 - r - INFO: - Episode: 614/2000, Reward: 14.531, Step: 300 +2023-04-14 22:40:54 - r - INFO: - Episode: 615/2000, Reward: -100.475, Step: 259 +2023-04-14 22:40:58 - r - INFO: - Episode: 616/2000, Reward: -129.272, Step: 176 +2023-04-14 22:41:05 - r - INFO: - Episode: 617/2000, Reward: 4.179, Step: 300 +2023-04-14 22:41:09 - r - INFO: - Episode: 618/2000, Reward: -90.178, Step: 174 +2023-04-14 22:41:13 - r - INFO: - Episode: 619/2000, Reward: -87.920, Step: 148 +2023-04-14 22:41:15 - r - INFO: - Episode: 620/2000, Reward: -97.940, Step: 101 +2023-04-14 22:41:24 - r - INFO: - Episode: 621/2000, Reward: -141.424, Step: 173 +2023-04-14 22:41:32 - r - INFO: - Episode: 622/2000, Reward: -3.468, Step: 300 +2023-04-14 22:41:39 - r - INFO: - Episode: 623/2000, Reward: -95.716, Step: 136 +2023-04-14 22:41:41 - r - INFO: - Episode: 624/2000, Reward: -105.132, Step: 72 +2023-04-14 22:41:44 - r - INFO: - Episode: 625/2000, Reward: -91.194, Step: 118 +2023-04-14 22:41:55 - r - INFO: - Episode: 626/2000, Reward: -24.811, Step: 300 +2023-04-14 22:41:58 - r - INFO: - Episode: 627/2000, Reward: -91.000, Step: 106 +2023-04-14 22:42:04 - r - INFO: - Episode: 628/2000, Reward: -88.905, Step: 224 +2023-04-14 22:42:07 - r - INFO: - Episode: 629/2000, Reward: -110.248, Step: 128 +2023-04-14 22:42:13 - r - INFO: - Episode: 630/2000, Reward: -9.651, Step: 300 +2023-04-14 22:42:16 - r - INFO: - Episode: 631/2000, Reward: -100.967, Step: 118 +2023-04-14 22:42:18 - r - INFO: - Episode: 632/2000, Reward: -102.751, Step: 110 +2023-04-14 22:42:23 - r - INFO: - Episode: 633/2000, Reward: -102.941, Step: 167 +2023-04-14 22:42:30 - r - INFO: - Episode: 634/2000, Reward: -18.258, Step: 300 +2023-04-14 22:42:37 - r - INFO: - Episode: 635/2000, Reward: -14.507, Step: 300 +2023-04-14 22:42:39 - r - INFO: - Episode: 636/2000, Reward: -106.032, Step: 108 +2023-04-14 22:42:44 - r - INFO: - Episode: 637/2000, Reward: -74.973, Step: 206 +2023-04-14 22:42:54 - r - INFO: - Episode: 638/2000, Reward: 1.251, Step: 300 +2023-04-14 22:43:03 - r - INFO: - Episode: 639/2000, Reward: -12.972, Step: 300 +2023-04-14 22:43:11 - r - INFO: - Episode: 640/2000, Reward: -19.759, Step: 300 +2023-04-14 22:43:30 - r - INFO: - Episode: 641/2000, Reward: -27.291, Step: 300 +2023-04-14 22:43:39 - r - INFO: - Episode: 642/2000, Reward: -24.548, Step: 300 +2023-04-14 22:43:47 - r - INFO: - Episode: 643/2000, Reward: -27.074, Step: 300 +2023-04-14 22:43:54 - r - INFO: - Episode: 644/2000, Reward: -27.967, Step: 300 +2023-04-14 22:44:02 - r - INFO: - Episode: 645/2000, Reward: -12.896, Step: 300 +2023-04-14 22:44:03 - r - INFO: - Episode: 646/2000, Reward: -121.157, Step: 57 +2023-04-14 22:44:12 - r - INFO: - Episode: 647/2000, Reward: -18.152, Step: 300 +2023-04-14 22:44:16 - r - INFO: - Episode: 648/2000, Reward: -140.197, Step: 177 +2023-04-14 22:44:22 - r - INFO: - Episode: 649/2000, Reward: -139.023, Step: 249 +2023-04-14 22:44:28 - r - INFO: - Episode: 650/2000, Reward: -135.843, Step: 194 +2023-04-14 22:44:35 - r - INFO: - Episode: 651/2000, Reward: -41.880, Step: 300 +2023-04-14 22:44:41 - r - INFO: - Episode: 652/2000, Reward: -112.651, Step: 241 +2023-04-14 22:44:46 - r - INFO: - Episode: 653/2000, Reward: -134.930, Step: 196 +2023-04-14 22:44:49 - r - INFO: - Episode: 654/2000, Reward: -115.894, Step: 86 +2023-04-14 22:44:51 - r - INFO: - Episode: 655/2000, Reward: -110.638, Step: 70 +2023-04-14 22:44:54 - r - INFO: - Episode: 656/2000, Reward: -109.468, Step: 153 +2023-04-14 22:44:56 - r - INFO: - Episode: 657/2000, Reward: -107.238, Step: 88 +2023-04-14 22:45:00 - r - INFO: - Episode: 658/2000, Reward: -111.913, Step: 116 +2023-04-14 22:45:03 - r - INFO: - Episode: 659/2000, Reward: -108.182, Step: 69 +2023-04-14 22:45:06 - r - INFO: - Episode: 660/2000, Reward: -83.345, Step: 114 +2023-04-14 22:45:12 - r - INFO: - Episode: 661/2000, Reward: -112.123, Step: 85 +2023-04-14 22:45:14 - r - INFO: - Episode: 662/2000, Reward: -103.055, Step: 85 +2023-04-14 22:45:18 - r - INFO: - Episode: 663/2000, Reward: -92.631, Step: 159 +2023-04-14 22:45:24 - r - INFO: - Episode: 664/2000, Reward: -96.356, Step: 214 +2023-04-14 22:45:29 - r - INFO: - Episode: 665/2000, Reward: -107.758, Step: 143 +2023-04-14 22:45:34 - r - INFO: - Episode: 666/2000, Reward: -119.437, Step: 129 +2023-04-14 22:45:39 - r - INFO: - Episode: 667/2000, Reward: -101.400, Step: 185 +2023-04-14 22:45:49 - r - INFO: - Episode: 668/2000, Reward: -0.478, Step: 300 +2023-04-14 22:45:58 - r - INFO: - Episode: 669/2000, Reward: 1.799, Step: 300 +2023-04-14 22:46:06 - r - INFO: - Episode: 670/2000, Reward: -7.655, Step: 300 +2023-04-14 22:46:12 - r - INFO: - Episode: 671/2000, Reward: -99.148, Step: 219 +2023-04-14 22:46:19 - r - INFO: - Episode: 672/2000, Reward: -35.466, Step: 300 +2023-04-14 22:46:22 - r - INFO: - Episode: 673/2000, Reward: -118.572, Step: 89 +2023-04-14 22:46:25 - r - INFO: - Episode: 674/2000, Reward: -109.019, Step: 116 +2023-04-14 22:46:33 - r - INFO: - Episode: 675/2000, Reward: 20.958, Step: 300 +2023-04-14 22:46:40 - r - INFO: - Episode: 676/2000, Reward: 31.429, Step: 300 +2023-04-14 22:46:48 - r - INFO: - Episode: 677/2000, Reward: -66.832, Step: 279 +2023-04-14 22:46:55 - r - INFO: - Episode: 678/2000, Reward: 21.515, Step: 300 +2023-04-14 22:47:02 - r - INFO: - Episode: 679/2000, Reward: 15.310, Step: 300 +2023-04-14 22:47:04 - r - INFO: - Episode: 680/2000, Reward: -123.405, Step: 92 +2023-04-14 22:47:16 - r - INFO: - Episode: 681/2000, Reward: 0.524, Step: 300 +2023-04-14 22:47:24 - r - INFO: - Episode: 682/2000, Reward: 35.969, Step: 300 +2023-04-14 22:47:32 - r - INFO: - Episode: 683/2000, Reward: -9.858, Step: 300 +2023-04-14 22:47:35 - r - INFO: - Episode: 684/2000, Reward: -87.654, Step: 134 +2023-04-14 22:47:41 - r - INFO: - Episode: 685/2000, Reward: -72.841, Step: 237 +2023-04-14 22:47:44 - r - INFO: - Episode: 686/2000, Reward: -76.318, Step: 124 +2023-04-14 22:47:50 - r - INFO: - Episode: 687/2000, Reward: 21.500, Step: 300 +2023-04-14 22:47:59 - r - INFO: - Episode: 688/2000, Reward: 22.707, Step: 300 +2023-04-14 22:48:01 - r - INFO: - Episode: 689/2000, Reward: -109.273, Step: 108 +2023-04-14 22:48:09 - r - INFO: - Episode: 690/2000, Reward: 11.130, Step: 300 +2023-04-14 22:48:16 - r - INFO: - Episode: 691/2000, Reward: -11.223, Step: 300 +2023-04-14 22:48:22 - r - INFO: - Episode: 692/2000, Reward: 17.899, Step: 300 +2023-04-14 22:48:31 - r - INFO: - Episode: 693/2000, Reward: 2.124, Step: 300 +2023-04-14 22:48:35 - r - INFO: - Episode: 694/2000, Reward: -98.751, Step: 159 +2023-04-14 22:48:42 - r - INFO: - Episode: 695/2000, Reward: -9.345, Step: 300 +2023-04-14 22:48:49 - r - INFO: - Episode: 696/2000, Reward: 2.439, Step: 300 +2023-04-14 22:48:55 - r - INFO: - Episode: 697/2000, Reward: 3.031, Step: 300 +2023-04-14 22:49:03 - r - INFO: - Episode: 698/2000, Reward: -13.047, Step: 300 +2023-04-14 22:49:09 - r - INFO: - Episode: 699/2000, Reward: -18.377, Step: 300 +2023-04-14 22:49:12 - r - INFO: - Episode: 700/2000, Reward: -104.498, Step: 93 +2023-04-14 22:49:18 - r - INFO: - Episode: 701/2000, Reward: -107.667, Step: 86 +2023-04-14 22:49:20 - r - INFO: - Episode: 702/2000, Reward: -104.048, Step: 107 +2023-04-14 22:49:23 - r - INFO: - Episode: 703/2000, Reward: -101.115, Step: 147 +2023-04-14 22:49:30 - r - INFO: - Episode: 704/2000, Reward: -23.682, Step: 300 +2023-04-14 22:49:34 - r - INFO: - Episode: 705/2000, Reward: -124.953, Step: 163 +2023-04-14 22:49:41 - r - INFO: - Episode: 706/2000, Reward: -12.355, Step: 300 +2023-04-14 22:49:43 - r - INFO: - Episode: 707/2000, Reward: -113.464, Step: 112 +2023-04-14 22:49:50 - r - INFO: - Episode: 708/2000, Reward: -5.759, Step: 300 +2023-04-14 22:49:57 - r - INFO: - Episode: 709/2000, Reward: 8.533, Step: 300 +2023-04-14 22:50:01 - r - INFO: - Episode: 710/2000, Reward: -85.131, Step: 174 +2023-04-14 22:50:05 - r - INFO: - Episode: 711/2000, Reward: -89.417, Step: 169 +2023-04-14 22:50:11 - r - INFO: - Episode: 712/2000, Reward: 14.458, Step: 300 +2023-04-14 22:50:14 - r - INFO: - Episode: 713/2000, Reward: -122.132, Step: 123 +2023-04-14 22:50:21 - r - INFO: - Episode: 714/2000, Reward: 19.988, Step: 300 +2023-04-14 22:50:25 - r - INFO: - Episode: 715/2000, Reward: -80.414, Step: 184 +2023-04-14 22:50:30 - r - INFO: - Episode: 716/2000, Reward: -95.670, Step: 191 +2023-04-14 22:50:32 - r - INFO: - Episode: 717/2000, Reward: -91.551, Step: 96 +2023-04-14 22:50:38 - r - INFO: - Episode: 718/2000, Reward: -85.730, Step: 184 +2023-04-14 22:50:41 - r - INFO: - Episode: 719/2000, Reward: -113.940, Step: 85 +2023-04-14 22:50:45 - r - INFO: - Episode: 720/2000, Reward: -91.913, Step: 107 +2023-04-14 22:50:50 - r - INFO: - Episode: 721/2000, Reward: -97.671, Step: 110 +2023-04-14 22:50:55 - r - INFO: - Episode: 722/2000, Reward: -81.294, Step: 241 +2023-04-14 22:51:02 - r - INFO: - Episode: 723/2000, Reward: 32.514, Step: 300 +2023-04-14 22:51:09 - r - INFO: - Episode: 724/2000, Reward: 18.011, Step: 300 +2023-04-14 22:51:14 - r - INFO: - Episode: 725/2000, Reward: -75.030, Step: 214 +2023-04-14 22:51:21 - r - INFO: - Episode: 726/2000, Reward: 25.294, Step: 300 +2023-04-14 22:51:28 - r - INFO: - Episode: 727/2000, Reward: 18.887, Step: 300 +2023-04-14 22:51:35 - r - INFO: - Episode: 728/2000, Reward: 13.518, Step: 300 +2023-04-14 22:51:43 - r - INFO: - Episode: 729/2000, Reward: 13.322, Step: 300 +2023-04-14 22:51:49 - r - INFO: - Episode: 730/2000, Reward: 10.878, Step: 300 +2023-04-14 22:51:53 - r - INFO: - Episode: 731/2000, Reward: -113.735, Step: 173 +2023-04-14 22:52:00 - r - INFO: - Episode: 732/2000, Reward: -18.569, Step: 300 +2023-04-14 22:52:07 - r - INFO: - Episode: 733/2000, Reward: -31.803, Step: 300 +2023-04-14 22:52:14 - r - INFO: - Episode: 734/2000, Reward: -19.255, Step: 300 +2023-04-14 22:52:21 - r - INFO: - Episode: 735/2000, Reward: -26.213, Step: 300 +2023-04-14 22:52:28 - r - INFO: - Episode: 736/2000, Reward: -18.068, Step: 300 +2023-04-14 22:52:35 - r - INFO: - Episode: 737/2000, Reward: -8.803, Step: 300 +2023-04-14 22:52:41 - r - INFO: - Episode: 738/2000, Reward: -2.386, Step: 300 +2023-04-14 22:52:44 - r - INFO: - Episode: 739/2000, Reward: -112.055, Step: 116 +2023-04-14 22:52:53 - r - INFO: - Episode: 740/2000, Reward: -4.976, Step: 300 +2023-04-14 22:53:04 - r - INFO: - Episode: 741/2000, Reward: -106.110, Step: 280 +2023-04-14 22:53:11 - r - INFO: - Episode: 742/2000, Reward: 9.559, Step: 300 +2023-04-14 22:53:17 - r - INFO: - Episode: 743/2000, Reward: -2.154, Step: 300 +2023-04-14 22:53:22 - r - INFO: - Episode: 744/2000, Reward: -105.158, Step: 229 +2023-04-14 22:53:26 - r - INFO: - Episode: 745/2000, Reward: -93.520, Step: 150 +2023-04-14 22:53:34 - r - INFO: - Episode: 746/2000, Reward: 9.796, Step: 300 +2023-04-14 22:53:42 - r - INFO: - Episode: 747/2000, Reward: 20.428, Step: 300 +2023-04-14 22:53:49 - r - INFO: - Episode: 748/2000, Reward: -77.742, Step: 247 +2023-04-14 22:53:53 - r - INFO: - Episode: 749/2000, Reward: -79.714, Step: 162 +2023-04-14 22:54:00 - r - INFO: - Episode: 750/2000, Reward: 11.843, Step: 300 +2023-04-14 22:54:07 - r - INFO: - Episode: 751/2000, Reward: 17.746, Step: 300 +2023-04-14 22:54:11 - r - INFO: - Episode: 752/2000, Reward: -83.242, Step: 152 +2023-04-14 22:54:17 - r - INFO: - Episode: 753/2000, Reward: -68.549, Step: 234 +2023-04-14 22:54:24 - r - INFO: - Episode: 754/2000, Reward: 21.695, Step: 300 +2023-04-14 22:54:26 - r - INFO: - Episode: 755/2000, Reward: -107.379, Step: 77 +2023-04-14 22:54:33 - r - INFO: - Episode: 756/2000, Reward: -0.439, Step: 300 +2023-04-14 22:54:41 - r - INFO: - Episode: 757/2000, Reward: -15.032, Step: 300 +2023-04-14 22:54:48 - r - INFO: - Episode: 758/2000, Reward: -10.704, Step: 300 +2023-04-14 22:54:55 - r - INFO: - Episode: 759/2000, Reward: -6.623, Step: 300 +2023-04-14 22:55:02 - r - INFO: - Episode: 760/2000, Reward: -5.784, Step: 300 +2023-04-14 22:55:15 - r - INFO: - Episode: 761/2000, Reward: -95.578, Step: 298 +2023-04-14 22:55:22 - r - INFO: - Episode: 762/2000, Reward: 7.207, Step: 300 +2023-04-14 22:55:24 - r - INFO: - Episode: 763/2000, Reward: -104.866, Step: 77 +2023-04-14 22:55:26 - r - INFO: - Episode: 764/2000, Reward: -97.809, Step: 107 +2023-04-14 22:55:28 - r - INFO: - Episode: 765/2000, Reward: -98.950, Step: 67 +2023-04-14 22:55:36 - r - INFO: - Episode: 766/2000, Reward: 38.510, Step: 300 +2023-04-14 22:55:37 - r - INFO: - Episode: 767/2000, Reward: -108.349, Step: 63 +2023-04-14 22:55:41 - r - INFO: - Episode: 768/2000, Reward: -85.671, Step: 143 +2023-04-14 22:55:48 - r - INFO: - Episode: 769/2000, Reward: -9.256, Step: 300 +2023-04-14 22:55:51 - r - INFO: - Episode: 770/2000, Reward: -96.755, Step: 131 +2023-04-14 22:55:54 - r - INFO: - Episode: 771/2000, Reward: -77.945, Step: 143 +2023-04-14 22:55:57 - r - INFO: - Episode: 772/2000, Reward: -101.966, Step: 113 +2023-04-14 22:56:00 - r - INFO: - Episode: 773/2000, Reward: -104.201, Step: 141 +2023-04-14 22:56:04 - r - INFO: - Episode: 774/2000, Reward: -83.863, Step: 176 +2023-04-14 22:56:11 - r - INFO: - Episode: 775/2000, Reward: -23.888, Step: 300 +2023-04-14 22:56:17 - r - INFO: - Episode: 776/2000, Reward: -111.473, Step: 274 +2023-04-14 22:56:22 - r - INFO: - Episode: 777/2000, Reward: -108.562, Step: 220 +2023-04-14 22:56:24 - r - INFO: - Episode: 778/2000, Reward: -103.102, Step: 109 +2023-04-14 22:56:31 - r - INFO: - Episode: 779/2000, Reward: -20.492, Step: 300 +2023-04-14 22:56:32 - r - INFO: - Episode: 780/2000, Reward: -125.577, Step: 71 +2023-04-14 22:56:39 - r - INFO: - Episode: 781/2000, Reward: -120.553, Step: 76 +2023-04-14 22:56:44 - r - INFO: - Episode: 782/2000, Reward: -147.412, Step: 214 +2023-04-14 22:56:46 - r - INFO: - Episode: 783/2000, Reward: -117.360, Step: 81 +2023-04-14 22:56:48 - r - INFO: - Episode: 784/2000, Reward: -128.931, Step: 118 +2023-04-14 22:56:54 - r - INFO: - Episode: 785/2000, Reward: -88.617, Step: 250 +2023-04-14 22:57:03 - r - INFO: - Episode: 786/2000, Reward: -100.007, Step: 286 +2023-04-14 22:57:06 - r - INFO: - Episode: 787/2000, Reward: -94.501, Step: 102 +2023-04-14 22:57:12 - r - INFO: - Episode: 788/2000, Reward: -68.989, Step: 263 +2023-04-14 22:57:15 - r - INFO: - Episode: 789/2000, Reward: -129.357, Step: 153 +2023-04-14 22:57:19 - r - INFO: - Episode: 790/2000, Reward: -93.380, Step: 162 +2023-04-14 22:57:23 - r - INFO: - Episode: 791/2000, Reward: -92.478, Step: 183 +2023-04-14 22:57:31 - r - INFO: - Episode: 792/2000, Reward: -86.031, Step: 290 +2023-04-14 22:57:34 - r - INFO: - Episode: 793/2000, Reward: -98.313, Step: 127 +2023-04-14 22:57:41 - r - INFO: - Episode: 794/2000, Reward: -99.514, Step: 258 +2023-04-14 22:57:49 - r - INFO: - Episode: 795/2000, Reward: -32.350, Step: 300 +2023-04-14 22:57:55 - r - INFO: - Episode: 796/2000, Reward: -118.482, Step: 245 +2023-04-14 22:58:01 - r - INFO: - Episode: 797/2000, Reward: -111.211, Step: 229 +2023-04-14 22:58:04 - r - INFO: - Episode: 798/2000, Reward: -112.294, Step: 105 +2023-04-14 22:58:05 - r - INFO: - Episode: 799/2000, Reward: -124.179, Step: 45 +2023-04-14 22:58:13 - r - INFO: - Episode: 800/2000, Reward: -31.865, Step: 300 +2023-04-14 22:58:19 - r - INFO: - Episode: 801/2000, Reward: -109.840, Step: 79 +2023-04-14 22:58:28 - r - INFO: - Episode: 802/2000, Reward: -15.290, Step: 300 +2023-04-14 22:58:29 - r - INFO: - Episode: 803/2000, Reward: -114.144, Step: 48 +2023-04-14 22:58:31 - r - INFO: - Episode: 804/2000, Reward: -123.328, Step: 53 +2023-04-14 22:58:32 - r - INFO: - Episode: 805/2000, Reward: -114.245, Step: 42 +2023-04-14 22:58:39 - r - INFO: - Episode: 806/2000, Reward: -29.994, Step: 300 +2023-04-14 22:58:49 - r - INFO: - Episode: 807/2000, Reward: -14.414, Step: 300 +2023-04-14 22:58:59 - r - INFO: - Episode: 808/2000, Reward: -19.690, Step: 300 +2023-04-14 22:59:01 - r - INFO: - Episode: 809/2000, Reward: -121.605, Step: 53 +2023-04-14 22:59:02 - r - INFO: - Episode: 810/2000, Reward: -116.045, Step: 52 +2023-04-14 22:59:09 - r - INFO: - Episode: 811/2000, Reward: -20.596, Step: 300 +2023-04-14 22:59:11 - r - INFO: - Episode: 812/2000, Reward: -123.491, Step: 55 +2023-04-14 22:59:15 - r - INFO: - Episode: 813/2000, Reward: -118.354, Step: 172 +2023-04-14 22:59:16 - r - INFO: - Episode: 814/2000, Reward: -116.801, Step: 41 +2023-04-14 22:59:23 - r - INFO: - Episode: 815/2000, Reward: -22.144, Step: 300 +2023-04-14 22:59:31 - r - INFO: - Episode: 816/2000, Reward: -7.144, Step: 300 +2023-04-14 22:59:39 - r - INFO: - Episode: 817/2000, Reward: -11.785, Step: 300 +2023-04-14 22:59:46 - r - INFO: - Episode: 818/2000, Reward: -10.094, Step: 300 +2023-04-14 22:59:52 - r - INFO: - Episode: 819/2000, Reward: -94.380, Step: 268 +2023-04-14 22:59:59 - r - INFO: - Episode: 820/2000, Reward: -19.939, Step: 300 +2023-04-14 23:00:05 - r - INFO: - Episode: 821/2000, Reward: -116.238, Step: 152 +2023-04-14 23:00:06 - r - INFO: - Episode: 822/2000, Reward: -110.370, Step: 53 +2023-04-14 23:00:10 - r - INFO: - Episode: 823/2000, Reward: -87.698, Step: 168 +2023-04-14 23:00:12 - r - INFO: - Episode: 824/2000, Reward: -103.481, Step: 63 +2023-04-14 23:00:14 - r - INFO: - Episode: 825/2000, Reward: -102.678, Step: 81 +2023-04-14 23:00:17 - r - INFO: - Episode: 826/2000, Reward: -104.059, Step: 146 +2023-04-14 23:00:21 - r - INFO: - Episode: 827/2000, Reward: -86.741, Step: 151 +2023-04-14 23:00:24 - r - INFO: - Episode: 828/2000, Reward: -94.374, Step: 109 +2023-04-14 23:00:27 - r - INFO: - Episode: 829/2000, Reward: -82.908, Step: 137 +2023-04-14 23:00:30 - r - INFO: - Episode: 830/2000, Reward: -78.090, Step: 139 +2023-04-14 23:00:37 - r - INFO: - Episode: 831/2000, Reward: -87.432, Step: 205 +2023-04-14 23:00:39 - r - INFO: - Episode: 832/2000, Reward: -103.993, Step: 85 +2023-04-14 23:00:49 - r - INFO: - Episode: 833/2000, Reward: 25.044, Step: 300 +2023-04-14 23:00:55 - r - INFO: - Episode: 834/2000, Reward: -87.360, Step: 267 +2023-04-14 23:01:02 - r - INFO: - Episode: 835/2000, Reward: 32.266, Step: 300 +2023-04-14 23:01:11 - r - INFO: - Episode: 836/2000, Reward: 21.561, Step: 300 +2023-04-14 23:01:16 - r - INFO: - Episode: 837/2000, Reward: -75.045, Step: 138 +2023-04-14 23:01:20 - r - INFO: - Episode: 838/2000, Reward: -106.789, Step: 160 +2023-04-14 23:01:21 - r - INFO: - Episode: 839/2000, Reward: -113.624, Step: 50 +2023-04-14 23:01:22 - r - INFO: - Episode: 840/2000, Reward: -112.705, Step: 48 +2023-04-14 23:01:25 - r - INFO: - Episode: 841/2000, Reward: -120.350, Step: 71 +2023-04-14 23:01:27 - r - INFO: - Episode: 842/2000, Reward: -125.245, Step: 100 +2023-04-14 23:01:34 - r - INFO: - Episode: 843/2000, Reward: -80.488, Step: 290 +2023-04-14 23:01:41 - r - INFO: - Episode: 844/2000, Reward: 38.437, Step: 300 +2023-04-14 23:01:46 - r - INFO: - Episode: 845/2000, Reward: -75.617, Step: 192 +2023-04-14 23:01:48 - r - INFO: - Episode: 846/2000, Reward: -125.194, Step: 84 +2023-04-14 23:01:51 - r - INFO: - Episode: 847/2000, Reward: -96.289, Step: 147 +2023-04-14 23:01:53 - r - INFO: - Episode: 848/2000, Reward: -119.931, Step: 77 +2023-04-14 23:01:55 - r - INFO: - Episode: 849/2000, Reward: -118.955, Step: 71 +2023-04-14 23:01:57 - r - INFO: - Episode: 850/2000, Reward: -92.421, Step: 111 +2023-04-14 23:01:59 - r - INFO: - Episode: 851/2000, Reward: -114.473, Step: 75 +2023-04-14 23:02:01 - r - INFO: - Episode: 852/2000, Reward: -119.281, Step: 83 +2023-04-14 23:02:02 - r - INFO: - Episode: 853/2000, Reward: -116.138, Step: 67 +2023-04-14 23:02:04 - r - INFO: - Episode: 854/2000, Reward: -114.424, Step: 72 +2023-04-14 23:02:06 - r - INFO: - Episode: 855/2000, Reward: -115.943, Step: 63 +2023-04-14 23:02:07 - r - INFO: - Episode: 856/2000, Reward: -116.635, Step: 66 +2023-04-14 23:02:09 - r - INFO: - Episode: 857/2000, Reward: -120.317, Step: 78 +2023-04-14 23:02:10 - r - INFO: - Episode: 858/2000, Reward: -115.245, Step: 65 +2023-04-14 23:02:12 - r - INFO: - Episode: 859/2000, Reward: -115.822, Step: 60 +2023-04-14 23:02:13 - r - INFO: - Episode: 860/2000, Reward: -111.030, Step: 49 +2023-04-14 23:02:16 - r - INFO: - Episode: 861/2000, Reward: -114.621, Step: 59 +2023-04-14 23:02:16 - r - INFO: - Episode: 862/2000, Reward: -108.063, Step: 41 +2023-04-14 23:02:18 - r - INFO: - Episode: 863/2000, Reward: -122.333, Step: 84 +2023-04-14 23:02:25 - r - INFO: - Episode: 864/2000, Reward: 25.381, Step: 300 +2023-04-14 23:02:27 - r - INFO: - Episode: 865/2000, Reward: -117.072, Step: 90 +2023-04-14 23:02:29 - r - INFO: - Episode: 866/2000, Reward: -117.133, Step: 72 +2023-04-14 23:02:30 - r - INFO: - Episode: 867/2000, Reward: -114.733, Step: 60 +2023-04-14 23:02:37 - r - INFO: - Episode: 868/2000, Reward: -24.689, Step: 300 +2023-04-14 23:02:39 - r - INFO: - Episode: 869/2000, Reward: -125.826, Step: 87 +2023-04-14 23:02:41 - r - INFO: - Episode: 870/2000, Reward: -111.365, Step: 54 +2023-04-14 23:02:49 - r - INFO: - Episode: 871/2000, Reward: -22.714, Step: 300 +2023-04-14 23:02:52 - r - INFO: - Episode: 872/2000, Reward: -114.423, Step: 80 +2023-04-14 23:02:58 - r - INFO: - Episode: 873/2000, Reward: -122.123, Step: 221 +2023-04-14 23:03:00 - r - INFO: - Episode: 874/2000, Reward: -114.110, Step: 73 +2023-04-14 23:03:08 - r - INFO: - Episode: 875/2000, Reward: -22.001, Step: 300 +2023-04-14 23:03:09 - r - INFO: - Episode: 876/2000, Reward: -109.293, Step: 49 +2023-04-14 23:03:11 - r - INFO: - Episode: 877/2000, Reward: -114.194, Step: 71 +2023-04-14 23:03:16 - r - INFO: - Episode: 878/2000, Reward: -119.527, Step: 235 +2023-04-14 23:03:23 - r - INFO: - Episode: 879/2000, Reward: -16.506, Step: 300 +2023-04-14 23:03:28 - r - INFO: - Episode: 880/2000, Reward: -129.152, Step: 191 +2023-04-14 23:03:33 - r - INFO: - Episode: 881/2000, Reward: -123.338, Step: 147 +2023-04-14 23:03:44 - r - INFO: - Episode: 882/2000, Reward: -29.342, Step: 300 +2023-04-14 23:03:46 - r - INFO: - Episode: 883/2000, Reward: -125.453, Step: 96 +2023-04-14 23:03:51 - r - INFO: - Episode: 884/2000, Reward: -141.932, Step: 167 +2023-04-14 23:03:53 - r - INFO: - Episode: 885/2000, Reward: -103.554, Step: 91 +2023-04-14 23:03:54 - r - INFO: - Episode: 886/2000, Reward: -113.680, Step: 53 +2023-04-14 23:03:57 - r - INFO: - Episode: 887/2000, Reward: -127.518, Step: 116 +2023-04-14 23:03:59 - r - INFO: - Episode: 888/2000, Reward: -126.488, Step: 80 +2023-04-14 23:04:00 - r - INFO: - Episode: 889/2000, Reward: -118.459, Step: 52 +2023-04-14 23:04:01 - r - INFO: - Episode: 890/2000, Reward: -118.289, Step: 48 +2023-04-14 23:04:03 - r - INFO: - Episode: 891/2000, Reward: -130.508, Step: 90 +2023-04-14 23:04:05 - r - INFO: - Episode: 892/2000, Reward: -123.968, Step: 86 +2023-04-14 23:04:07 - r - INFO: - Episode: 893/2000, Reward: -120.609, Step: 83 +2023-04-14 23:04:09 - r - INFO: - Episode: 894/2000, Reward: -121.235, Step: 66 +2023-04-14 23:04:14 - r - INFO: - Episode: 895/2000, Reward: -130.727, Step: 153 +2023-04-14 23:04:17 - r - INFO: - Episode: 896/2000, Reward: -131.004, Step: 147 +2023-04-14 23:04:19 - r - INFO: - Episode: 897/2000, Reward: -124.092, Step: 68 +2023-04-14 23:04:22 - r - INFO: - Episode: 898/2000, Reward: -127.586, Step: 117 +2023-04-14 23:04:23 - r - INFO: - Episode: 899/2000, Reward: -118.229, Step: 73 +2023-04-14 23:04:25 - r - INFO: - Episode: 900/2000, Reward: -114.351, Step: 51 +2023-04-14 23:04:28 - r - INFO: - Episode: 901/2000, Reward: -122.128, Step: 94 +2023-04-14 23:04:31 - r - INFO: - Episode: 902/2000, Reward: -128.521, Step: 95 +2023-04-14 23:04:35 - r - INFO: - Episode: 903/2000, Reward: -140.424, Step: 188 +2023-04-14 23:04:43 - r - INFO: - Episode: 904/2000, Reward: -31.118, Step: 300 +2023-04-14 23:04:52 - r - INFO: - Episode: 905/2000, Reward: -29.896, Step: 300 +2023-04-14 23:04:56 - r - INFO: - Episode: 906/2000, Reward: -128.014, Step: 166 +2023-04-14 23:05:03 - r - INFO: - Episode: 907/2000, Reward: -26.803, Step: 300 +2023-04-14 23:05:10 - r - INFO: - Episode: 908/2000, Reward: -18.593, Step: 300 +2023-04-14 23:05:12 - r - INFO: - Episode: 909/2000, Reward: -106.720, Step: 84 +2023-04-14 23:05:19 - r - INFO: - Episode: 910/2000, Reward: -33.963, Step: 300 +2023-04-14 23:05:20 - r - INFO: - Episode: 911/2000, Reward: -108.662, Step: 61 +2023-04-14 23:05:25 - r - INFO: - Episode: 912/2000, Reward: -93.990, Step: 209 +2023-04-14 23:05:32 - r - INFO: - Episode: 913/2000, Reward: -24.538, Step: 300 +2023-04-14 23:05:39 - r - INFO: - Episode: 914/2000, Reward: -15.354, Step: 300 +2023-04-14 23:05:40 - r - INFO: - Episode: 915/2000, Reward: -110.980, Step: 51 +2023-04-14 23:05:43 - r - INFO: - Episode: 916/2000, Reward: -134.207, Step: 126 +2023-04-14 23:05:50 - r - INFO: - Episode: 917/2000, Reward: -23.580, Step: 300 +2023-04-14 23:05:57 - r - INFO: - Episode: 918/2000, Reward: -19.745, Step: 300 +2023-04-14 23:06:05 - r - INFO: - Episode: 919/2000, Reward: -21.849, Step: 300 +2023-04-14 23:06:12 - r - INFO: - Episode: 920/2000, Reward: -21.701, Step: 300 +2023-04-14 23:06:24 - r - INFO: - Episode: 921/2000, Reward: -24.841, Step: 300 +2023-04-14 23:06:31 - r - INFO: - Episode: 922/2000, Reward: -20.734, Step: 300 +2023-04-14 23:06:38 - r - INFO: - Episode: 923/2000, Reward: -14.688, Step: 300 +2023-04-14 23:06:45 - r - INFO: - Episode: 924/2000, Reward: -23.787, Step: 300 +2023-04-14 23:06:47 - r - INFO: - Episode: 925/2000, Reward: -117.572, Step: 73 +2023-04-14 23:06:48 - r - INFO: - Episode: 926/2000, Reward: -119.216, Step: 78 +2023-04-14 23:06:51 - r - INFO: - Episode: 927/2000, Reward: -129.251, Step: 105 +2023-04-14 23:06:53 - r - INFO: - Episode: 928/2000, Reward: -123.160, Step: 87 +2023-04-14 23:07:01 - r - INFO: - Episode: 929/2000, Reward: -24.502, Step: 300 +2023-04-14 23:07:03 - r - INFO: - Episode: 930/2000, Reward: -123.163, Step: 94 +2023-04-14 23:07:05 - r - INFO: - Episode: 931/2000, Reward: -123.495, Step: 71 +2023-04-14 23:07:06 - r - INFO: - Episode: 932/2000, Reward: -112.632, Step: 52 +2023-04-14 23:07:08 - r - INFO: - Episode: 933/2000, Reward: -124.848, Step: 75 +2023-04-14 23:07:10 - r - INFO: - Episode: 934/2000, Reward: -122.094, Step: 78 +2023-04-14 23:07:13 - r - INFO: - Episode: 935/2000, Reward: -142.675, Step: 119 +2023-04-14 23:07:14 - r - INFO: - Episode: 936/2000, Reward: -119.940, Step: 70 +2023-04-14 23:07:16 - r - INFO: - Episode: 937/2000, Reward: -120.188, Step: 89 +2023-04-14 23:07:17 - r - INFO: - Episode: 938/2000, Reward: -111.917, Step: 48 +2023-04-14 23:07:19 - r - INFO: - Episode: 939/2000, Reward: -118.059, Step: 67 +2023-04-14 23:07:21 - r - INFO: - Episode: 940/2000, Reward: -127.742, Step: 96 +2023-04-14 23:07:24 - r - INFO: - Episode: 941/2000, Reward: -120.634, Step: 79 +2023-04-14 23:07:26 - r - INFO: - Episode: 942/2000, Reward: -120.067, Step: 76 +2023-04-14 23:07:28 - r - INFO: - Episode: 943/2000, Reward: -118.489, Step: 67 +2023-04-14 23:07:30 - r - INFO: - Episode: 944/2000, Reward: -131.052, Step: 88 +2023-04-14 23:07:32 - r - INFO: - Episode: 945/2000, Reward: -126.105, Step: 89 +2023-04-14 23:07:33 - r - INFO: - Episode: 946/2000, Reward: -128.112, Step: 71 +2023-04-14 23:07:35 - r - INFO: - Episode: 947/2000, Reward: -121.884, Step: 77 +2023-04-14 23:07:39 - r - INFO: - Episode: 948/2000, Reward: -143.213, Step: 163 +2023-04-14 23:07:41 - r - INFO: - Episode: 949/2000, Reward: -121.533, Step: 64 +2023-04-14 23:07:42 - r - INFO: - Episode: 950/2000, Reward: -122.558, Step: 64 +2023-04-14 23:07:44 - r - INFO: - Episode: 951/2000, Reward: -124.543, Step: 70 +2023-04-14 23:07:46 - r - INFO: - Episode: 952/2000, Reward: -139.658, Step: 95 +2023-04-14 23:07:48 - r - INFO: - Episode: 953/2000, Reward: -124.665, Step: 78 +2023-04-14 23:07:49 - r - INFO: - Episode: 954/2000, Reward: -121.017, Step: 74 +2023-04-14 23:07:58 - r - INFO: - Episode: 955/2000, Reward: -17.048, Step: 300 +2023-04-14 23:08:00 - r - INFO: - Episode: 956/2000, Reward: -122.533, Step: 70 +2023-04-14 23:08:01 - r - INFO: - Episode: 957/2000, Reward: -106.544, Step: 59 +2023-04-14 23:08:03 - r - INFO: - Episode: 958/2000, Reward: -108.349, Step: 59 +2023-04-14 23:08:04 - r - INFO: - Episode: 959/2000, Reward: -123.622, Step: 60 +2023-04-14 23:08:11 - r - INFO: - Episode: 960/2000, Reward: -30.974, Step: 300 +2023-04-14 23:08:17 - r - INFO: - Episode: 961/2000, Reward: -110.404, Step: 70 +2023-04-14 23:08:20 - r - INFO: - Episode: 962/2000, Reward: -128.460, Step: 100 +2023-04-14 23:08:21 - r - INFO: - Episode: 963/2000, Reward: -117.947, Step: 71 +2023-04-14 23:08:23 - r - INFO: - Episode: 964/2000, Reward: -120.271, Step: 86 +2023-04-14 23:08:25 - r - INFO: - Episode: 965/2000, Reward: -128.497, Step: 86 +2023-04-14 23:08:28 - r - INFO: - Episode: 966/2000, Reward: -130.830, Step: 88 +2023-04-14 23:08:30 - r - INFO: - Episode: 967/2000, Reward: -120.280, Step: 81 +2023-04-14 23:08:37 - r - INFO: - Episode: 968/2000, Reward: -103.548, Step: 261 +2023-04-14 23:08:38 - r - INFO: - Episode: 969/2000, Reward: -114.969, Step: 69 +2023-04-14 23:08:41 - r - INFO: - Episode: 970/2000, Reward: -132.032, Step: 92 +2023-04-14 23:08:43 - r - INFO: - Episode: 971/2000, Reward: -121.469, Step: 76 +2023-04-14 23:08:45 - r - INFO: - Episode: 972/2000, Reward: -126.400, Step: 79 +2023-04-14 23:08:47 - r - INFO: - Episode: 973/2000, Reward: -127.905, Step: 87 +2023-04-14 23:08:49 - r - INFO: - Episode: 974/2000, Reward: -132.928, Step: 109 +2023-04-14 23:08:54 - r - INFO: - Episode: 975/2000, Reward: -111.007, Step: 206 +2023-04-14 23:08:57 - r - INFO: - Episode: 976/2000, Reward: -128.938, Step: 96 +2023-04-14 23:08:59 - r - INFO: - Episode: 977/2000, Reward: -129.454, Step: 107 +2023-04-14 23:09:02 - r - INFO: - Episode: 978/2000, Reward: -104.660, Step: 124 +2023-04-14 23:09:04 - r - INFO: - Episode: 979/2000, Reward: -118.271, Step: 71 +2023-04-14 23:09:10 - r - INFO: - Episode: 980/2000, Reward: -148.851, Step: 270 +2023-04-14 23:09:19 - r - INFO: - Episode: 981/2000, Reward: -96.834, Step: 189 +2023-04-14 23:09:21 - r - INFO: - Episode: 982/2000, Reward: -119.483, Step: 89 +2023-04-14 23:09:28 - r - INFO: - Episode: 983/2000, Reward: -2.818, Step: 300 +2023-04-14 23:09:31 - r - INFO: - Episode: 984/2000, Reward: -100.582, Step: 161 +2023-04-14 23:09:36 - r - INFO: - Episode: 985/2000, Reward: -96.156, Step: 208 +2023-04-14 23:09:43 - r - INFO: - Episode: 986/2000, Reward: -85.931, Step: 298 +2023-04-14 23:09:48 - r - INFO: - Episode: 987/2000, Reward: -95.208, Step: 224 +2023-04-14 23:09:51 - r - INFO: - Episode: 988/2000, Reward: -101.291, Step: 93 +2023-04-14 23:09:52 - r - INFO: - Episode: 989/2000, Reward: -103.767, Step: 61 +2023-04-14 23:09:54 - r - INFO: - Episode: 990/2000, Reward: -101.811, Step: 57 +2023-04-14 23:09:59 - r - INFO: - Episode: 991/2000, Reward: -95.113, Step: 199 +2023-04-14 23:10:04 - r - INFO: - Episode: 992/2000, Reward: -85.808, Step: 212 +2023-04-14 23:10:05 - r - INFO: - Episode: 993/2000, Reward: -108.721, Step: 38 +2023-04-14 23:10:12 - r - INFO: - Episode: 994/2000, Reward: 41.557, Step: 300 +2023-04-14 23:10:15 - r - INFO: - Episode: 995/2000, Reward: -86.283, Step: 143 +2023-04-14 23:10:20 - r - INFO: - Episode: 996/2000, Reward: -96.202, Step: 220 +2023-04-14 23:10:26 - r - INFO: - Episode: 997/2000, Reward: -139.947, Step: 281 +2023-04-14 23:10:28 - r - INFO: - Episode: 998/2000, Reward: -107.014, Step: 63 +2023-04-14 23:10:34 - r - INFO: - Episode: 999/2000, Reward: -79.229, Step: 247 +2023-04-14 23:10:37 - r - INFO: - Episode: 1000/2000, Reward: -134.678, Step: 121 +2023-04-14 23:10:40 - r - INFO: - Episode: 1001/2000, Reward: -109.934, Step: 36 +2023-04-14 23:10:42 - r - INFO: - Episode: 1002/2000, Reward: -113.965, Step: 60 +2023-04-14 23:10:43 - r - INFO: - Episode: 1003/2000, Reward: -109.424, Step: 69 +2023-04-14 23:10:45 - r - INFO: - Episode: 1004/2000, Reward: -133.280, Step: 76 +2023-04-14 23:10:49 - r - INFO: - Episode: 1005/2000, Reward: -123.751, Step: 175 +2023-04-14 23:10:56 - r - INFO: - Episode: 1006/2000, Reward: -127.027, Step: 252 +2023-04-14 23:11:02 - r - INFO: - Episode: 1007/2000, Reward: -123.955, Step: 252 +2023-04-14 23:11:05 - r - INFO: - Episode: 1008/2000, Reward: -121.892, Step: 114 +2023-04-14 23:11:08 - r - INFO: - Episode: 1009/2000, Reward: -118.969, Step: 145 +2023-04-14 23:11:11 - r - INFO: - Episode: 1010/2000, Reward: -119.797, Step: 102 +2023-04-14 23:11:13 - r - INFO: - Episode: 1011/2000, Reward: -121.380, Step: 98 +2023-04-14 23:11:16 - r - INFO: - Episode: 1012/2000, Reward: -127.433, Step: 106 +2023-04-14 23:11:20 - r - INFO: - Episode: 1013/2000, Reward: -124.296, Step: 94 +2023-04-14 23:11:25 - r - INFO: - Episode: 1014/2000, Reward: -137.743, Step: 108 +2023-04-14 23:11:27 - r - INFO: - Episode: 1015/2000, Reward: -126.186, Step: 86 +2023-04-14 23:11:28 - r - INFO: - Episode: 1016/2000, Reward: -122.671, Step: 63 +2023-04-14 23:11:31 - r - INFO: - Episode: 1017/2000, Reward: -140.276, Step: 104 +2023-04-14 23:11:32 - r - INFO: - Episode: 1018/2000, Reward: -113.200, Step: 42 +2023-04-14 23:11:39 - r - INFO: - Episode: 1019/2000, Reward: -149.953, Step: 279 +2023-04-14 23:11:42 - r - INFO: - Episode: 1020/2000, Reward: -133.428, Step: 113 +2023-04-14 23:11:44 - r - INFO: - Episode: 1021/2000, Reward: -109.011, Step: 37 +2023-04-14 23:11:45 - r - INFO: - Episode: 1022/2000, Reward: -123.632, Step: 60 +2023-04-14 23:11:50 - r - INFO: - Episode: 1023/2000, Reward: -123.965, Step: 180 +2023-04-14 23:11:53 - r - INFO: - Episode: 1024/2000, Reward: -112.048, Step: 134 +2023-04-14 23:11:54 - r - INFO: - Episode: 1025/2000, Reward: -120.322, Step: 53 +2023-04-14 23:12:01 - r - INFO: - Episode: 1026/2000, Reward: -139.577, Step: 263 +2023-04-14 23:12:08 - r - INFO: - Episode: 1027/2000, Reward: 1.003, Step: 300 +2023-04-14 23:12:10 - r - INFO: - Episode: 1028/2000, Reward: -122.627, Step: 71 +2023-04-14 23:12:13 - r - INFO: - Episode: 1029/2000, Reward: -126.732, Step: 154 +2023-04-14 23:12:16 - r - INFO: - Episode: 1030/2000, Reward: -122.600, Step: 96 +2023-04-14 23:12:19 - r - INFO: - Episode: 1031/2000, Reward: -99.523, Step: 138 +2023-04-14 23:12:24 - r - INFO: - Episode: 1032/2000, Reward: -136.347, Step: 191 +2023-04-14 23:12:29 - r - INFO: - Episode: 1033/2000, Reward: -96.404, Step: 215 +2023-04-14 23:12:30 - r - INFO: - Episode: 1034/2000, Reward: -106.703, Step: 77 +2023-04-14 23:12:35 - r - INFO: - Episode: 1035/2000, Reward: -140.996, Step: 220 +2023-04-14 23:12:37 - r - INFO: - Episode: 1036/2000, Reward: -128.225, Step: 64 +2023-04-14 23:12:39 - r - INFO: - Episode: 1037/2000, Reward: -95.370, Step: 116 +2023-04-14 23:12:42 - r - INFO: - Episode: 1038/2000, Reward: -96.904, Step: 117 +2023-04-14 23:12:44 - r - INFO: - Episode: 1039/2000, Reward: -105.768, Step: 73 +2023-04-14 23:12:49 - r - INFO: - Episode: 1040/2000, Reward: -96.987, Step: 241 +2023-04-14 23:13:00 - r - INFO: - Episode: 1041/2000, Reward: -95.867, Step: 222 +2023-04-14 23:13:02 - r - INFO: - Episode: 1042/2000, Reward: -101.218, Step: 86 +2023-04-14 23:13:06 - r - INFO: - Episode: 1043/2000, Reward: -99.547, Step: 150 +2023-04-14 23:13:08 - r - INFO: - Episode: 1044/2000, Reward: -127.823, Step: 83 +2023-04-14 23:13:13 - r - INFO: - Episode: 1045/2000, Reward: -110.475, Step: 190 +2023-04-14 23:13:20 - r - INFO: - Episode: 1046/2000, Reward: -27.377, Step: 300 +2023-04-14 23:13:26 - r - INFO: - Episode: 1047/2000, Reward: -119.875, Step: 299 +2023-04-14 23:13:30 - r - INFO: - Episode: 1048/2000, Reward: -126.247, Step: 105 +2023-04-14 23:13:34 - r - INFO: - Episode: 1049/2000, Reward: -131.617, Step: 91 +2023-04-14 23:13:37 - r - INFO: - Episode: 1050/2000, Reward: -131.735, Step: 116 +2023-04-14 23:13:39 - r - INFO: - Episode: 1051/2000, Reward: -108.245, Step: 108 +2023-04-14 23:13:42 - r - INFO: - Episode: 1052/2000, Reward: -136.772, Step: 111 +2023-04-14 23:13:50 - r - INFO: - Episode: 1053/2000, Reward: -10.996, Step: 300 +2023-04-14 23:13:52 - r - INFO: - Episode: 1054/2000, Reward: -110.835, Step: 82 +2023-04-14 23:13:55 - r - INFO: - Episode: 1055/2000, Reward: -125.631, Step: 93 +2023-04-14 23:13:59 - r - INFO: - Episode: 1056/2000, Reward: -123.035, Step: 163 +2023-04-14 23:14:02 - r - INFO: - Episode: 1057/2000, Reward: -132.722, Step: 97 +2023-04-14 23:14:04 - r - INFO: - Episode: 1058/2000, Reward: -109.708, Step: 79 +2023-04-14 23:14:05 - r - INFO: - Episode: 1059/2000, Reward: -115.461, Step: 77 +2023-04-14 23:14:11 - r - INFO: - Episode: 1060/2000, Reward: -141.900, Step: 201 +2023-04-14 23:14:21 - r - INFO: - Episode: 1061/2000, Reward: -33.908, Step: 300 +2023-04-14 23:14:23 - r - INFO: - Episode: 1062/2000, Reward: -119.403, Step: 55 +2023-04-14 23:14:24 - r - INFO: - Episode: 1063/2000, Reward: -114.271, Step: 69 +2023-04-14 23:14:27 - r - INFO: - Episode: 1064/2000, Reward: -108.222, Step: 86 +2023-04-14 23:14:30 - r - INFO: - Episode: 1065/2000, Reward: -107.141, Step: 120 +2023-04-14 23:14:32 - r - INFO: - Episode: 1066/2000, Reward: -106.381, Step: 76 +2023-04-14 23:14:34 - r - INFO: - Episode: 1067/2000, Reward: -105.449, Step: 73 +2023-04-14 23:14:39 - r - INFO: - Episode: 1068/2000, Reward: -79.645, Step: 211 +2023-04-14 23:14:41 - r - INFO: - Episode: 1069/2000, Reward: -131.168, Step: 89 +2023-04-14 23:14:43 - r - INFO: - Episode: 1070/2000, Reward: -120.763, Step: 78 +2023-04-14 23:14:44 - r - INFO: - Episode: 1071/2000, Reward: -111.911, Step: 71 +2023-04-14 23:14:48 - r - INFO: - Episode: 1072/2000, Reward: -107.469, Step: 146 +2023-04-14 23:14:50 - r - INFO: - Episode: 1073/2000, Reward: -124.821, Step: 108 +2023-04-14 23:14:53 - r - INFO: - Episode: 1074/2000, Reward: -111.164, Step: 132 +2023-04-14 23:14:56 - r - INFO: - Episode: 1075/2000, Reward: -111.712, Step: 131 +2023-04-14 23:14:59 - r - INFO: - Episode: 1076/2000, Reward: -124.004, Step: 91 +2023-04-14 23:15:06 - r - INFO: - Episode: 1077/2000, Reward: -22.455, Step: 300 +2023-04-14 23:15:08 - r - INFO: - Episode: 1078/2000, Reward: -115.227, Step: 67 +2023-04-14 23:15:11 - r - INFO: - Episode: 1079/2000, Reward: -98.656, Step: 120 +2023-04-14 23:15:12 - r - INFO: - Episode: 1080/2000, Reward: -116.513, Step: 75 +2023-04-14 23:15:15 - r - INFO: - Episode: 1081/2000, Reward: -113.865, Step: 64 +2023-04-14 23:15:17 - r - INFO: - Episode: 1082/2000, Reward: -122.328, Step: 79 +2023-04-14 23:15:24 - r - INFO: - Episode: 1083/2000, Reward: -16.864, Step: 300 +2023-04-14 23:15:26 - r - INFO: - Episode: 1084/2000, Reward: -133.800, Step: 91 +2023-04-14 23:15:28 - r - INFO: - Episode: 1085/2000, Reward: -137.750, Step: 110 +2023-04-14 23:15:31 - r - INFO: - Episode: 1086/2000, Reward: -134.965, Step: 113 +2023-04-14 23:15:33 - r - INFO: - Episode: 1087/2000, Reward: -123.913, Step: 67 +2023-04-14 23:15:35 - r - INFO: - Episode: 1088/2000, Reward: -105.761, Step: 95 +2023-04-14 23:15:42 - r - INFO: - Episode: 1089/2000, Reward: -28.101, Step: 300 +2023-04-14 23:15:45 - r - INFO: - Episode: 1090/2000, Reward: -99.170, Step: 167 +2023-04-14 23:15:46 - r - INFO: - Episode: 1091/2000, Reward: -110.372, Step: 44 +2023-04-14 23:15:49 - r - INFO: - Episode: 1092/2000, Reward: -140.165, Step: 126 +2023-04-14 23:15:56 - r - INFO: - Episode: 1093/2000, Reward: -27.469, Step: 300 +2023-04-14 23:16:04 - r - INFO: - Episode: 1094/2000, Reward: -22.809, Step: 300 +2023-04-14 23:16:11 - r - INFO: - Episode: 1095/2000, Reward: -29.512, Step: 300 +2023-04-14 23:16:18 - r - INFO: - Episode: 1096/2000, Reward: -17.419, Step: 300 +2023-04-14 23:16:25 - r - INFO: - Episode: 1097/2000, Reward: -23.032, Step: 300 +2023-04-14 23:16:27 - r - INFO: - Episode: 1098/2000, Reward: -111.076, Step: 93 +2023-04-14 23:16:34 - r - INFO: - Episode: 1099/2000, Reward: -27.056, Step: 300 +2023-04-14 23:16:41 - r - INFO: - Episode: 1100/2000, Reward: -23.415, Step: 300 +2023-04-14 23:16:52 - r - INFO: - Episode: 1101/2000, Reward: -23.411, Step: 300 +2023-04-14 23:16:54 - r - INFO: - Episode: 1102/2000, Reward: -95.043, Step: 88 +2023-04-14 23:17:01 - r - INFO: - Episode: 1103/2000, Reward: -29.636, Step: 300 +2023-04-14 23:17:05 - r - INFO: - Episode: 1104/2000, Reward: -80.218, Step: 155 +2023-04-14 23:17:13 - r - INFO: - Episode: 1105/2000, Reward: -20.491, Step: 300 +2023-04-14 23:17:21 - r - INFO: - Episode: 1106/2000, Reward: -23.441, Step: 300 +2023-04-14 23:17:30 - r - INFO: - Episode: 1107/2000, Reward: -16.867, Step: 300 +2023-04-14 23:17:38 - r - INFO: - Episode: 1108/2000, Reward: -30.469, Step: 300 +2023-04-14 23:17:46 - r - INFO: - Episode: 1109/2000, Reward: -9.263, Step: 300 +2023-04-14 23:17:53 - r - INFO: - Episode: 1110/2000, Reward: -100.869, Step: 279 +2023-04-14 23:17:58 - r - INFO: - Episode: 1111/2000, Reward: -104.252, Step: 177 +2023-04-14 23:18:08 - r - INFO: - Episode: 1112/2000, Reward: -25.886, Step: 300 +2023-04-14 23:18:13 - r - INFO: - Episode: 1113/2000, Reward: -114.779, Step: 232 +2023-04-14 23:18:18 - r - INFO: - Episode: 1114/2000, Reward: -82.311, Step: 209 +2023-04-14 23:18:23 - r - INFO: - Episode: 1115/2000, Reward: -78.954, Step: 221 +2023-04-14 23:18:26 - r - INFO: - Episode: 1116/2000, Reward: -100.670, Step: 110 +2023-04-14 23:18:32 - r - INFO: - Episode: 1117/2000, Reward: -70.599, Step: 223 +2023-04-14 23:18:33 - r - INFO: - Episode: 1118/2000, Reward: -103.613, Step: 55 +2023-04-14 23:18:38 - r - INFO: - Episode: 1119/2000, Reward: -100.778, Step: 148 +2023-04-14 23:18:41 - r - INFO: - Episode: 1120/2000, Reward: -97.280, Step: 125 +2023-04-14 23:18:49 - r - INFO: - Episode: 1121/2000, Reward: -80.809, Step: 207 +2023-04-14 23:18:56 - r - INFO: - Episode: 1122/2000, Reward: -25.731, Step: 300 +2023-04-14 23:19:02 - r - INFO: - Episode: 1123/2000, Reward: -96.874, Step: 193 +2023-04-14 23:19:06 - r - INFO: - Episode: 1124/2000, Reward: -89.984, Step: 149 +2023-04-14 23:19:08 - r - INFO: - Episode: 1125/2000, Reward: -116.171, Step: 114 +2023-04-14 23:19:13 - r - INFO: - Episode: 1126/2000, Reward: -73.963, Step: 168 +2023-04-14 23:19:15 - r - INFO: - Episode: 1127/2000, Reward: -112.334, Step: 114 +2023-04-14 23:19:19 - r - INFO: - Episode: 1128/2000, Reward: -90.490, Step: 171 +2023-04-14 23:19:24 - r - INFO: - Episode: 1129/2000, Reward: -110.219, Step: 182 +2023-04-14 23:19:31 - r - INFO: - Episode: 1130/2000, Reward: 23.286, Step: 300 +2023-04-14 23:19:38 - r - INFO: - Episode: 1131/2000, Reward: 9.866, Step: 300 +2023-04-14 23:19:45 - r - INFO: - Episode: 1132/2000, Reward: 40.539, Step: 300 +2023-04-14 23:19:53 - r - INFO: - Episode: 1133/2000, Reward: 29.305, Step: 300 +2023-04-14 23:19:59 - r - INFO: - Episode: 1134/2000, Reward: 12.277, Step: 300 +2023-04-14 23:20:04 - r - INFO: - Episode: 1135/2000, Reward: -93.562, Step: 188 +2023-04-14 23:20:10 - r - INFO: - Episode: 1136/2000, Reward: -75.366, Step: 239 +2023-04-14 23:20:13 - r - INFO: - Episode: 1137/2000, Reward: -104.148, Step: 132 +2023-04-14 23:20:19 - r - INFO: - Episode: 1138/2000, Reward: 27.955, Step: 300 +2023-04-14 23:20:23 - r - INFO: - Episode: 1139/2000, Reward: -94.206, Step: 167 +2023-04-14 23:20:30 - r - INFO: - Episode: 1140/2000, Reward: -8.888, Step: 300 +2023-04-14 23:20:36 - r - INFO: - Episode: 1141/2000, Reward: -81.982, Step: 146 +2023-04-14 23:20:41 - r - INFO: - Episode: 1142/2000, Reward: -68.897, Step: 191 +2023-04-14 23:20:45 - r - INFO: - Episode: 1143/2000, Reward: -67.897, Step: 182 +2023-04-14 23:20:51 - r - INFO: - Episode: 1144/2000, Reward: -53.381, Step: 294 +2023-04-14 23:20:59 - r - INFO: - Episode: 1145/2000, Reward: 35.384, Step: 300 +2023-04-14 23:21:07 - r - INFO: - Episode: 1146/2000, Reward: -48.452, Step: 285 +2023-04-14 23:21:13 - r - INFO: - Episode: 1147/2000, Reward: -70.618, Step: 212 +2023-04-14 23:21:17 - r - INFO: - Episode: 1148/2000, Reward: -79.787, Step: 158 +2023-04-14 23:21:21 - r - INFO: - Episode: 1149/2000, Reward: -75.091, Step: 159 +2023-04-14 23:21:28 - r - INFO: - Episode: 1150/2000, Reward: -67.415, Step: 269 +2023-04-14 23:21:32 - r - INFO: - Episode: 1151/2000, Reward: -75.237, Step: 152 +2023-04-14 23:21:39 - r - INFO: - Episode: 1152/2000, Reward: 31.208, Step: 300 +2023-04-14 23:21:47 - r - INFO: - Episode: 1153/2000, Reward: 17.079, Step: 300 +2023-04-14 23:21:54 - r - INFO: - Episode: 1154/2000, Reward: 13.694, Step: 300 +2023-04-14 23:21:56 - r - INFO: - Episode: 1155/2000, Reward: -92.184, Step: 87 +2023-04-14 23:22:03 - r - INFO: - Episode: 1156/2000, Reward: 5.266, Step: 300 +2023-04-14 23:22:11 - r - INFO: - Episode: 1157/2000, Reward: 19.428, Step: 300 +2023-04-14 23:22:20 - r - INFO: - Episode: 1158/2000, Reward: 13.319, Step: 300 +2023-04-14 23:22:27 - r - INFO: - Episode: 1159/2000, Reward: 20.473, Step: 300 +2023-04-14 23:22:35 - r - INFO: - Episode: 1160/2000, Reward: -7.907, Step: 300 +2023-04-14 23:22:51 - r - INFO: - Episode: 1161/2000, Reward: 3.270, Step: 300 +2023-04-14 23:23:00 - r - INFO: - Episode: 1162/2000, Reward: 3.324, Step: 300 +2023-04-14 23:23:08 - r - INFO: - Episode: 1163/2000, Reward: 7.185, Step: 300 +2023-04-14 23:23:15 - r - INFO: - Episode: 1164/2000, Reward: 6.277, Step: 300 +2023-04-14 23:23:22 - r - INFO: - Episode: 1165/2000, Reward: 1.122, Step: 300 +2023-04-14 23:23:28 - r - INFO: - Episode: 1166/2000, Reward: -95.477, Step: 250 +2023-04-14 23:23:31 - r - INFO: - Episode: 1167/2000, Reward: -87.831, Step: 121 +2023-04-14 23:23:40 - r - INFO: - Episode: 1168/2000, Reward: 25.185, Step: 300 +2023-04-14 23:23:47 - r - INFO: - Episode: 1169/2000, Reward: -77.789, Step: 221 +2023-04-14 23:23:50 - r - INFO: - Episode: 1170/2000, Reward: -92.155, Step: 160 +2023-04-14 23:23:58 - r - INFO: - Episode: 1171/2000, Reward: -66.537, Step: 295 +2023-04-14 23:24:06 - r - INFO: - Episode: 1172/2000, Reward: -62.810, Step: 273 +2023-04-14 23:24:10 - r - INFO: - Episode: 1173/2000, Reward: -106.821, Step: 152 +2023-04-14 23:24:14 - r - INFO: - Episode: 1174/2000, Reward: -85.504, Step: 169 +2023-04-14 23:24:18 - r - INFO: - Episode: 1175/2000, Reward: -91.321, Step: 147 +2023-04-14 23:24:21 - r - INFO: - Episode: 1176/2000, Reward: -103.038, Step: 110 +2023-04-14 23:24:24 - r - INFO: - Episode: 1177/2000, Reward: -96.160, Step: 112 +2023-04-14 23:24:28 - r - INFO: - Episode: 1178/2000, Reward: -99.421, Step: 125 +2023-04-14 23:24:32 - r - INFO: - Episode: 1179/2000, Reward: -95.756, Step: 176 +2023-04-14 23:24:36 - r - INFO: - Episode: 1180/2000, Reward: -87.080, Step: 134 +2023-04-14 23:24:48 - r - INFO: - Episode: 1181/2000, Reward: -140.673, Step: 190 +2023-04-14 23:24:57 - r - INFO: - Episode: 1182/2000, Reward: -82.484, Step: 297 +2023-04-14 23:25:05 - r - INFO: - Episode: 1183/2000, Reward: -16.981, Step: 300 +2023-04-14 23:25:13 - r - INFO: - Episode: 1184/2000, Reward: -2.079, Step: 300 +2023-04-14 23:25:20 - r - INFO: - Episode: 1185/2000, Reward: -34.987, Step: 300 +2023-04-14 23:25:28 - r - INFO: - Episode: 1186/2000, Reward: -18.021, Step: 300 +2023-04-14 23:25:35 - r - INFO: - Episode: 1187/2000, Reward: -27.799, Step: 300 +2023-04-14 23:25:43 - r - INFO: - Episode: 1188/2000, Reward: 34.974, Step: 300 +2023-04-14 23:25:48 - r - INFO: - Episode: 1189/2000, Reward: -139.666, Step: 232 +2023-04-14 23:25:55 - r - INFO: - Episode: 1190/2000, Reward: -0.713, Step: 300 +2023-04-14 23:26:03 - r - INFO: - Episode: 1191/2000, Reward: -12.751, Step: 300 +2023-04-14 23:26:05 - r - INFO: - Episode: 1192/2000, Reward: -103.613, Step: 78 +2023-04-14 23:26:08 - r - INFO: - Episode: 1193/2000, Reward: -101.301, Step: 103 +2023-04-14 23:26:16 - r - INFO: - Episode: 1194/2000, Reward: -0.840, Step: 300 +2023-04-14 23:26:23 - r - INFO: - Episode: 1195/2000, Reward: -49.758, Step: 298 +2023-04-14 23:26:26 - r - INFO: - Episode: 1196/2000, Reward: -102.925, Step: 125 +2023-04-14 23:26:29 - r - INFO: - Episode: 1197/2000, Reward: -108.321, Step: 114 +2023-04-14 23:26:34 - r - INFO: - Episode: 1198/2000, Reward: -99.596, Step: 173 +2023-04-14 23:26:36 - r - INFO: - Episode: 1199/2000, Reward: -96.534, Step: 116 +2023-04-14 23:26:40 - r - INFO: - Episode: 1200/2000, Reward: -100.006, Step: 144 +2023-04-14 23:26:47 - r - INFO: - Episode: 1201/2000, Reward: -94.046, Step: 177 +2023-04-14 23:26:54 - r - INFO: - Episode: 1202/2000, Reward: 39.233, Step: 300 +2023-04-14 23:26:55 - r - INFO: - Episode: 1203/2000, Reward: -112.379, Step: 55 +2023-04-14 23:26:58 - r - INFO: - Episode: 1204/2000, Reward: -99.375, Step: 127 +2023-04-14 23:27:01 - r - INFO: - Episode: 1205/2000, Reward: -110.924, Step: 110 +2023-04-14 23:27:03 - r - INFO: - Episode: 1206/2000, Reward: -112.982, Step: 66 +2023-04-14 23:27:05 - r - INFO: - Episode: 1207/2000, Reward: -97.337, Step: 123 +2023-04-14 23:27:07 - r - INFO: - Episode: 1208/2000, Reward: -127.499, Step: 72 +2023-04-14 23:27:09 - r - INFO: - Episode: 1209/2000, Reward: -104.303, Step: 80 +2023-04-14 23:27:12 - r - INFO: - Episode: 1210/2000, Reward: -99.124, Step: 105 +2023-04-14 23:27:15 - r - INFO: - Episode: 1211/2000, Reward: -97.943, Step: 107 +2023-04-14 23:27:20 - r - INFO: - Episode: 1212/2000, Reward: -79.893, Step: 209 +2023-04-14 23:27:27 - r - INFO: - Episode: 1213/2000, Reward: 26.092, Step: 300 +2023-04-14 23:27:31 - r - INFO: - Episode: 1214/2000, Reward: -100.473, Step: 152 +2023-04-14 23:27:33 - r - INFO: - Episode: 1215/2000, Reward: -111.607, Step: 114 +2023-04-14 23:27:35 - r - INFO: - Episode: 1216/2000, Reward: -121.697, Step: 62 +2023-04-14 23:27:36 - r - INFO: - Episode: 1217/2000, Reward: -123.836, Step: 64 +2023-04-14 23:27:39 - r - INFO: - Episode: 1218/2000, Reward: -113.891, Step: 118 +2023-04-14 23:27:45 - r - INFO: - Episode: 1219/2000, Reward: 4.738, Step: 300 +2023-04-14 23:27:47 - r - INFO: - Episode: 1220/2000, Reward: -116.863, Step: 48 +2023-04-14 23:27:57 - r - INFO: - Episode: 1221/2000, Reward: 22.106, Step: 300 +2023-04-14 23:27:58 - r - INFO: - Episode: 1222/2000, Reward: -116.664, Step: 47 +2023-04-14 23:28:00 - r - INFO: - Episode: 1223/2000, Reward: -124.949, Step: 69 +2023-04-14 23:28:02 - r - INFO: - Episode: 1224/2000, Reward: -117.799, Step: 53 +2023-04-14 23:28:04 - r - INFO: - Episode: 1225/2000, Reward: -138.147, Step: 98 +2023-04-14 23:28:06 - r - INFO: - Episode: 1226/2000, Reward: -130.019, Step: 73 +2023-04-14 23:28:08 - r - INFO: - Episode: 1227/2000, Reward: -133.804, Step: 85 +2023-04-14 23:28:10 - r - INFO: - Episode: 1228/2000, Reward: -133.384, Step: 77 +2023-04-14 23:28:14 - r - INFO: - Episode: 1229/2000, Reward: -102.023, Step: 177 +2023-04-14 23:28:17 - r - INFO: - Episode: 1230/2000, Reward: -133.552, Step: 113 +2023-04-14 23:28:23 - r - INFO: - Episode: 1231/2000, Reward: 24.599, Step: 300 +2023-04-14 23:28:24 - r - INFO: - Episode: 1232/2000, Reward: -114.121, Step: 43 +2023-04-14 23:28:26 - r - INFO: - Episode: 1233/2000, Reward: -120.434, Step: 68 +2023-04-14 23:28:28 - r - INFO: - Episode: 1234/2000, Reward: -132.424, Step: 96 +2023-04-14 23:28:29 - r - INFO: - Episode: 1235/2000, Reward: -124.290, Step: 68 +2023-04-14 23:28:31 - r - INFO: - Episode: 1236/2000, Reward: -120.895, Step: 74 +2023-04-14 23:28:33 - r - INFO: - Episode: 1237/2000, Reward: -121.481, Step: 58 +2023-04-14 23:28:35 - r - INFO: - Episode: 1238/2000, Reward: -126.100, Step: 77 +2023-04-14 23:28:37 - r - INFO: - Episode: 1239/2000, Reward: -134.183, Step: 88 +2023-04-14 23:28:39 - r - INFO: - Episode: 1240/2000, Reward: -125.324, Step: 79 +2023-04-14 23:28:42 - r - INFO: - Episode: 1241/2000, Reward: -129.809, Step: 90 +2023-04-14 23:28:44 - r - INFO: - Episode: 1242/2000, Reward: -125.807, Step: 65 +2023-04-14 23:28:47 - r - INFO: - Episode: 1243/2000, Reward: -141.158, Step: 123 +2023-04-14 23:28:48 - r - INFO: - Episode: 1244/2000, Reward: -116.503, Step: 50 +2023-04-14 23:28:51 - r - INFO: - Episode: 1245/2000, Reward: -137.312, Step: 123 +2023-04-14 23:28:53 - r - INFO: - Episode: 1246/2000, Reward: -130.109, Step: 75 +2023-04-14 23:28:54 - r - INFO: - Episode: 1247/2000, Reward: -122.455, Step: 71 +2023-04-14 23:28:55 - r - INFO: - Episode: 1248/2000, Reward: -105.179, Step: 58 +2023-04-14 23:28:57 - r - INFO: - Episode: 1249/2000, Reward: -128.076, Step: 71 +2023-04-14 23:29:01 - r - INFO: - Episode: 1250/2000, Reward: -136.421, Step: 170 +2023-04-14 23:29:04 - r - INFO: - Episode: 1251/2000, Reward: -137.003, Step: 151 +2023-04-14 23:29:06 - r - INFO: - Episode: 1252/2000, Reward: -111.689, Step: 77 +2023-04-14 23:29:09 - r - INFO: - Episode: 1253/2000, Reward: -97.895, Step: 146 +2023-04-14 23:29:11 - r - INFO: - Episode: 1254/2000, Reward: -129.071, Step: 107 +2023-04-14 23:29:16 - r - INFO: - Episode: 1255/2000, Reward: -130.459, Step: 174 +2023-04-14 23:29:18 - r - INFO: - Episode: 1256/2000, Reward: -111.020, Step: 90 +2023-04-14 23:29:21 - r - INFO: - Episode: 1257/2000, Reward: -125.185, Step: 120 +2023-04-14 23:29:23 - r - INFO: - Episode: 1258/2000, Reward: -118.608, Step: 75 +2023-04-14 23:29:25 - r - INFO: - Episode: 1259/2000, Reward: -123.879, Step: 97 +2023-04-14 23:29:27 - r - INFO: - Episode: 1260/2000, Reward: -135.509, Step: 94 +2023-04-14 23:29:31 - r - INFO: - Episode: 1261/2000, Reward: -126.490, Step: 113 +2023-04-14 23:29:33 - r - INFO: - Episode: 1262/2000, Reward: -128.191, Step: 68 +2023-04-14 23:29:40 - r - INFO: - Episode: 1263/2000, Reward: -19.499, Step: 300 +2023-04-14 23:29:46 - r - INFO: - Episode: 1264/2000, Reward: -18.977, Step: 300 +2023-04-14 23:29:54 - r - INFO: - Episode: 1265/2000, Reward: -22.799, Step: 300 +2023-04-14 23:30:01 - r - INFO: - Episode: 1266/2000, Reward: -24.661, Step: 300 +2023-04-14 23:30:09 - r - INFO: - Episode: 1267/2000, Reward: -25.001, Step: 300 +2023-04-14 23:30:17 - r - INFO: - Episode: 1268/2000, Reward: -23.811, Step: 300 +2023-04-14 23:30:24 - r - INFO: - Episode: 1269/2000, Reward: -13.859, Step: 300 +2023-04-14 23:30:30 - r - INFO: - Episode: 1270/2000, Reward: -129.014, Step: 249 +2023-04-14 23:30:32 - r - INFO: - Episode: 1271/2000, Reward: -107.268, Step: 63 +2023-04-14 23:30:36 - r - INFO: - Episode: 1272/2000, Reward: -105.071, Step: 158 +2023-04-14 23:30:38 - r - INFO: - Episode: 1273/2000, Reward: -125.899, Step: 84 +2023-04-14 23:30:42 - r - INFO: - Episode: 1274/2000, Reward: -115.864, Step: 171 +2023-04-14 23:30:45 - r - INFO: - Episode: 1275/2000, Reward: -97.976, Step: 124 +2023-04-14 23:30:48 - r - INFO: - Episode: 1276/2000, Reward: -98.976, Step: 137 +2023-04-14 23:30:51 - r - INFO: - Episode: 1277/2000, Reward: -99.077, Step: 97 +2023-04-14 23:30:53 - r - INFO: - Episode: 1278/2000, Reward: -131.328, Step: 88 +2023-04-14 23:30:55 - r - INFO: - Episode: 1279/2000, Reward: -124.756, Step: 91 +2023-04-14 23:30:58 - r - INFO: - Episode: 1280/2000, Reward: -139.748, Step: 145 +2023-04-14 23:31:06 - r - INFO: - Episode: 1281/2000, Reward: -115.585, Step: 160 +2023-04-14 23:31:15 - r - INFO: - Episode: 1282/2000, Reward: 58.381, Step: 300 +2023-04-14 23:31:23 - r - INFO: - Episode: 1283/2000, Reward: 60.660, Step: 300 +2023-04-14 23:31:28 - r - INFO: - Episode: 1284/2000, Reward: -86.092, Step: 204 +2023-04-14 23:31:33 - r - INFO: - Episode: 1285/2000, Reward: -115.583, Step: 199 +2023-04-14 23:31:40 - r - INFO: - Episode: 1286/2000, Reward: -27.411, Step: 300 +2023-04-14 23:31:47 - r - INFO: - Episode: 1287/2000, Reward: -33.875, Step: 300 +2023-04-14 23:31:54 - r - INFO: - Episode: 1288/2000, Reward: -32.934, Step: 300 +2023-04-14 23:32:01 - r - INFO: - Episode: 1289/2000, Reward: -37.839, Step: 300 +2023-04-14 23:32:09 - r - INFO: - Episode: 1290/2000, Reward: -32.036, Step: 300 +2023-04-14 23:32:16 - r - INFO: - Episode: 1291/2000, Reward: -27.863, Step: 300 +2023-04-14 23:32:19 - r - INFO: - Episode: 1292/2000, Reward: -113.433, Step: 78 +2023-04-14 23:32:22 - r - INFO: - Episode: 1293/2000, Reward: -134.461, Step: 129 +2023-04-14 23:32:24 - r - INFO: - Episode: 1294/2000, Reward: -86.639, Step: 121 +2023-04-14 23:32:27 - r - INFO: - Episode: 1295/2000, Reward: -100.422, Step: 116 +2023-04-14 23:32:34 - r - INFO: - Episode: 1296/2000, Reward: -18.552, Step: 300 +2023-04-14 23:32:41 - r - INFO: - Episode: 1297/2000, Reward: -23.260, Step: 300 +2023-04-14 23:32:48 - r - INFO: - Episode: 1298/2000, Reward: -19.504, Step: 300 +2023-04-14 23:32:51 - r - INFO: - Episode: 1299/2000, Reward: -126.011, Step: 150 +2023-04-14 23:32:58 - r - INFO: - Episode: 1300/2000, Reward: -25.081, Step: 300 +2023-04-14 23:33:11 - r - INFO: - Episode: 1301/2000, Reward: -10.573, Step: 300 +2023-04-14 23:33:13 - r - INFO: - Episode: 1302/2000, Reward: -99.195, Step: 89 +2023-04-14 23:33:15 - r - INFO: - Episode: 1303/2000, Reward: -99.651, Step: 111 +2023-04-14 23:33:23 - r - INFO: - Episode: 1304/2000, Reward: -22.245, Step: 300 +2023-04-14 23:33:27 - r - INFO: - Episode: 1305/2000, Reward: -96.381, Step: 191 +2023-04-14 23:33:30 - r - INFO: - Episode: 1306/2000, Reward: -101.638, Step: 109 +2023-04-14 23:33:31 - r - INFO: - Episode: 1307/2000, Reward: -101.102, Step: 59 +2023-04-14 23:33:39 - r - INFO: - Episode: 1308/2000, Reward: -137.498, Step: 255 +2023-04-14 23:33:45 - r - INFO: - Episode: 1309/2000, Reward: -14.349, Step: 300 +2023-04-14 23:33:53 - r - INFO: - Episode: 1310/2000, Reward: -8.653, Step: 300 +2023-04-14 23:33:59 - r - INFO: - Episode: 1311/2000, Reward: -21.739, Step: 300 +2023-04-14 23:34:04 - r - INFO: - Episode: 1312/2000, Reward: -99.929, Step: 180 +2023-04-14 23:34:05 - r - INFO: - Episode: 1313/2000, Reward: -106.698, Step: 48 +2023-04-14 23:34:06 - r - INFO: - Episode: 1314/2000, Reward: -104.556, Step: 44 +2023-04-14 23:34:08 - r - INFO: - Episode: 1315/2000, Reward: -100.291, Step: 79 +2023-04-14 23:34:10 - r - INFO: - Episode: 1316/2000, Reward: -103.948, Step: 74 +2023-04-14 23:34:11 - r - INFO: - Episode: 1317/2000, Reward: -99.910, Step: 73 +2023-04-14 23:34:22 - r - INFO: - Episode: 1318/2000, Reward: -0.222, Step: 300 +2023-04-14 23:34:26 - r - INFO: - Episode: 1319/2000, Reward: -86.630, Step: 100 +2023-04-14 23:34:32 - r - INFO: - Episode: 1320/2000, Reward: -70.101, Step: 208 +2023-04-14 23:34:42 - r - INFO: - Episode: 1321/2000, Reward: -74.397, Step: 212 +2023-04-14 23:34:45 - r - INFO: - Episode: 1322/2000, Reward: -115.434, Step: 133 +2023-04-14 23:34:52 - r - INFO: - Episode: 1323/2000, Reward: 18.171, Step: 300 +2023-04-14 23:35:00 - r - INFO: - Episode: 1324/2000, Reward: -0.234, Step: 300 +2023-04-14 23:35:06 - r - INFO: - Episode: 1325/2000, Reward: -110.254, Step: 221 +2023-04-14 23:35:13 - r - INFO: - Episode: 1326/2000, Reward: -115.264, Step: 168 +2023-04-14 23:35:21 - r - INFO: - Episode: 1327/2000, Reward: -4.720, Step: 300 +2023-04-14 23:35:25 - r - INFO: - Episode: 1328/2000, Reward: -133.552, Step: 132 +2023-04-14 23:35:27 - r - INFO: - Episode: 1329/2000, Reward: -114.229, Step: 77 +2023-04-14 23:35:29 - r - INFO: - Episode: 1330/2000, Reward: -131.157, Step: 103 +2023-04-14 23:35:31 - r - INFO: - Episode: 1331/2000, Reward: -129.529, Step: 78 +2023-04-14 23:35:36 - r - INFO: - Episode: 1332/2000, Reward: -96.241, Step: 178 +2023-04-14 23:35:38 - r - INFO: - Episode: 1333/2000, Reward: -117.346, Step: 65 +2023-04-14 23:35:43 - r - INFO: - Episode: 1334/2000, Reward: -87.999, Step: 194 +2023-04-14 23:35:44 - r - INFO: - Episode: 1335/2000, Reward: -117.072, Step: 64 +2023-04-14 23:35:46 - r - INFO: - Episode: 1336/2000, Reward: -116.638, Step: 67 +2023-04-14 23:35:48 - r - INFO: - Episode: 1337/2000, Reward: -113.873, Step: 65 +2023-04-14 23:35:50 - r - INFO: - Episode: 1338/2000, Reward: -113.145, Step: 55 +2023-04-14 23:35:52 - r - INFO: - Episode: 1339/2000, Reward: -115.767, Step: 64 +2023-04-14 23:35:53 - r - INFO: - Episode: 1340/2000, Reward: -113.802, Step: 53 +2023-04-14 23:35:56 - r - INFO: - Episode: 1341/2000, Reward: -111.741, Step: 60 +2023-04-14 23:35:58 - r - INFO: - Episode: 1342/2000, Reward: -121.628, Step: 87 +2023-04-14 23:35:59 - r - INFO: - Episode: 1343/2000, Reward: -114.328, Step: 54 +2023-04-14 23:36:01 - r - INFO: - Episode: 1344/2000, Reward: -113.223, Step: 56 +2023-04-14 23:36:02 - r - INFO: - Episode: 1345/2000, Reward: -116.138, Step: 57 +2023-04-14 23:36:05 - r - INFO: - Episode: 1346/2000, Reward: -125.385, Step: 92 +2023-04-14 23:36:06 - r - INFO: - Episode: 1347/2000, Reward: -115.124, Step: 57 +2023-04-14 23:36:07 - r - INFO: - Episode: 1348/2000, Reward: -116.059, Step: 54 +2023-04-14 23:36:09 - r - INFO: - Episode: 1349/2000, Reward: -116.591, Step: 64 +2023-04-14 23:36:11 - r - INFO: - Episode: 1350/2000, Reward: -121.798, Step: 77 +2023-04-14 23:36:14 - r - INFO: - Episode: 1351/2000, Reward: -122.187, Step: 100 +2023-04-14 23:36:16 - r - INFO: - Episode: 1352/2000, Reward: -122.295, Step: 67 +2023-04-14 23:36:19 - r - INFO: - Episode: 1353/2000, Reward: -123.317, Step: 84 +2023-04-14 23:36:22 - r - INFO: - Episode: 1354/2000, Reward: -127.417, Step: 90 +2023-04-14 23:36:26 - r - INFO: - Episode: 1355/2000, Reward: -132.939, Step: 110 +2023-04-14 23:36:30 - r - INFO: - Episode: 1356/2000, Reward: -136.415, Step: 174 +2023-04-14 23:36:35 - r - INFO: - Episode: 1357/2000, Reward: -140.753, Step: 164 +2023-04-14 23:36:43 - r - INFO: - Episode: 1358/2000, Reward: 19.578, Step: 300 +2023-04-14 23:36:48 - r - INFO: - Episode: 1359/2000, Reward: -86.615, Step: 211 +2023-04-14 23:36:56 - r - INFO: - Episode: 1360/2000, Reward: -11.702, Step: 300 +2023-04-14 23:37:01 - r - INFO: - Episode: 1361/2000, Reward: -135.531, Step: 151 +2023-04-14 23:37:04 - r - INFO: - Episode: 1362/2000, Reward: -104.160, Step: 94 +2023-04-14 23:37:12 - r - INFO: - Episode: 1363/2000, Reward: -23.578, Step: 300 +2023-04-14 23:37:16 - r - INFO: - Episode: 1364/2000, Reward: -104.381, Step: 166 +2023-04-14 23:37:22 - r - INFO: - Episode: 1365/2000, Reward: -87.264, Step: 237 +2023-04-14 23:37:26 - r - INFO: - Episode: 1366/2000, Reward: -93.256, Step: 145 +2023-04-14 23:37:29 - r - INFO: - Episode: 1367/2000, Reward: -103.935, Step: 90 +2023-04-14 23:37:33 - r - INFO: - Episode: 1368/2000, Reward: -126.192, Step: 170 +2023-04-14 23:37:43 - r - INFO: - Episode: 1369/2000, Reward: 35.276, Step: 300 +2023-04-14 23:37:56 - r - INFO: - Episode: 1370/2000, Reward: -0.889, Step: 300 +2023-04-14 23:38:07 - r - INFO: - Episode: 1371/2000, Reward: -104.684, Step: 101 +2023-04-14 23:38:19 - r - INFO: - Episode: 1372/2000, Reward: -112.018, Step: 194 +2023-04-14 23:38:24 - r - INFO: - Episode: 1373/2000, Reward: -69.723, Step: 213 +2023-04-14 23:38:29 - r - INFO: - Episode: 1374/2000, Reward: -108.704, Step: 186 +2023-04-14 23:38:34 - r - INFO: - Episode: 1375/2000, Reward: -91.788, Step: 141 +2023-04-14 23:38:39 - r - INFO: - Episode: 1376/2000, Reward: -121.087, Step: 161 +2023-04-14 23:38:45 - r - INFO: - Episode: 1377/2000, Reward: -91.231, Step: 234 +2023-04-14 23:38:50 - r - INFO: - Episode: 1378/2000, Reward: -79.006, Step: 185 +2023-04-14 23:38:53 - r - INFO: - Episode: 1379/2000, Reward: -141.827, Step: 122 +2023-04-14 23:38:56 - r - INFO: - Episode: 1380/2000, Reward: -110.454, Step: 114 +2023-04-14 23:39:04 - r - INFO: - Episode: 1381/2000, Reward: -139.588, Step: 194 +2023-04-14 23:39:07 - r - INFO: - Episode: 1382/2000, Reward: -109.302, Step: 148 +2023-04-14 23:39:10 - r - INFO: - Episode: 1383/2000, Reward: -124.375, Step: 107 +2023-04-14 23:39:12 - r - INFO: - Episode: 1384/2000, Reward: -119.234, Step: 64 +2023-04-14 23:39:17 - r - INFO: - Episode: 1385/2000, Reward: -127.216, Step: 227 +2023-04-14 23:39:19 - r - INFO: - Episode: 1386/2000, Reward: -116.982, Step: 70 +2023-04-14 23:39:21 - r - INFO: - Episode: 1387/2000, Reward: -94.051, Step: 91 +2023-04-14 23:39:24 - r - INFO: - Episode: 1388/2000, Reward: -91.076, Step: 133 +2023-04-14 23:39:28 - r - INFO: - Episode: 1389/2000, Reward: -94.101, Step: 96 +2023-04-14 23:39:34 - r - INFO: - Episode: 1390/2000, Reward: -105.008, Step: 279 +2023-04-14 23:39:40 - r - INFO: - Episode: 1391/2000, Reward: -85.668, Step: 242 +2023-04-14 23:39:43 - r - INFO: - Episode: 1392/2000, Reward: -87.797, Step: 134 +2023-04-14 23:39:50 - r - INFO: - Episode: 1393/2000, Reward: 24.796, Step: 300 +2023-04-14 23:39:57 - r - INFO: - Episode: 1394/2000, Reward: 5.943, Step: 300 +2023-04-14 23:40:05 - r - INFO: - Episode: 1395/2000, Reward: -5.758, Step: 300 +2023-04-14 23:40:08 - r - INFO: - Episode: 1396/2000, Reward: -108.725, Step: 92 +2023-04-14 23:40:10 - r - INFO: - Episode: 1397/2000, Reward: -120.491, Step: 73 +2023-04-14 23:40:16 - r - INFO: - Episode: 1398/2000, Reward: -131.839, Step: 227 +2023-04-14 23:40:20 - r - INFO: - Episode: 1399/2000, Reward: -117.692, Step: 165 +2023-04-14 23:40:27 - r - INFO: - Episode: 1400/2000, Reward: -123.747, Step: 259 +2023-04-14 23:40:37 - r - INFO: - Episode: 1401/2000, Reward: -110.153, Step: 131 +2023-04-14 23:40:49 - r - INFO: - Episode: 1402/2000, Reward: 8.364, Step: 300 +2023-04-14 23:40:52 - r - INFO: - Episode: 1403/2000, Reward: -92.425, Step: 128 +2023-04-14 23:40:54 - r - INFO: - Episode: 1404/2000, Reward: -120.444, Step: 81 +2023-04-14 23:41:04 - r - INFO: - Episode: 1405/2000, Reward: 21.941, Step: 300 +2023-04-14 23:41:12 - r - INFO: - Episode: 1406/2000, Reward: -8.672, Step: 300 +2023-04-14 23:41:16 - r - INFO: - Episode: 1407/2000, Reward: -130.186, Step: 129 +2023-04-14 23:41:23 - r - INFO: - Episode: 1408/2000, Reward: -75.557, Step: 260 +2023-04-14 23:41:32 - r - INFO: - Episode: 1409/2000, Reward: 11.967, Step: 300 +2023-04-14 23:41:38 - r - INFO: - Episode: 1410/2000, Reward: -99.667, Step: 254 +2023-04-14 23:41:42 - r - INFO: - Episode: 1411/2000, Reward: -100.144, Step: 150 +2023-04-14 23:41:48 - r - INFO: - Episode: 1412/2000, Reward: -94.119, Step: 251 +2023-04-14 23:41:53 - r - INFO: - Episode: 1413/2000, Reward: -114.746, Step: 229 +2023-04-14 23:41:57 - r - INFO: - Episode: 1414/2000, Reward: -101.113, Step: 169 +2023-04-14 23:42:00 - r - INFO: - Episode: 1415/2000, Reward: -102.843, Step: 123 +2023-04-14 23:42:04 - r - INFO: - Episode: 1416/2000, Reward: -109.107, Step: 175 +2023-04-14 23:42:11 - r - INFO: - Episode: 1417/2000, Reward: -102.857, Step: 279 +2023-04-14 23:42:17 - r - INFO: - Episode: 1418/2000, Reward: -68.144, Step: 228 +2023-04-14 23:42:23 - r - INFO: - Episode: 1419/2000, Reward: -79.536, Step: 250 +2023-04-14 23:42:31 - r - INFO: - Episode: 1420/2000, Reward: 28.307, Step: 300 +2023-04-14 23:42:41 - r - INFO: - Episode: 1421/2000, Reward: -101.723, Step: 249 +2023-04-14 23:42:48 - r - INFO: - Episode: 1422/2000, Reward: -81.656, Step: 265 +2023-04-14 23:42:52 - r - INFO: - Episode: 1423/2000, Reward: -121.929, Step: 200 +2023-04-14 23:42:57 - r - INFO: - Episode: 1424/2000, Reward: -99.803, Step: 185 +2023-04-14 23:42:59 - r - INFO: - Episode: 1425/2000, Reward: -113.119, Step: 71 +2023-04-14 23:43:02 - r - INFO: - Episode: 1426/2000, Reward: -106.726, Step: 119 +2023-04-14 23:43:07 - r - INFO: - Episode: 1427/2000, Reward: -69.820, Step: 167 +2023-04-14 23:43:11 - r - INFO: - Episode: 1428/2000, Reward: -93.626, Step: 152 +2023-04-14 23:43:14 - r - INFO: - Episode: 1429/2000, Reward: -100.359, Step: 110 +2023-04-14 23:43:22 - r - INFO: - Episode: 1430/2000, Reward: 36.459, Step: 300 +2023-04-14 23:43:30 - r - INFO: - Episode: 1431/2000, Reward: -28.096, Step: 300 +2023-04-14 23:43:33 - r - INFO: - Episode: 1432/2000, Reward: -97.266, Step: 114 +2023-04-14 23:43:41 - r - INFO: - Episode: 1433/2000, Reward: -12.703, Step: 300 +2023-04-14 23:43:49 - r - INFO: - Episode: 1434/2000, Reward: 3.788, Step: 300 +2023-04-14 23:43:56 - r - INFO: - Episode: 1435/2000, Reward: -16.072, Step: 300 +2023-04-14 23:43:57 - r - INFO: - Episode: 1436/2000, Reward: -110.289, Step: 45 +2023-04-14 23:44:04 - r - INFO: - Episode: 1437/2000, Reward: -26.409, Step: 300 +2023-04-14 23:44:12 - r - INFO: - Episode: 1438/2000, Reward: -28.730, Step: 300 +2023-04-14 23:44:19 - r - INFO: - Episode: 1439/2000, Reward: -26.687, Step: 300 +2023-04-14 23:44:26 - r - INFO: - Episode: 1440/2000, Reward: -27.394, Step: 300 +2023-04-14 23:44:38 - r - INFO: - Episode: 1441/2000, Reward: -21.980, Step: 300 +2023-04-14 23:44:53 - r - INFO: - Episode: 1442/2000, Reward: -27.046, Step: 300 +2023-04-14 23:45:07 - r - INFO: - Episode: 1443/2000, Reward: -26.475, Step: 300 +2023-04-14 23:45:14 - r - INFO: - Episode: 1444/2000, Reward: -26.489, Step: 300 +2023-04-14 23:45:16 - r - INFO: - Episode: 1445/2000, Reward: -115.270, Step: 66 +2023-04-14 23:45:24 - r - INFO: - Episode: 1446/2000, Reward: 6.981, Step: 300 +2023-04-14 23:45:27 - r - INFO: - Episode: 1447/2000, Reward: -96.328, Step: 112 +2023-04-14 23:45:36 - r - INFO: - Episode: 1448/2000, Reward: -144.455, Step: 264 +2023-04-14 23:45:39 - r - INFO: - Episode: 1449/2000, Reward: -130.940, Step: 104 +2023-04-14 23:45:40 - r - INFO: - Episode: 1450/2000, Reward: -110.621, Step: 51 +2023-04-14 23:45:43 - r - INFO: - Episode: 1451/2000, Reward: -130.829, Step: 112 +2023-04-14 23:45:45 - r - INFO: - Episode: 1452/2000, Reward: -126.768, Step: 74 +2023-04-14 23:45:46 - r - INFO: - Episode: 1453/2000, Reward: -102.838, Step: 59 +2023-04-14 23:45:51 - r - INFO: - Episode: 1454/2000, Reward: -60.158, Step: 227 +2023-04-14 23:45:53 - r - INFO: - Episode: 1455/2000, Reward: -112.175, Step: 72 +2023-04-14 23:45:55 - r - INFO: - Episode: 1456/2000, Reward: -104.491, Step: 73 +2023-04-14 23:46:01 - r - INFO: - Episode: 1457/2000, Reward: -90.119, Step: 231 +2023-04-14 23:46:08 - r - INFO: - Episode: 1458/2000, Reward: -151.743, Step: 273 +2023-04-14 23:46:09 - r - INFO: - Episode: 1459/2000, Reward: -124.872, Step: 76 +2023-04-14 23:46:17 - r - INFO: - Episode: 1460/2000, Reward: -35.180, Step: 300 +2023-04-14 23:46:24 - r - INFO: - Episode: 1461/2000, Reward: -89.136, Step: 135 +2023-04-14 23:46:28 - r - INFO: - Episode: 1462/2000, Reward: -115.367, Step: 154 +2023-04-14 23:46:31 - r - INFO: - Episode: 1463/2000, Reward: -130.445, Step: 117 +2023-04-14 23:46:33 - r - INFO: - Episode: 1464/2000, Reward: -110.058, Step: 80 +2023-04-14 23:46:36 - r - INFO: - Episode: 1465/2000, Reward: -113.482, Step: 129 +2023-04-14 23:46:43 - r - INFO: - Episode: 1466/2000, Reward: -78.262, Step: 277 +2023-04-14 23:46:46 - r - INFO: - Episode: 1467/2000, Reward: -94.361, Step: 151 +2023-04-14 23:46:53 - r - INFO: - Episode: 1468/2000, Reward: -6.307, Step: 300 +2023-04-14 23:47:00 - r - INFO: - Episode: 1469/2000, Reward: -26.937, Step: 300 +2023-04-14 23:47:06 - r - INFO: - Episode: 1470/2000, Reward: -108.564, Step: 248 +2023-04-14 23:47:13 - r - INFO: - Episode: 1471/2000, Reward: -12.307, Step: 300 +2023-04-14 23:47:15 - r - INFO: - Episode: 1472/2000, Reward: -120.352, Step: 68 +2023-04-14 23:47:18 - r - INFO: - Episode: 1473/2000, Reward: -99.203, Step: 138 +2023-04-14 23:47:25 - r - INFO: - Episode: 1474/2000, Reward: 7.513, Step: 300 +2023-04-14 23:47:33 - r - INFO: - Episode: 1475/2000, Reward: -20.347, Step: 300 +2023-04-14 23:47:41 - r - INFO: - Episode: 1476/2000, Reward: 24.684, Step: 300 +2023-04-14 23:47:48 - r - INFO: - Episode: 1477/2000, Reward: -10.917, Step: 300 +2023-04-14 23:47:50 - r - INFO: - Episode: 1478/2000, Reward: -93.704, Step: 86 +2023-04-14 23:47:54 - r - INFO: - Episode: 1479/2000, Reward: -137.090, Step: 189 +2023-04-14 23:47:55 - r - INFO: - Episode: 1480/2000, Reward: -109.207, Step: 37 +2023-04-14 23:48:11 - r - INFO: - Episode: 1481/2000, Reward: -123.502, Step: 196 +2023-04-14 23:48:15 - r - INFO: - Episode: 1482/2000, Reward: -120.611, Step: 123 +2023-04-14 23:48:18 - r - INFO: - Episode: 1483/2000, Reward: -104.594, Step: 139 +2023-04-14 23:48:25 - r - INFO: - Episode: 1484/2000, Reward: -24.900, Step: 300 +2023-04-14 23:48:26 - r - INFO: - Episode: 1485/2000, Reward: -111.124, Step: 46 +2023-04-14 23:48:34 - r - INFO: - Episode: 1486/2000, Reward: -29.352, Step: 300 +2023-04-14 23:48:40 - r - INFO: - Episode: 1487/2000, Reward: -126.124, Step: 172 +2023-04-14 23:48:48 - r - INFO: - Episode: 1488/2000, Reward: -30.163, Step: 300 +2023-04-14 23:48:49 - r - INFO: - Episode: 1489/2000, Reward: -107.217, Step: 47 +2023-04-14 23:48:56 - r - INFO: - Episode: 1490/2000, Reward: -27.840, Step: 300 +2023-04-14 23:49:04 - r - INFO: - Episode: 1491/2000, Reward: -27.757, Step: 300 +2023-04-14 23:49:11 - r - INFO: - Episode: 1492/2000, Reward: -35.620, Step: 300 +2023-04-14 23:49:18 - r - INFO: - Episode: 1493/2000, Reward: -26.313, Step: 300 +2023-04-14 23:49:25 - r - INFO: - Episode: 1494/2000, Reward: -29.084, Step: 300 +2023-04-14 23:49:29 - r - INFO: - Episode: 1495/2000, Reward: -100.774, Step: 130 +2023-04-14 23:49:31 - r - INFO: - Episode: 1496/2000, Reward: -110.390, Step: 124 +2023-04-14 23:49:34 - r - INFO: - Episode: 1497/2000, Reward: -96.995, Step: 108 +2023-04-14 23:49:42 - r - INFO: - Episode: 1498/2000, Reward: -26.958, Step: 300 +2023-04-14 23:49:49 - r - INFO: - Episode: 1499/2000, Reward: -23.830, Step: 300 +2023-04-14 23:49:54 - r - INFO: - Episode: 1500/2000, Reward: -94.691, Step: 249 +2023-04-14 23:50:02 - r - INFO: - Episode: 1501/2000, Reward: -90.474, Step: 221 +2023-04-14 23:50:04 - r - INFO: - Episode: 1502/2000, Reward: -101.707, Step: 64 +2023-04-14 23:50:07 - r - INFO: - Episode: 1503/2000, Reward: -103.440, Step: 121 +2023-04-14 23:50:10 - r - INFO: - Episode: 1504/2000, Reward: -111.129, Step: 164 +2023-04-14 23:50:17 - r - INFO: - Episode: 1505/2000, Reward: -8.921, Step: 300 +2023-04-14 23:50:19 - r - INFO: - Episode: 1506/2000, Reward: -106.919, Step: 85 +2023-04-14 23:50:26 - r - INFO: - Episode: 1507/2000, Reward: -21.918, Step: 300 +2023-04-14 23:50:33 - r - INFO: - Episode: 1508/2000, Reward: -25.144, Step: 300 +2023-04-14 23:50:39 - r - INFO: - Episode: 1509/2000, Reward: -108.917, Step: 216 +2023-04-14 23:50:45 - r - INFO: - Episode: 1510/2000, Reward: -120.677, Step: 250 +2023-04-14 23:50:48 - r - INFO: - Episode: 1511/2000, Reward: -90.249, Step: 128 +2023-04-14 23:50:50 - r - INFO: - Episode: 1512/2000, Reward: -121.198, Step: 75 +2023-04-14 23:50:54 - r - INFO: - Episode: 1513/2000, Reward: -92.324, Step: 167 +2023-04-14 23:51:01 - r - INFO: - Episode: 1514/2000, Reward: -17.429, Step: 300 +2023-04-14 23:51:09 - r - INFO: - Episode: 1515/2000, Reward: -17.293, Step: 300 +2023-04-14 23:51:16 - r - INFO: - Episode: 1516/2000, Reward: -11.126, Step: 300 +2023-04-14 23:51:23 - r - INFO: - Episode: 1517/2000, Reward: -98.308, Step: 272 +2023-04-14 23:51:30 - r - INFO: - Episode: 1518/2000, Reward: -105.741, Step: 249 +2023-04-14 23:51:34 - r - INFO: - Episode: 1519/2000, Reward: -98.929, Step: 124 +2023-04-14 23:51:37 - r - INFO: - Episode: 1520/2000, Reward: -92.464, Step: 89 +2023-04-14 23:51:46 - r - INFO: - Episode: 1521/2000, Reward: -91.097, Step: 145 +2023-04-14 23:51:49 - r - INFO: - Episode: 1522/2000, Reward: -100.993, Step: 106 +2023-04-14 23:51:56 - r - INFO: - Episode: 1523/2000, Reward: -27.736, Step: 300 +2023-04-14 23:52:02 - r - INFO: - Episode: 1524/2000, Reward: -28.411, Step: 300 +2023-04-14 23:52:10 - r - INFO: - Episode: 1525/2000, Reward: -25.794, Step: 300 +2023-04-14 23:52:15 - r - INFO: - Episode: 1526/2000, Reward: -70.260, Step: 194 +2023-04-14 23:52:20 - r - INFO: - Episode: 1527/2000, Reward: -92.581, Step: 205 +2023-04-14 23:52:23 - r - INFO: - Episode: 1528/2000, Reward: -89.538, Step: 96 +2023-04-14 23:52:26 - r - INFO: - Episode: 1529/2000, Reward: -85.707, Step: 102 +2023-04-14 23:52:30 - r - INFO: - Episode: 1530/2000, Reward: -111.472, Step: 175 +2023-04-14 23:52:35 - r - INFO: - Episode: 1531/2000, Reward: -80.385, Step: 199 +2023-04-14 23:52:37 - r - INFO: - Episode: 1532/2000, Reward: -122.058, Step: 85 +2023-04-14 23:52:39 - r - INFO: - Episode: 1533/2000, Reward: -99.985, Step: 60 +2023-04-14 23:52:40 - r - INFO: - Episode: 1534/2000, Reward: -101.820, Step: 66 +2023-04-14 23:52:43 - r - INFO: - Episode: 1535/2000, Reward: -93.442, Step: 84 +2023-04-14 23:52:51 - r - INFO: - Episode: 1536/2000, Reward: 40.521, Step: 300 +2023-04-14 23:52:53 - r - INFO: - Episode: 1537/2000, Reward: -96.273, Step: 96 +2023-04-14 23:52:58 - r - INFO: - Episode: 1538/2000, Reward: -62.491, Step: 190 +2023-04-14 23:53:02 - r - INFO: - Episode: 1539/2000, Reward: -102.136, Step: 131 +2023-04-14 23:53:03 - r - INFO: - Episode: 1540/2000, Reward: -102.943, Step: 73 +2023-04-14 23:53:14 - r - INFO: - Episode: 1541/2000, Reward: -17.531, Step: 300 +2023-04-14 23:53:22 - r - INFO: - Episode: 1542/2000, Reward: -16.506, Step: 300 +2023-04-14 23:53:23 - r - INFO: - Episode: 1543/2000, Reward: -101.381, Step: 55 +2023-04-14 23:53:30 - r - INFO: - Episode: 1544/2000, Reward: -14.754, Step: 300 +2023-04-14 23:53:41 - r - INFO: - Episode: 1545/2000, Reward: -18.075, Step: 300 +2023-04-14 23:53:53 - r - INFO: - Episode: 1546/2000, Reward: 0.976, Step: 300 +2023-04-14 23:53:57 - r - INFO: - Episode: 1547/2000, Reward: -96.930, Step: 119 +2023-04-14 23:54:01 - r - INFO: - Episode: 1548/2000, Reward: -80.451, Step: 154 +2023-04-14 23:54:04 - r - INFO: - Episode: 1549/2000, Reward: -96.790, Step: 122 +2023-04-14 23:54:06 - r - INFO: - Episode: 1550/2000, Reward: -98.520, Step: 84 +2023-04-14 23:54:10 - r - INFO: - Episode: 1551/2000, Reward: -76.315, Step: 167 +2023-04-14 23:54:13 - r - INFO: - Episode: 1552/2000, Reward: -103.994, Step: 74 +2023-04-14 23:54:21 - r - INFO: - Episode: 1553/2000, Reward: -20.657, Step: 300 +2023-04-14 23:54:23 - r - INFO: - Episode: 1554/2000, Reward: -106.576, Step: 81 +2023-04-14 23:54:31 - r - INFO: - Episode: 1555/2000, Reward: -27.538, Step: 300 +2023-04-14 23:54:32 - r - INFO: - Episode: 1556/2000, Reward: -108.435, Step: 49 +2023-04-14 23:54:39 - r - INFO: - Episode: 1557/2000, Reward: -28.201, Step: 300 +2023-04-14 23:54:42 - r - INFO: - Episode: 1558/2000, Reward: -107.543, Step: 64 +2023-04-14 23:54:44 - r - INFO: - Episode: 1559/2000, Reward: -110.711, Step: 64 +2023-04-14 23:54:52 - r - INFO: - Episode: 1560/2000, Reward: -29.906, Step: 300 +2023-04-14 23:54:58 - r - INFO: - Episode: 1561/2000, Reward: -97.336, Step: 106 +2023-04-14 23:55:04 - r - INFO: - Episode: 1562/2000, Reward: -142.824, Step: 203 +2023-04-14 23:55:12 - r - INFO: - Episode: 1563/2000, Reward: -76.496, Step: 286 +2023-04-14 23:55:16 - r - INFO: - Episode: 1564/2000, Reward: -94.201, Step: 174 +2023-04-14 23:55:21 - r - INFO: - Episode: 1565/2000, Reward: -81.168, Step: 197 +2023-04-14 23:55:25 - r - INFO: - Episode: 1566/2000, Reward: -93.734, Step: 120 +2023-04-14 23:55:28 - r - INFO: - Episode: 1567/2000, Reward: -87.253, Step: 112 +2023-04-14 23:55:32 - r - INFO: - Episode: 1568/2000, Reward: -98.948, Step: 159 +2023-04-14 23:55:38 - r - INFO: - Episode: 1569/2000, Reward: -90.996, Step: 189 +2023-04-14 23:55:44 - r - INFO: - Episode: 1570/2000, Reward: -91.357, Step: 216 +2023-04-14 23:55:48 - r - INFO: - Episode: 1571/2000, Reward: -115.689, Step: 166 +2023-04-14 23:55:53 - r - INFO: - Episode: 1572/2000, Reward: -88.660, Step: 174 +2023-04-14 23:55:57 - r - INFO: - Episode: 1573/2000, Reward: -85.937, Step: 150 +2023-04-14 23:56:05 - r - INFO: - Episode: 1574/2000, Reward: -140.531, Step: 293 +2023-04-14 23:56:08 - r - INFO: - Episode: 1575/2000, Reward: -131.591, Step: 130 +2023-04-14 23:56:16 - r - INFO: - Episode: 1576/2000, Reward: -22.750, Step: 300 +2023-04-14 23:56:18 - r - INFO: - Episode: 1577/2000, Reward: -105.302, Step: 74 +2023-04-14 23:56:24 - r - INFO: - Episode: 1578/2000, Reward: -141.659, Step: 204 +2023-04-14 23:56:26 - r - INFO: - Episode: 1579/2000, Reward: -108.212, Step: 66 +2023-04-14 23:56:30 - r - INFO: - Episode: 1580/2000, Reward: -105.870, Step: 140 +2023-04-14 23:56:37 - r - INFO: - Episode: 1581/2000, Reward: -94.862, Step: 152 +2023-04-14 23:56:40 - r - INFO: - Episode: 1582/2000, Reward: -104.274, Step: 78 +2023-04-14 23:56:44 - r - INFO: - Episode: 1583/2000, Reward: -97.254, Step: 134 +2023-04-14 23:56:47 - r - INFO: - Episode: 1584/2000, Reward: -102.063, Step: 75 +2023-04-14 23:56:51 - r - INFO: - Episode: 1585/2000, Reward: -85.375, Step: 122 +2023-04-14 23:56:54 - r - INFO: - Episode: 1586/2000, Reward: -84.339, Step: 131 +2023-04-14 23:56:57 - r - INFO: - Episode: 1587/2000, Reward: -112.335, Step: 98 +2023-04-14 23:57:02 - r - INFO: - Episode: 1588/2000, Reward: -132.734, Step: 182 +2023-04-14 23:57:07 - r - INFO: - Episode: 1589/2000, Reward: -91.823, Step: 126 +2023-04-14 23:57:10 - r - INFO: - Episode: 1590/2000, Reward: -87.269, Step: 95 +2023-04-14 23:57:12 - r - INFO: - Episode: 1591/2000, Reward: -102.590, Step: 59 +2023-04-14 23:57:17 - r - INFO: - Episode: 1592/2000, Reward: -89.026, Step: 204 +2023-04-14 23:57:22 - r - INFO: - Episode: 1593/2000, Reward: -90.021, Step: 151 +2023-04-14 23:57:24 - r - INFO: - Episode: 1594/2000, Reward: -109.770, Step: 98 +2023-04-14 23:57:27 - r - INFO: - Episode: 1595/2000, Reward: -109.446, Step: 105 +2023-04-14 23:57:38 - r - INFO: - Episode: 1596/2000, Reward: -24.777, Step: 300 +2023-04-14 23:57:43 - r - INFO: - Episode: 1597/2000, Reward: -83.534, Step: 181 +2023-04-14 23:57:49 - r - INFO: - Episode: 1598/2000, Reward: -78.544, Step: 158 +2023-04-14 23:57:53 - r - INFO: - Episode: 1599/2000, Reward: -103.939, Step: 161 +2023-04-14 23:57:57 - r - INFO: - Episode: 1600/2000, Reward: -93.903, Step: 109 +2023-04-14 23:58:03 - r - INFO: - Episode: 1601/2000, Reward: -93.734, Step: 100 +2023-04-14 23:58:09 - r - INFO: - Episode: 1602/2000, Reward: -132.966, Step: 212 +2023-04-14 23:58:15 - r - INFO: - Episode: 1603/2000, Reward: -77.269, Step: 205 +2023-04-14 23:58:16 - r - INFO: - Episode: 1604/2000, Reward: -104.734, Step: 60 +2023-04-14 23:58:18 - r - INFO: - Episode: 1605/2000, Reward: -103.968, Step: 73 +2023-04-14 23:58:21 - r - INFO: - Episode: 1606/2000, Reward: -99.054, Step: 103 +2023-04-14 23:58:23 - r - INFO: - Episode: 1607/2000, Reward: -105.919, Step: 83 +2023-04-14 23:58:26 - r - INFO: - Episode: 1608/2000, Reward: -81.760, Step: 124 +2023-04-14 23:58:29 - r - INFO: - Episode: 1609/2000, Reward: -128.163, Step: 97 +2023-04-14 23:58:31 - r - INFO: - Episode: 1610/2000, Reward: -100.209, Step: 66 +2023-04-14 23:58:33 - r - INFO: - Episode: 1611/2000, Reward: -100.396, Step: 70 +2023-04-14 23:58:35 - r - INFO: - Episode: 1612/2000, Reward: -99.005, Step: 71 +2023-04-14 23:58:37 - r - INFO: - Episode: 1613/2000, Reward: -101.131, Step: 55 +2023-04-14 23:58:42 - r - INFO: - Episode: 1614/2000, Reward: -128.337, Step: 101 +2023-04-14 23:58:44 - r - INFO: - Episode: 1615/2000, Reward: -102.811, Step: 51 +2023-04-14 23:58:46 - r - INFO: - Episode: 1616/2000, Reward: -97.280, Step: 96 +2023-04-14 23:58:48 - r - INFO: - Episode: 1617/2000, Reward: -96.040, Step: 74 +2023-04-14 23:58:50 - r - INFO: - Episode: 1618/2000, Reward: -109.229, Step: 53 +2023-04-14 23:58:52 - r - INFO: - Episode: 1619/2000, Reward: -108.900, Step: 70 +2023-04-14 23:58:55 - r - INFO: - Episode: 1620/2000, Reward: -80.517, Step: 127 +2023-04-14 23:58:57 - r - INFO: - Episode: 1621/2000, Reward: -107.586, Step: 62 +2023-04-14 23:58:59 - r - INFO: - Episode: 1622/2000, Reward: -107.888, Step: 77 +2023-04-14 23:59:02 - r - INFO: - Episode: 1623/2000, Reward: -104.494, Step: 120 +2023-04-14 23:59:07 - r - INFO: - Episode: 1624/2000, Reward: -84.380, Step: 180 +2023-04-14 23:59:13 - r - INFO: - Episode: 1625/2000, Reward: -133.468, Step: 240 +2023-04-14 23:59:20 - r - INFO: - Episode: 1626/2000, Reward: -5.745, Step: 300 +2023-04-14 23:59:28 - r - INFO: - Episode: 1627/2000, Reward: -40.514, Step: 300 +2023-04-14 23:59:31 - r - INFO: - Episode: 1628/2000, Reward: -132.635, Step: 138 +2023-04-14 23:59:33 - r - INFO: - Episode: 1629/2000, Reward: -114.595, Step: 80 +2023-04-14 23:59:41 - r - INFO: - Episode: 1630/2000, Reward: -26.309, Step: 300 +2023-04-14 23:59:46 - r - INFO: - Episode: 1631/2000, Reward: -132.440, Step: 175 +2023-04-14 23:59:50 - r - INFO: - Episode: 1632/2000, Reward: -96.398, Step: 150 +2023-04-14 23:59:58 - r - INFO: - Episode: 1633/2000, Reward: -29.424, Step: 300 +2023-04-15 00:00:01 - r - INFO: - Episode: 1634/2000, Reward: -95.548, Step: 123 +2023-04-15 00:00:09 - r - INFO: - Episode: 1635/2000, Reward: 1.055, Step: 300 +2023-04-15 00:00:11 - r - INFO: - Episode: 1636/2000, Reward: -111.914, Step: 61 +2023-04-15 00:00:13 - r - INFO: - Episode: 1637/2000, Reward: -117.578, Step: 65 +2023-04-15 00:00:17 - r - INFO: - Episode: 1638/2000, Reward: -97.123, Step: 175 +2023-04-15 00:00:25 - r - INFO: - Episode: 1639/2000, Reward: -44.868, Step: 300 +2023-04-15 00:00:27 - r - INFO: - Episode: 1640/2000, Reward: -116.705, Step: 84 +2023-04-15 00:00:34 - r - INFO: - Episode: 1641/2000, Reward: -87.088, Step: 157 +2023-04-15 00:00:35 - r - INFO: - Episode: 1642/2000, Reward: -103.817, Step: 73 +2023-04-15 00:00:40 - r - INFO: - Episode: 1643/2000, Reward: -79.229, Step: 179 +2023-04-15 00:00:46 - r - INFO: - Episode: 1644/2000, Reward: -142.435, Step: 262 +2023-04-15 00:00:49 - r - INFO: - Episode: 1645/2000, Reward: -90.249, Step: 130 +2023-04-15 00:00:56 - r - INFO: - Episode: 1646/2000, Reward: -122.482, Step: 247 +2023-04-15 00:00:58 - r - INFO: - Episode: 1647/2000, Reward: -108.038, Step: 80 +2023-04-15 00:01:00 - r - INFO: - Episode: 1648/2000, Reward: -108.627, Step: 83 +2023-04-15 00:01:03 - r - INFO: - Episode: 1649/2000, Reward: -123.984, Step: 109 +2023-04-15 00:01:06 - r - INFO: - Episode: 1650/2000, Reward: -105.913, Step: 120 +2023-04-15 00:01:14 - r - INFO: - Episode: 1651/2000, Reward: -11.996, Step: 300 +2023-04-15 00:01:20 - r - INFO: - Episode: 1652/2000, Reward: -103.465, Step: 236 +2023-04-15 00:01:28 - r - INFO: - Episode: 1653/2000, Reward: -12.042, Step: 300 +2023-04-15 00:01:32 - r - INFO: - Episode: 1654/2000, Reward: -92.671, Step: 159 +2023-04-15 00:01:33 - r - INFO: - Episode: 1655/2000, Reward: -104.026, Step: 49 +2023-04-15 00:01:41 - r - INFO: - Episode: 1656/2000, Reward: -16.799, Step: 300 +2023-04-15 00:01:51 - r - INFO: - Episode: 1657/2000, Reward: -2.646, Step: 300 +2023-04-15 00:01:55 - r - INFO: - Episode: 1658/2000, Reward: -81.524, Step: 137 +2023-04-15 00:01:58 - r - INFO: - Episode: 1659/2000, Reward: -97.058, Step: 89 +2023-04-15 00:02:00 - r - INFO: - Episode: 1660/2000, Reward: -92.083, Step: 95 +2023-04-15 00:02:04 - r - INFO: - Episode: 1661/2000, Reward: -87.654, Step: 92 +2023-04-15 00:02:07 - r - INFO: - Episode: 1662/2000, Reward: -92.996, Step: 130 +2023-04-15 00:02:09 - r - INFO: - Episode: 1663/2000, Reward: -107.783, Step: 83 +2023-04-15 00:02:10 - r - INFO: - Episode: 1664/2000, Reward: -104.025, Step: 65 +2023-04-15 00:02:18 - r - INFO: - Episode: 1665/2000, Reward: -9.261, Step: 300 +2023-04-15 00:02:26 - r - INFO: - Episode: 1666/2000, Reward: 47.409, Step: 300 +2023-04-15 00:02:32 - r - INFO: - Episode: 1667/2000, Reward: -50.270, Step: 276 +2023-04-15 00:02:35 - r - INFO: - Episode: 1668/2000, Reward: -88.053, Step: 122 +2023-04-15 00:02:37 - r - INFO: - Episode: 1669/2000, Reward: -103.125, Step: 70 +2023-04-15 00:02:39 - r - INFO: - Episode: 1670/2000, Reward: -88.872, Step: 98 +2023-04-15 00:02:42 - r - INFO: - Episode: 1671/2000, Reward: -88.886, Step: 99 +2023-04-15 00:02:46 - r - INFO: - Episode: 1672/2000, Reward: -69.494, Step: 167 +2023-04-15 00:02:48 - r - INFO: - Episode: 1673/2000, Reward: -105.505, Step: 60 +2023-04-15 00:02:50 - r - INFO: - Episode: 1674/2000, Reward: -98.870, Step: 67 +2023-04-15 00:02:52 - r - INFO: - Episode: 1675/2000, Reward: -97.579, Step: 114 +2023-04-15 00:02:56 - r - INFO: - Episode: 1676/2000, Reward: -88.303, Step: 117 +2023-04-15 00:03:01 - r - INFO: - Episode: 1677/2000, Reward: -64.094, Step: 207 +2023-04-15 00:03:03 - r - INFO: - Episode: 1678/2000, Reward: -102.119, Step: 63 +2023-04-15 00:03:05 - r - INFO: - Episode: 1679/2000, Reward: -99.614, Step: 69 +2023-04-15 00:03:09 - r - INFO: - Episode: 1680/2000, Reward: -59.398, Step: 172 +2023-04-15 00:03:15 - r - INFO: - Current episode 1680 has the best eval reward: 53.515 +2023-04-15 00:03:18 - r - INFO: - Episode: 1681/2000, Reward: -89.579, Step: 123 +2023-04-15 00:03:22 - r - INFO: - Episode: 1682/2000, Reward: -62.235, Step: 161 +2023-04-15 00:03:24 - r - INFO: - Episode: 1683/2000, Reward: -102.677, Step: 58 +2023-04-15 00:03:26 - r - INFO: - Episode: 1684/2000, Reward: -97.238, Step: 78 +2023-04-15 00:03:27 - r - INFO: - Episode: 1685/2000, Reward: -106.710, Step: 50 +2023-04-15 00:03:30 - r - INFO: - Episode: 1686/2000, Reward: -97.837, Step: 95 +2023-04-15 00:03:31 - r - INFO: - Episode: 1687/2000, Reward: -102.352, Step: 55 +2023-04-15 00:03:35 - r - INFO: - Episode: 1688/2000, Reward: -85.641, Step: 114 +2023-04-15 00:03:38 - r - INFO: - Episode: 1689/2000, Reward: -96.939, Step: 98 +2023-04-15 00:03:46 - r - INFO: - Episode: 1690/2000, Reward: -61.829, Step: 240 +2023-04-15 00:03:53 - r - INFO: - Episode: 1691/2000, Reward: -68.257, Step: 253 +2023-04-15 00:03:57 - r - INFO: - Episode: 1692/2000, Reward: -86.823, Step: 137 +2023-04-15 00:04:01 - r - INFO: - Episode: 1693/2000, Reward: -89.768, Step: 136 +2023-04-15 00:04:02 - r - INFO: - Episode: 1694/2000, Reward: -104.367, Step: 58 +2023-04-15 00:04:11 - r - INFO: - Episode: 1695/2000, Reward: 41.988, Step: 300 +2023-04-15 00:04:18 - r - INFO: - Episode: 1696/2000, Reward: -70.874, Step: 226 +2023-04-15 00:04:22 - r - INFO: - Episode: 1697/2000, Reward: -94.115, Step: 146 +2023-04-15 00:04:29 - r - INFO: - Episode: 1698/2000, Reward: 57.150, Step: 300 +2023-04-15 00:04:31 - r - INFO: - Episode: 1699/2000, Reward: -104.660, Step: 64 +2023-04-15 00:04:33 - r - INFO: - Episode: 1700/2000, Reward: -99.207, Step: 74 +2023-04-15 00:04:46 - r - INFO: - Episode: 1701/2000, Reward: 30.261, Step: 300 +2023-04-15 00:04:49 - r - INFO: - Episode: 1702/2000, Reward: -82.093, Step: 140 +2023-04-15 00:04:51 - r - INFO: - Episode: 1703/2000, Reward: -104.120, Step: 64 +2023-04-15 00:04:53 - r - INFO: - Episode: 1704/2000, Reward: -105.247, Step: 49 +2023-04-15 00:04:56 - r - INFO: - Episode: 1705/2000, Reward: -100.014, Step: 79 +2023-04-15 00:04:59 - r - INFO: - Episode: 1706/2000, Reward: -86.230, Step: 144 +2023-04-15 00:05:02 - r - INFO: - Episode: 1707/2000, Reward: -86.790, Step: 102 +2023-04-15 00:05:04 - r - INFO: - Episode: 1708/2000, Reward: -112.001, Step: 75 +2023-04-15 00:05:06 - r - INFO: - Episode: 1709/2000, Reward: -110.014, Step: 80 +2023-04-15 00:05:10 - r - INFO: - Episode: 1710/2000, Reward: -86.361, Step: 179 +2023-04-15 00:05:15 - r - INFO: - Episode: 1711/2000, Reward: -64.496, Step: 196 +2023-04-15 00:05:17 - r - INFO: - Episode: 1712/2000, Reward: -100.213, Step: 92 +2023-04-15 00:05:19 - r - INFO: - Episode: 1713/2000, Reward: -96.016, Step: 85 +2023-04-15 00:05:21 - r - INFO: - Episode: 1714/2000, Reward: -86.865, Step: 100 +2023-04-15 00:05:24 - r - INFO: - Episode: 1715/2000, Reward: -95.740, Step: 126 +2023-04-15 00:05:26 - r - INFO: - Episode: 1716/2000, Reward: -101.461, Step: 64 +2023-04-15 00:05:28 - r - INFO: - Episode: 1717/2000, Reward: -91.661, Step: 117 +2023-04-15 00:05:33 - r - INFO: - Episode: 1718/2000, Reward: -98.622, Step: 182 +2023-04-15 00:05:40 - r - INFO: - Episode: 1719/2000, Reward: 39.557, Step: 300 +2023-04-15 00:05:45 - r - INFO: - Episode: 1720/2000, Reward: -70.805, Step: 212 +2023-04-15 00:05:49 - r - INFO: - Current episode 1720 has the best eval reward: 56.122 +2023-04-15 00:05:53 - r - INFO: - Episode: 1721/2000, Reward: -89.341, Step: 161 +2023-04-15 00:05:59 - r - INFO: - Episode: 1722/2000, Reward: -101.353, Step: 201 +2023-04-15 00:06:00 - r - INFO: - Episode: 1723/2000, Reward: -100.031, Step: 69 +2023-04-15 00:06:06 - r - INFO: - Episode: 1724/2000, Reward: -68.393, Step: 255 +2023-04-15 00:06:09 - r - INFO: - Episode: 1725/2000, Reward: -76.573, Step: 127 +2023-04-15 00:06:11 - r - INFO: - Episode: 1726/2000, Reward: -112.782, Step: 74 +2023-04-15 00:06:13 - r - INFO: - Episode: 1727/2000, Reward: -106.070, Step: 97 +2023-04-15 00:06:16 - r - INFO: - Episode: 1728/2000, Reward: -99.428, Step: 106 +2023-04-15 00:06:19 - r - INFO: - Episode: 1729/2000, Reward: -107.613, Step: 114 +2023-04-15 00:06:26 - r - INFO: - Episode: 1730/2000, Reward: 41.436, Step: 300 +2023-04-15 00:06:31 - r - INFO: - Episode: 1731/2000, Reward: -87.882, Step: 207 +2023-04-15 00:06:37 - r - INFO: - Episode: 1732/2000, Reward: -74.285, Step: 258 +2023-04-15 00:06:40 - r - INFO: - Episode: 1733/2000, Reward: -93.796, Step: 133 +2023-04-15 00:06:42 - r - INFO: - Episode: 1734/2000, Reward: -83.045, Step: 107 +2023-04-15 00:06:48 - r - INFO: - Episode: 1735/2000, Reward: -73.644, Step: 238 +2023-04-15 00:06:50 - r - INFO: - Episode: 1736/2000, Reward: -104.826, Step: 60 +2023-04-15 00:06:55 - r - INFO: - Episode: 1737/2000, Reward: -102.129, Step: 255 +2023-04-15 00:06:59 - r - INFO: - Episode: 1738/2000, Reward: -99.139, Step: 129 +2023-04-15 00:07:00 - r - INFO: - Episode: 1739/2000, Reward: -103.643, Step: 52 +2023-04-15 00:07:07 - r - INFO: - Episode: 1740/2000, Reward: 29.909, Step: 300 +2023-04-15 00:07:16 - r - INFO: - Episode: 1741/2000, Reward: -73.207, Step: 170 +2023-04-15 00:07:23 - r - INFO: - Episode: 1742/2000, Reward: -20.468, Step: 300 +2023-04-15 00:07:26 - r - INFO: - Episode: 1743/2000, Reward: -92.957, Step: 93 +2023-04-15 00:07:33 - r - INFO: - Episode: 1744/2000, Reward: 3.198, Step: 300 +2023-04-15 00:07:40 - r - INFO: - Episode: 1745/2000, Reward: 15.467, Step: 300 +2023-04-15 00:07:44 - r - INFO: - Episode: 1746/2000, Reward: -92.289, Step: 183 +2023-04-15 00:07:49 - r - INFO: - Episode: 1747/2000, Reward: -119.665, Step: 207 +2023-04-15 00:07:56 - r - INFO: - Episode: 1748/2000, Reward: -73.254, Step: 290 +2023-04-15 00:08:01 - r - INFO: - Episode: 1749/2000, Reward: -90.237, Step: 211 +2023-04-15 00:08:03 - r - INFO: - Episode: 1750/2000, Reward: -120.427, Step: 84 +2023-04-15 00:08:05 - r - INFO: - Episode: 1751/2000, Reward: -128.761, Step: 84 +2023-04-15 00:08:12 - r - INFO: - Episode: 1752/2000, Reward: -94.312, Step: 248 +2023-04-15 00:08:17 - r - INFO: - Episode: 1753/2000, Reward: -91.325, Step: 196 +2023-04-15 00:08:23 - r - INFO: - Episode: 1754/2000, Reward: -62.706, Step: 260 +2023-04-15 00:08:30 - r - INFO: - Episode: 1755/2000, Reward: -65.807, Step: 290 +2023-04-15 00:08:34 - r - INFO: - Episode: 1756/2000, Reward: -71.119, Step: 184 +2023-04-15 00:08:37 - r - INFO: - Episode: 1757/2000, Reward: -80.588, Step: 121 +2023-04-15 00:08:41 - r - INFO: - Episode: 1758/2000, Reward: -80.505, Step: 161 +2023-04-15 00:08:46 - r - INFO: - Episode: 1759/2000, Reward: -65.625, Step: 183 +2023-04-15 00:08:50 - r - INFO: - Episode: 1760/2000, Reward: -90.121, Step: 130 +2023-04-15 00:08:56 - r - INFO: - Episode: 1761/2000, Reward: -80.820, Step: 121 +2023-04-15 00:09:03 - r - INFO: - Episode: 1762/2000, Reward: -80.713, Step: 271 +2023-04-15 00:09:06 - r - INFO: - Episode: 1763/2000, Reward: -100.195, Step: 102 +2023-04-15 00:09:07 - r - INFO: - Episode: 1764/2000, Reward: -100.942, Step: 86 +2023-04-15 00:09:13 - r - INFO: - Episode: 1765/2000, Reward: -102.758, Step: 223 +2023-04-15 00:09:21 - r - INFO: - Episode: 1766/2000, Reward: -29.068, Step: 300 +2023-04-15 00:09:27 - r - INFO: - Episode: 1767/2000, Reward: -141.406, Step: 234 +2023-04-15 00:09:30 - r - INFO: - Episode: 1768/2000, Reward: -82.553, Step: 160 +2023-04-15 00:09:35 - r - INFO: - Episode: 1769/2000, Reward: -120.422, Step: 196 +2023-04-15 00:09:42 - r - INFO: - Episode: 1770/2000, Reward: 22.906, Step: 300 +2023-04-15 00:09:45 - r - INFO: - Episode: 1771/2000, Reward: -100.440, Step: 105 +2023-04-15 00:09:52 - r - INFO: - Episode: 1772/2000, Reward: -5.520, Step: 300 +2023-04-15 00:09:59 - r - INFO: - Episode: 1773/2000, Reward: -24.523, Step: 300 +2023-04-15 00:10:06 - r - INFO: - Episode: 1774/2000, Reward: -9.319, Step: 300 +2023-04-15 00:10:13 - r - INFO: - Episode: 1775/2000, Reward: -21.726, Step: 300 +2023-04-15 00:10:21 - r - INFO: - Episode: 1776/2000, Reward: -9.632, Step: 300 +2023-04-15 00:10:28 - r - INFO: - Episode: 1777/2000, Reward: -7.421, Step: 300 +2023-04-15 00:10:35 - r - INFO: - Episode: 1778/2000, Reward: -11.634, Step: 300 +2023-04-15 00:10:42 - r - INFO: - Episode: 1779/2000, Reward: -2.312, Step: 300 +2023-04-15 00:10:49 - r - INFO: - Episode: 1780/2000, Reward: -11.084, Step: 300 +2023-04-15 00:11:00 - r - INFO: - Episode: 1781/2000, Reward: -104.991, Step: 281 +2023-04-15 00:11:08 - r - INFO: - Episode: 1782/2000, Reward: -86.802, Step: 289 +2023-04-15 00:11:12 - r - INFO: - Episode: 1783/2000, Reward: -127.547, Step: 185 +2023-04-15 00:11:16 - r - INFO: - Episode: 1784/2000, Reward: -107.168, Step: 127 +2023-04-15 00:11:20 - r - INFO: - Episode: 1785/2000, Reward: -98.529, Step: 142 +2023-04-15 00:11:23 - r - INFO: - Episode: 1786/2000, Reward: -94.260, Step: 142 +2023-04-15 00:11:26 - r - INFO: - Episode: 1787/2000, Reward: -110.836, Step: 87 +2023-04-15 00:11:33 - r - INFO: - Episode: 1788/2000, Reward: 28.547, Step: 300 +2023-04-15 00:11:35 - r - INFO: - Episode: 1789/2000, Reward: -97.462, Step: 83 +2023-04-15 00:11:43 - r - INFO: - Episode: 1790/2000, Reward: 29.889, Step: 300 +2023-04-15 00:11:49 - r - INFO: - Episode: 1791/2000, Reward: -70.733, Step: 242 +2023-04-15 00:11:56 - r - INFO: - Episode: 1792/2000, Reward: 16.171, Step: 300 +2023-04-15 00:12:04 - r - INFO: - Episode: 1793/2000, Reward: 31.374, Step: 300 +2023-04-15 00:12:11 - r - INFO: - Episode: 1794/2000, Reward: -86.126, Step: 282 +2023-04-15 00:12:15 - r - INFO: - Episode: 1795/2000, Reward: -84.053, Step: 179 +2023-04-15 00:12:23 - r - INFO: - Episode: 1796/2000, Reward: -17.374, Step: 300 +2023-04-15 00:12:30 - r - INFO: - Episode: 1797/2000, Reward: -24.306, Step: 300 +2023-04-15 00:12:37 - r - INFO: - Episode: 1798/2000, Reward: -24.019, Step: 300 +2023-04-15 00:12:44 - r - INFO: - Episode: 1799/2000, Reward: -22.826, Step: 300 +2023-04-15 00:12:51 - r - INFO: - Episode: 1800/2000, Reward: -20.573, Step: 300 +2023-04-15 00:13:02 - r - INFO: - Episode: 1801/2000, Reward: -23.272, Step: 300 +2023-04-15 00:13:11 - r - INFO: - Episode: 1802/2000, Reward: -17.575, Step: 300 +2023-04-15 00:13:15 - r - INFO: - Episode: 1803/2000, Reward: -73.650, Step: 171 +2023-04-15 00:13:18 - r - INFO: - Episode: 1804/2000, Reward: -97.588, Step: 131 +2023-04-15 00:13:21 - r - INFO: - Episode: 1805/2000, Reward: -93.526, Step: 118 +2023-04-15 00:13:29 - r - INFO: - Episode: 1806/2000, Reward: 48.846, Step: 300 +2023-04-15 00:13:33 - r - INFO: - Episode: 1807/2000, Reward: -82.671, Step: 122 +2023-04-15 00:13:41 - r - INFO: - Episode: 1808/2000, Reward: -56.194, Step: 288 +2023-04-15 00:13:50 - r - INFO: - Episode: 1809/2000, Reward: -61.554, Step: 259 +2023-04-15 00:13:53 - r - INFO: - Episode: 1810/2000, Reward: -96.778, Step: 109 +2023-04-15 00:13:59 - r - INFO: - Episode: 1811/2000, Reward: -75.172, Step: 253 +2023-04-15 00:14:04 - r - INFO: - Episode: 1812/2000, Reward: -87.951, Step: 156 +2023-04-15 00:14:08 - r - INFO: - Episode: 1813/2000, Reward: -89.942, Step: 143 +2023-04-15 00:14:12 - r - INFO: - Episode: 1814/2000, Reward: -90.063, Step: 141 +2023-04-15 00:14:20 - r - INFO: - Episode: 1815/2000, Reward: 54.694, Step: 300 +2023-04-15 00:14:26 - r - INFO: - Episode: 1816/2000, Reward: -59.706, Step: 217 +2023-04-15 00:14:35 - r - INFO: - Episode: 1817/2000, Reward: 45.015, Step: 300 +2023-04-15 00:14:46 - r - INFO: - Episode: 1818/2000, Reward: 21.661, Step: 300 +2023-04-15 00:14:51 - r - INFO: - Episode: 1819/2000, Reward: -95.580, Step: 100 +2023-04-15 00:14:57 - r - INFO: - Episode: 1820/2000, Reward: -74.743, Step: 219 +2023-04-15 00:15:06 - r - INFO: - Episode: 1821/2000, Reward: -109.499, Step: 120 +2023-04-15 00:15:09 - r - INFO: - Episode: 1822/2000, Reward: -112.940, Step: 87 +2023-04-15 00:15:14 - r - INFO: - Episode: 1823/2000, Reward: -112.801, Step: 217 +2023-04-15 00:15:17 - r - INFO: - Episode: 1824/2000, Reward: -126.671, Step: 127 +2023-04-15 00:15:20 - r - INFO: - Episode: 1825/2000, Reward: -119.509, Step: 69 +2023-04-15 00:15:25 - r - INFO: - Episode: 1826/2000, Reward: -77.625, Step: 165 +2023-04-15 00:15:28 - r - INFO: - Episode: 1827/2000, Reward: -115.646, Step: 106 +2023-04-15 00:15:36 - r - INFO: - Episode: 1828/2000, Reward: -27.810, Step: 300 +2023-04-15 00:15:38 - r - INFO: - Episode: 1829/2000, Reward: -111.801, Step: 70 +2023-04-15 00:15:45 - r - INFO: - Episode: 1830/2000, Reward: 11.284, Step: 300 +2023-04-15 00:15:48 - r - INFO: - Episode: 1831/2000, Reward: -109.162, Step: 95 +2023-04-15 00:15:50 - r - INFO: - Episode: 1832/2000, Reward: -113.146, Step: 79 +2023-04-15 00:15:57 - r - INFO: - Episode: 1833/2000, Reward: 28.371, Step: 300 +2023-04-15 00:16:04 - r - INFO: - Episode: 1834/2000, Reward: -2.006, Step: 300 +2023-04-15 00:16:12 - r - INFO: - Episode: 1835/2000, Reward: -9.489, Step: 300 +2023-04-15 00:16:18 - r - INFO: - Episode: 1836/2000, Reward: -137.604, Step: 242 +2023-04-15 00:16:24 - r - INFO: - Episode: 1837/2000, Reward: -105.284, Step: 243 +2023-04-15 00:16:31 - r - INFO: - Episode: 1838/2000, Reward: -33.606, Step: 300 +2023-04-15 00:16:39 - r - INFO: - Episode: 1839/2000, Reward: -33.458, Step: 300 +2023-04-15 00:16:42 - r - INFO: - Episode: 1840/2000, Reward: -117.226, Step: 135 +2023-04-15 00:16:57 - r - INFO: - Episode: 1841/2000, Reward: -28.383, Step: 300 +2023-04-15 00:17:09 - r - INFO: - Episode: 1842/2000, Reward: -28.678, Step: 300 +2023-04-15 00:17:13 - r - INFO: - Episode: 1843/2000, Reward: -87.896, Step: 158 +2023-04-15 00:17:21 - r - INFO: - Episode: 1844/2000, Reward: -25.369, Step: 300 +2023-04-15 00:17:27 - r - INFO: - Episode: 1845/2000, Reward: -72.367, Step: 208 +2023-04-15 00:17:32 - r - INFO: - Episode: 1846/2000, Reward: -70.552, Step: 236 +2023-04-15 00:17:34 - r - INFO: - Episode: 1847/2000, Reward: -104.235, Step: 53 +2023-04-15 00:17:41 - r - INFO: - Episode: 1848/2000, Reward: -15.926, Step: 300 +2023-04-15 00:17:47 - r - INFO: - Episode: 1849/2000, Reward: -83.208, Step: 253 +2023-04-15 00:17:52 - r - INFO: - Episode: 1850/2000, Reward: -83.722, Step: 214 +2023-04-15 00:17:53 - r - INFO: - Episode: 1851/2000, Reward: -102.572, Step: 51 +2023-04-15 00:17:57 - r - INFO: - Episode: 1852/2000, Reward: -98.576, Step: 182 +2023-04-15 00:18:00 - r - INFO: - Episode: 1853/2000, Reward: -93.727, Step: 108 +2023-04-15 00:18:05 - r - INFO: - Episode: 1854/2000, Reward: -82.409, Step: 212 +2023-04-15 00:18:12 - r - INFO: - Episode: 1855/2000, Reward: -106.064, Step: 238 +2023-04-15 00:18:15 - r - INFO: - Episode: 1856/2000, Reward: -108.820, Step: 123 +2023-04-15 00:18:22 - r - INFO: - Episode: 1857/2000, Reward: 27.341, Step: 300 +2023-04-15 00:18:30 - r - INFO: - Episode: 1858/2000, Reward: 2.342, Step: 300 +2023-04-15 00:18:34 - r - INFO: - Episode: 1859/2000, Reward: -112.361, Step: 202 +2023-04-15 00:18:36 - r - INFO: - Episode: 1860/2000, Reward: -122.982, Step: 81 +2023-04-15 00:18:43 - r - INFO: - Episode: 1861/2000, Reward: -93.719, Step: 115 +2023-04-15 00:18:52 - r - INFO: - Episode: 1862/2000, Reward: 12.245, Step: 300 +2023-04-15 00:18:57 - r - INFO: - Episode: 1863/2000, Reward: -78.408, Step: 172 +2023-04-15 00:18:58 - r - INFO: - Episode: 1864/2000, Reward: -113.425, Step: 63 +2023-04-15 00:19:02 - r - INFO: - Episode: 1865/2000, Reward: -79.532, Step: 156 +2023-04-15 00:19:07 - r - INFO: - Episode: 1866/2000, Reward: -90.992, Step: 217 +2023-04-15 00:19:09 - r - INFO: - Episode: 1867/2000, Reward: -111.185, Step: 61 +2023-04-15 00:19:16 - r - INFO: - Episode: 1868/2000, Reward: 24.979, Step: 300 +2023-04-15 00:19:18 - r - INFO: - Episode: 1869/2000, Reward: -116.713, Step: 66 +2023-04-15 00:19:22 - r - INFO: - Episode: 1870/2000, Reward: -83.690, Step: 156 +2023-04-15 00:19:25 - r - INFO: - Episode: 1871/2000, Reward: -88.495, Step: 149 +2023-04-15 00:19:29 - r - INFO: - Episode: 1872/2000, Reward: -99.786, Step: 173 +2023-04-15 00:19:33 - r - INFO: - Episode: 1873/2000, Reward: -84.629, Step: 167 +2023-04-15 00:19:36 - r - INFO: - Episode: 1874/2000, Reward: -92.375, Step: 108 +2023-04-15 00:19:37 - r - INFO: - Episode: 1875/2000, Reward: -119.302, Step: 83 +2023-04-15 00:19:40 - r - INFO: - Episode: 1876/2000, Reward: -88.128, Step: 114 +2023-04-15 00:19:43 - r - INFO: - Episode: 1877/2000, Reward: -84.368, Step: 135 +2023-04-15 00:19:48 - r - INFO: - Episode: 1878/2000, Reward: -70.568, Step: 184 +2023-04-15 00:19:51 - r - INFO: - Episode: 1879/2000, Reward: -77.444, Step: 152 +2023-04-15 00:19:53 - r - INFO: - Episode: 1880/2000, Reward: -96.379, Step: 97 +2023-04-15 00:19:58 - r - INFO: - Episode: 1881/2000, Reward: -90.598, Step: 112 +2023-04-15 00:20:02 - r - INFO: - Episode: 1882/2000, Reward: -81.191, Step: 165 +2023-04-15 00:20:07 - r - INFO: - Episode: 1883/2000, Reward: -88.334, Step: 209 +2023-04-15 00:20:15 - r - INFO: - Episode: 1884/2000, Reward: -5.908, Step: 300 +2023-04-15 00:20:17 - r - INFO: - Episode: 1885/2000, Reward: -107.063, Step: 124 +2023-04-15 00:20:22 - r - INFO: - Episode: 1886/2000, Reward: -86.158, Step: 203 +2023-04-15 00:20:25 - r - INFO: - Episode: 1887/2000, Reward: -99.931, Step: 111 +2023-04-15 00:20:32 - r - INFO: - Episode: 1888/2000, Reward: 43.878, Step: 300 +2023-04-15 00:20:41 - r - INFO: - Episode: 1889/2000, Reward: -4.982, Step: 300 +2023-04-15 00:20:48 - r - INFO: - Episode: 1890/2000, Reward: 31.698, Step: 300 +2023-04-15 00:20:55 - r - INFO: - Episode: 1891/2000, Reward: 19.363, Step: 300 +2023-04-15 00:20:58 - r - INFO: - Episode: 1892/2000, Reward: -97.358, Step: 150 +2023-04-15 00:21:06 - r - INFO: - Episode: 1893/2000, Reward: 24.101, Step: 300 +2023-04-15 00:21:13 - r - INFO: - Episode: 1894/2000, Reward: 19.375, Step: 300 +2023-04-15 00:21:20 - r - INFO: - Episode: 1895/2000, Reward: -84.976, Step: 283 +2023-04-15 00:21:27 - r - INFO: - Episode: 1896/2000, Reward: 5.222, Step: 300 +2023-04-15 00:21:30 - r - INFO: - Episode: 1897/2000, Reward: -104.016, Step: 117 +2023-04-15 00:21:33 - r - INFO: - Episode: 1898/2000, Reward: -107.411, Step: 115 +2023-04-15 00:21:40 - r - INFO: - Episode: 1899/2000, Reward: 20.589, Step: 300 +2023-04-15 00:21:47 - r - INFO: - Episode: 1900/2000, Reward: 18.925, Step: 300 +2023-04-15 00:21:58 - r - INFO: - Episode: 1901/2000, Reward: 40.796, Step: 300 +2023-04-15 00:22:00 - r - INFO: - Episode: 1902/2000, Reward: -99.232, Step: 53 +2023-04-15 00:22:04 - r - INFO: - Episode: 1903/2000, Reward: -79.466, Step: 170 +2023-04-15 00:22:07 - r - INFO: - Episode: 1904/2000, Reward: -140.518, Step: 150 +2023-04-15 00:22:14 - r - INFO: - Episode: 1905/2000, Reward: 35.585, Step: 300 +2023-04-15 00:22:16 - r - INFO: - Episode: 1906/2000, Reward: -95.462, Step: 68 +2023-04-15 00:22:22 - r - INFO: - Episode: 1907/2000, Reward: -113.558, Step: 214 +2023-04-15 00:22:26 - r - INFO: - Episode: 1908/2000, Reward: -59.705, Step: 200 +2023-04-15 00:22:28 - r - INFO: - Episode: 1909/2000, Reward: -99.061, Step: 73 +2023-04-15 00:22:31 - r - INFO: - Episode: 1910/2000, Reward: -85.955, Step: 102 +2023-04-15 00:22:32 - r - INFO: - Episode: 1911/2000, Reward: -103.713, Step: 82 +2023-04-15 00:22:40 - r - INFO: - Episode: 1912/2000, Reward: -32.004, Step: 300 +2023-04-15 00:22:43 - r - INFO: - Episode: 1913/2000, Reward: -92.880, Step: 135 +2023-04-15 00:22:48 - r - INFO: - Episode: 1914/2000, Reward: -81.249, Step: 191 +2023-04-15 00:22:49 - r - INFO: - Episode: 1915/2000, Reward: -117.298, Step: 72 +2023-04-15 00:22:56 - r - INFO: - Episode: 1916/2000, Reward: 22.757, Step: 300 +2023-04-15 00:23:04 - r - INFO: - Episode: 1917/2000, Reward: -24.917, Step: 300 +2023-04-15 00:23:06 - r - INFO: - Episode: 1918/2000, Reward: -91.492, Step: 113 +2023-04-15 00:23:09 - r - INFO: - Episode: 1919/2000, Reward: -96.764, Step: 105 +2023-04-15 00:23:16 - r - INFO: - Episode: 1920/2000, Reward: -66.921, Step: 291 +2023-04-15 00:23:24 - r - INFO: - Episode: 1921/2000, Reward: -97.792, Step: 112 +2023-04-15 00:23:25 - r - INFO: - Episode: 1922/2000, Reward: -104.801, Step: 77 +2023-04-15 00:23:28 - r - INFO: - Episode: 1923/2000, Reward: -106.061, Step: 129 +2023-04-15 00:23:32 - r - INFO: - Episode: 1924/2000, Reward: -125.207, Step: 145 +2023-04-15 00:23:33 - r - INFO: - Episode: 1925/2000, Reward: -104.846, Step: 60 +2023-04-15 00:23:34 - r - INFO: - Episode: 1926/2000, Reward: -103.344, Step: 51 +2023-04-15 00:23:35 - r - INFO: - Episode: 1927/2000, Reward: -106.077, Step: 39 +2023-04-15 00:23:37 - r - INFO: - Episode: 1928/2000, Reward: -107.637, Step: 56 +2023-04-15 00:23:38 - r - INFO: - Episode: 1929/2000, Reward: -105.056, Step: 62 +2023-04-15 00:23:40 - r - INFO: - Episode: 1930/2000, Reward: -100.809, Step: 67 +2023-04-15 00:23:43 - r - INFO: - Episode: 1931/2000, Reward: -91.044, Step: 96 +2023-04-15 00:23:44 - r - INFO: - Episode: 1932/2000, Reward: -103.815, Step: 61 +2023-04-15 00:23:46 - r - INFO: - Episode: 1933/2000, Reward: -106.313, Step: 67 +2023-04-15 00:23:49 - r - INFO: - Episode: 1934/2000, Reward: -111.342, Step: 117 +2023-04-15 00:23:57 - r - INFO: - Episode: 1935/2000, Reward: 2.419, Step: 300 +2023-04-15 00:24:01 - r - INFO: - Episode: 1936/2000, Reward: -133.669, Step: 196 +2023-04-15 00:24:03 - r - INFO: - Episode: 1937/2000, Reward: -115.942, Step: 93 +2023-04-15 00:24:09 - r - INFO: - Episode: 1938/2000, Reward: -90.170, Step: 247 +2023-04-15 00:24:12 - r - INFO: - Episode: 1939/2000, Reward: -128.075, Step: 105 +2023-04-15 00:24:13 - r - INFO: - Episode: 1940/2000, Reward: -118.033, Step: 62 +2023-04-15 00:24:22 - r - INFO: - Episode: 1941/2000, Reward: -113.835, Step: 256 +2023-04-15 00:24:24 - r - INFO: - Episode: 1942/2000, Reward: -127.133, Step: 99 +2023-04-15 00:24:28 - r - INFO: - Episode: 1943/2000, Reward: -133.533, Step: 157 +2023-04-15 00:24:31 - r - INFO: - Episode: 1944/2000, Reward: -128.072, Step: 130 +2023-04-15 00:24:38 - r - INFO: - Episode: 1945/2000, Reward: -16.534, Step: 300 +2023-04-15 00:24:42 - r - INFO: - Episode: 1946/2000, Reward: -124.647, Step: 185 +2023-04-15 00:24:44 - r - INFO: - Episode: 1947/2000, Reward: -125.097, Step: 75 +2023-04-15 00:24:47 - r - INFO: - Episode: 1948/2000, Reward: -126.182, Step: 100 +2023-04-15 00:24:53 - r - INFO: - Episode: 1949/2000, Reward: -146.611, Step: 248 +2023-04-15 00:24:57 - r - INFO: - Episode: 1950/2000, Reward: -136.372, Step: 165 +2023-04-15 00:25:02 - r - INFO: - Episode: 1951/2000, Reward: -142.721, Step: 201 +2023-04-15 00:25:09 - r - INFO: - Episode: 1952/2000, Reward: -19.246, Step: 300 +2023-04-15 00:25:15 - r - INFO: - Episode: 1953/2000, Reward: -145.115, Step: 244 +2023-04-15 00:25:18 - r - INFO: - Episode: 1954/2000, Reward: -136.893, Step: 143 +2023-04-15 00:25:25 - r - INFO: - Episode: 1955/2000, Reward: -17.870, Step: 300 +2023-04-15 00:25:29 - r - INFO: - Episode: 1956/2000, Reward: -133.889, Step: 167 +2023-04-15 00:25:31 - r - INFO: - Episode: 1957/2000, Reward: -119.605, Step: 68 +2023-04-15 00:25:35 - r - INFO: - Episode: 1958/2000, Reward: -137.157, Step: 149 +2023-04-15 00:25:42 - r - INFO: - Episode: 1959/2000, Reward: 5.694, Step: 300 +2023-04-15 00:25:49 - r - INFO: - Episode: 1960/2000, Reward: -13.490, Step: 300 +2023-04-15 00:26:00 - r - INFO: - Episode: 1961/2000, Reward: 20.111, Step: 300 +2023-04-15 00:26:04 - r - INFO: - Episode: 1962/2000, Reward: -100.362, Step: 158 +2023-04-15 00:26:11 - r - INFO: - Episode: 1963/2000, Reward: 21.665, Step: 300 +2023-04-15 00:26:18 - r - INFO: - Episode: 1964/2000, Reward: -83.637, Step: 274 +2023-04-15 00:26:24 - r - INFO: - Episode: 1965/2000, Reward: -78.260, Step: 222 +2023-04-15 00:26:29 - r - INFO: - Episode: 1966/2000, Reward: -94.409, Step: 215 +2023-04-15 00:26:32 - r - INFO: - Episode: 1967/2000, Reward: -97.257, Step: 112 +2023-04-15 00:26:33 - r - INFO: - Episode: 1968/2000, Reward: -104.430, Step: 74 +2023-04-15 00:26:37 - r - INFO: - Episode: 1969/2000, Reward: -101.835, Step: 169 +2023-04-15 00:26:40 - r - INFO: - Episode: 1970/2000, Reward: -83.899, Step: 138 +2023-04-15 00:26:43 - r - INFO: - Episode: 1971/2000, Reward: -90.541, Step: 102 +2023-04-15 00:26:49 - r - INFO: - Episode: 1972/2000, Reward: -95.098, Step: 249 +2023-04-15 00:26:52 - r - INFO: - Episode: 1973/2000, Reward: -76.394, Step: 141 +2023-04-15 00:26:59 - r - INFO: - Episode: 1974/2000, Reward: -10.388, Step: 300 +2023-04-15 00:27:03 - r - INFO: - Episode: 1975/2000, Reward: -98.810, Step: 152 +2023-04-15 00:27:10 - r - INFO: - Episode: 1976/2000, Reward: -23.613, Step: 300 +2023-04-15 00:27:17 - r - INFO: - Episode: 1977/2000, Reward: -12.328, Step: 300 +2023-04-15 00:27:25 - r - INFO: - Episode: 1978/2000, Reward: -16.276, Step: 300 +2023-04-15 00:27:32 - r - INFO: - Episode: 1979/2000, Reward: 23.668, Step: 300 +2023-04-15 00:27:39 - r - INFO: - Episode: 1980/2000, Reward: -14.854, Step: 300 +2023-04-15 00:27:45 - r - INFO: - Episode: 1981/2000, Reward: -99.773, Step: 69 +2023-04-15 00:27:53 - r - INFO: - Episode: 1982/2000, Reward: -14.126, Step: 300 +2023-04-15 00:27:55 - r - INFO: - Episode: 1983/2000, Reward: -106.883, Step: 100 +2023-04-15 00:28:01 - r - INFO: - Episode: 1984/2000, Reward: -50.502, Step: 264 +2023-04-15 00:28:08 - r - INFO: - Episode: 1985/2000, Reward: 18.040, Step: 300 +2023-04-15 00:28:14 - r - INFO: - Episode: 1986/2000, Reward: -89.386, Step: 213 +2023-04-15 00:28:22 - r - INFO: - Episode: 1987/2000, Reward: 34.395, Step: 300 +2023-04-15 00:28:23 - r - INFO: - Episode: 1988/2000, Reward: -104.402, Step: 70 +2023-04-15 00:28:27 - r - INFO: - Episode: 1989/2000, Reward: -88.273, Step: 136 +2023-04-15 00:28:30 - r - INFO: - Episode: 1990/2000, Reward: -76.312, Step: 137 +2023-04-15 00:28:35 - r - INFO: - Episode: 1991/2000, Reward: -60.010, Step: 228 +2023-04-15 00:28:40 - r - INFO: - Episode: 1992/2000, Reward: -86.316, Step: 220 +2023-04-15 00:28:43 - r - INFO: - Episode: 1993/2000, Reward: -103.353, Step: 123 +2023-04-15 00:28:51 - r - INFO: - Episode: 1994/2000, Reward: 53.291, Step: 300 +2023-04-15 00:28:54 - r - INFO: - Episode: 1995/2000, Reward: -107.525, Step: 112 +2023-04-15 00:28:56 - r - INFO: - Episode: 1996/2000, Reward: -96.918, Step: 88 +2023-04-15 00:28:59 - r - INFO: - Episode: 1997/2000, Reward: -132.547, Step: 151 +2023-04-15 00:29:01 - r - INFO: - Episode: 1998/2000, Reward: -102.451, Step: 79 +2023-04-15 00:29:08 - r - INFO: - Episode: 1999/2000, Reward: -70.587, Step: 255 +2023-04-15 00:29:14 - r - INFO: - Episode: 2000/2000, Reward: -122.970, Step: 286 +2023-04-15 00:29:18 - r - INFO: - Finish training! diff --git a/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/models/actor_checkpoint.pt b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..694d1ed28c84f81d1ed05ea93f0c141b7f68dda6 --- /dev/null +++ b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e21820818072463e981ee39f73418521857414fbb150ee3e9de8a70b4a29cc5c +size 83167 diff --git a/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/results/learning_curve.png b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..e182a3a18babbd93db21e62162a882258357b01c Binary files /dev/null and b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/results/learning_curve.png differ diff --git a/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/results/res.csv b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..512329f463da9c08870668890a71a4d0b68c0aa8 --- /dev/null +++ b/BipedalWalker-v3/Train_BipedalWalker-v3_DDPG_20230414-214211/results/res.csv @@ -0,0 +1,2001 @@ +episodes,rewards,steps +0,-109.11185204309866,185 +1,-100.859559127591,78 +2,-106.09376264985087,57 +3,-122.9722144182229,105 +4,-115.26161700012817,44 +5,-13.241716707537329,300 +6,-121.54154714582812,42 +7,-110.97043974205731,105 +8,-115.35854705357008,127 +9,-130.54968083802024,236 +10,-33.805138635623386,300 +11,-118.99251868949528,153 +12,-17.5964688792741,300 +13,-117.66033614552161,117 +14,-16.64067621982546,300 +15,-23.54733610941334,300 +16,-21.801729081663904,300 +17,-22.08400689168785,300 +18,-26.890465092543472,300 +19,-19.62837555174513,300 +20,-113.43385034311059,282 +21,-16.4130707026274,300 +22,-24.657937721894214,300 +23,-19.758873601438623,300 +24,-18.33937962644963,300 +25,-20.7635321663012,300 +26,-25.0385494843162,300 +27,-21.535200948722306,300 +28,-118.79501500681927,224 +29,-115.9835185938125,67 +30,-115.90116891469499,67 +31,-112.12760858188543,50 +32,-111.15452048163772,44 +33,-104.33570058815761,93 +34,-114.6172397745379,62 +35,-113.15776223020072,52 +36,-111.61226648018538,41 +37,-110.22594567646169,46 +38,-114.13388820088797,64 +39,-29.079015607414288,300 +40,-34.99407111933747,300 +41,-28.210119428083562,300 +42,-26.79316245540075,300 +43,-29.962274088025726,300 +44,-28.750788938137916,300 +45,-28.50498341337539,300 +46,-114.12171200817687,36 +47,-114.0880741387608,41 +48,-25.577407920712414,300 +49,-23.632973268625726,300 +50,-113.77224051879072,42 +51,-125.20573683484565,116 +52,-21.012680065040495,300 +53,-117.21475807832438,48 +54,-117.23379348314413,52 +55,-29.327139896244024,300 +56,-117.0630939376803,47 +57,-113.74180772545664,39 +58,-123.29420191502233,89 +59,-23.20163773871781,300 +60,-33.77095364442136,300 +61,-108.1610900284436,64 +62,-29.419410691452697,300 +63,-101.8114187551688,122 +64,-108.05470844407421,124 +65,-26.284881354403694,300 +66,-107.42224795245076,75 +67,-111.80706211350483,129 +68,-101.43446366775383,131 +69,-112.77549112773788,128 +70,-110.31058664878003,71 +71,-99.56263389483982,114 +72,-109.4225350680979,126 +73,-31.711387683435206,300 +74,-17.547319381338387,300 +75,-119.46000689918114,64 +76,-22.546470756861748,300 +77,-106.00555146081408,163 +78,-30.912633530927316,300 +79,-20.37261209710873,300 +80,-25.56483960903847,300 +81,-28.357303632971636,300 +82,-26.50601908593725,300 +83,-27.159666957664548,300 +84,-27.436278139009772,300 +85,-28.296634682205728,300 +86,-25.462001735555802,300 +87,-28.484208205151923,300 +88,-28.666923924670186,300 +89,-27.734628348067012,300 +90,-26.746685967617797,300 +91,-18.251639766377973,300 +92,-106.64067979258652,124 +93,-22.98437101370958,300 +94,-110.76700078019829,55 +95,-122.38880448715871,58 +96,-113.43826525729492,144 +97,-117.84151202770558,118 +98,-104.38671938108921,96 +99,-129.94180411434084,119 +100,-121.97374535830193,69 +101,-119.78595986125839,78 +102,-23.826507147498937,300 +103,-127.49220354068004,99 +104,-21.96442547320076,300 +105,-25.954872565720187,300 +106,-18.989323624318853,300 +107,-24.1169862712159,300 +108,-118.6448623585997,53 +109,-121.28233258934995,72 +110,-15.121557632665617,300 +111,-128.64945622232722,101 +112,-20.965322025339304,300 +113,-19.885842080632077,300 +114,-114.62959181969694,73 +115,-28.69441890679498,300 +116,-22.583823915397385,300 +117,-25.955827056023306,300 +118,-26.00541600319325,300 +119,-20.101959813204928,300 +120,-15.634346928412901,300 +121,-18.24228575043798,300 +122,-12.301579839035066,300 +123,-115.3541102591679,281 +124,-16.131959326870707,300 +125,-12.939329921953826,300 +126,-17.10467067753301,300 +127,-22.028221422140472,300 +128,-111.8016330295104,53 +129,-12.520158501759596,300 +130,-13.98301475027718,300 +131,-112.49445843202241,48 +132,-16.83052874114646,300 +133,-17.93731578182926,300 +134,-13.941647421935574,300 +135,-109.47598550884334,40 +136,-20.929132320811984,300 +137,-104.72577786002233,126 +138,-102.11493732807445,138 +139,-107.00375899953455,67 +140,-17.9281462147919,300 +141,-111.1731222019038,147 +142,-106.46622613296948,48 +143,-116.86847608334486,51 +144,-12.773333252718343,300 +145,-106.85283924573622,162 +146,-131.27749305955032,175 +147,-106.48127694050045,80 +148,-20.06629723439549,300 +149,-104.68683313126616,151 +150,-107.5931728876056,62 +151,-19.554580736912957,300 +152,-23.64514144793627,300 +153,-12.602674343061127,300 +154,-19.999035469728675,300 +155,-13.25677053481807,300 +156,-110.93614199123743,249 +157,-16.87875657610622,300 +158,-15.017095411381641,300 +159,-25.627418908774796,300 +160,-16.12802764977084,300 +161,-14.815059965975296,300 +162,-112.88600317306965,221 +163,-19.76231501778003,300 +164,-24.05083249714645,300 +165,-17.533284262491247,300 +166,-14.292237631693666,300 +167,-13.065914483813781,300 +168,-8.492163626295815,300 +169,-11.666789199565505,300 +170,-17.48480187520755,300 +171,-18.128183762534977,300 +172,-13.669471235495624,300 +173,-8.72227585983258,300 +174,-16.47575083355433,300 +175,-6.4025779242094,300 +176,-100.83302144428039,200 +177,-132.506334957742,104 +178,-98.93974364904489,84 +179,-101.24470800965445,69 +180,-100.23371275005745,80 +181,-107.10795490293285,117 +182,-99.62689529798558,59 +183,-99.01127746130516,80 +184,-118.45157106708804,53 +185,-136.91177261830393,123 +186,-140.03702691042082,175 +187,-101.00247533984319,94 +188,-123.93375419800233,81 +189,-108.26964223713246,72 +190,-103.41283527982735,79 +191,-102.7223460218484,61 +192,-105.44898646143305,133 +193,-103.77843859423132,52 +194,-141.93229830631927,162 +195,-101.54846976548062,86 +196,-103.06729484717606,64 +197,-101.69553622196379,84 +198,-103.4462868333207,113 +199,-124.96543278436502,106 +200,-108.29862765029972,88 +201,-126.02628262840103,90 +202,-103.31356915914809,51 +203,-124.90384506826823,76 +204,-124.5133483791021,99 +205,-125.79276652280818,59 +206,-124.36546753165753,56 +207,-120.56763078390676,74 +208,-126.89202782188053,61 +209,-135.58805804088087,128 +210,-128.08715591424289,71 +211,-130.14952233671528,105 +212,-124.4398909383741,70 +213,-27.89249871219596,300 +214,-26.540936522394922,300 +215,-123.66739202547132,86 +216,-37.20917789612793,300 +217,-14.759490547872462,300 +218,-20.736892973622794,300 +219,-29.672750657172813,300 +220,-33.92802098953746,300 +221,-23.772763502712156,300 +222,-26.534589606049707,300 +223,-26.4886962121316,300 +224,-113.93879167016584,118 +225,-114.98545104844766,261 +226,-109.47991061217361,203 +227,-104.62306169381137,160 +228,-114.1872607663813,102 +229,-104.97867722334828,123 +230,-107.17439144845729,151 +231,-110.84447470383869,115 +232,-108.18611496216009,150 +233,-114.26894110639394,128 +234,-122.51006896033286,98 +235,-112.23197812314926,114 +236,-104.01271117326559,113 +237,-113.43227014783048,123 +238,-104.16991750713251,134 +239,-108.7096489632806,147 +240,-108.7120072515448,157 +241,-105.90306210810944,176 +242,-5.8033810467832785,300 +243,-118.06803061662211,288 +244,-106.93594395998386,115 +245,-108.4241687336773,296 +246,-16.542907966054983,300 +247,-16.00700786370893,300 +248,-25.236874875942984,300 +249,-26.478383521539037,300 +250,-26.721884775128476,300 +251,-19.99644687043744,300 +252,-38.0161896210571,300 +253,-141.58855242764173,273 +254,-106.8630409169427,195 +255,-100.87353462165669,77 +256,-99.88517312799507,75 +257,-100.53190496400302,102 +258,-28.615338331974236,300 +259,-9.07422977897895,300 +260,-110.71763378244171,61 +261,-101.96490521102048,144 +262,-107.55235681551348,75 +263,-99.24254476029407,64 +264,-111.67621809334975,60 +265,-95.96309859499362,183 +266,-7.738181585981171,300 +267,-111.0716186333995,58 +268,-122.07261925296426,78 +269,-128.79600839024806,118 +270,-10.650955927904812,300 +271,-18.4779841827909,300 +272,-107.6530064640985,75 +273,-13.71798290981904,300 +274,-102.53397494588256,62 +275,-104.84975329606432,94 +276,-104.58496897185869,74 +277,-20.520496397431856,300 +278,-25.097816203587954,300 +279,-31.481857823264246,300 +280,-25.9830928470694,300 +281,-18.614510877245234,300 +282,-21.483906975727894,300 +283,-18.972408938986963,300 +284,-100.92678451328715,92 +285,-26.80882517605343,300 +286,-25.90828001686666,300 +287,-24.72891580897829,300 +288,-25.59296221911117,300 +289,-21.192478176907564,300 +290,-100.5260442671854,76 +291,-29.540363453523266,300 +292,-23.734263741914052,300 +293,-38.09492887015359,300 +294,-115.28994153075001,46 +295,-23.176591194890914,300 +296,-113.36532804960908,101 +297,-118.09310234544509,196 +298,-142.04831626581813,190 +299,-112.20797879271012,122 +300,-114.12556760183479,109 +301,-112.33373054548485,111 +302,-26.616136555072853,300 +303,-6.405623824120353,300 +304,-115.12536111558495,172 +305,-133.17657994747225,130 +306,-21.432435373301395,300 +307,-106.48730168851498,161 +308,-108.1446901652668,184 +309,-13.788421627158007,300 +310,-109.35659857939989,165 +311,-14.325140104625527,300 +312,-107.73957528195835,141 +313,-19.35524116541461,300 +314,-11.45077825396447,300 +315,-17.64983596817575,300 +316,-118.74425030493171,142 +317,-41.64369430030203,300 +318,-126.15276406459733,250 +319,-105.3985490635618,242 +320,-100.89284303279861,257 +321,-111.04239559521898,252 +322,-106.81228794685023,159 +323,-7.8256837072752345,300 +324,-37.343393193667175,300 +325,-121.39932500197924,67 +326,-107.2558488849774,148 +327,-108.01764622798274,135 +328,-126.52425773229231,121 +329,-117.28626377635115,84 +330,-99.33524355528552,114 +331,-128.46861109134557,167 +332,-125.34227022070914,117 +333,-101.52678369629317,63 +334,-102.12056224288483,183 +335,-106.51696289591908,130 +336,-104.04790837267657,105 +337,-105.70662502100873,130 +338,-103.93959977617543,94 +339,-100.59702831476675,235 +340,-111.7493229170737,185 +341,-102.82436385779532,139 +342,4.364014099110492,300 +343,-105.9859830474937,92 +344,-0.48746045758975276,300 +345,-23.064846743774158,300 +346,-28.013337935561687,300 +347,-103.63136071788668,148 +348,-93.96775408744432,137 +349,-1.3003233262258844,300 +350,-105.38867581477366,139 +351,-108.77272118913588,147 +352,-0.3731389023723865,300 +353,8.800714000732656,300 +354,-95.03164996703606,241 +355,-2.5646681113526077,300 +356,8.098767915982698,300 +357,-107.43174777838382,120 +358,-101.84471738189885,149 +359,18.247632077300597,300 +360,-98.74784100580312,99 +361,-95.93932386349097,196 +362,-106.79684063097099,91 +363,-89.97131093239753,300 +364,-100.81594478700511,118 +365,-24.075206460471332,300 +366,-106.83521316489994,83 +367,8.413537396698931,300 +368,-103.14268925033193,101 +369,-87.69404993053064,139 +370,-96.28084762340305,190 +371,-6.460616644116512,300 +372,-112.1436933901454,115 +373,-91.01585205650099,182 +374,-107.39688514478013,60 +375,-95.47125689336549,189 +376,-103.09036888102324,67 +377,-107.55277892719207,119 +378,-88.94747106777321,196 +379,-99.5640448184945,193 +380,-107.20127182578725,99 +381,-11.114108358485353,300 +382,-102.62883077916167,218 +383,-15.80537564393088,300 +384,-122.98921248607488,72 +385,-99.10347076241295,146 +386,-97.79987972871015,99 +387,-122.38393325768963,78 +388,-127.8502209894942,83 +389,-123.96807497846031,73 +390,-96.54738161119779,166 +391,-100.42781868050639,106 +392,-88.34923643078972,224 +393,-96.9136512042203,104 +394,-97.01792213823593,184 +395,-22.14761711951888,300 +396,-101.6547446652376,51 +397,-30.572181563792316,300 +398,-89.32521261601339,230 +399,-24.98202131994664,300 +400,-35.87629159244838,300 +401,-103.95936051577002,56 +402,-20.680467780690066,300 +403,-29.67859446155294,300 +404,-14.902078089355856,300 +405,-16.148504712206766,300 +406,-29.861359137477706,300 +407,2.07931467874399,300 +408,-90.66313549124541,233 +409,-31.26685707398534,300 +410,7.242721982745276,300 +411,-113.02254928391572,114 +412,-33.593873248890546,300 +413,-32.559885414563375,300 +414,-116.98669138787908,71 +415,-27.54079516535132,300 +416,8.136192789796656,300 +417,-100.19031162945534,239 +418,5.091458451397397,300 +419,-24.964655332148897,300 +420,-30.804880404338643,300 +421,-129.88317168349403,131 +422,-17.67054356502954,300 +423,-32.11030154895786,300 +424,9.918283492036592,300 +425,-2.8967139006322244,300 +426,-38.354164574707,300 +427,-88.92888634315482,245 +428,-84.61188086892743,254 +429,21.768103954464483,300 +430,-104.16549929723661,259 +431,20.208296560082676,300 +432,-28.827463054789355,300 +433,-113.64187241417757,126 +434,-107.97263895706647,47 +435,-95.10915141571013,123 +436,-119.97483287513639,137 +437,-77.77272347714873,188 +438,-22.61576348195729,300 +439,-123.25538231366266,203 +440,9.307992794392822,300 +441,-105.12372419832798,130 +442,-116.2305730626124,152 +443,-83.7444801866415,159 +444,-15.60597023432958,300 +445,-99.1078697231271,144 +446,2.312619919606507,300 +447,27.422923452085953,300 +448,-76.63485279813875,201 +449,-23.146951661826726,300 +450,15.33592092954697,300 +451,16.752726914669307,300 +452,13.256875245349026,300 +453,-96.74914911387485,181 +454,-105.63264447600537,64 +455,-102.02975788522534,82 +456,4.277032460358136,300 +457,1.7378004354795942,300 +458,-5.711868839031692,300 +459,-18.36971592931633,300 +460,-22.800765096002227,300 +461,33.39436800083838,300 +462,-13.768765717186108,300 +463,-100.12270985308066,133 +464,-89.47993073912707,116 +465,-89.06355529790017,127 +466,-95.62924337030014,108 +467,-99.95281404545561,173 +468,-95.82220243735024,120 +469,-113.19217834411198,167 +470,-98.20667550361236,141 +471,-113.95817628700921,295 +472,-88.16206767134679,151 +473,-16.5048987919806,300 +474,-93.5401194052477,142 +475,-111.65080978876618,85 +476,-93.04319260300791,120 +477,-96.3470794772965,109 +478,-78.84973024419624,187 +479,-109.34246839468236,42 +480,-63.81304440164346,285 +481,-124.40663607039166,68 +482,-127.75235105965619,114 +483,-87.25637083908558,144 +484,-76.89504898647611,144 +485,-130.56861019027977,110 +486,-128.63321447559713,113 +487,-105.62036690990152,52 +488,-95.27985361426131,141 +489,-122.46130472011531,138 +490,-123.62922938156137,77 +491,-126.02952033798863,99 +492,-116.16923257343902,124 +493,-138.2740116964136,265 +494,-7.868709191868777,300 +495,4.875650940350886,300 +496,-7.277886262331133,300 +497,-117.14622280604348,62 +498,15.529619137869496,300 +499,23.71779604158043,300 +500,22.24711247095728,300 +501,31.01980994947624,300 +502,-84.09472135289823,254 +503,-103.54973461351646,80 +504,13.072636884936026,300 +505,44.73838904905698,300 +506,-72.73327505763721,253 +507,-124.50982919751354,85 +508,-69.95806900834725,253 +509,7.527668764300226,300 +510,-18.035620851405678,300 +511,-0.22941557863977902,300 +512,-128.10215703682178,125 +513,27.66692190789002,300 +514,-117.38786072244658,85 +515,24.091698577422065,300 +516,-101.98961650749875,190 +517,1.1616554731405413,300 +518,-13.124345570159921,300 +519,-131.2015778056868,299 +520,-83.27009905912662,197 +521,-91.8388002132451,127 +522,-6.724471139051984,300 +523,7.276042834728989,300 +524,-99.51340355930327,132 +525,-100.4006414367413,111 +526,-103.5883241700575,69 +527,-103.97255922494544,84 +528,-80.15821539432545,213 +529,-107.1771638094454,70 +530,-107.57456952215148,46 +531,-97.8900435871302,239 +532,-81.03817194302954,177 +533,-99.90242936257673,61 +534,-93.5849678574542,127 +535,21.766293228370277,300 +536,30.343529074411286,300 +537,30.996341950150327,300 +538,-85.71419471451597,222 +539,-110.87399157035182,43 +540,39.435164890992745,300 +541,-95.66029079972766,169 +542,-108.55901346146001,99 +543,15.236740105515134,300 +544,-107.61811571270206,194 +545,-78.93668266620188,176 +546,36.82681798685156,300 +547,-82.97394599673504,133 +548,-89.00974938247631,232 +549,7.72408071333994,300 +550,8.609359463893785,300 +551,8.294689478152412,300 +552,31.49871676687111,300 +553,1.1207606227292997,300 +554,33.333632335671375,300 +555,-82.81728462553377,296 +556,29.726352578827573,300 +557,-86.92845783658534,183 +558,-64.62913598889978,221 +559,-83.51071510665105,194 +560,19.64382755225955,300 +561,-104.48458259870476,92 +562,-108.05901706990284,177 +563,-64.37438982248193,265 +564,-84.66725670113112,174 +565,-130.7357696342724,149 +566,-90.95745200463001,161 +567,-3.7635048610493143,300 +568,16.05803129150905,300 +569,-96.84581894846433,120 +570,-81.24683281979208,250 +571,-97.39691541456352,174 +572,-124.63521624398567,110 +573,-124.7411114423694,95 +574,-114.46388072825842,167 +575,-120.34089607344697,80 +576,-120.21296669853983,96 +577,-88.38282645040839,128 +578,-83.4910904195192,123 +579,-104.11958506836844,73 +580,12.367845575952753,300 +581,-87.13349859012398,157 +582,-101.3352526480472,70 +583,-96.50890924395439,74 +584,-98.5753518401505,68 +585,-98.5534755457858,74 +586,-97.97709270280545,64 +587,-123.44122933275429,75 +588,-100.6988278070123,62 +589,-108.13071344503814,82 +590,-96.45661485208906,74 +591,-116.81603018065252,61 +592,-112.31135538575876,87 +593,-119.6073926631567,69 +594,3.5320976551381036,300 +595,-96.11537001263035,116 +596,-114.40159111301182,85 +597,-107.90995036648539,97 +598,-8.218514093410283,300 +599,-115.68286167565581,293 +600,-19.558190822574495,300 +601,-100.15011826793744,105 +602,33.392386664603926,300 +603,-107.8792148800253,102 +604,-133.5401200583355,174 +605,-121.88641574437311,98 +606,-99.97998068519797,293 +607,-101.12173150044147,98 +608,-125.61066792158172,209 +609,-3.2145811845069887,300 +610,-113.46090058468533,178 +611,-14.623353191396728,300 +612,17.712937918904245,300 +613,14.531168818783629,300 +614,-100.4750909104727,259 +615,-129.27241279555264,176 +616,4.179348647183998,300 +617,-90.17792742448474,174 +618,-87.91978775565758,148 +619,-97.93993763205745,101 +620,-141.4241348702202,173 +621,-3.46816304971656,300 +622,-95.71591807580224,136 +623,-105.13170639025995,72 +624,-91.19360592420345,118 +625,-24.810673928446842,300 +626,-90.9996570317421,106 +627,-88.90516416976945,224 +628,-110.24786203905133,128 +629,-9.651078781698763,300 +630,-100.96660348380145,118 +631,-102.75136746353492,110 +632,-102.9407189736412,167 +633,-18.257687406044244,300 +634,-14.507386000507763,300 +635,-106.0316250693619,108 +636,-74.97281006508666,206 +637,1.2510048611040292,300 +638,-12.97194744845502,300 +639,-19.759112106933244,300 +640,-27.290830991710198,300 +641,-24.547908419026665,300 +642,-27.073549904509363,300 +643,-27.966660651457786,300 +644,-12.895977469017522,300 +645,-121.15725618723317,57 +646,-18.152074954760096,300 +647,-140.1965104887945,177 +648,-139.0233715445858,249 +649,-135.84337245590865,194 +650,-41.88004011344173,300 +651,-112.65082275450435,241 +652,-134.9300438691306,196 +653,-115.8937972790995,86 +654,-110.63770585353052,70 +655,-109.46755184924923,153 +656,-107.23830198101739,88 +657,-111.91255241378309,116 +658,-108.1821471850027,69 +659,-83.34452320576275,114 +660,-112.12343380365934,85 +661,-103.05543289999531,85 +662,-92.63093190111906,159 +663,-96.35643058973639,214 +664,-107.75837373425944,143 +665,-119.43726579250719,129 +666,-101.39988729072877,185 +667,-0.4782919989115768,300 +668,1.7989609348520164,300 +669,-7.654857592629131,300 +670,-99.14811997888228,219 +671,-35.466475419351646,300 +672,-118.57231505282678,89 +673,-109.01866769501882,116 +674,20.958417463950557,300 +675,31.42901238400426,300 +676,-66.83237275952622,279 +677,21.514554840251826,300 +678,15.310253113911179,300 +679,-123.40542782999127,92 +680,0.5241512336898737,300 +681,35.968520180395494,300 +682,-9.857786851359853,300 +683,-87.65444990464616,134 +684,-72.84082588517516,237 +685,-76.3176960562146,124 +686,21.500464009617776,300 +687,22.706702385631772,300 +688,-109.2731808064419,108 +689,11.130470385801832,300 +690,-11.223139338685943,300 +691,17.899120612728492,300 +692,2.124010527479436,300 +693,-98.7511756698098,159 +694,-9.34542470529864,300 +695,2.4393492111870962,300 +696,3.03119073193769,300 +697,-13.046747899520804,300 +698,-18.37667427049098,300 +699,-104.4984989896746,93 +700,-107.66675619130507,86 +701,-104.04812951950119,107 +702,-101.11525053801844,147 +703,-23.68189730261614,300 +704,-124.95258205314389,163 +705,-12.354915695401234,300 +706,-113.46396964163515,112 +707,-5.75894482105924,300 +708,8.533360197568303,300 +709,-85.13114747898506,174 +710,-89.41722258859008,169 +711,14.458223655171677,300 +712,-122.13205534535975,123 +713,19.987611528957586,300 +714,-80.41432473371798,184 +715,-95.67002730765441,191 +716,-91.55060818679462,96 +717,-85.73013498952977,184 +718,-113.93988979403392,85 +719,-91.91261354929897,107 +720,-97.67108877586544,110 +721,-81.29392845868807,241 +722,32.51415786463349,300 +723,18.010627290249104,300 +724,-75.02999143842652,214 +725,25.294033875554085,300 +726,18.886919167932856,300 +727,13.517551791239455,300 +728,13.322187073083896,300 +729,10.87766136951899,300 +730,-113.73524924543283,173 +731,-18.569474159183045,300 +732,-31.802741266912037,300 +733,-19.255441487742104,300 +734,-26.213425269440375,300 +735,-18.06795721674884,300 +736,-8.803421238113474,300 +737,-2.3860186299198194,300 +738,-112.05462998688853,116 +739,-4.975764984981652,300 +740,-106.10954344132409,280 +741,9.55906523030371,300 +742,-2.153519761942185,300 +743,-105.15835571813258,229 +744,-93.51952601243019,150 +745,9.795695350659209,300 +746,20.428365741409053,300 +747,-77.74245806030865,247 +748,-79.71371502558958,162 +749,11.84262591510613,300 +750,17.746445568186143,300 +751,-83.24152597292972,152 +752,-68.54853836548364,234 +753,21.695402433956364,300 +754,-107.37864138196795,77 +755,-0.43936668600398404,300 +756,-15.032478743212138,300 +757,-10.703755439585391,300 +758,-6.622865617183805,300 +759,-5.784068317656319,300 +760,-95.57823989889167,298 +761,7.206895652563995,300 +762,-104.865784211225,77 +763,-97.80858551980714,107 +764,-98.95008576709644,67 +765,38.509956447699146,300 +766,-108.34871384481964,63 +767,-85.67082673933304,143 +768,-9.25638789532254,300 +769,-96.75532727429893,131 +770,-77.94498236727611,143 +771,-101.96594608755687,113 +772,-104.20145028266691,141 +773,-83.86343550714888,176 +774,-23.88827842291661,300 +775,-111.47323433394462,274 +776,-108.56172980564396,220 +777,-103.10230212307063,109 +778,-20.492319010361893,300 +779,-125.57737924582479,71 +780,-120.55308185528355,76 +781,-147.41184389441122,214 +782,-117.35989534587767,81 +783,-128.93126154501385,118 +784,-88.61664384222738,250 +785,-100.00715338474595,286 +786,-94.50085902690002,102 +787,-68.9893079059249,263 +788,-129.35684444483343,153 +789,-93.37977848599523,162 +790,-92.47785520842534,183 +791,-86.03061939184629,290 +792,-98.31276819703706,127 +793,-99.51432732429188,258 +794,-32.349538283002204,300 +795,-118.48177143647612,245 +796,-111.21129853668005,229 +797,-112.29377319180351,105 +798,-124.17930508205964,45 +799,-31.86537785601242,300 +800,-109.84022033785314,79 +801,-15.289613412392338,300 +802,-114.14416799447098,48 +803,-123.32774083607025,53 +804,-114.2449700992279,42 +805,-29.994127835313606,300 +806,-14.414125446609034,300 +807,-19.690479245701827,300 +808,-121.60539641792536,53 +809,-116.04521608028062,52 +810,-20.595689360205792,300 +811,-123.49094120179096,55 +812,-118.3535127853475,172 +813,-116.80064594766296,41 +814,-22.144160007172026,300 +815,-7.144228518728778,300 +816,-11.784912973261658,300 +817,-10.093596231965606,300 +818,-94.38017624018435,268 +819,-19.939353774761376,300 +820,-116.23800102042426,152 +821,-110.36995681008746,53 +822,-87.69804657063878,168 +823,-103.48089509626496,63 +824,-102.67771105712697,81 +825,-104.05941832685028,146 +826,-86.7405081608156,151 +827,-94.37375417360222,109 +828,-82.90750507057737,137 +829,-78.08971518914056,139 +830,-87.4319642762638,205 +831,-103.99252929047832,85 +832,25.044417768257198,300 +833,-87.360485818464,267 +834,32.2656905474822,300 +835,21.561222903101157,300 +836,-75.04534474253214,138 +837,-106.78928344406226,160 +838,-113.62358219548275,50 +839,-112.7047850442267,48 +840,-120.34996949894614,71 +841,-125.2453067021103,100 +842,-80.4875111014824,290 +843,38.437083916411204,300 +844,-75.61733815324176,192 +845,-125.19418497390228,84 +846,-96.28861084301914,147 +847,-119.93053963906497,77 +848,-118.95511161981258,71 +849,-92.42091850291193,111 +850,-114.47257102248159,75 +851,-119.28129556603157,83 +852,-116.13811223130814,67 +853,-114.42407339855708,72 +854,-115.94346491399259,63 +855,-116.63502496959939,66 +856,-120.31663305473796,78 +857,-115.2446732593288,65 +858,-115.82230476321385,60 +859,-111.03026910759506,49 +860,-114.62072272835294,59 +861,-108.06271944967654,41 +862,-122.33304017778562,84 +863,25.38126493735771,300 +864,-117.07211839833761,90 +865,-117.13297166276998,72 +866,-114.7328534438085,60 +867,-24.68850534374477,300 +868,-125.82603101664297,87 +869,-111.36489348735626,54 +870,-22.714108195581954,300 +871,-114.42304035943481,80 +872,-122.12273850789717,221 +873,-114.11020453295197,73 +874,-22.001155895417227,300 +875,-109.29288292149002,49 +876,-114.19440410989526,71 +877,-119.52682797139198,235 +878,-16.506412769397564,300 +879,-129.1515484272252,191 +880,-123.33772144236929,147 +881,-29.34191431953867,300 +882,-125.45264411401084,96 +883,-141.93214090738593,167 +884,-103.55412855561107,91 +885,-113.679787299206,53 +886,-127.51783095920284,116 +887,-126.48751070275948,80 +888,-118.45890908618507,52 +889,-118.2889484060797,48 +890,-130.50777386806712,90 +891,-123.96844794092438,86 +892,-120.60913120091675,83 +893,-121.23532651072475,66 +894,-130.72651222433376,153 +895,-131.0037875098062,147 +896,-124.0921708369478,68 +897,-127.58559922849906,117 +898,-118.22949190919371,73 +899,-114.35116456086735,51 +900,-122.12847561699459,94 +901,-128.5212561466546,95 +902,-140.42355389022953,188 +903,-31.118226769596223,300 +904,-29.896079953372595,300 +905,-128.01440756995507,166 +906,-26.80283436278899,300 +907,-18.592568028882532,300 +908,-106.72002157408274,84 +909,-33.96279102892081,300 +910,-108.6618684971294,61 +911,-93.99047313922759,209 +912,-24.5381553465043,300 +913,-15.3540685727268,300 +914,-110.9803617582985,51 +915,-134.20725761729955,126 +916,-23.58003396804055,300 +917,-19.74500832430629,300 +918,-21.848724947508266,300 +919,-21.700885559590105,300 +920,-24.840549149350323,300 +921,-20.733928084207232,300 +922,-14.68826094810861,300 +923,-23.78712569793284,300 +924,-117.57209350783253,73 +925,-119.21648705296784,78 +926,-129.25096733204404,105 +927,-123.16021062633507,87 +928,-24.501528078488867,300 +929,-123.16299670905224,94 +930,-123.49539600845054,71 +931,-112.63247062133459,52 +932,-124.84795838122155,75 +933,-122.09410316474685,78 +934,-142.675071011506,119 +935,-119.94047789145809,70 +936,-120.18806566310055,89 +937,-111.91680479311495,48 +938,-118.05878238865125,67 +939,-127.74187553437498,96 +940,-120.63387035442913,79 +941,-120.0673235742442,76 +942,-118.48912564063286,67 +943,-131.0517955361523,88 +944,-126.10508338181847,89 +945,-128.11236806332724,71 +946,-121.8843001753732,77 +947,-143.21327472747726,163 +948,-121.53281038945106,64 +949,-122.55803626646582,64 +950,-124.54300688996653,70 +951,-139.65805555542195,95 +952,-124.66467090992552,78 +953,-121.01704865136455,74 +954,-17.04755516115298,300 +955,-122.53258694150786,70 +956,-106.5438543660255,59 +957,-108.34896731252357,59 +958,-123.62215524605884,60 +959,-30.97430853248044,300 +960,-110.40404868703186,70 +961,-128.46010609292483,100 +962,-117.94689725214603,71 +963,-120.2711432335233,86 +964,-128.4965695092691,86 +965,-130.83017878710808,88 +966,-120.28005049605827,81 +967,-103.54825914944156,261 +968,-114.9690910423953,69 +969,-132.03235931757354,92 +970,-121.46889695125067,76 +971,-126.39967093165679,79 +972,-127.90491480576702,87 +973,-132.92835209961714,109 +974,-111.00716039339996,206 +975,-128.93817705577626,96 +976,-129.45439311505348,107 +977,-104.65971012012771,124 +978,-118.27100431989227,71 +979,-148.85074823765595,270 +980,-96.833658493108,189 +981,-119.48307284573869,89 +982,-2.8181178607288544,300 +983,-100.58184253942468,161 +984,-96.15636274713307,208 +985,-85.93079470260737,298 +986,-95.20787259974763,224 +987,-101.29071216081759,93 +988,-103.7665754072377,61 +989,-101.81138045332601,57 +990,-95.113300880068,199 +991,-85.80769112694142,212 +992,-108.72109785482695,38 +993,41.55706819354516,300 +994,-86.28343283014908,143 +995,-96.20156137327764,220 +996,-139.9474753089365,281 +997,-107.01384878605496,63 +998,-79.22901911176474,247 +999,-134.6784218246021,121 +1000,-109.93357402492714,36 +1001,-113.96506838574128,60 +1002,-109.42379052504768,69 +1003,-133.2801595810628,76 +1004,-123.7510376795037,175 +1005,-127.02682472625725,252 +1006,-123.9547043490951,252 +1007,-121.89240762085133,114 +1008,-118.96931210707315,145 +1009,-119.79737713882209,102 +1010,-121.38009282421385,98 +1011,-127.43345484745021,106 +1012,-124.29642885396315,94 +1013,-137.7431248654903,108 +1014,-126.18593999407162,86 +1015,-122.67060560220567,63 +1016,-140.27573039041545,104 +1017,-113.20010237396598,42 +1018,-149.95294709385922,279 +1019,-133.4276490491046,113 +1020,-109.01125109530184,37 +1021,-123.63174456406371,60 +1022,-123.96524929261435,180 +1023,-112.04815402972605,134 +1024,-120.32158137012375,53 +1025,-139.5770361424139,263 +1026,1.0025058766400181,300 +1027,-122.6272022068301,71 +1028,-126.73178390122972,154 +1029,-122.60011316618208,96 +1030,-99.52269262342179,138 +1031,-136.34715598985142,191 +1032,-96.40356205129103,215 +1033,-106.70261999435861,77 +1034,-140.99629812873056,220 +1035,-128.22470279462902,64 +1036,-95.37010233596175,116 +1037,-96.90375354045838,117 +1038,-105.767899388966,73 +1039,-96.9870937247966,241 +1040,-95.86695523999165,222 +1041,-101.21841802752938,86 +1042,-99.54693324033418,150 +1043,-127.8225942729851,83 +1044,-110.4750414873481,190 +1045,-27.377239619465417,300 +1046,-119.87492984539085,299 +1047,-126.24749524235224,105 +1048,-131.61692423450694,91 +1049,-131.73472251128925,116 +1050,-108.24504856030174,108 +1051,-136.77222347850991,111 +1052,-10.99564095062553,300 +1053,-110.83479227707554,82 +1054,-125.63130797487969,93 +1055,-123.03535540041895,163 +1056,-132.7215823306144,97 +1057,-109.70829060828692,79 +1058,-115.46149933300094,77 +1059,-141.89985336973376,201 +1060,-33.907659749391435,300 +1061,-119.40272467393777,55 +1062,-114.27060967954075,69 +1063,-108.22193414628866,86 +1064,-107.14105435407124,120 +1065,-106.38074985000382,76 +1066,-105.44920055586765,73 +1067,-79.6449613497389,211 +1068,-131.1675879158053,89 +1069,-120.76306659086504,78 +1070,-111.91055165360699,71 +1071,-107.46889287642253,146 +1072,-124.82070624101242,108 +1073,-111.16372509092758,132 +1074,-111.7117695166643,131 +1075,-124.00380573991967,91 +1076,-22.454587169772104,300 +1077,-115.22715846865827,67 +1078,-98.65602585611285,120 +1079,-116.51311963355148,75 +1080,-113.86503900577989,64 +1081,-122.32763375210531,79 +1082,-16.863803627988844,300 +1083,-133.79959390607826,91 +1084,-137.74990479997894,110 +1085,-134.96488241076298,113 +1086,-123.91306370996334,67 +1087,-105.76143631543057,95 +1088,-28.100833939646613,300 +1089,-99.16950119410441,167 +1090,-110.37238313679441,44 +1091,-140.16517422088043,126 +1092,-27.469297689837497,300 +1093,-22.80903206076735,300 +1094,-29.511800204066432,300 +1095,-17.418847917517557,300 +1096,-23.032291883854803,300 +1097,-111.07562580231958,93 +1098,-27.05550679286618,300 +1099,-23.414768094346343,300 +1100,-23.41094740585313,300 +1101,-95.04313529264498,88 +1102,-29.63617887560205,300 +1103,-80.2177424578367,155 +1104,-20.490546203898788,300 +1105,-23.4407668195953,300 +1106,-16.86747460432368,300 +1107,-30.469262306051924,300 +1108,-9.262900002064399,300 +1109,-100.8689376512762,279 +1110,-104.25179053402196,177 +1111,-25.886349200347688,300 +1112,-114.77854381322379,232 +1113,-82.31108715469006,209 +1114,-78.95384198573218,221 +1115,-100.66985964689967,110 +1116,-70.59902468157598,223 +1117,-103.61251762246943,55 +1118,-100.77837697600876,148 +1119,-97.28011144028743,125 +1120,-80.80933640576875,207 +1121,-25.731393171420617,300 +1122,-96.8740380685618,193 +1123,-89.98376211068368,149 +1124,-116.17087790657314,114 +1125,-73.96281121850174,168 +1126,-112.33412575666053,114 +1127,-90.49031462718187,171 +1128,-110.21862820947132,182 +1129,23.28630670721159,300 +1130,9.865844342780079,300 +1131,40.53880121636868,300 +1132,29.30475271818658,300 +1133,12.277270731821504,300 +1134,-93.56186084589805,188 +1135,-75.36568636027025,239 +1136,-104.14822974744744,132 +1137,27.954903900114886,300 +1138,-94.20564012050235,167 +1139,-8.888230397560994,300 +1140,-81.98198993423242,146 +1141,-68.8971313743823,191 +1142,-67.89695197213638,182 +1143,-53.38112878687703,294 +1144,35.38413909813069,300 +1145,-48.45199347534103,285 +1146,-70.61837391998968,212 +1147,-79.78709978682319,158 +1148,-75.0912166016349,159 +1149,-67.41542692818682,269 +1150,-75.2374343050827,152 +1151,31.208319031248656,300 +1152,17.0790219643718,300 +1153,13.694230769563822,300 +1154,-92.18433638457736,87 +1155,5.2661790700481745,300 +1156,19.42821222606421,300 +1157,13.318548103867059,300 +1158,20.47338662041545,300 +1159,-7.906613427529936,300 +1160,3.2695580702422085,300 +1161,3.3241608772787203,300 +1162,7.18543825971187,300 +1163,6.276538749084168,300 +1164,1.122253023715625,300 +1165,-95.47685640393578,250 +1166,-87.8312622655644,121 +1167,25.185142441950326,300 +1168,-77.78941733519416,221 +1169,-92.15528773916893,160 +1170,-66.53658668823505,295 +1171,-62.81028095189288,273 +1172,-106.82051612625986,152 +1173,-85.50364731850586,169 +1174,-91.3211841299191,147 +1175,-103.03825447897829,110 +1176,-96.1603642981766,112 +1177,-99.42068954788655,125 +1178,-95.75622637413423,176 +1179,-87.07985821584185,134 +1180,-140.6732751441201,190 +1181,-82.48407820302992,297 +1182,-16.980816614401974,300 +1183,-2.0788759770687997,300 +1184,-34.9866950921074,300 +1185,-18.021374258164087,300 +1186,-27.799431264181433,300 +1187,34.974481032046555,300 +1188,-139.66590333730784,232 +1189,-0.7130845122842242,300 +1190,-12.75088655555797,300 +1191,-103.61344066047604,78 +1192,-101.30110780463124,103 +1193,-0.8401668781198006,300 +1194,-49.75814161298713,298 +1195,-102.92468436884755,125 +1196,-108.32112541432132,114 +1197,-99.59643333795643,173 +1198,-96.5338547499381,116 +1199,-100.00563587771194,144 +1200,-94.04648534265905,177 +1201,39.2329945270467,300 +1202,-112.37851106165179,55 +1203,-99.37483069450428,127 +1204,-110.92399368312856,110 +1205,-112.98154690960968,66 +1206,-97.3372872737728,123 +1207,-127.49855938571086,72 +1208,-104.30320064510248,80 +1209,-99.12394430499315,105 +1210,-97.9425088815369,107 +1211,-79.8934909643544,209 +1212,26.092020645502092,300 +1213,-100.47296709262663,152 +1214,-111.60665928530248,114 +1215,-121.69668135103495,62 +1216,-123.83552020258992,64 +1217,-113.8906228238515,118 +1218,4.737594308657923,300 +1219,-116.86282710117523,48 +1220,22.1059490729484,300 +1221,-116.66373840441162,47 +1222,-124.94862634017161,69 +1223,-117.79915932602876,53 +1224,-138.14719559898745,98 +1225,-130.01875531896576,73 +1226,-133.80356499263414,85 +1227,-133.38396443466738,77 +1228,-102.02344838198748,177 +1229,-133.55213494746195,113 +1230,24.5987787864789,300 +1231,-114.12071644567168,43 +1232,-120.43364745925712,68 +1233,-132.42419378342868,96 +1234,-124.29005667892068,68 +1235,-120.89450071838311,74 +1236,-121.48057115225623,58 +1237,-126.09976209794547,77 +1238,-134.18334438297356,88 +1239,-125.32354843390763,79 +1240,-129.8085304114222,90 +1241,-125.8071432479806,65 +1242,-141.15783471323664,123 +1243,-116.50318640762025,50 +1244,-137.3117065358546,123 +1245,-130.10934906001165,75 +1246,-122.45489275794662,71 +1247,-105.17856287559651,58 +1248,-128.0764827919066,71 +1249,-136.42089730491273,170 +1250,-137.0033783743139,151 +1251,-111.68912660873636,77 +1252,-97.89481833328021,146 +1253,-129.07144468608038,107 +1254,-130.45876113184295,174 +1255,-111.01963446400588,90 +1256,-125.18512406241783,120 +1257,-118.60834089644109,75 +1258,-123.8789227381738,97 +1259,-135.50870329564512,94 +1260,-126.49041032434482,113 +1261,-128.19121941653265,68 +1262,-19.499321736095517,300 +1263,-18.977441275755826,300 +1264,-22.799229223268483,300 +1265,-24.66130644268445,300 +1266,-25.001466662337766,300 +1267,-23.811158172218946,300 +1268,-13.858646138545376,300 +1269,-129.0144053512069,249 +1270,-107.26781676050555,63 +1271,-105.07102393141541,158 +1272,-125.89898973606847,84 +1273,-115.86406016939793,171 +1274,-97.9764697677471,124 +1275,-98.97563626733077,137 +1276,-99.07702526833633,97 +1277,-131.32824794413355,88 +1278,-124.75620690854191,91 +1279,-139.74766979650104,145 +1280,-115.58517306886972,160 +1281,58.38117468336908,300 +1282,60.6600315512171,300 +1283,-86.0919412917045,204 +1284,-115.5826080601779,199 +1285,-27.411374676254056,300 +1286,-33.874966993273645,300 +1287,-32.93365969185131,300 +1288,-37.83947297670829,300 +1289,-32.036281553970625,300 +1290,-27.86317615719241,300 +1291,-113.43263344147432,78 +1292,-134.4611462923279,129 +1293,-86.63853199681331,121 +1294,-100.42227735652064,116 +1295,-18.552151302504456,300 +1296,-23.259916748160716,300 +1297,-19.5041918432662,300 +1298,-126.01121349258474,150 +1299,-25.081139935496946,300 +1300,-10.572995617777169,300 +1301,-99.19518482647723,89 +1302,-99.65133972291093,111 +1303,-22.244802985729564,300 +1304,-96.38079278617505,191 +1305,-101.63785883304423,109 +1306,-101.1017409053303,59 +1307,-137.49822818572855,255 +1308,-14.349189369046675,300 +1309,-8.65285453599675,300 +1310,-21.73932330710014,300 +1311,-99.92864632112266,180 +1312,-106.69799086295394,48 +1313,-104.55646442388671,44 +1314,-100.29080867166259,79 +1315,-103.94802478752247,74 +1316,-99.90991844674593,73 +1317,-0.22245462055590925,300 +1318,-86.62982595445051,100 +1319,-70.10115890166414,208 +1320,-74.39710562589967,212 +1321,-115.43364612460037,133 +1322,18.171151703331514,300 +1323,-0.2337849790765257,300 +1324,-110.25362791411372,221 +1325,-115.26413364940741,168 +1326,-4.719505472281039,300 +1327,-133.55153273023515,132 +1328,-114.22909655534245,77 +1329,-131.1573259172101,103 +1330,-129.52850478715882,78 +1331,-96.24106092174942,178 +1332,-117.34633062409843,65 +1333,-87.9992868253349,194 +1334,-117.07245816971064,64 +1335,-116.63767727364156,67 +1336,-113.87347973939107,65 +1337,-113.14496778091055,55 +1338,-115.7669669296295,64 +1339,-113.80216355849726,53 +1340,-111.74051533960643,60 +1341,-121.62809512945469,87 +1342,-114.3283757371832,54 +1343,-113.22269539058779,56 +1344,-116.13813278921602,57 +1345,-125.3854643869949,92 +1346,-115.12427881915075,57 +1347,-116.05878557610941,54 +1348,-116.59050147523334,64 +1349,-121.79772177670371,77 +1350,-122.18733366685353,100 +1351,-122.29497120938493,67 +1352,-123.31651169969837,84 +1353,-127.41733999013108,90 +1354,-132.93901688600175,110 +1355,-136.41474861546683,174 +1356,-140.75330838086558,164 +1357,19.577604911841583,300 +1358,-86.61456892008329,211 +1359,-11.701686622037982,300 +1360,-135.5312365509115,151 +1361,-104.16034820017268,94 +1362,-23.577973851466588,300 +1363,-104.38094824669382,166 +1364,-87.26431581487407,237 +1365,-93.25610190835377,145 +1366,-103.93520287704388,90 +1367,-126.19224229756715,170 +1368,35.276377284904726,300 +1369,-0.8886486258864458,300 +1370,-104.6838956873505,101 +1371,-112.01782498017461,194 +1372,-69.72253457380071,213 +1373,-108.70431028461252,186 +1374,-91.78765800247366,141 +1375,-121.08655497850407,161 +1376,-91.23137855034572,234 +1377,-79.00588885138863,185 +1378,-141.82725074288683,122 +1379,-110.45385655553451,114 +1380,-139.5875371420568,194 +1381,-109.30215013987595,148 +1382,-124.37495902320336,107 +1383,-119.23432669561184,64 +1384,-127.21639745038152,227 +1385,-116.98193896123398,70 +1386,-94.05054484390863,91 +1387,-91.07559809126406,133 +1388,-94.10097373446825,96 +1389,-105.00767522826959,279 +1390,-85.66788834861904,242 +1391,-87.79698716271334,134 +1392,24.796417062895724,300 +1393,5.943006266343277,300 +1394,-5.757540464252924,300 +1395,-108.72526442356278,92 +1396,-120.49066397333277,73 +1397,-131.8388084490508,227 +1398,-117.69210636484831,165 +1399,-123.74688434078232,259 +1400,-110.15298708671843,131 +1401,8.364035150729181,300 +1402,-92.42546431848314,128 +1403,-120.44449059266745,81 +1404,21.94098982935812,300 +1405,-8.672094471952121,300 +1406,-130.18584597000816,129 +1407,-75.5570059954132,260 +1408,11.967316948056403,300 +1409,-99.66663380711732,254 +1410,-100.14406414935536,150 +1411,-94.11939191964453,251 +1412,-114.7460776717536,229 +1413,-101.11341589189195,169 +1414,-102.84323451849252,123 +1415,-109.10653471507712,175 +1416,-102.85669588924455,279 +1417,-68.14440974303237,228 +1418,-79.53559019475202,250 +1419,28.306874324466303,300 +1420,-101.72312233236289,249 +1421,-81.65600627882648,265 +1422,-121.92924489507301,200 +1423,-99.8034576215067,185 +1424,-113.11884779682,71 +1425,-106.72587711778536,119 +1426,-69.8200165613972,167 +1427,-93.62556019865502,152 +1428,-100.35928422428066,110 +1429,36.45906571330618,300 +1430,-28.09602829908819,300 +1431,-97.26639988560767,114 +1432,-12.703156137875103,300 +1433,3.7884872966083383,300 +1434,-16.071825559397436,300 +1435,-110.28875877389362,45 +1436,-26.408949607573355,300 +1437,-28.72965191925097,300 +1438,-26.686719567149193,300 +1439,-27.394177884539825,300 +1440,-21.97961117061791,300 +1441,-27.04647455985963,300 +1442,-26.475393243103404,300 +1443,-26.489170907865528,300 +1444,-115.26975881248457,66 +1445,6.980968137202574,300 +1446,-96.32843566508284,112 +1447,-144.45474301010665,264 +1448,-130.94013858965653,104 +1449,-110.62064426622163,51 +1450,-130.8294976171977,112 +1451,-126.7684387189335,74 +1452,-102.8384981221465,59 +1453,-60.157919799374156,227 +1454,-112.17547397145692,72 +1455,-104.4906078520267,73 +1456,-90.11887911250106,231 +1457,-151.74327922173518,273 +1458,-124.87186834777555,76 +1459,-35.179794834966735,300 +1460,-89.13609128251446,135 +1461,-115.36669775534592,154 +1462,-130.44532733685713,117 +1463,-110.05847325933873,80 +1464,-113.48170676462028,129 +1465,-78.26174374236348,277 +1466,-94.3611226165236,151 +1467,-6.307265478592488,300 +1468,-26.937279984089432,300 +1469,-108.56433138366256,248 +1470,-12.306801400498381,300 +1471,-120.35181707537035,68 +1472,-99.2029818039395,138 +1473,7.513482607313089,300 +1474,-20.347195819199584,300 +1475,24.68363039311862,300 +1476,-10.917486680397102,300 +1477,-93.70438579338119,86 +1478,-137.08984255780197,189 +1479,-109.2071060865213,37 +1480,-123.5016514135426,196 +1481,-120.61096553952589,123 +1482,-104.59353283002726,139 +1483,-24.900488909746564,300 +1484,-111.12403199793042,46 +1485,-29.352001409992464,300 +1486,-126.12366215899846,172 +1487,-30.163493394770413,300 +1488,-107.21726876711641,47 +1489,-27.84003777998455,300 +1490,-27.756585983960587,300 +1491,-35.62016163707726,300 +1492,-26.313486862548718,300 +1493,-29.083517157656022,300 +1494,-100.77431513694575,130 +1495,-110.39045758831675,124 +1496,-96.99506172595524,108 +1497,-26.957930028901192,300 +1498,-23.830114323283667,300 +1499,-94.69108018117345,249 +1500,-90.4736906368112,221 +1501,-101.70706132306081,64 +1502,-103.43987751768677,121 +1503,-111.1293417883243,164 +1504,-8.920874212558006,300 +1505,-106.91907963577736,85 +1506,-21.91827156633117,300 +1507,-25.143967362584643,300 +1508,-108.91683913438862,216 +1509,-120.6770884564009,250 +1510,-90.2486484204843,128 +1511,-121.19810844817111,75 +1512,-92.32397997479863,167 +1513,-17.429414928563357,300 +1514,-17.292654938436737,300 +1515,-11.125962069058726,300 +1516,-98.30845083360218,272 +1517,-105.74115769355984,249 +1518,-98.92922132568116,124 +1519,-92.46404097157871,89 +1520,-91.09739921220327,145 +1521,-100.99300787884913,106 +1522,-27.73642686855128,300 +1523,-28.410615563816343,300 +1524,-25.794048659418632,300 +1525,-70.25983800639371,194 +1526,-92.58098305099041,205 +1527,-89.53766011886688,96 +1528,-85.70730645235918,102 +1529,-111.47160533972719,175 +1530,-80.38463634376454,199 +1531,-122.05812538193463,85 +1532,-99.98541401871562,60 +1533,-101.81957135050503,66 +1534,-93.44217029218768,84 +1535,40.520850399516284,300 +1536,-96.27306058285602,96 +1537,-62.49054518621705,190 +1538,-102.13572517122057,131 +1539,-102.94329974086483,73 +1540,-17.53087601765934,300 +1541,-16.506114693980646,300 +1542,-101.38117829033588,55 +1543,-14.754265242003783,300 +1544,-18.075358153470315,300 +1545,0.9762185744379555,300 +1546,-96.93045228164857,119 +1547,-80.45135742900453,154 +1548,-96.78953113051557,122 +1549,-98.52009573717937,84 +1550,-76.3147436520876,167 +1551,-103.99393776461349,74 +1552,-20.657377942731944,300 +1553,-106.57560742033597,81 +1554,-27.53801759239966,300 +1555,-108.43523874577907,49 +1556,-28.20132194343558,300 +1557,-107.54315509493644,64 +1558,-110.71061241372846,64 +1559,-29.906420956739304,300 +1560,-97.33571676256052,106 +1561,-142.8244772889718,203 +1562,-76.49595029698736,286 +1563,-94.20112524563552,174 +1564,-81.1684394269981,197 +1565,-93.73424975803954,120 +1566,-87.25328908627029,112 +1567,-98.94836005693095,159 +1568,-90.99643321913759,189 +1569,-91.35688513830911,216 +1570,-115.68879915983852,166 +1571,-88.6602556894059,174 +1572,-85.93716230660456,150 +1573,-140.5309315257159,293 +1574,-131.5905214512323,130 +1575,-22.750393973750167,300 +1576,-105.30159794895803,74 +1577,-141.65916900045713,204 +1578,-108.21172526633082,66 +1579,-105.86951059421612,140 +1580,-94.86205470550212,152 +1581,-104.27432088354385,78 +1582,-97.2540534855855,134 +1583,-102.06321265176896,75 +1584,-85.37486678639786,122 +1585,-84.33883787458886,131 +1586,-112.33497564138553,98 +1587,-132.73420840502482,182 +1588,-91.82297051108011,126 +1589,-87.26855717963849,95 +1590,-102.58981607065115,59 +1591,-89.02601268515272,204 +1592,-90.0206672490185,151 +1593,-109.76975272990946,98 +1594,-109.4462469177088,105 +1595,-24.776636104196452,300 +1596,-83.53376043840959,181 +1597,-78.54400964007499,158 +1598,-103.93936527476404,161 +1599,-93.90323417550975,109 +1600,-93.73426891468958,100 +1601,-132.9662150803336,212 +1602,-77.26861786190382,205 +1603,-104.73394472662498,60 +1604,-103.96774672799832,73 +1605,-99.05417873270616,103 +1606,-105.91920197309094,83 +1607,-81.7597221599176,124 +1608,-128.1627852341883,97 +1609,-100.20935762649991,66 +1610,-100.39645850033803,70 +1611,-99.00460613517352,71 +1612,-101.13135262923467,55 +1613,-128.3371132033409,101 +1614,-102.81087775414821,51 +1615,-97.28021587368657,96 +1616,-96.04036592679851,74 +1617,-109.22872204914023,53 +1618,-108.89956579836254,70 +1619,-80.51652707433328,127 +1620,-107.58605774089027,62 +1621,-107.88837215401102,77 +1622,-104.49380367788415,120 +1623,-84.3801238828986,180 +1624,-133.4681798508476,240 +1625,-5.74456301352846,300 +1626,-40.51442592821985,300 +1627,-132.634771421638,138 +1628,-114.59452189630883,80 +1629,-26.308874036175837,300 +1630,-132.43995704957055,175 +1631,-96.39760134168604,150 +1632,-29.424094687512227,300 +1633,-95.54775650187311,123 +1634,1.0549453670746423,300 +1635,-111.91352210168607,61 +1636,-117.57830537800693,65 +1637,-97.12267971886973,175 +1638,-44.86782208850687,300 +1639,-116.70450422191188,84 +1640,-87.0883860159972,157 +1641,-103.81692599210211,73 +1642,-79.22867402899584,179 +1643,-142.43477411367698,262 +1644,-90.24931158948746,130 +1645,-122.48169175928675,247 +1646,-108.03782398995136,80 +1647,-108.62724734066424,83 +1648,-123.98411037223224,109 +1649,-105.91266757812942,120 +1650,-11.996462706673492,300 +1651,-103.46502525729052,236 +1652,-12.042452657317256,300 +1653,-92.67078768898055,159 +1654,-104.02567295047017,49 +1655,-16.79864413977033,300 +1656,-2.6458647070298262,300 +1657,-81.52416222186785,137 +1658,-97.05793998531153,89 +1659,-92.08261763012567,95 +1660,-87.65368845967282,92 +1661,-92.99626067104279,130 +1662,-107.7826709843128,83 +1663,-104.02537168045363,65 +1664,-9.260900015836034,300 +1665,47.40885757135002,300 +1666,-50.27041895804351,276 +1667,-88.05307923402393,122 +1668,-103.12511597106368,70 +1669,-88.87221623099484,98 +1670,-88.88589273322816,99 +1671,-69.49354035173886,167 +1672,-105.5051114287835,60 +1673,-98.87042052149376,67 +1674,-97.57870673797989,114 +1675,-88.303120678822,117 +1676,-64.09427439525828,207 +1677,-102.11929178847744,63 +1678,-99.61355214028193,69 +1679,-59.39835768691558,172 +1680,-89.57872166720468,123 +1681,-62.235054662223995,161 +1682,-102.67741146519624,58 +1683,-97.23828447750297,78 +1684,-106.70978320305666,50 +1685,-97.83689737989252,95 +1686,-102.35245946332813,55 +1687,-85.64082523991412,114 +1688,-96.9392334978448,98 +1689,-61.82878423432708,240 +1690,-68.25653975959534,253 +1691,-86.82260152998494,137 +1692,-89.76808849857538,136 +1693,-104.36705593926011,58 +1694,41.98789594570748,300 +1695,-70.87428370676825,226 +1696,-94.11458372571335,146 +1697,57.14977797375887,300 +1698,-104.65960991069784,64 +1699,-99.20673646277693,74 +1700,30.26119859652976,300 +1701,-82.0927375916277,140 +1702,-104.12007924097219,64 +1703,-105.24695189747888,49 +1704,-100.01419360820127,79 +1705,-86.23004738613896,144 +1706,-86.78967630330145,102 +1707,-112.00101389783539,75 +1708,-110.01446920228838,80 +1709,-86.36119889143919,179 +1710,-64.4957249624311,196 +1711,-100.21283899739073,92 +1712,-96.01606666405931,85 +1713,-86.86485943295044,100 +1714,-95.73971871843996,126 +1715,-101.46114875173822,64 +1716,-91.66140200576879,117 +1717,-98.62190397662923,182 +1718,39.557222037785365,300 +1719,-70.80481926741074,212 +1720,-89.34110345785857,161 +1721,-101.35322125991144,201 +1722,-100.03055187010433,69 +1723,-68.39291989917838,255 +1724,-76.57276347647502,127 +1725,-112.78221399874305,74 +1726,-106.0703759387329,97 +1727,-99.42764182367121,106 +1728,-107.61336193540848,114 +1729,41.435751572024095,300 +1730,-87.88188703333621,207 +1731,-74.28537211672696,258 +1732,-93.7955784836667,133 +1733,-83.04531889172843,107 +1734,-73.64398945850627,238 +1735,-104.82580749072531,60 +1736,-102.12891330171699,255 +1737,-99.13941960307424,129 +1738,-103.6432658043103,52 +1739,29.909224908255034,300 +1740,-73.20705423518835,170 +1741,-20.46796948192848,300 +1742,-92.95722732166665,93 +1743,3.198438110558193,300 +1744,15.46718270308864,300 +1745,-92.28868731498723,183 +1746,-119.66513474025376,207 +1747,-73.25397246086098,290 +1748,-90.23732162617156,211 +1749,-120.42724043949453,84 +1750,-128.7610430392199,84 +1751,-94.31200087112008,248 +1752,-91.32458988076522,196 +1753,-62.706354089549635,260 +1754,-65.80678008048531,290 +1755,-71.11900335710163,184 +1756,-80.58769193303331,121 +1757,-80.50472284396659,161 +1758,-65.62506200242343,183 +1759,-90.12069882009447,130 +1760,-80.8199766409553,121 +1761,-80.71282320204192,271 +1762,-100.19452403627102,102 +1763,-100.94196384788374,86 +1764,-102.7577834454058,223 +1765,-29.067833518646612,300 +1766,-141.40587780300734,234 +1767,-82.55301321204652,160 +1768,-120.42203299321407,196 +1769,22.906498953982936,300 +1770,-100.43952416834306,105 +1771,-5.520314751899216,300 +1772,-24.522732785142313,300 +1773,-9.318731163107069,300 +1774,-21.72580038485454,300 +1775,-9.631853854375347,300 +1776,-7.4206348050122655,300 +1777,-11.633653890765908,300 +1778,-2.3122884111999937,300 +1779,-11.083632593538852,300 +1780,-104.99087260065384,281 +1781,-86.80158789696948,289 +1782,-127.54656568061546,185 +1783,-107.16835852657232,127 +1784,-98.52943563231975,142 +1785,-94.26005718997112,142 +1786,-110.83626698453749,87 +1787,28.546645760327024,300 +1788,-97.461518155608,83 +1789,29.889129335348276,300 +1790,-70.73256537388916,242 +1791,16.17144268508023,300 +1792,31.373593491890613,300 +1793,-86.12644870646565,282 +1794,-84.05340568938556,179 +1795,-17.37445471198517,300 +1796,-24.30553442279879,300 +1797,-24.019272000928954,300 +1798,-22.825704993220903,300 +1799,-20.572756605842997,300 +1800,-23.272228505741825,300 +1801,-17.57530932099975,300 +1802,-73.64986816026115,171 +1803,-97.58829783115597,131 +1804,-93.52604647594171,118 +1805,48.8458718358512,300 +1806,-82.67101169778707,122 +1807,-56.19415641903898,288 +1808,-61.553888662154854,259 +1809,-96.7781499923814,109 +1810,-75.17169267963376,253 +1811,-87.95116790995114,156 +1812,-89.94168738693536,143 +1813,-90.06292232117856,141 +1814,54.69439374757619,300 +1815,-59.70615549020118,217 +1816,45.01517290824402,300 +1817,21.661396427842888,300 +1818,-95.58034202898317,100 +1819,-74.74291491224226,219 +1820,-109.49851486889618,120 +1821,-112.94045258303359,87 +1822,-112.8012657008456,217 +1823,-126.67130068365634,127 +1824,-119.50862088529948,69 +1825,-77.62528297263461,165 +1826,-115.64558374428918,106 +1827,-27.809769783743622,300 +1828,-111.80067487785661,70 +1829,11.283998689563102,300 +1830,-109.16169774823095,95 +1831,-113.14559159712469,79 +1832,28.37051473337969,300 +1833,-2.0057105959732295,300 +1834,-9.489401095430964,300 +1835,-137.6043342720309,242 +1836,-105.2844785624434,243 +1837,-33.606303794630136,300 +1838,-33.45792667136889,300 +1839,-117.2260390382795,135 +1840,-28.3833428339147,300 +1841,-28.677530167783278,300 +1842,-87.89559370748515,158 +1843,-25.36870007225689,300 +1844,-72.36742357205883,208 +1845,-70.55153122419847,236 +1846,-104.23530254254933,53 +1847,-15.925885282478657,300 +1848,-83.2078302191059,253 +1849,-83.7221293528065,214 +1850,-102.57150554323941,51 +1851,-98.57613939908975,182 +1852,-93.72663153642988,108 +1853,-82.40949362196027,212 +1854,-106.06433725889815,238 +1855,-108.81951758424583,123 +1856,27.341174630734248,300 +1857,2.342350663608562,300 +1858,-112.36071782915218,202 +1859,-122.98178761149993,81 +1860,-93.71871585380592,115 +1861,12.245315034124753,300 +1862,-78.407893261726,172 +1863,-113.42468608200836,63 +1864,-79.53189993822514,156 +1865,-90.9920997166856,217 +1866,-111.18518371919363,61 +1867,24.97864381131255,300 +1868,-116.71312699570359,66 +1869,-83.69016630853268,156 +1870,-88.49453776689842,149 +1871,-99.78625302962132,173 +1872,-84.62949305501851,167 +1873,-92.37507113640429,108 +1874,-119.30197354175553,83 +1875,-88.12816332558548,114 +1876,-84.36781481305384,135 +1877,-70.56797182630845,184 +1878,-77.44437959898016,152 +1879,-96.37877608941218,97 +1880,-90.59810545524516,112 +1881,-81.1912041084589,165 +1882,-88.33371542349391,209 +1883,-5.908020012434981,300 +1884,-107.06326776981871,124 +1885,-86.15807689815983,203 +1886,-99.93051633414285,111 +1887,43.87765254165442,300 +1888,-4.982098434396563,300 +1889,31.697937221138673,300 +1890,19.36285881024788,300 +1891,-97.35808699999373,150 +1892,24.101053012774,300 +1893,19.374837353404867,300 +1894,-84.97580488678503,283 +1895,5.222198303477463,300 +1896,-104.01611373859934,117 +1897,-107.41149435922502,115 +1898,20.589027804491412,300 +1899,18.92533018882382,300 +1900,40.79554968610718,300 +1901,-99.23161009784351,53 +1902,-79.46581908817619,170 +1903,-140.51798219847788,150 +1904,35.58547894489389,300 +1905,-95.46210733748703,68 +1906,-113.55782358139595,214 +1907,-59.704834756276114,200 +1908,-99.06109722884833,73 +1909,-85.95546957917824,102 +1910,-103.71259391589821,82 +1911,-32.00439691047921,300 +1912,-92.88025528059774,135 +1913,-81.24935817990728,191 +1914,-117.29832193744168,72 +1915,22.756547819015655,300 +1916,-24.917031004086667,300 +1917,-91.49169674110705,113 +1918,-96.76360278248238,105 +1919,-66.92102277957602,291 +1920,-97.79214445059287,112 +1921,-104.80141359817807,77 +1922,-106.06075141097416,129 +1923,-125.20685542772527,145 +1924,-104.84610449753319,60 +1925,-103.34354451237905,51 +1926,-106.0774841819872,39 +1927,-107.63711607548558,56 +1928,-105.05565316776084,62 +1929,-100.80922051612933,67 +1930,-91.04423603652884,96 +1931,-103.81529978213945,61 +1932,-106.31305594726487,67 +1933,-111.34205670053446,117 +1934,2.4186890698436896,300 +1935,-133.6686674904689,196 +1936,-115.94152601980338,93 +1937,-90.17021390356895,247 +1938,-128.0750578884215,105 +1939,-118.03277181077341,62 +1940,-113.83525459943341,256 +1941,-127.13314560429295,99 +1942,-133.53297517446353,157 +1943,-128.0723872900887,130 +1944,-16.533700915904266,300 +1945,-124.6465701163529,185 +1946,-125.09707821206516,75 +1947,-126.18246824685197,100 +1948,-146.61075127399414,248 +1949,-136.37176490832633,165 +1950,-142.72066704659056,201 +1951,-19.246191124043435,300 +1952,-145.11533661899477,244 +1953,-136.89317000251242,143 +1954,-17.869668093158683,300 +1955,-133.8886438794713,167 +1956,-119.60545786945491,68 +1957,-137.15718394360488,149 +1958,5.693915502815032,300 +1959,-13.489908573495127,300 +1960,20.110970761042058,300 +1961,-100.36169436963567,158 +1962,21.665047123831872,300 +1963,-83.63737199934786,274 +1964,-78.25952011267813,222 +1965,-94.40931645470815,215 +1966,-97.25661794363,112 +1967,-104.42977531345683,74 +1968,-101.83534466012448,169 +1969,-83.89860512462081,138 +1970,-90.54132170797192,102 +1971,-95.09822572198296,249 +1972,-76.3941150155611,141 +1973,-10.387786142524716,300 +1974,-98.81035287228656,152 +1975,-23.613236573468573,300 +1976,-12.32816729449793,300 +1977,-16.276371189958176,300 +1978,23.668332940486643,300 +1979,-14.853740357856523,300 +1980,-99.7727822203581,69 +1981,-14.126458342510448,300 +1982,-106.88276809902827,100 +1983,-50.50219248987632,264 +1984,18.04013206587545,300 +1985,-89.38649216889678,213 +1986,34.39470504502386,300 +1987,-104.40155043703443,70 +1988,-88.27299408116787,136 +1989,-76.31167311402558,137 +1990,-60.01031926111059,228 +1991,-86.3157556916386,220 +1992,-103.35286409594681,123 +1993,53.29052260995296,300 +1994,-107.52549939246276,112 +1995,-96.9176105720994,88 +1996,-132.54672066429697,151 +1997,-102.45132853350071,79 +1998,-70.58700460295375,255 +1999,-122.96991517137837,286 diff --git a/BipedalWalker-v3/Train_gym_SAC_20230415-140928/config.yaml b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d8c3c790d7e92a55e3b54ac48775acc515e412f --- /dev/null +++ b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/config.yaml @@ -0,0 +1,38 @@ +general_cfg: + algo_name: SAC + continous: true + device: cuda + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_BipedalWalker-v3_SAC + max_steps: 500 + mode: train + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 666 + show_fig: false + test_eps: 20 + train_eps: 2000 + wrapper: null +algo_cfg: + alpha: 0.01 + automatic_entropy_tuning: false + batch_size: 64 + buffer_size: 1000000 + gamma: 0.98 + hidden_dim: 256 + lr: 0.0004 + n_epochs: 1 + policy_type: Gaussian + start_steps: 10000 + target_update_fre: 1 + tau: 0.01 +env_cfg: + id: BipedalWalker-v3 + new_step_api: true + render_mode: null diff --git a/BipedalWalker-v3/Train_gym_SAC_20230415-140928/logs/log.txt b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4639654ed3b843ea4f66c352b1af608f033f124f --- /dev/null +++ b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/logs/log.txt @@ -0,0 +1,2095 @@ +2023-04-15 14:09:28 - r - INFO: - Hyperparameters: +2023-04-15 14:09:28 - r - INFO: - ================================================================================ +2023-04-15 14:09:28 - r - INFO: - Name Value Type +2023-04-15 14:09:28 - r - INFO: - env_name gym +2023-04-15 14:09:28 - r - INFO: - new_step_api 1 +2023-04-15 14:09:28 - r - INFO: - wrapper None +2023-04-15 14:09:28 - r - INFO: - render 0 +2023-04-15 14:09:28 - r - INFO: - render_mode None +2023-04-15 14:09:28 - r - INFO: - algo_name SAC +2023-04-15 14:09:28 - r - INFO: - mode train +2023-04-15 14:09:28 - r - INFO: - mp_backend mp +2023-04-15 14:09:28 - r - INFO: - seed 666 +2023-04-15 14:09:28 - r - INFO: - device cuda +2023-04-15 14:09:28 - r - INFO: - train_eps 2000 +2023-04-15 14:09:28 - r - INFO: - test_eps 20 +2023-04-15 14:09:28 - r - INFO: - eval_eps 10 +2023-04-15 14:09:28 - r - INFO: - eval_per_episode 5 +2023-04-15 14:09:28 - r - INFO: - max_steps 500 +2023-04-15 14:09:28 - r - INFO: - load_checkpoint 0 +2023-04-15 14:09:28 - r - INFO: - load_path Train_BipedalWalker-v3_SAC +2023-04-15 14:09:28 - r - INFO: - show_fig 0 +2023-04-15 14:09:28 - r - INFO: - save_fig 1 +2023-04-15 14:09:28 - r - INFO: - continous 1 +2023-04-15 14:09:28 - r - INFO: - policy_type Gaussian +2023-04-15 14:09:28 - r - INFO: - lr 0.0004 +2023-04-15 14:09:28 - r - INFO: - gamma 0.98 +2023-04-15 14:09:28 - r - INFO: - tau 0.01 +2023-04-15 14:09:28 - r - INFO: - alpha 0.01 +2023-04-15 14:09:28 - r - INFO: - automatic_entropy_tuning 0 +2023-04-15 14:09:28 - r - INFO: - batch_size 64 +2023-04-15 14:09:28 - r - INFO: - hidden_dim 256 +2023-04-15 14:09:28 - r - INFO: - n_epochs 1 +2023-04-15 14:09:28 - r - INFO: - start_steps 10000 +2023-04-15 14:09:28 - r - INFO: - target_update_fre 1 +2023-04-15 14:09:28 - r - INFO: - buffer_size 1000000 +2023-04-15 14:09:28 - r - INFO: - id BipedalWalker-v3 +2023-04-15 14:09:28 - r - INFO: - task_dir D:\lab\joyrl-offline/tasks/Train_gym_SAC_20230415-140928 +2023-04-15 14:09:28 - r - INFO: - res_dir D:\lab\joyrl-offline/tasks/Train_gym_SAC_20230415-140928/results +2023-04-15 14:09:28 - r - INFO: - log_dir D:\lab\joyrl-offline/tasks/Train_gym_SAC_20230415-140928/logs +2023-04-15 14:09:28 - r - INFO: - traj_dir D:\lab\joyrl-offline/tasks/Train_gym_SAC_20230415-140928/traj +2023-04-15 14:09:28 - r - INFO: - video_dir D:\lab\joyrl-offline/tasks/Train_gym_SAC_20230415-140928/videos +2023-04-15 14:09:28 - r - INFO: - ================================================================================ +2023-04-15 14:09:28 - r - INFO: - action_bound: 1.0 +2023-04-15 14:09:28 - r - INFO: - n_states: 24, n_actions: 4 +2023-04-15 14:09:29 - r - INFO: - Start training! +2023-04-15 14:09:29 - r - INFO: - Env: gym, Algorithm: SAC, Device: cuda +2023-04-15 14:09:30 - r - INFO: - Episode: 1/2000, Reward: -109.176, Step: 75 +2023-04-15 14:09:30 - r - INFO: - Episode: 2/2000, Reward: -100.016, Step: 93 +2023-04-15 14:09:31 - r - INFO: - Episode: 3/2000, Reward: -101.040, Step: 91 +2023-04-15 14:09:32 - r - INFO: - Episode: 4/2000, Reward: -108.710, Step: 141 +2023-04-15 14:09:36 - r - INFO: - Episode: 5/2000, Reward: -27.626, Step: 500 +2023-04-15 14:09:43 - r - INFO: - Current episode 5 has the best eval reward: -56.780 +2023-04-15 14:09:43 - r - INFO: - Episode: 6/2000, Reward: -103.044, Step: 71 +2023-04-15 14:09:47 - r - INFO: - Episode: 7/2000, Reward: -27.145, Step: 500 +2023-04-15 14:09:48 - r - INFO: - Episode: 8/2000, Reward: -107.599, Step: 47 +2023-04-15 14:09:48 - r - INFO: - Episode: 9/2000, Reward: -99.189, Step: 98 +2023-04-15 14:09:52 - r - INFO: - Episode: 10/2000, Reward: -26.911, Step: 500 +2023-04-15 14:09:53 - r - INFO: - Episode: 11/2000, Reward: -114.284, Step: 66 +2023-04-15 14:09:57 - r - INFO: - Episode: 12/2000, Reward: -35.446, Step: 500 +2023-04-15 14:10:01 - r - INFO: - Episode: 13/2000, Reward: -25.799, Step: 500 +2023-04-15 14:10:02 - r - INFO: - Episode: 14/2000, Reward: -114.296, Step: 89 +2023-04-15 14:10:02 - r - INFO: - Episode: 15/2000, Reward: -97.413, Step: 84 +2023-04-15 14:10:09 - r - INFO: - Episode: 16/2000, Reward: -35.194, Step: 500 +2023-04-15 14:10:12 - r - INFO: - Episode: 17/2000, Reward: -26.215, Step: 500 +2023-04-15 14:10:13 - r - INFO: - Episode: 18/2000, Reward: -98.702, Step: 80 +2023-04-15 14:10:14 - r - INFO: - Episode: 19/2000, Reward: -106.983, Step: 57 +2023-04-15 14:10:17 - r - INFO: - Episode: 20/2000, Reward: -26.971, Step: 500 +2023-04-15 14:10:20 - r - INFO: - Episode: 21/2000, Reward: -109.315, Step: 149 +2023-04-15 14:10:21 - r - INFO: - Episode: 22/2000, Reward: -100.052, Step: 86 +2023-04-15 14:10:21 - r - INFO: - Episode: 23/2000, Reward: -109.197, Step: 41 +2023-04-15 14:10:25 - r - INFO: - Episode: 24/2000, Reward: -24.714, Step: 500 +2023-04-15 14:10:29 - r - INFO: - Episode: 25/2000, Reward: -25.429, Step: 500 +2023-04-15 14:10:32 - r - INFO: - Episode: 26/2000, Reward: -106.015, Step: 62 +2023-04-15 14:10:32 - r - INFO: - Episode: 27/2000, Reward: -101.945, Step: 76 +2023-04-15 14:10:33 - r - INFO: - Episode: 28/2000, Reward: -101.864, Step: 106 +2023-04-15 14:10:34 - r - INFO: - Episode: 29/2000, Reward: -116.630, Step: 70 +2023-04-15 14:10:34 - r - INFO: - Episode: 30/2000, Reward: -110.272, Step: 46 +2023-04-15 14:10:36 - r - INFO: - Episode: 31/2000, Reward: -98.832, Step: 110 +2023-04-15 14:10:37 - r - INFO: - Episode: 32/2000, Reward: -107.261, Step: 93 +2023-04-15 14:10:38 - r - INFO: - Episode: 33/2000, Reward: -115.480, Step: 90 +2023-04-15 14:10:39 - r - INFO: - Episode: 34/2000, Reward: -101.451, Step: 85 +2023-04-15 14:10:39 - r - INFO: - Episode: 35/2000, Reward: -122.465, Step: 80 +2023-04-15 14:10:41 - r - INFO: - Episode: 36/2000, Reward: -99.986, Step: 58 +2023-04-15 14:10:41 - r - INFO: - Episode: 37/2000, Reward: -100.194, Step: 80 +2023-04-15 14:10:42 - r - INFO: - Episode: 38/2000, Reward: -118.440, Step: 76 +2023-04-15 14:10:42 - r - INFO: - Episode: 39/2000, Reward: -109.708, Step: 42 +2023-04-15 14:10:43 - r - INFO: - Episode: 40/2000, Reward: -108.763, Step: 42 +2023-04-15 14:10:46 - r - INFO: - Episode: 41/2000, Reward: -99.495, Step: 81 +2023-04-15 14:10:47 - r - INFO: - Episode: 42/2000, Reward: -113.156, Step: 81 +2023-04-15 14:10:51 - r - INFO: - Episode: 43/2000, Reward: -23.712, Step: 500 +2023-04-15 14:10:55 - r - INFO: - Episode: 44/2000, Reward: -25.025, Step: 500 +2023-04-15 14:10:55 - r - INFO: - Episode: 45/2000, Reward: -97.836, Step: 70 +2023-04-15 14:11:01 - r - INFO: - Episode: 46/2000, Reward: -21.998, Step: 500 +2023-04-15 14:11:01 - r - INFO: - Episode: 47/2000, Reward: -110.373, Step: 66 +2023-04-15 14:11:05 - r - INFO: - Episode: 48/2000, Reward: -22.774, Step: 500 +2023-04-15 14:11:06 - r - INFO: - Episode: 49/2000, Reward: -99.666, Step: 65 +2023-04-15 14:11:10 - r - INFO: - Episode: 50/2000, Reward: -36.209, Step: 500 +2023-04-15 14:11:13 - r - INFO: - Episode: 51/2000, Reward: -111.627, Step: 248 +2023-04-15 14:11:17 - r - INFO: - Episode: 52/2000, Reward: -23.537, Step: 500 +2023-04-15 14:11:18 - r - INFO: - Episode: 53/2000, Reward: -124.570, Step: 80 +2023-04-15 14:11:18 - r - INFO: - Episode: 54/2000, Reward: -98.712, Step: 81 +2023-04-15 14:11:19 - r - INFO: - Episode: 55/2000, Reward: -125.502, Step: 83 +2023-04-15 14:11:26 - r - INFO: - Episode: 56/2000, Reward: -28.698, Step: 500 +2023-04-15 14:11:30 - r - INFO: - Episode: 57/2000, Reward: -33.805, Step: 500 +2023-04-15 14:11:31 - r - INFO: - Episode: 58/2000, Reward: -107.623, Step: 72 +2023-04-15 14:11:32 - r - INFO: - Episode: 59/2000, Reward: -102.412, Step: 109 +2023-04-15 14:11:33 - r - INFO: - Episode: 60/2000, Reward: -104.690, Step: 124 +2023-04-15 14:11:38 - r - INFO: - Episode: 61/2000, Reward: -121.320, Step: 99 +2023-04-15 14:11:38 - r - INFO: - Episode: 62/2000, Reward: -105.057, Step: 78 +2023-04-15 14:11:43 - r - INFO: - Episode: 63/2000, Reward: -31.169, Step: 500 +2023-04-15 14:11:48 - r - INFO: - Episode: 64/2000, Reward: -150.023, Step: 466 +2023-04-15 14:11:52 - r - INFO: - Episode: 65/2000, Reward: -22.506, Step: 500 +2023-04-15 14:11:57 - r - INFO: - Current episode 65 has the best eval reward: -45.491 +2023-04-15 14:12:02 - r - INFO: - Episode: 66/2000, Reward: -8.337, Step: 500 +2023-04-15 14:12:06 - r - INFO: - Episode: 67/2000, Reward: -17.253, Step: 500 +2023-04-15 14:12:08 - r - INFO: - Episode: 68/2000, Reward: -111.360, Step: 182 +2023-04-15 14:12:12 - r - INFO: - Episode: 69/2000, Reward: -24.281, Step: 500 +2023-04-15 14:12:13 - r - INFO: - Episode: 70/2000, Reward: -98.758, Step: 77 +2023-04-15 14:12:24 - r - INFO: - Episode: 71/2000, Reward: -20.299, Step: 500 +2023-04-15 14:12:28 - r - INFO: - Episode: 72/2000, Reward: -15.941, Step: 500 +2023-04-15 14:12:33 - r - INFO: - Episode: 73/2000, Reward: -14.262, Step: 500 +2023-04-15 14:12:38 - r - INFO: - Episode: 74/2000, Reward: -32.541, Step: 500 +2023-04-15 14:12:43 - r - INFO: - Episode: 75/2000, Reward: -26.718, Step: 500 +2023-04-15 14:12:50 - r - INFO: - Current episode 75 has the best eval reward: -28.600 +2023-04-15 14:12:55 - r - INFO: - Episode: 76/2000, Reward: -32.423, Step: 500 +2023-04-15 14:12:59 - r - INFO: - Episode: 77/2000, Reward: -28.474, Step: 500 +2023-04-15 14:13:04 - r - INFO: - Episode: 78/2000, Reward: -24.279, Step: 500 +2023-04-15 14:13:09 - r - INFO: - Episode: 79/2000, Reward: -19.191, Step: 500 +2023-04-15 14:13:13 - r - INFO: - Episode: 80/2000, Reward: -23.844, Step: 500 +2023-04-15 14:13:21 - r - INFO: - Current episode 80 has the best eval reward: -15.836 +2023-04-15 14:13:25 - r - INFO: - Episode: 81/2000, Reward: -26.297, Step: 500 +2023-04-15 14:13:30 - r - INFO: - Episode: 82/2000, Reward: -21.796, Step: 500 +2023-04-15 14:13:34 - r - INFO: - Episode: 83/2000, Reward: -25.648, Step: 500 +2023-04-15 14:13:39 - r - INFO: - Episode: 84/2000, Reward: -25.688, Step: 500 +2023-04-15 14:13:41 - r - INFO: - Episode: 85/2000, Reward: -97.206, Step: 235 +2023-04-15 14:13:53 - r - INFO: - Episode: 86/2000, Reward: -15.133, Step: 500 +2023-04-15 14:13:57 - r - INFO: - Episode: 87/2000, Reward: -36.641, Step: 500 +2023-04-15 14:14:02 - r - INFO: - Episode: 88/2000, Reward: -38.778, Step: 500 +2023-04-15 14:14:06 - r - INFO: - Episode: 89/2000, Reward: -40.080, Step: 500 +2023-04-15 14:14:11 - r - INFO: - Episode: 90/2000, Reward: -28.345, Step: 500 +2023-04-15 14:14:19 - r - INFO: - Episode: 91/2000, Reward: -117.497, Step: 75 +2023-04-15 14:14:23 - r - INFO: - Episode: 92/2000, Reward: -28.844, Step: 500 +2023-04-15 14:14:28 - r - INFO: - Episode: 93/2000, Reward: -26.978, Step: 500 +2023-04-15 14:14:32 - r - INFO: - Episode: 94/2000, Reward: -32.589, Step: 500 +2023-04-15 14:14:37 - r - INFO: - Episode: 95/2000, Reward: -27.702, Step: 500 +2023-04-15 14:14:49 - r - INFO: - Episode: 96/2000, Reward: -17.246, Step: 500 +2023-04-15 14:14:53 - r - INFO: - Episode: 97/2000, Reward: -16.413, Step: 500 +2023-04-15 14:14:58 - r - INFO: - Episode: 98/2000, Reward: -23.937, Step: 500 +2023-04-15 14:15:02 - r - INFO: - Episode: 99/2000, Reward: -18.236, Step: 500 +2023-04-15 14:15:06 - r - INFO: - Episode: 100/2000, Reward: -20.872, Step: 500 +2023-04-15 14:15:18 - r - INFO: - Episode: 101/2000, Reward: -21.802, Step: 500 +2023-04-15 14:15:22 - r - INFO: - Episode: 102/2000, Reward: -26.185, Step: 500 +2023-04-15 14:15:27 - r - INFO: - Episode: 103/2000, Reward: -26.587, Step: 500 +2023-04-15 14:15:31 - r - INFO: - Episode: 104/2000, Reward: -27.466, Step: 500 +2023-04-15 14:15:36 - r - INFO: - Episode: 105/2000, Reward: -24.475, Step: 500 +2023-04-15 14:15:44 - r - INFO: - Episode: 106/2000, Reward: -23.655, Step: 500 +2023-04-15 14:15:48 - r - INFO: - Episode: 107/2000, Reward: -21.924, Step: 500 +2023-04-15 14:15:52 - r - INFO: - Episode: 108/2000, Reward: -26.563, Step: 500 +2023-04-15 14:15:53 - r - INFO: - Episode: 109/2000, Reward: -97.671, Step: 86 +2023-04-15 14:15:54 - r - INFO: - Episode: 110/2000, Reward: -113.126, Step: 134 +2023-04-15 14:16:02 - r - INFO: - Episode: 111/2000, Reward: -99.384, Step: 152 +2023-04-15 14:16:07 - r - INFO: - Episode: 112/2000, Reward: -15.169, Step: 500 +2023-04-15 14:16:11 - r - INFO: - Episode: 113/2000, Reward: -17.825, Step: 500 +2023-04-15 14:16:15 - r - INFO: - Episode: 114/2000, Reward: -20.768, Step: 500 +2023-04-15 14:16:20 - r - INFO: - Episode: 115/2000, Reward: -14.061, Step: 500 +2023-04-15 14:16:27 - r - INFO: - Current episode 115 has the best eval reward: -15.694 +2023-04-15 14:16:31 - r - INFO: - Episode: 116/2000, Reward: -15.811, Step: 500 +2023-04-15 14:16:36 - r - INFO: - Episode: 117/2000, Reward: -27.770, Step: 500 +2023-04-15 14:16:40 - r - INFO: - Episode: 118/2000, Reward: -26.572, Step: 500 +2023-04-15 14:16:44 - r - INFO: - Episode: 119/2000, Reward: -25.633, Step: 500 +2023-04-15 14:16:49 - r - INFO: - Episode: 120/2000, Reward: -26.570, Step: 500 +2023-04-15 14:17:01 - r - INFO: - Episode: 121/2000, Reward: -21.238, Step: 500 +2023-04-15 14:17:06 - r - INFO: - Episode: 122/2000, Reward: -10.052, Step: 500 +2023-04-15 14:17:06 - r - INFO: - Episode: 123/2000, Reward: -121.545, Step: 74 +2023-04-15 14:17:11 - r - INFO: - Episode: 124/2000, Reward: -23.983, Step: 500 +2023-04-15 14:17:15 - r - INFO: - Episode: 125/2000, Reward: -19.305, Step: 500 +2023-04-15 14:17:24 - r - INFO: - Episode: 126/2000, Reward: -134.314, Step: 163 +2023-04-15 14:17:29 - r - INFO: - Episode: 127/2000, Reward: -27.831, Step: 500 +2023-04-15 14:17:34 - r - INFO: - Episode: 128/2000, Reward: -32.146, Step: 500 +2023-04-15 14:17:38 - r - INFO: - Episode: 129/2000, Reward: -31.763, Step: 500 +2023-04-15 14:17:43 - r - INFO: - Episode: 130/2000, Reward: -29.210, Step: 500 +2023-04-15 14:17:54 - r - INFO: - Episode: 131/2000, Reward: -20.699, Step: 500 +2023-04-15 14:17:59 - r - INFO: - Episode: 132/2000, Reward: -32.027, Step: 500 +2023-04-15 14:18:03 - r - INFO: - Episode: 133/2000, Reward: -26.395, Step: 500 +2023-04-15 14:18:08 - r - INFO: - Episode: 134/2000, Reward: -32.675, Step: 500 +2023-04-15 14:18:12 - r - INFO: - Episode: 135/2000, Reward: -36.649, Step: 500 +2023-04-15 14:18:24 - r - INFO: - Episode: 136/2000, Reward: -15.826, Step: 500 +2023-04-15 14:18:28 - r - INFO: - Episode: 137/2000, Reward: -15.980, Step: 500 +2023-04-15 14:18:33 - r - INFO: - Episode: 138/2000, Reward: -12.321, Step: 500 +2023-04-15 14:18:38 - r - INFO: - Episode: 139/2000, Reward: -4.984, Step: 500 +2023-04-15 14:18:42 - r - INFO: - Episode: 140/2000, Reward: -11.359, Step: 500 +2023-04-15 14:18:54 - r - INFO: - Episode: 141/2000, Reward: -15.741, Step: 500 +2023-04-15 14:18:58 - r - INFO: - Episode: 142/2000, Reward: -5.929, Step: 500 +2023-04-15 14:19:02 - r - INFO: - Episode: 143/2000, Reward: -11.564, Step: 500 +2023-04-15 14:19:07 - r - INFO: - Episode: 144/2000, Reward: -15.421, Step: 500 +2023-04-15 14:19:11 - r - INFO: - Episode: 145/2000, Reward: -20.924, Step: 500 +2023-04-15 14:19:23 - r - INFO: - Episode: 146/2000, Reward: -14.137, Step: 500 +2023-04-15 14:19:28 - r - INFO: - Episode: 147/2000, Reward: -21.478, Step: 500 +2023-04-15 14:19:32 - r - INFO: - Episode: 148/2000, Reward: -16.386, Step: 500 +2023-04-15 14:19:36 - r - INFO: - Episode: 149/2000, Reward: -17.580, Step: 500 +2023-04-15 14:19:41 - r - INFO: - Episode: 150/2000, Reward: -7.455, Step: 500 +2023-04-15 14:19:52 - r - INFO: - Episode: 151/2000, Reward: -10.209, Step: 500 +2023-04-15 14:19:57 - r - INFO: - Episode: 152/2000, Reward: -41.640, Step: 500 +2023-04-15 14:20:01 - r - INFO: - Episode: 153/2000, Reward: -26.294, Step: 500 +2023-04-15 14:20:05 - r - INFO: - Episode: 154/2000, Reward: -19.839, Step: 500 +2023-04-15 14:20:10 - r - INFO: - Episode: 155/2000, Reward: -30.129, Step: 500 +2023-04-15 14:20:21 - r - INFO: - Episode: 156/2000, Reward: -27.877, Step: 500 +2023-04-15 14:21:23 - r - INFO: - Episode: 157/2000, Reward: -28.419, Step: 500 +2023-04-15 14:21:27 - r - INFO: - Episode: 158/2000, Reward: -26.450, Step: 500 +2023-04-15 14:21:32 - r - INFO: - Episode: 159/2000, Reward: -29.323, Step: 500 +2023-04-15 14:21:36 - r - INFO: - Episode: 160/2000, Reward: -22.723, Step: 500 +2023-04-15 14:21:44 - r - INFO: - Current episode 160 has the best eval reward: -15.562 +2023-04-15 14:21:48 - r - INFO: - Episode: 161/2000, Reward: -12.634, Step: 500 +2023-04-15 14:21:53 - r - INFO: - Episode: 162/2000, Reward: -20.114, Step: 500 +2023-04-15 14:21:57 - r - INFO: - Episode: 163/2000, Reward: -14.455, Step: 500 +2023-04-15 14:22:02 - r - INFO: - Episode: 164/2000, Reward: -19.219, Step: 500 +2023-04-15 14:22:06 - r - INFO: - Episode: 165/2000, Reward: -17.960, Step: 500 +2023-04-15 14:22:17 - r - INFO: - Episode: 166/2000, Reward: -22.411, Step: 500 +2023-04-15 14:22:22 - r - INFO: - Episode: 167/2000, Reward: -24.840, Step: 500 +2023-04-15 14:22:26 - r - INFO: - Episode: 168/2000, Reward: -28.984, Step: 500 +2023-04-15 14:22:26 - r - INFO: - Episode: 169/2000, Reward: -114.682, Step: 46 +2023-04-15 14:22:31 - r - INFO: - Episode: 170/2000, Reward: -18.947, Step: 500 +2023-04-15 14:22:43 - r - INFO: - Episode: 171/2000, Reward: -18.739, Step: 500 +2023-04-15 14:22:47 - r - INFO: - Episode: 172/2000, Reward: -14.314, Step: 500 +2023-04-15 14:22:52 - r - INFO: - Episode: 173/2000, Reward: -19.592, Step: 500 +2023-04-15 14:22:57 - r - INFO: - Episode: 174/2000, Reward: -33.014, Step: 500 +2023-04-15 14:23:01 - r - INFO: - Episode: 175/2000, Reward: -20.059, Step: 500 +2023-04-15 14:23:13 - r - INFO: - Episode: 176/2000, Reward: -15.571, Step: 500 +2023-04-15 14:23:18 - r - INFO: - Episode: 177/2000, Reward: -10.026, Step: 500 +2023-04-15 14:23:22 - r - INFO: - Episode: 178/2000, Reward: -6.414, Step: 500 +2023-04-15 14:23:27 - r - INFO: - Episode: 179/2000, Reward: -23.352, Step: 500 +2023-04-15 14:23:31 - r - INFO: - Episode: 180/2000, Reward: -27.634, Step: 500 +2023-04-15 14:23:37 - r - INFO: - Episode: 181/2000, Reward: -104.611, Step: 158 +2023-04-15 14:23:41 - r - INFO: - Episode: 182/2000, Reward: -17.799, Step: 500 +2023-04-15 14:23:46 - r - INFO: - Episode: 183/2000, Reward: -16.625, Step: 500 +2023-04-15 14:23:50 - r - INFO: - Episode: 184/2000, Reward: -12.714, Step: 500 +2023-04-15 14:23:55 - r - INFO: - Episode: 185/2000, Reward: -8.524, Step: 500 +2023-04-15 14:24:07 - r - INFO: - Episode: 186/2000, Reward: -23.866, Step: 500 +2023-04-15 14:24:12 - r - INFO: - Episode: 187/2000, Reward: -20.091, Step: 500 +2023-04-15 14:24:16 - r - INFO: - Episode: 188/2000, Reward: -23.364, Step: 500 +2023-04-15 14:24:21 - r - INFO: - Episode: 189/2000, Reward: -1.868, Step: 500 +2023-04-15 14:24:25 - r - INFO: - Episode: 190/2000, Reward: -15.354, Step: 500 +2023-04-15 14:24:34 - r - INFO: - Episode: 191/2000, Reward: -116.035, Step: 84 +2023-04-15 14:24:38 - r - INFO: - Episode: 192/2000, Reward: -15.563, Step: 500 +2023-04-15 14:24:43 - r - INFO: - Episode: 193/2000, Reward: -18.838, Step: 500 +2023-04-15 14:24:47 - r - INFO: - Episode: 194/2000, Reward: -11.037, Step: 500 +2023-04-15 14:24:52 - r - INFO: - Episode: 195/2000, Reward: -17.946, Step: 500 +2023-04-15 14:25:04 - r - INFO: - Episode: 196/2000, Reward: -16.631, Step: 500 +2023-04-15 14:25:09 - r - INFO: - Episode: 197/2000, Reward: -10.618, Step: 500 +2023-04-15 14:25:13 - r - INFO: - Episode: 198/2000, Reward: -17.573, Step: 500 +2023-04-15 14:25:18 - r - INFO: - Episode: 199/2000, Reward: -15.369, Step: 500 +2023-04-15 14:25:22 - r - INFO: - Episode: 200/2000, Reward: -11.963, Step: 500 +2023-04-15 14:25:30 - r - INFO: - Current episode 200 has the best eval reward: -4.124 +2023-04-15 14:25:35 - r - INFO: - Episode: 201/2000, Reward: -27.458, Step: 500 +2023-04-15 14:25:39 - r - INFO: - Episode: 202/2000, Reward: -16.494, Step: 500 +2023-04-15 14:25:44 - r - INFO: - Episode: 203/2000, Reward: -9.793, Step: 500 +2023-04-15 14:25:49 - r - INFO: - Episode: 204/2000, Reward: -8.005, Step: 500 +2023-04-15 14:25:53 - r - INFO: - Episode: 205/2000, Reward: -21.846, Step: 500 +2023-04-15 14:26:05 - r - INFO: - Episode: 206/2000, Reward: -17.838, Step: 500 +2023-04-15 14:26:10 - r - INFO: - Episode: 207/2000, Reward: -17.027, Step: 500 +2023-04-15 14:26:14 - r - INFO: - Episode: 208/2000, Reward: -17.976, Step: 500 +2023-04-15 14:26:19 - r - INFO: - Episode: 209/2000, Reward: -13.785, Step: 500 +2023-04-15 14:26:23 - r - INFO: - Episode: 210/2000, Reward: -9.319, Step: 500 +2023-04-15 14:26:35 - r - INFO: - Episode: 211/2000, Reward: -14.770, Step: 500 +2023-04-15 14:26:40 - r - INFO: - Episode: 212/2000, Reward: -13.773, Step: 500 +2023-04-15 14:26:45 - r - INFO: - Episode: 213/2000, Reward: -17.484, Step: 500 +2023-04-15 14:26:49 - r - INFO: - Episode: 214/2000, Reward: -26.864, Step: 500 +2023-04-15 14:26:54 - r - INFO: - Episode: 215/2000, Reward: -12.291, Step: 500 +2023-04-15 14:27:06 - r - INFO: - Episode: 216/2000, Reward: -5.674, Step: 500 +2023-04-15 14:27:10 - r - INFO: - Episode: 217/2000, Reward: -5.626, Step: 500 +2023-04-15 14:27:15 - r - INFO: - Episode: 218/2000, Reward: -10.133, Step: 500 +2023-04-15 14:27:20 - r - INFO: - Episode: 219/2000, Reward: -16.108, Step: 500 +2023-04-15 14:27:24 - r - INFO: - Episode: 220/2000, Reward: -15.000, Step: 500 +2023-04-15 14:27:35 - r - INFO: - Episode: 221/2000, Reward: -18.994, Step: 500 +2023-04-15 14:27:40 - r - INFO: - Episode: 222/2000, Reward: -14.804, Step: 500 +2023-04-15 14:27:44 - r - INFO: - Episode: 223/2000, Reward: -16.160, Step: 500 +2023-04-15 14:27:45 - r - INFO: - Episode: 224/2000, Reward: -106.238, Step: 112 +2023-04-15 14:27:46 - r - INFO: - Episode: 225/2000, Reward: -104.904, Step: 108 +2023-04-15 14:27:58 - r - INFO: - Episode: 226/2000, Reward: -8.520, Step: 500 +2023-04-15 14:28:03 - r - INFO: - Episode: 227/2000, Reward: 0.132, Step: 500 +2023-04-15 14:28:08 - r - INFO: - Episode: 228/2000, Reward: -11.456, Step: 500 +2023-04-15 14:28:12 - r - INFO: - Episode: 229/2000, Reward: 1.346, Step: 500 +2023-04-15 14:28:17 - r - INFO: - Episode: 230/2000, Reward: 4.278, Step: 500 +2023-04-15 14:28:29 - r - INFO: - Episode: 231/2000, Reward: -19.499, Step: 500 +2023-04-15 14:28:33 - r - INFO: - Episode: 232/2000, Reward: -6.463, Step: 500 +2023-04-15 14:28:34 - r - INFO: - Episode: 233/2000, Reward: -107.033, Step: 66 +2023-04-15 14:28:35 - r - INFO: - Episode: 234/2000, Reward: -117.288, Step: 81 +2023-04-15 14:28:36 - r - INFO: - Episode: 235/2000, Reward: -124.235, Step: 103 +2023-04-15 14:28:40 - r - INFO: - Episode: 236/2000, Reward: -119.860, Step: 148 +2023-04-15 14:28:45 - r - INFO: - Episode: 237/2000, Reward: 6.657, Step: 500 +2023-04-15 14:28:49 - r - INFO: - Episode: 238/2000, Reward: -3.263, Step: 500 +2023-04-15 14:28:54 - r - INFO: - Episode: 239/2000, Reward: -17.619, Step: 500 +2023-04-15 14:28:58 - r - INFO: - Episode: 240/2000, Reward: -5.574, Step: 500 +2023-04-15 14:29:11 - r - INFO: - Episode: 241/2000, Reward: -4.529, Step: 500 +2023-04-15 14:29:15 - r - INFO: - Episode: 242/2000, Reward: -15.192, Step: 500 +2023-04-15 14:29:20 - r - INFO: - Episode: 243/2000, Reward: -15.513, Step: 500 +2023-04-15 14:29:20 - r - INFO: - Episode: 244/2000, Reward: -119.448, Step: 53 +2023-04-15 14:29:25 - r - INFO: - Episode: 245/2000, Reward: -13.981, Step: 500 +2023-04-15 14:29:37 - r - INFO: - Episode: 246/2000, Reward: -13.005, Step: 500 +2023-04-15 14:29:42 - r - INFO: - Episode: 247/2000, Reward: -18.202, Step: 500 +2023-04-15 14:29:46 - r - INFO: - Episode: 248/2000, Reward: -21.412, Step: 500 +2023-04-15 14:29:51 - r - INFO: - Episode: 249/2000, Reward: -16.894, Step: 500 +2023-04-15 14:29:55 - r - INFO: - Episode: 250/2000, Reward: -18.058, Step: 500 +2023-04-15 14:30:08 - r - INFO: - Episode: 251/2000, Reward: -11.733, Step: 500 +2023-04-15 14:30:08 - r - INFO: - Episode: 252/2000, Reward: -104.115, Step: 92 +2023-04-15 14:30:13 - r - INFO: - Episode: 253/2000, Reward: -12.756, Step: 500 +2023-04-15 14:30:18 - r - INFO: - Episode: 254/2000, Reward: -12.749, Step: 500 +2023-04-15 14:30:22 - r - INFO: - Episode: 255/2000, Reward: -17.662, Step: 500 +2023-04-15 14:30:35 - r - INFO: - Episode: 256/2000, Reward: -10.871, Step: 500 +2023-04-15 14:30:39 - r - INFO: - Episode: 257/2000, Reward: -6.162, Step: 500 +2023-04-15 14:30:44 - r - INFO: - Episode: 258/2000, Reward: -15.871, Step: 500 +2023-04-15 14:30:48 - r - INFO: - Episode: 259/2000, Reward: -9.629, Step: 500 +2023-04-15 14:30:53 - r - INFO: - Episode: 260/2000, Reward: -9.856, Step: 500 +2023-04-15 14:31:05 - r - INFO: - Episode: 261/2000, Reward: -10.799, Step: 500 +2023-04-15 14:31:10 - r - INFO: - Episode: 262/2000, Reward: -14.233, Step: 500 +2023-04-15 14:31:11 - r - INFO: - Episode: 263/2000, Reward: -117.476, Step: 74 +2023-04-15 14:31:15 - r - INFO: - Episode: 264/2000, Reward: -9.245, Step: 500 +2023-04-15 14:31:20 - r - INFO: - Episode: 265/2000, Reward: -6.391, Step: 500 +2023-04-15 14:31:32 - r - INFO: - Episode: 266/2000, Reward: -5.099, Step: 500 +2023-04-15 14:31:37 - r - INFO: - Episode: 267/2000, Reward: -12.592, Step: 500 +2023-04-15 14:31:41 - r - INFO: - Episode: 268/2000, Reward: -11.618, Step: 500 +2023-04-15 14:31:46 - r - INFO: - Episode: 269/2000, Reward: -4.881, Step: 500 +2023-04-15 14:31:50 - r - INFO: - Episode: 270/2000, Reward: -10.688, Step: 500 +2023-04-15 14:31:58 - r - INFO: - Current episode 270 has the best eval reward: -1.565 +2023-04-15 14:32:03 - r - INFO: - Episode: 271/2000, Reward: 3.810, Step: 500 +2023-04-15 14:32:07 - r - INFO: - Episode: 272/2000, Reward: -8.231, Step: 500 +2023-04-15 14:32:12 - r - INFO: - Episode: 273/2000, Reward: -11.890, Step: 500 +2023-04-15 14:32:16 - r - INFO: - Episode: 274/2000, Reward: -11.666, Step: 500 +2023-04-15 14:32:21 - r - INFO: - Episode: 275/2000, Reward: -9.386, Step: 500 +2023-04-15 14:32:33 - r - INFO: - Episode: 276/2000, Reward: -7.016, Step: 500 +2023-04-15 14:32:37 - r - INFO: - Episode: 277/2000, Reward: -12.341, Step: 500 +2023-04-15 14:32:42 - r - INFO: - Episode: 278/2000, Reward: -8.582, Step: 500 +2023-04-15 14:32:46 - r - INFO: - Episode: 279/2000, Reward: -12.316, Step: 500 +2023-04-15 14:32:50 - r - INFO: - Episode: 280/2000, Reward: -8.333, Step: 500 +2023-04-15 14:33:02 - r - INFO: - Episode: 281/2000, Reward: -13.787, Step: 500 +2023-04-15 14:33:07 - r - INFO: - Episode: 282/2000, Reward: -12.311, Step: 500 +2023-04-15 14:33:11 - r - INFO: - Episode: 283/2000, Reward: -9.030, Step: 500 +2023-04-15 14:33:15 - r - INFO: - Episode: 284/2000, Reward: -2.341, Step: 500 +2023-04-15 14:33:20 - r - INFO: - Episode: 285/2000, Reward: -20.646, Step: 500 +2023-04-15 14:33:32 - r - INFO: - Episode: 286/2000, Reward: -11.440, Step: 500 +2023-04-15 14:33:36 - r - INFO: - Episode: 287/2000, Reward: -9.507, Step: 500 +2023-04-15 14:33:40 - r - INFO: - Episode: 288/2000, Reward: -7.311, Step: 500 +2023-04-15 14:33:45 - r - INFO: - Episode: 289/2000, Reward: -8.184, Step: 500 +2023-04-15 14:33:49 - r - INFO: - Episode: 290/2000, Reward: -11.066, Step: 500 +2023-04-15 14:34:01 - r - INFO: - Episode: 291/2000, Reward: -8.237, Step: 500 +2023-04-15 14:34:05 - r - INFO: - Episode: 292/2000, Reward: -12.022, Step: 500 +2023-04-15 14:34:07 - r - INFO: - Episode: 293/2000, Reward: -103.549, Step: 223 +2023-04-15 14:34:12 - r - INFO: - Episode: 294/2000, Reward: -11.441, Step: 500 +2023-04-15 14:34:16 - r - INFO: - Episode: 295/2000, Reward: -9.514, Step: 500 +2023-04-15 14:34:26 - r - INFO: - Episode: 296/2000, Reward: -14.984, Step: 500 +2023-04-15 14:34:31 - r - INFO: - Episode: 297/2000, Reward: -12.714, Step: 500 +2023-04-15 14:34:35 - r - INFO: - Episode: 298/2000, Reward: -10.479, Step: 500 +2023-04-15 14:34:40 - r - INFO: - Episode: 299/2000, Reward: -14.816, Step: 500 +2023-04-15 14:34:44 - r - INFO: - Episode: 300/2000, Reward: -13.524, Step: 500 +2023-04-15 14:34:56 - r - INFO: - Episode: 301/2000, Reward: -8.391, Step: 500 +2023-04-15 14:35:00 - r - INFO: - Episode: 302/2000, Reward: -7.577, Step: 500 +2023-04-15 14:35:04 - r - INFO: - Episode: 303/2000, Reward: -11.212, Step: 500 +2023-04-15 14:35:09 - r - INFO: - Episode: 304/2000, Reward: -8.348, Step: 500 +2023-04-15 14:35:13 - r - INFO: - Episode: 305/2000, Reward: -7.506, Step: 500 +2023-04-15 14:35:24 - r - INFO: - Episode: 306/2000, Reward: -10.062, Step: 500 +2023-04-15 14:35:29 - r - INFO: - Episode: 307/2000, Reward: -14.075, Step: 500 +2023-04-15 14:35:33 - r - INFO: - Episode: 308/2000, Reward: -10.752, Step: 500 +2023-04-15 14:35:37 - r - INFO: - Episode: 309/2000, Reward: -11.910, Step: 500 +2023-04-15 14:35:42 - r - INFO: - Episode: 310/2000, Reward: -10.660, Step: 500 +2023-04-15 14:35:53 - r - INFO: - Episode: 311/2000, Reward: -8.369, Step: 500 +2023-04-15 14:35:58 - r - INFO: - Episode: 312/2000, Reward: -6.390, Step: 500 +2023-04-15 14:36:02 - r - INFO: - Episode: 313/2000, Reward: -9.037, Step: 500 +2023-04-15 14:36:06 - r - INFO: - Episode: 314/2000, Reward: -9.121, Step: 500 +2023-04-15 14:36:11 - r - INFO: - Episode: 315/2000, Reward: -7.941, Step: 500 +2023-04-15 14:36:22 - r - INFO: - Episode: 316/2000, Reward: -11.645, Step: 500 +2023-04-15 14:36:27 - r - INFO: - Episode: 317/2000, Reward: -11.019, Step: 500 +2023-04-15 14:36:31 - r - INFO: - Episode: 318/2000, Reward: -6.296, Step: 500 +2023-04-15 14:36:35 - r - INFO: - Episode: 319/2000, Reward: -8.146, Step: 500 +2023-04-15 14:36:40 - r - INFO: - Episode: 320/2000, Reward: -11.198, Step: 500 +2023-04-15 14:36:53 - r - INFO: - Episode: 321/2000, Reward: -5.290, Step: 500 +2023-04-15 14:36:53 - r - INFO: - Episode: 322/2000, Reward: -112.793, Step: 58 +2023-04-15 14:36:58 - r - INFO: - Episode: 323/2000, Reward: -6.790, Step: 500 +2023-04-15 14:37:03 - r - INFO: - Episode: 324/2000, Reward: -5.742, Step: 500 +2023-04-15 14:37:07 - r - INFO: - Episode: 325/2000, Reward: -11.808, Step: 500 +2023-04-15 14:37:14 - r - INFO: - Current episode 325 has the best eval reward: -0.381 +2023-04-15 14:37:19 - r - INFO: - Episode: 326/2000, Reward: -12.122, Step: 500 +2023-04-15 14:37:23 - r - INFO: - Episode: 327/2000, Reward: -15.917, Step: 500 +2023-04-15 14:37:27 - r - INFO: - Episode: 328/2000, Reward: -9.736, Step: 500 +2023-04-15 15:00:20 - r - INFO: - Episode: 329/2000, Reward: -5.095, Step: 500 +2023-04-15 15:00:24 - r - INFO: - Episode: 330/2000, Reward: -15.521, Step: 500 +2023-04-15 15:00:36 - r - INFO: - Episode: 331/2000, Reward: -9.908, Step: 500 +2023-04-15 15:00:40 - r - INFO: - Episode: 332/2000, Reward: -6.518, Step: 500 +2023-04-15 15:00:45 - r - INFO: - Episode: 333/2000, Reward: -23.382, Step: 500 +2023-04-15 15:00:49 - r - INFO: - Episode: 334/2000, Reward: -13.618, Step: 500 +2023-04-15 15:00:53 - r - INFO: - Episode: 335/2000, Reward: -9.759, Step: 500 +2023-04-15 15:01:04 - r - INFO: - Episode: 336/2000, Reward: -13.495, Step: 500 +2023-04-15 15:01:09 - r - INFO: - Episode: 337/2000, Reward: -12.344, Step: 500 +2023-04-15 15:01:13 - r - INFO: - Episode: 338/2000, Reward: -8.386, Step: 500 +2023-04-15 15:01:18 - r - INFO: - Episode: 339/2000, Reward: -12.025, Step: 500 +2023-04-15 15:01:22 - r - INFO: - Episode: 340/2000, Reward: -12.333, Step: 500 +2023-04-15 15:01:33 - r - INFO: - Episode: 341/2000, Reward: -9.670, Step: 500 +2023-04-15 15:01:38 - r - INFO: - Episode: 342/2000, Reward: -10.534, Step: 500 +2023-04-15 15:01:42 - r - INFO: - Episode: 343/2000, Reward: -8.565, Step: 500 +2023-04-15 15:01:46 - r - INFO: - Episode: 344/2000, Reward: -9.202, Step: 500 +2023-04-15 15:01:51 - r - INFO: - Episode: 345/2000, Reward: -5.240, Step: 500 +2023-04-15 15:02:02 - r - INFO: - Episode: 346/2000, Reward: -1.922, Step: 500 +2023-04-15 15:02:07 - r - INFO: - Episode: 347/2000, Reward: -7.598, Step: 500 +2023-04-15 15:02:11 - r - INFO: - Episode: 348/2000, Reward: -9.928, Step: 500 +2023-04-15 15:02:15 - r - INFO: - Episode: 349/2000, Reward: -11.220, Step: 500 +2023-04-15 15:02:20 - r - INFO: - Episode: 350/2000, Reward: -1.992, Step: 500 +2023-04-15 15:02:31 - r - INFO: - Episode: 351/2000, Reward: -7.596, Step: 500 +2023-04-15 15:02:36 - r - INFO: - Episode: 352/2000, Reward: -12.468, Step: 500 +2023-04-15 15:02:40 - r - INFO: - Episode: 353/2000, Reward: -8.078, Step: 500 +2023-04-15 15:02:45 - r - INFO: - Episode: 354/2000, Reward: -7.661, Step: 500 +2023-04-15 15:02:49 - r - INFO: - Episode: 355/2000, Reward: -6.741, Step: 500 +2023-04-15 15:03:00 - r - INFO: - Episode: 356/2000, Reward: -9.661, Step: 500 +2023-04-15 15:03:05 - r - INFO: - Episode: 357/2000, Reward: -7.812, Step: 500 +2023-04-15 15:03:09 - r - INFO: - Episode: 358/2000, Reward: -10.593, Step: 500 +2023-04-15 15:03:14 - r - INFO: - Episode: 359/2000, Reward: -12.706, Step: 500 +2023-04-15 15:03:18 - r - INFO: - Episode: 360/2000, Reward: -12.909, Step: 500 +2023-04-15 15:03:29 - r - INFO: - Episode: 361/2000, Reward: -11.772, Step: 500 +2023-04-15 15:03:34 - r - INFO: - Episode: 362/2000, Reward: -11.705, Step: 500 +2023-04-15 15:03:38 - r - INFO: - Episode: 363/2000, Reward: -12.034, Step: 500 +2023-04-15 15:03:43 - r - INFO: - Episode: 364/2000, Reward: -10.023, Step: 500 +2023-04-15 15:03:47 - r - INFO: - Episode: 365/2000, Reward: -13.375, Step: 500 +2023-04-15 15:03:59 - r - INFO: - Episode: 366/2000, Reward: -7.635, Step: 500 +2023-04-15 15:04:03 - r - INFO: - Episode: 367/2000, Reward: -5.359, Step: 500 +2023-04-15 15:04:07 - r - INFO: - Episode: 368/2000, Reward: -4.910, Step: 500 +2023-04-15 15:04:12 - r - INFO: - Episode: 369/2000, Reward: -2.854, Step: 500 +2023-04-15 15:04:16 - r - INFO: - Episode: 370/2000, Reward: -3.151, Step: 500 +2023-04-15 15:04:28 - r - INFO: - Episode: 371/2000, Reward: -2.779, Step: 500 +2023-04-15 15:04:32 - r - INFO: - Episode: 372/2000, Reward: -0.543, Step: 500 +2023-04-15 15:04:37 - r - INFO: - Episode: 373/2000, Reward: -4.335, Step: 500 +2023-04-15 15:04:41 - r - INFO: - Episode: 374/2000, Reward: -3.400, Step: 500 +2023-04-15 15:04:46 - r - INFO: - Episode: 375/2000, Reward: -3.608, Step: 500 +2023-04-15 15:04:53 - r - INFO: - Current episode 375 has the best eval reward: 0.719 +2023-04-15 15:04:54 - r - INFO: - Episode: 376/2000, Reward: -129.068, Step: 138 +2023-04-15 15:04:55 - r - INFO: - Episode: 377/2000, Reward: -116.293, Step: 63 +2023-04-15 15:04:59 - r - INFO: - Episode: 378/2000, Reward: -6.746, Step: 500 +2023-04-15 15:05:04 - r - INFO: - Episode: 379/2000, Reward: -26.474, Step: 500 +2023-04-15 15:05:08 - r - INFO: - Episode: 380/2000, Reward: -6.486, Step: 500 +2023-04-15 15:05:20 - r - INFO: - Episode: 381/2000, Reward: -30.548, Step: 500 +2023-04-15 15:05:24 - r - INFO: - Episode: 382/2000, Reward: -8.921, Step: 500 +2023-04-15 15:05:28 - r - INFO: - Episode: 383/2000, Reward: -7.345, Step: 500 +2023-04-15 15:05:33 - r - INFO: - Episode: 384/2000, Reward: -3.583, Step: 500 +2023-04-15 15:05:37 - r - INFO: - Episode: 385/2000, Reward: -6.887, Step: 500 +2023-04-15 15:05:49 - r - INFO: - Episode: 386/2000, Reward: -2.709, Step: 500 +2023-04-15 15:05:54 - r - INFO: - Episode: 387/2000, Reward: -3.978, Step: 500 +2023-04-15 15:05:58 - r - INFO: - Episode: 388/2000, Reward: -5.675, Step: 500 +2023-04-15 15:06:02 - r - INFO: - Episode: 389/2000, Reward: -5.425, Step: 500 +2023-04-15 15:06:07 - r - INFO: - Episode: 390/2000, Reward: -14.964, Step: 500 +2023-04-15 15:06:18 - r - INFO: - Episode: 391/2000, Reward: 1.567, Step: 500 +2023-04-15 15:06:23 - r - INFO: - Episode: 392/2000, Reward: -18.503, Step: 500 +2023-04-15 15:06:27 - r - INFO: - Episode: 393/2000, Reward: 0.128, Step: 500 +2023-04-15 15:06:31 - r - INFO: - Episode: 394/2000, Reward: -6.233, Step: 500 +2023-04-15 15:06:36 - r - INFO: - Episode: 395/2000, Reward: -22.645, Step: 500 +2023-04-15 15:06:48 - r - INFO: - Episode: 396/2000, Reward: -7.545, Step: 500 +2023-04-15 15:06:52 - r - INFO: - Episode: 397/2000, Reward: -2.951, Step: 500 +2023-04-15 15:06:54 - r - INFO: - Episode: 398/2000, Reward: -118.123, Step: 294 +2023-04-15 15:06:59 - r - INFO: - Episode: 399/2000, Reward: -3.112, Step: 500 +2023-04-15 15:07:03 - r - INFO: - Episode: 400/2000, Reward: -27.356, Step: 500 +2023-04-15 15:07:15 - r - INFO: - Episode: 401/2000, Reward: -8.459, Step: 500 +2023-04-15 15:07:19 - r - INFO: - Episode: 402/2000, Reward: -12.448, Step: 500 +2023-04-15 15:07:24 - r - INFO: - Episode: 403/2000, Reward: -8.394, Step: 500 +2023-04-15 15:07:28 - r - INFO: - Episode: 404/2000, Reward: -6.495, Step: 500 +2023-04-15 15:07:32 - r - INFO: - Episode: 405/2000, Reward: -5.858, Step: 500 +2023-04-15 15:07:44 - r - INFO: - Episode: 406/2000, Reward: -7.144, Step: 500 +2023-04-15 15:07:48 - r - INFO: - Episode: 407/2000, Reward: -6.441, Step: 500 +2023-04-15 15:07:53 - r - INFO: - Episode: 408/2000, Reward: -12.861, Step: 500 +2023-04-15 15:07:57 - r - INFO: - Episode: 409/2000, Reward: -14.663, Step: 500 +2023-04-15 15:08:01 - r - INFO: - Episode: 410/2000, Reward: -15.694, Step: 500 +2023-04-15 15:08:13 - r - INFO: - Episode: 411/2000, Reward: -15.418, Step: 500 +2023-04-15 15:08:18 - r - INFO: - Episode: 412/2000, Reward: -11.272, Step: 500 +2023-04-15 15:08:22 - r - INFO: - Episode: 413/2000, Reward: -8.752, Step: 500 +2023-04-15 15:08:26 - r - INFO: - Episode: 414/2000, Reward: -9.359, Step: 500 +2023-04-15 15:08:31 - r - INFO: - Episode: 415/2000, Reward: -10.377, Step: 500 +2023-04-15 15:08:43 - r - INFO: - Episode: 416/2000, Reward: -15.091, Step: 500 +2023-04-15 15:08:47 - r - INFO: - Episode: 417/2000, Reward: -19.415, Step: 500 +2023-04-15 15:08:52 - r - INFO: - Episode: 418/2000, Reward: -1.792, Step: 500 +2023-04-15 15:08:56 - r - INFO: - Episode: 419/2000, Reward: -4.055, Step: 500 +2023-04-15 15:09:00 - r - INFO: - Episode: 420/2000, Reward: -2.523, Step: 500 +2023-04-15 15:09:12 - r - INFO: - Episode: 421/2000, Reward: -7.988, Step: 500 +2023-04-15 15:09:17 - r - INFO: - Episode: 422/2000, Reward: -0.490, Step: 500 +2023-04-15 15:09:21 - r - INFO: - Episode: 423/2000, Reward: -2.923, Step: 500 +2023-04-15 15:09:22 - r - INFO: - Episode: 424/2000, Reward: -131.185, Step: 137 +2023-04-15 15:09:27 - r - INFO: - Episode: 425/2000, Reward: -11.489, Step: 500 +2023-04-15 15:09:38 - r - INFO: - Episode: 426/2000, Reward: -7.105, Step: 500 +2023-04-15 15:09:43 - r - INFO: - Episode: 427/2000, Reward: -18.105, Step: 500 +2023-04-15 15:09:47 - r - INFO: - Episode: 428/2000, Reward: -2.879, Step: 500 +2023-04-15 15:09:51 - r - INFO: - Episode: 429/2000, Reward: -8.960, Step: 500 +2023-04-15 15:09:56 - r - INFO: - Episode: 430/2000, Reward: 1.855, Step: 500 +2023-04-15 15:10:07 - r - INFO: - Episode: 431/2000, Reward: 4.996, Step: 500 +2023-04-15 15:10:12 - r - INFO: - Episode: 432/2000, Reward: 1.671, Step: 500 +2023-04-15 15:10:12 - r - INFO: - Episode: 433/2000, Reward: -107.073, Step: 46 +2023-04-15 15:10:13 - r - INFO: - Episode: 434/2000, Reward: -110.941, Step: 41 +2023-04-15 15:10:17 - r - INFO: - Episode: 435/2000, Reward: -4.010, Step: 500 +2023-04-15 15:10:24 - r - INFO: - Current episode 435 has the best eval reward: 10.455 +2023-04-15 15:10:29 - r - INFO: - Episode: 436/2000, Reward: 6.711, Step: 500 +2023-04-15 15:10:33 - r - INFO: - Episode: 437/2000, Reward: -2.684, Step: 500 +2023-04-15 15:10:38 - r - INFO: - Episode: 438/2000, Reward: 1.495, Step: 500 +2023-04-15 15:10:42 - r - INFO: - Episode: 439/2000, Reward: 3.369, Step: 500 +2023-04-15 15:10:46 - r - INFO: - Episode: 440/2000, Reward: 2.574, Step: 500 +2023-04-15 15:10:58 - r - INFO: - Episode: 441/2000, Reward: 2.783, Step: 500 +2023-04-15 15:11:03 - r - INFO: - Episode: 442/2000, Reward: -0.809, Step: 500 +2023-04-15 15:11:07 - r - INFO: - Episode: 443/2000, Reward: -2.536, Step: 500 +2023-04-15 15:11:11 - r - INFO: - Episode: 444/2000, Reward: -0.451, Step: 500 +2023-04-15 15:11:16 - r - INFO: - Episode: 445/2000, Reward: -1.170, Step: 500 +2023-04-15 15:11:28 - r - INFO: - Episode: 446/2000, Reward: -1.196, Step: 500 +2023-04-15 15:11:29 - r - INFO: - Episode: 447/2000, Reward: -102.125, Step: 165 +2023-04-15 15:11:33 - r - INFO: - Episode: 448/2000, Reward: 7.795, Step: 500 +2023-04-15 15:11:38 - r - INFO: - Episode: 449/2000, Reward: 5.973, Step: 500 +2023-04-15 15:11:42 - r - INFO: - Episode: 450/2000, Reward: 8.617, Step: 500 +2023-04-15 15:11:54 - r - INFO: - Episode: 451/2000, Reward: 4.561, Step: 500 +2023-04-15 15:11:58 - r - INFO: - Episode: 452/2000, Reward: -0.806, Step: 500 +2023-04-15 15:12:03 - r - INFO: - Episode: 453/2000, Reward: -8.220, Step: 500 +2023-04-15 15:12:07 - r - INFO: - Episode: 454/2000, Reward: -0.426, Step: 500 +2023-04-15 15:12:12 - r - INFO: - Episode: 455/2000, Reward: -2.246, Step: 500 +2023-04-15 15:12:20 - r - INFO: - Episode: 456/2000, Reward: -103.839, Step: 170 +2023-04-15 15:12:24 - r - INFO: - Episode: 457/2000, Reward: 0.922, Step: 500 +2023-04-15 15:12:29 - r - INFO: - Episode: 458/2000, Reward: 8.421, Step: 500 +2023-04-15 15:12:33 - r - INFO: - Episode: 459/2000, Reward: 0.354, Step: 500 +2023-04-15 15:12:38 - r - INFO: - Episode: 460/2000, Reward: 3.612, Step: 500 +2023-04-15 15:12:50 - r - INFO: - Episode: 461/2000, Reward: 6.724, Step: 500 +2023-04-15 15:12:54 - r - INFO: - Episode: 462/2000, Reward: 4.491, Step: 500 +2023-04-15 15:12:58 - r - INFO: - Episode: 463/2000, Reward: 3.498, Step: 500 +2023-04-15 15:13:03 - r - INFO: - Episode: 464/2000, Reward: -2.396, Step: 500 +2023-04-15 15:13:07 - r - INFO: - Episode: 465/2000, Reward: -6.797, Step: 500 +2023-04-15 15:13:18 - r - INFO: - Episode: 466/2000, Reward: 3.225, Step: 500 +2023-04-15 15:13:23 - r - INFO: - Episode: 467/2000, Reward: -6.997, Step: 500 +2023-04-15 15:13:27 - r - INFO: - Episode: 468/2000, Reward: -2.784, Step: 500 +2023-04-15 15:13:32 - r - INFO: - Episode: 469/2000, Reward: -12.632, Step: 500 +2023-04-15 15:13:36 - r - INFO: - Episode: 470/2000, Reward: -10.808, Step: 500 +2023-04-15 15:13:48 - r - INFO: - Episode: 471/2000, Reward: -12.809, Step: 500 +2023-04-15 15:13:52 - r - INFO: - Episode: 472/2000, Reward: -2.334, Step: 500 +2023-04-15 15:13:57 - r - INFO: - Episode: 473/2000, Reward: -4.627, Step: 500 +2023-04-15 15:14:01 - r - INFO: - Episode: 474/2000, Reward: -11.216, Step: 500 +2023-04-15 15:14:06 - r - INFO: - Episode: 475/2000, Reward: -3.440, Step: 500 +2023-04-15 15:14:18 - r - INFO: - Episode: 476/2000, Reward: -2.878, Step: 500 +2023-04-15 15:14:22 - r - INFO: - Episode: 477/2000, Reward: -6.830, Step: 500 +2023-04-15 15:14:26 - r - INFO: - Episode: 478/2000, Reward: -1.278, Step: 500 +2023-04-15 15:14:31 - r - INFO: - Episode: 479/2000, Reward: -3.508, Step: 500 +2023-04-15 15:14:35 - r - INFO: - Episode: 480/2000, Reward: -2.850, Step: 500 +2023-04-15 15:14:47 - r - INFO: - Episode: 481/2000, Reward: -1.148, Step: 500 +2023-04-15 15:14:51 - r - INFO: - Episode: 482/2000, Reward: 7.965, Step: 500 +2023-04-15 15:14:56 - r - INFO: - Episode: 483/2000, Reward: -11.033, Step: 500 +2023-04-15 15:15:00 - r - INFO: - Episode: 484/2000, Reward: -1.277, Step: 500 +2023-04-15 15:15:05 - r - INFO: - Episode: 485/2000, Reward: 1.413, Step: 500 +2023-04-15 15:15:17 - r - INFO: - Episode: 486/2000, Reward: -0.778, Step: 500 +2023-04-15 15:15:21 - r - INFO: - Episode: 487/2000, Reward: 5.616, Step: 500 +2023-04-15 15:15:25 - r - INFO: - Episode: 488/2000, Reward: 2.638, Step: 500 +2023-04-15 15:15:29 - r - INFO: - Episode: 489/2000, Reward: -101.534, Step: 448 +2023-04-15 15:15:34 - r - INFO: - Episode: 490/2000, Reward: -12.626, Step: 500 +2023-04-15 15:15:46 - r - INFO: - Episode: 491/2000, Reward: -12.375, Step: 500 +2023-04-15 15:15:50 - r - INFO: - Episode: 492/2000, Reward: -11.265, Step: 500 +2023-04-15 15:15:55 - r - INFO: - Episode: 493/2000, Reward: -0.004, Step: 500 +2023-04-15 15:15:59 - r - INFO: - Episode: 494/2000, Reward: 2.848, Step: 500 +2023-04-15 15:16:03 - r - INFO: - Episode: 495/2000, Reward: 5.061, Step: 500 +2023-04-15 15:16:15 - r - INFO: - Episode: 496/2000, Reward: 5.545, Step: 500 +2023-04-15 15:16:20 - r - INFO: - Episode: 497/2000, Reward: 0.724, Step: 500 +2023-04-15 15:16:24 - r - INFO: - Episode: 498/2000, Reward: -0.548, Step: 500 +2023-04-15 15:16:28 - r - INFO: - Episode: 499/2000, Reward: 3.575, Step: 500 +2023-04-15 15:16:33 - r - INFO: - Episode: 500/2000, Reward: 2.161, Step: 500 +2023-04-15 15:16:44 - r - INFO: - Episode: 501/2000, Reward: 6.460, Step: 500 +2023-04-15 15:16:49 - r - INFO: - Episode: 502/2000, Reward: -2.085, Step: 500 +2023-04-15 15:16:53 - r - INFO: - Episode: 503/2000, Reward: 8.528, Step: 500 +2023-04-15 15:16:58 - r - INFO: - Episode: 504/2000, Reward: 11.349, Step: 500 +2023-04-15 15:17:02 - r - INFO: - Episode: 505/2000, Reward: -0.190, Step: 500 +2023-04-15 15:17:14 - r - INFO: - Episode: 506/2000, Reward: 2.364, Step: 500 +2023-04-15 15:17:18 - r - INFO: - Episode: 507/2000, Reward: -13.312, Step: 500 +2023-04-15 15:17:23 - r - INFO: - Episode: 508/2000, Reward: -14.233, Step: 500 +2023-04-15 15:17:27 - r - INFO: - Episode: 509/2000, Reward: -12.696, Step: 500 +2023-04-15 15:17:32 - r - INFO: - Episode: 510/2000, Reward: 6.034, Step: 500 +2023-04-15 15:17:44 - r - INFO: - Episode: 511/2000, Reward: -14.897, Step: 500 +2023-04-15 15:17:48 - r - INFO: - Episode: 512/2000, Reward: -3.032, Step: 500 +2023-04-15 15:17:52 - r - INFO: - Episode: 513/2000, Reward: 5.359, Step: 500 +2023-04-15 15:17:57 - r - INFO: - Episode: 514/2000, Reward: -2.764, Step: 500 +2023-04-15 15:18:01 - r - INFO: - Episode: 515/2000, Reward: -0.383, Step: 500 +2023-04-15 15:18:09 - r - INFO: - Episode: 516/2000, Reward: -120.613, Step: 88 +2023-04-15 15:18:14 - r - INFO: - Episode: 517/2000, Reward: 3.263, Step: 500 +2023-04-15 15:18:18 - r - INFO: - Episode: 518/2000, Reward: 6.691, Step: 500 +2023-04-15 15:18:23 - r - INFO: - Episode: 519/2000, Reward: 11.978, Step: 500 +2023-04-15 15:18:27 - r - INFO: - Episode: 520/2000, Reward: 5.179, Step: 500 +2023-04-15 15:18:39 - r - INFO: - Episode: 521/2000, Reward: 1.583, Step: 500 +2023-04-15 15:18:43 - r - INFO: - Episode: 522/2000, Reward: 10.076, Step: 500 +2023-04-15 15:18:48 - r - INFO: - Episode: 523/2000, Reward: 5.094, Step: 500 +2023-04-15 15:18:52 - r - INFO: - Episode: 524/2000, Reward: 4.144, Step: 500 +2023-04-15 15:18:56 - r - INFO: - Episode: 525/2000, Reward: 8.624, Step: 500 +2023-04-15 15:19:04 - r - INFO: - Current episode 525 has the best eval reward: 11.811 +2023-04-15 15:19:08 - r - INFO: - Episode: 526/2000, Reward: 6.502, Step: 500 +2023-04-15 15:19:13 - r - INFO: - Episode: 527/2000, Reward: -26.375, Step: 500 +2023-04-15 15:19:17 - r - INFO: - Episode: 528/2000, Reward: -15.480, Step: 500 +2023-04-15 15:19:22 - r - INFO: - Episode: 529/2000, Reward: -8.949, Step: 500 +2023-04-15 15:19:26 - r - INFO: - Episode: 530/2000, Reward: -13.200, Step: 500 +2023-04-15 15:19:38 - r - INFO: - Episode: 531/2000, Reward: -12.305, Step: 500 +2023-04-15 15:19:43 - r - INFO: - Episode: 532/2000, Reward: -14.720, Step: 500 +2023-04-15 15:19:47 - r - INFO: - Episode: 533/2000, Reward: -15.949, Step: 500 +2023-04-15 15:19:52 - r - INFO: - Episode: 534/2000, Reward: -10.007, Step: 500 +2023-04-15 15:19:57 - r - INFO: - Episode: 535/2000, Reward: -14.726, Step: 500 +2023-04-15 15:20:08 - r - INFO: - Episode: 536/2000, Reward: -22.917, Step: 500 +2023-04-15 15:20:13 - r - INFO: - Episode: 537/2000, Reward: 9.529, Step: 500 +2023-04-15 15:20:18 - r - INFO: - Episode: 538/2000, Reward: 12.094, Step: 500 +2023-04-15 15:20:22 - r - INFO: - Episode: 539/2000, Reward: 5.354, Step: 500 +2023-04-15 15:20:27 - r - INFO: - Episode: 540/2000, Reward: 9.353, Step: 500 +2023-04-15 15:20:39 - r - INFO: - Episode: 541/2000, Reward: 6.630, Step: 500 +2023-04-15 15:20:43 - r - INFO: - Episode: 542/2000, Reward: 15.758, Step: 500 +2023-04-15 15:20:48 - r - INFO: - Episode: 543/2000, Reward: 15.401, Step: 500 +2023-04-15 15:20:52 - r - INFO: - Episode: 544/2000, Reward: 9.888, Step: 500 +2023-04-15 15:20:57 - r - INFO: - Episode: 545/2000, Reward: 5.234, Step: 500 +2023-04-15 15:21:04 - r - INFO: - Current episode 545 has the best eval reward: 25.821 +2023-04-15 15:21:08 - r - INFO: - Episode: 546/2000, Reward: 11.365, Step: 500 +2023-04-15 15:21:13 - r - INFO: - Episode: 547/2000, Reward: 8.068, Step: 500 +2023-04-15 15:21:17 - r - INFO: - Episode: 548/2000, Reward: 6.835, Step: 500 +2023-04-15 15:21:22 - r - INFO: - Episode: 549/2000, Reward: 9.446, Step: 500 +2023-04-15 15:21:26 - r - INFO: - Episode: 550/2000, Reward: 11.188, Step: 500 +2023-04-15 15:21:37 - r - INFO: - Episode: 551/2000, Reward: 4.882, Step: 500 +2023-04-15 15:21:38 - r - INFO: - Episode: 552/2000, Reward: -114.720, Step: 75 +2023-04-15 15:21:42 - r - INFO: - Episode: 553/2000, Reward: 10.181, Step: 500 +2023-04-15 15:21:47 - r - INFO: - Episode: 554/2000, Reward: 14.423, Step: 500 +2023-04-15 15:21:51 - r - INFO: - Episode: 555/2000, Reward: 5.541, Step: 500 +2023-04-15 15:22:03 - r - INFO: - Episode: 556/2000, Reward: 9.005, Step: 500 +2023-04-15 15:22:07 - r - INFO: - Episode: 557/2000, Reward: 10.035, Step: 500 +2023-04-15 15:22:12 - r - INFO: - Episode: 558/2000, Reward: 9.210, Step: 500 +2023-04-15 15:22:16 - r - INFO: - Episode: 559/2000, Reward: 7.383, Step: 500 +2023-04-15 15:22:21 - r - INFO: - Episode: 560/2000, Reward: 11.617, Step: 500 +2023-04-15 15:22:33 - r - INFO: - Episode: 561/2000, Reward: 11.470, Step: 500 +2023-04-15 15:22:37 - r - INFO: - Episode: 562/2000, Reward: 7.642, Step: 500 +2023-04-15 15:22:42 - r - INFO: - Episode: 563/2000, Reward: 9.006, Step: 500 +2023-04-15 15:22:46 - r - INFO: - Episode: 564/2000, Reward: 10.711, Step: 500 +2023-04-15 15:22:50 - r - INFO: - Episode: 565/2000, Reward: 8.975, Step: 500 +2023-04-15 15:23:02 - r - INFO: - Episode: 566/2000, Reward: 19.734, Step: 500 +2023-04-15 15:23:06 - r - INFO: - Episode: 567/2000, Reward: 19.418, Step: 500 +2023-04-15 15:23:11 - r - INFO: - Episode: 568/2000, Reward: 11.406, Step: 500 +2023-04-15 15:23:15 - r - INFO: - Episode: 569/2000, Reward: 14.070, Step: 500 +2023-04-15 15:23:19 - r - INFO: - Episode: 570/2000, Reward: 10.250, Step: 500 +2023-04-15 15:23:31 - r - INFO: - Episode: 571/2000, Reward: 10.881, Step: 500 +2023-04-15 15:23:35 - r - INFO: - Episode: 572/2000, Reward: 11.689, Step: 500 +2023-04-15 15:23:39 - r - INFO: - Episode: 573/2000, Reward: 15.494, Step: 500 +2023-04-15 15:23:44 - r - INFO: - Episode: 574/2000, Reward: 15.168, Step: 500 +2023-04-15 15:23:48 - r - INFO: - Episode: 575/2000, Reward: 16.677, Step: 500 +2023-04-15 15:23:59 - r - INFO: - Episode: 576/2000, Reward: 7.421, Step: 500 +2023-04-15 15:24:04 - r - INFO: - Episode: 577/2000, Reward: 11.725, Step: 500 +2023-04-15 15:24:08 - r - INFO: - Episode: 578/2000, Reward: -24.252, Step: 500 +2023-04-15 15:24:12 - r - INFO: - Episode: 579/2000, Reward: -20.686, Step: 500 +2023-04-15 15:24:16 - r - INFO: - Episode: 580/2000, Reward: -18.206, Step: 500 +2023-04-15 15:24:28 - r - INFO: - Episode: 581/2000, Reward: -19.995, Step: 500 +2023-04-15 15:24:32 - r - INFO: - Episode: 582/2000, Reward: -18.656, Step: 500 +2023-04-15 15:24:37 - r - INFO: - Episode: 583/2000, Reward: -15.285, Step: 500 +2023-04-15 15:24:38 - r - INFO: - Episode: 584/2000, Reward: -109.621, Step: 185 +2023-04-15 15:24:43 - r - INFO: - Episode: 585/2000, Reward: 12.779, Step: 500 +2023-04-15 15:24:54 - r - INFO: - Episode: 586/2000, Reward: -12.092, Step: 500 +2023-04-15 15:24:58 - r - INFO: - Episode: 587/2000, Reward: 20.772, Step: 500 +2023-04-15 15:25:03 - r - INFO: - Episode: 588/2000, Reward: 14.499, Step: 500 +2023-04-15 15:25:07 - r - INFO: - Episode: 589/2000, Reward: 6.674, Step: 500 +2023-04-15 15:25:11 - r - INFO: - Episode: 590/2000, Reward: 6.450, Step: 500 +2023-04-15 15:25:23 - r - INFO: - Episode: 591/2000, Reward: 12.601, Step: 500 +2023-04-15 15:25:27 - r - INFO: - Episode: 592/2000, Reward: 10.868, Step: 500 +2023-04-15 15:25:31 - r - INFO: - Episode: 593/2000, Reward: 10.013, Step: 500 +2023-04-15 15:25:36 - r - INFO: - Episode: 594/2000, Reward: 3.672, Step: 500 +2023-04-15 15:25:40 - r - INFO: - Episode: 595/2000, Reward: 6.987, Step: 500 +2023-04-15 15:25:52 - r - INFO: - Episode: 596/2000, Reward: 16.274, Step: 500 +2023-04-15 15:25:56 - r - INFO: - Episode: 597/2000, Reward: 11.265, Step: 500 +2023-04-15 15:26:00 - r - INFO: - Episode: 598/2000, Reward: 10.189, Step: 500 +2023-04-15 15:26:04 - r - INFO: - Episode: 599/2000, Reward: 10.319, Step: 500 +2023-04-15 15:26:09 - r - INFO: - Episode: 600/2000, Reward: 18.888, Step: 500 +2023-04-15 15:26:20 - r - INFO: - Episode: 601/2000, Reward: -2.053, Step: 500 +2023-04-15 15:26:25 - r - INFO: - Episode: 602/2000, Reward: 6.030, Step: 500 +2023-04-15 15:26:29 - r - INFO: - Episode: 603/2000, Reward: 10.703, Step: 500 +2023-04-15 15:26:33 - r - INFO: - Episode: 604/2000, Reward: 9.976, Step: 500 +2023-04-15 15:26:34 - r - INFO: - Episode: 605/2000, Reward: -115.956, Step: 108 +2023-04-15 15:26:44 - r - INFO: - Episode: 606/2000, Reward: -21.730, Step: 500 +2023-04-15 15:26:49 - r - INFO: - Episode: 607/2000, Reward: 16.452, Step: 500 +2023-04-15 15:26:53 - r - INFO: - Episode: 608/2000, Reward: 11.830, Step: 500 +2023-04-15 15:26:57 - r - INFO: - Episode: 609/2000, Reward: 3.733, Step: 500 +2023-04-15 15:27:01 - r - INFO: - Episode: 610/2000, Reward: 9.610, Step: 500 +2023-04-15 15:27:13 - r - INFO: - Episode: 611/2000, Reward: 12.698, Step: 500 +2023-04-15 15:27:17 - r - INFO: - Episode: 612/2000, Reward: 11.539, Step: 500 +2023-04-15 15:27:21 - r - INFO: - Episode: 613/2000, Reward: 13.384, Step: 500 +2023-04-15 15:27:26 - r - INFO: - Episode: 614/2000, Reward: 11.951, Step: 500 +2023-04-15 15:27:30 - r - INFO: - Episode: 615/2000, Reward: 11.511, Step: 500 +2023-04-15 15:27:42 - r - INFO: - Episode: 616/2000, Reward: 7.633, Step: 500 +2023-04-15 15:27:46 - r - INFO: - Episode: 617/2000, Reward: 7.308, Step: 500 +2023-04-15 15:27:50 - r - INFO: - Episode: 618/2000, Reward: 14.279, Step: 500 +2023-04-15 15:27:55 - r - INFO: - Episode: 619/2000, Reward: 12.519, Step: 500 +2023-04-15 15:27:59 - r - INFO: - Episode: 620/2000, Reward: 11.558, Step: 500 +2023-04-15 15:28:06 - r - INFO: - Current episode 620 has the best eval reward: 36.108 +2023-04-15 15:28:11 - r - INFO: - Episode: 621/2000, Reward: 16.553, Step: 500 +2023-04-15 15:28:15 - r - INFO: - Episode: 622/2000, Reward: 13.818, Step: 500 +2023-04-15 15:28:20 - r - INFO: - Episode: 623/2000, Reward: 12.585, Step: 500 +2023-04-15 15:28:24 - r - INFO: - Episode: 624/2000, Reward: 15.298, Step: 500 +2023-04-15 15:28:29 - r - INFO: - Episode: 625/2000, Reward: 9.067, Step: 500 +2023-04-15 15:28:41 - r - INFO: - Episode: 626/2000, Reward: 8.335, Step: 500 +2023-04-15 15:28:46 - r - INFO: - Episode: 627/2000, Reward: 12.433, Step: 500 +2023-04-15 15:28:50 - r - INFO: - Episode: 628/2000, Reward: -7.863, Step: 500 +2023-04-15 15:28:54 - r - INFO: - Episode: 629/2000, Reward: 13.251, Step: 500 +2023-04-15 15:28:59 - r - INFO: - Episode: 630/2000, Reward: 7.225, Step: 500 +2023-04-15 15:29:10 - r - INFO: - Episode: 631/2000, Reward: 26.818, Step: 500 +2023-04-15 15:29:15 - r - INFO: - Episode: 632/2000, Reward: 16.066, Step: 500 +2023-04-15 15:29:19 - r - INFO: - Episode: 633/2000, Reward: 16.948, Step: 500 +2023-04-15 15:29:23 - r - INFO: - Episode: 634/2000, Reward: 20.841, Step: 500 +2023-04-15 15:29:27 - r - INFO: - Episode: 635/2000, Reward: 13.545, Step: 500 +2023-04-15 15:29:39 - r - INFO: - Episode: 636/2000, Reward: 18.802, Step: 500 +2023-04-15 15:29:43 - r - INFO: - Episode: 637/2000, Reward: 18.330, Step: 500 +2023-04-15 15:29:48 - r - INFO: - Episode: 638/2000, Reward: 23.574, Step: 500 +2023-04-15 15:29:52 - r - INFO: - Episode: 639/2000, Reward: 13.446, Step: 500 +2023-04-15 15:29:56 - r - INFO: - Episode: 640/2000, Reward: 11.189, Step: 500 +2023-04-15 15:30:08 - r - INFO: - Episode: 641/2000, Reward: 18.788, Step: 500 +2023-04-15 15:30:12 - r - INFO: - Episode: 642/2000, Reward: 12.258, Step: 500 +2023-04-15 15:30:16 - r - INFO: - Episode: 643/2000, Reward: 20.476, Step: 500 +2023-04-15 15:30:21 - r - INFO: - Episode: 644/2000, Reward: -10.047, Step: 500 +2023-04-15 15:30:25 - r - INFO: - Episode: 645/2000, Reward: -8.869, Step: 500 +2023-04-15 15:30:36 - r - INFO: - Episode: 646/2000, Reward: 9.157, Step: 500 +2023-04-15 15:30:41 - r - INFO: - Episode: 647/2000, Reward: 18.844, Step: 500 +2023-04-15 15:30:45 - r - INFO: - Episode: 648/2000, Reward: 15.015, Step: 500 +2023-04-15 15:30:49 - r - INFO: - Episode: 649/2000, Reward: 14.495, Step: 500 +2023-04-15 15:30:54 - r - INFO: - Episode: 650/2000, Reward: 12.126, Step: 500 +2023-04-15 15:31:05 - r - INFO: - Episode: 651/2000, Reward: 11.521, Step: 500 +2023-04-15 15:33:43 - r - INFO: - Episode: 652/2000, Reward: 15.619, Step: 500 +2023-04-15 15:33:47 - r - INFO: - Episode: 653/2000, Reward: 17.102, Step: 500 +2023-04-15 15:33:51 - r - INFO: - Episode: 654/2000, Reward: 1.025, Step: 500 +2023-04-15 15:33:55 - r - INFO: - Episode: 655/2000, Reward: 11.629, Step: 500 +2023-04-15 15:34:06 - r - INFO: - Episode: 656/2000, Reward: 20.305, Step: 500 +2023-04-15 15:34:10 - r - INFO: - Episode: 657/2000, Reward: 17.683, Step: 500 +2023-04-15 15:34:15 - r - INFO: - Episode: 658/2000, Reward: 26.588, Step: 500 +2023-04-15 15:34:19 - r - INFO: - Episode: 659/2000, Reward: 16.232, Step: 500 +2023-04-15 15:34:23 - r - INFO: - Episode: 660/2000, Reward: 22.258, Step: 500 +2023-04-15 15:34:34 - r - INFO: - Episode: 661/2000, Reward: 11.311, Step: 500 +2023-04-15 15:34:38 - r - INFO: - Episode: 662/2000, Reward: -87.910, Step: 497 +2023-04-15 15:34:42 - r - INFO: - Episode: 663/2000, Reward: 16.312, Step: 500 +2023-04-15 15:34:47 - r - INFO: - Episode: 664/2000, Reward: 23.847, Step: 500 +2023-04-15 15:34:51 - r - INFO: - Episode: 665/2000, Reward: 14.506, Step: 500 +2023-04-15 15:35:02 - r - INFO: - Episode: 666/2000, Reward: 4.532, Step: 500 +2023-04-15 15:35:06 - r - INFO: - Episode: 667/2000, Reward: 17.960, Step: 500 +2023-04-15 15:35:10 - r - INFO: - Episode: 668/2000, Reward: 22.432, Step: 500 +2023-04-15 15:35:14 - r - INFO: - Episode: 669/2000, Reward: 21.416, Step: 500 +2023-04-15 15:35:19 - r - INFO: - Episode: 670/2000, Reward: -1.833, Step: 500 +2023-04-15 15:35:28 - r - INFO: - Episode: 671/2000, Reward: 14.517, Step: 500 +2023-04-15 15:35:32 - r - INFO: - Episode: 672/2000, Reward: 16.577, Step: 500 +2023-04-15 15:35:36 - r - INFO: - Episode: 673/2000, Reward: 12.015, Step: 500 +2023-04-15 15:35:41 - r - INFO: - Episode: 674/2000, Reward: 17.430, Step: 500 +2023-04-15 15:35:45 - r - INFO: - Episode: 675/2000, Reward: 23.088, Step: 500 +2023-04-15 15:35:56 - r - INFO: - Episode: 676/2000, Reward: 19.497, Step: 500 +2023-04-15 15:36:00 - r - INFO: - Episode: 677/2000, Reward: 24.454, Step: 500 +2023-04-15 15:36:05 - r - INFO: - Episode: 678/2000, Reward: 22.333, Step: 500 +2023-04-15 15:36:09 - r - INFO: - Episode: 679/2000, Reward: 15.555, Step: 500 +2023-04-15 15:36:13 - r - INFO: - Episode: 680/2000, Reward: 20.126, Step: 500 +2023-04-15 15:36:23 - r - INFO: - Episode: 681/2000, Reward: -97.039, Step: 395 +2023-04-15 15:36:27 - r - INFO: - Episode: 682/2000, Reward: 22.007, Step: 500 +2023-04-15 15:36:31 - r - INFO: - Episode: 683/2000, Reward: 20.586, Step: 500 +2023-04-15 15:36:36 - r - INFO: - Episode: 684/2000, Reward: 11.818, Step: 500 +2023-04-15 15:36:40 - r - INFO: - Episode: 685/2000, Reward: 15.060, Step: 500 +2023-04-15 15:36:51 - r - INFO: - Episode: 686/2000, Reward: 16.215, Step: 500 +2023-04-15 15:36:56 - r - INFO: - Episode: 687/2000, Reward: 8.027, Step: 500 +2023-04-15 15:37:00 - r - INFO: - Episode: 688/2000, Reward: 24.204, Step: 500 +2023-04-15 15:37:04 - r - INFO: - Episode: 689/2000, Reward: -100.586, Step: 461 +2023-04-15 15:37:08 - r - INFO: - Episode: 690/2000, Reward: 24.722, Step: 500 +2023-04-15 15:37:19 - r - INFO: - Episode: 691/2000, Reward: 20.825, Step: 500 +2023-04-15 15:37:23 - r - INFO: - Episode: 692/2000, Reward: 29.186, Step: 500 +2023-04-15 15:37:28 - r - INFO: - Episode: 693/2000, Reward: 21.663, Step: 500 +2023-04-15 15:37:32 - r - INFO: - Episode: 694/2000, Reward: 21.201, Step: 500 +2023-04-15 15:37:36 - r - INFO: - Episode: 695/2000, Reward: 19.888, Step: 500 +2023-04-15 15:37:48 - r - INFO: - Episode: 696/2000, Reward: 8.296, Step: 500 +2023-04-15 15:37:52 - r - INFO: - Episode: 697/2000, Reward: 13.201, Step: 500 +2023-04-15 15:37:56 - r - INFO: - Episode: 698/2000, Reward: 23.525, Step: 500 +2023-04-15 15:38:00 - r - INFO: - Episode: 699/2000, Reward: 14.086, Step: 500 +2023-04-15 15:38:05 - r - INFO: - Episode: 700/2000, Reward: 19.185, Step: 500 +2023-04-15 15:38:16 - r - INFO: - Episode: 701/2000, Reward: 13.815, Step: 500 +2023-04-15 15:38:20 - r - INFO: - Episode: 702/2000, Reward: 18.297, Step: 500 +2023-04-15 15:38:24 - r - INFO: - Episode: 703/2000, Reward: 14.803, Step: 500 +2023-04-15 15:38:29 - r - INFO: - Episode: 704/2000, Reward: 14.726, Step: 500 +2023-04-15 15:38:33 - r - INFO: - Episode: 705/2000, Reward: 21.331, Step: 500 +2023-04-15 15:38:44 - r - INFO: - Episode: 706/2000, Reward: 23.328, Step: 500 +2023-04-15 15:38:49 - r - INFO: - Episode: 707/2000, Reward: 20.353, Step: 500 +2023-04-15 15:38:53 - r - INFO: - Episode: 708/2000, Reward: 26.936, Step: 500 +2023-04-15 15:38:57 - r - INFO: - Episode: 709/2000, Reward: 26.883, Step: 500 +2023-04-15 15:39:01 - r - INFO: - Episode: 710/2000, Reward: 18.027, Step: 500 +2023-04-15 15:39:13 - r - INFO: - Episode: 711/2000, Reward: 20.143, Step: 500 +2023-04-15 15:39:17 - r - INFO: - Episode: 712/2000, Reward: 21.360, Step: 500 +2023-04-15 15:39:21 - r - INFO: - Episode: 713/2000, Reward: 21.011, Step: 500 +2023-04-15 15:39:26 - r - INFO: - Episode: 714/2000, Reward: 10.665, Step: 500 +2023-04-15 15:39:30 - r - INFO: - Episode: 715/2000, Reward: 22.602, Step: 500 +2023-04-15 15:41:02 - r - INFO: - Episode: 716/2000, Reward: 27.262, Step: 500 +2023-04-15 15:41:06 - r - INFO: - Episode: 717/2000, Reward: 33.106, Step: 500 +2023-04-15 15:41:11 - r - INFO: - Episode: 718/2000, Reward: 22.352, Step: 500 +2023-04-15 15:41:15 - r - INFO: - Episode: 719/2000, Reward: 34.139, Step: 500 +2023-04-15 15:41:19 - r - INFO: - Episode: 720/2000, Reward: 35.638, Step: 500 +2023-04-15 15:41:26 - r - INFO: - Current episode 720 has the best eval reward: 36.487 +2023-04-15 15:41:30 - r - INFO: - Episode: 721/2000, Reward: 29.883, Step: 500 +2023-04-15 15:41:34 - r - INFO: - Episode: 722/2000, Reward: 36.865, Step: 500 +2023-04-15 15:41:39 - r - INFO: - Episode: 723/2000, Reward: 34.763, Step: 500 +2023-04-15 15:41:43 - r - INFO: - Episode: 724/2000, Reward: 33.087, Step: 500 +2023-04-15 15:41:47 - r - INFO: - Episode: 725/2000, Reward: 38.525, Step: 500 +2023-04-15 15:41:54 - r - INFO: - Current episode 725 has the best eval reward: 43.019 +2023-04-15 15:41:58 - r - INFO: - Episode: 726/2000, Reward: 39.805, Step: 500 +2023-04-15 15:42:03 - r - INFO: - Episode: 727/2000, Reward: 27.953, Step: 500 +2023-04-15 15:42:07 - r - INFO: - Episode: 728/2000, Reward: 36.580, Step: 500 +2023-04-15 15:42:11 - r - INFO: - Episode: 729/2000, Reward: 35.151, Step: 500 +2023-04-15 15:42:15 - r - INFO: - Episode: 730/2000, Reward: 18.733, Step: 500 +2023-04-15 15:42:27 - r - INFO: - Episode: 731/2000, Reward: 35.844, Step: 500 +2023-04-15 15:42:31 - r - INFO: - Episode: 732/2000, Reward: 29.151, Step: 500 +2023-04-15 15:42:35 - r - INFO: - Episode: 733/2000, Reward: 31.567, Step: 500 +2023-04-15 15:42:39 - r - INFO: - Episode: 734/2000, Reward: 23.993, Step: 500 +2023-04-15 15:42:44 - r - INFO: - Episode: 735/2000, Reward: 28.343, Step: 500 +2023-04-15 15:42:51 - r - INFO: - Current episode 735 has the best eval reward: 51.858 +2023-04-15 15:42:55 - r - INFO: - Episode: 736/2000, Reward: 34.663, Step: 500 +2023-04-15 15:42:59 - r - INFO: - Episode: 737/2000, Reward: 40.771, Step: 500 +2023-04-15 15:43:03 - r - INFO: - Episode: 738/2000, Reward: 29.827, Step: 500 +2023-04-15 15:43:08 - r - INFO: - Episode: 739/2000, Reward: 42.813, Step: 500 +2023-04-15 15:43:12 - r - INFO: - Episode: 740/2000, Reward: 38.556, Step: 500 +2023-04-15 15:43:23 - r - INFO: - Episode: 741/2000, Reward: 27.165, Step: 500 +2023-04-15 15:43:28 - r - INFO: - Episode: 742/2000, Reward: 40.703, Step: 500 +2023-04-15 15:43:32 - r - INFO: - Episode: 743/2000, Reward: 32.478, Step: 500 +2023-04-15 15:43:36 - r - INFO: - Episode: 744/2000, Reward: 27.484, Step: 500 +2023-04-15 15:43:41 - r - INFO: - Episode: 745/2000, Reward: 41.910, Step: 500 +2023-04-15 15:43:48 - r - INFO: - Current episode 745 has the best eval reward: 52.711 +2023-04-15 15:43:52 - r - INFO: - Episode: 746/2000, Reward: 46.422, Step: 500 +2023-04-15 15:43:56 - r - INFO: - Episode: 747/2000, Reward: 40.829, Step: 500 +2023-04-15 15:44:00 - r - INFO: - Episode: 748/2000, Reward: 34.301, Step: 500 +2023-04-15 15:44:05 - r - INFO: - Episode: 749/2000, Reward: 29.765, Step: 500 +2023-04-15 15:44:09 - r - INFO: - Episode: 750/2000, Reward: 29.566, Step: 500 +2023-04-15 15:44:20 - r - INFO: - Episode: 751/2000, Reward: 37.683, Step: 500 +2023-04-15 15:44:25 - r - INFO: - Episode: 752/2000, Reward: 28.498, Step: 500 +2023-04-15 15:44:29 - r - INFO: - Episode: 753/2000, Reward: 47.237, Step: 500 +2023-04-15 15:44:33 - r - INFO: - Episode: 754/2000, Reward: 43.990, Step: 500 +2023-04-15 15:44:38 - r - INFO: - Episode: 755/2000, Reward: 20.420, Step: 500 +2023-04-15 15:44:48 - r - INFO: - Episode: 756/2000, Reward: 53.713, Step: 500 +2023-04-15 15:44:52 - r - INFO: - Episode: 757/2000, Reward: -95.599, Step: 471 +2023-04-15 15:44:56 - r - INFO: - Episode: 758/2000, Reward: 28.404, Step: 500 +2023-04-15 15:45:00 - r - INFO: - Episode: 759/2000, Reward: 44.021, Step: 500 +2023-04-15 15:45:05 - r - INFO: - Episode: 760/2000, Reward: 47.921, Step: 500 +2023-04-15 15:45:16 - r - INFO: - Episode: 761/2000, Reward: 43.741, Step: 500 +2023-04-15 15:45:20 - r - INFO: - Episode: 762/2000, Reward: 40.391, Step: 500 +2023-04-15 15:45:25 - r - INFO: - Episode: 763/2000, Reward: 38.224, Step: 500 +2023-04-15 15:45:29 - r - INFO: - Episode: 764/2000, Reward: 32.843, Step: 500 +2023-04-15 15:45:33 - r - INFO: - Episode: 765/2000, Reward: 29.753, Step: 500 +2023-04-15 15:45:44 - r - INFO: - Episode: 766/2000, Reward: 25.846, Step: 500 +2023-04-15 15:45:49 - r - INFO: - Episode: 767/2000, Reward: 52.902, Step: 500 +2023-04-15 15:45:53 - r - INFO: - Episode: 768/2000, Reward: 29.694, Step: 500 +2023-04-15 15:45:57 - r - INFO: - Episode: 769/2000, Reward: 42.852, Step: 500 +2023-04-15 15:46:01 - r - INFO: - Episode: 770/2000, Reward: 56.689, Step: 500 +2023-04-15 15:46:13 - r - INFO: - Episode: 771/2000, Reward: 36.622, Step: 500 +2023-04-15 15:46:17 - r - INFO: - Episode: 772/2000, Reward: 39.374, Step: 500 +2023-04-15 15:46:22 - r - INFO: - Episode: 773/2000, Reward: 31.738, Step: 500 +2023-04-15 15:46:26 - r - INFO: - Episode: 774/2000, Reward: 35.431, Step: 500 +2023-04-15 15:46:30 - r - INFO: - Episode: 775/2000, Reward: 52.736, Step: 500 +2023-04-15 15:46:41 - r - INFO: - Episode: 776/2000, Reward: 39.405, Step: 500 +2023-04-15 15:46:46 - r - INFO: - Episode: 777/2000, Reward: 35.169, Step: 500 +2023-04-15 15:46:50 - r - INFO: - Episode: 778/2000, Reward: 42.586, Step: 500 +2023-04-15 15:46:54 - r - INFO: - Episode: 779/2000, Reward: 49.042, Step: 500 +2023-04-15 15:46:58 - r - INFO: - Episode: 780/2000, Reward: 55.692, Step: 500 +2023-04-15 15:47:06 - r - INFO: - Current episode 780 has the best eval reward: 63.370 +2023-04-15 15:47:09 - r - INFO: - Episode: 781/2000, Reward: -87.272, Step: 380 +2023-04-15 15:47:13 - r - INFO: - Episode: 782/2000, Reward: 54.921, Step: 500 +2023-04-15 15:47:18 - r - INFO: - Episode: 783/2000, Reward: 59.670, Step: 500 +2023-04-15 15:47:22 - r - INFO: - Episode: 784/2000, Reward: 46.411, Step: 500 +2023-04-15 15:47:26 - r - INFO: - Episode: 785/2000, Reward: 46.253, Step: 500 +2023-04-15 15:47:33 - r - INFO: - Current episode 785 has the best eval reward: 66.669 +2023-04-15 15:47:38 - r - INFO: - Episode: 786/2000, Reward: 60.769, Step: 500 +2023-04-15 15:47:42 - r - INFO: - Episode: 787/2000, Reward: 58.926, Step: 500 +2023-04-15 15:47:46 - r - INFO: - Episode: 788/2000, Reward: 62.135, Step: 500 +2023-04-15 15:47:51 - r - INFO: - Episode: 789/2000, Reward: 55.031, Step: 500 +2023-04-15 15:47:55 - r - INFO: - Episode: 790/2000, Reward: 62.248, Step: 500 +2023-04-15 15:48:02 - r - INFO: - Current episode 790 has the best eval reward: 71.395 +2023-04-15 15:48:07 - r - INFO: - Episode: 791/2000, Reward: 51.461, Step: 500 +2023-04-15 15:48:08 - r - INFO: - Episode: 792/2000, Reward: -76.646, Step: 195 +2023-04-15 15:48:13 - r - INFO: - Episode: 793/2000, Reward: 66.507, Step: 500 +2023-04-15 15:48:17 - r - INFO: - Episode: 794/2000, Reward: 47.660, Step: 500 +2023-04-15 15:48:21 - r - INFO: - Episode: 795/2000, Reward: 49.431, Step: 500 +2023-04-15 15:48:31 - r - INFO: - Episode: 796/2000, Reward: 50.893, Step: 500 +2023-04-15 15:48:32 - r - INFO: - Episode: 797/2000, Reward: -106.550, Step: 75 +2023-04-15 15:48:36 - r - INFO: - Episode: 798/2000, Reward: 63.720, Step: 500 +2023-04-15 15:48:40 - r - INFO: - Episode: 799/2000, Reward: 52.738, Step: 500 +2023-04-15 15:48:45 - r - INFO: - Episode: 800/2000, Reward: 44.015, Step: 500 +2023-04-15 15:48:56 - r - INFO: - Episode: 801/2000, Reward: 49.737, Step: 500 +2023-04-15 15:49:01 - r - INFO: - Episode: 802/2000, Reward: 38.073, Step: 500 +2023-04-15 15:49:05 - r - INFO: - Episode: 803/2000, Reward: 50.792, Step: 500 +2023-04-15 15:49:09 - r - INFO: - Episode: 804/2000, Reward: 49.642, Step: 500 +2023-04-15 15:49:14 - r - INFO: - Episode: 805/2000, Reward: 64.169, Step: 500 +2023-04-15 15:49:25 - r - INFO: - Episode: 806/2000, Reward: 50.460, Step: 500 +2023-04-15 15:49:29 - r - INFO: - Episode: 807/2000, Reward: 36.265, Step: 500 +2023-04-15 15:49:34 - r - INFO: - Episode: 808/2000, Reward: 45.741, Step: 500 +2023-04-15 15:49:38 - r - INFO: - Episode: 809/2000, Reward: 42.382, Step: 500 +2023-04-15 15:49:42 - r - INFO: - Episode: 810/2000, Reward: 60.790, Step: 500 +2023-04-15 15:49:54 - r - INFO: - Episode: 811/2000, Reward: 71.121, Step: 500 +2023-04-15 15:49:58 - r - INFO: - Episode: 812/2000, Reward: 73.183, Step: 500 +2023-04-15 15:50:03 - r - INFO: - Episode: 813/2000, Reward: 65.241, Step: 500 +2023-04-15 15:50:05 - r - INFO: - Episode: 814/2000, Reward: -70.454, Step: 269 +2023-04-15 15:50:09 - r - INFO: - Episode: 815/2000, Reward: 51.758, Step: 500 +2023-04-15 15:50:16 - r - INFO: - Current episode 815 has the best eval reward: 79.873 +2023-04-15 15:50:21 - r - INFO: - Episode: 816/2000, Reward: 63.901, Step: 500 +2023-04-15 15:50:25 - r - INFO: - Episode: 817/2000, Reward: 63.710, Step: 500 +2023-04-15 15:50:29 - r - INFO: - Episode: 818/2000, Reward: 73.578, Step: 500 +2023-04-15 15:50:34 - r - INFO: - Episode: 819/2000, Reward: 70.338, Step: 500 +2023-04-15 15:50:38 - r - INFO: - Episode: 820/2000, Reward: 52.383, Step: 500 +2023-04-15 15:50:49 - r - INFO: - Episode: 821/2000, Reward: 66.952, Step: 500 +2023-04-15 15:50:53 - r - INFO: - Episode: 822/2000, Reward: 60.301, Step: 500 +2023-04-15 15:50:58 - r - INFO: - Episode: 823/2000, Reward: 56.452, Step: 500 +2023-04-15 15:51:02 - r - INFO: - Episode: 824/2000, Reward: 48.269, Step: 500 +2023-04-15 15:51:06 - r - INFO: - Episode: 825/2000, Reward: 77.623, Step: 500 +2023-04-15 15:51:18 - r - INFO: - Episode: 826/2000, Reward: 66.086, Step: 500 +2023-04-15 15:51:22 - r - INFO: - Episode: 827/2000, Reward: 59.025, Step: 500 +2023-04-15 15:51:26 - r - INFO: - Episode: 828/2000, Reward: 78.841, Step: 500 +2023-04-15 15:51:31 - r - INFO: - Episode: 829/2000, Reward: 80.898, Step: 500 +2023-04-15 15:51:35 - r - INFO: - Episode: 830/2000, Reward: 75.058, Step: 500 +2023-04-15 15:51:46 - r - INFO: - Episode: 831/2000, Reward: 82.089, Step: 500 +2023-04-15 15:51:51 - r - INFO: - Episode: 832/2000, Reward: 82.586, Step: 500 +2023-04-15 15:51:52 - r - INFO: - Episode: 833/2000, Reward: -80.467, Step: 169 +2023-04-15 15:51:57 - r - INFO: - Episode: 834/2000, Reward: 77.167, Step: 500 +2023-04-15 15:52:01 - r - INFO: - Episode: 835/2000, Reward: 57.074, Step: 500 +2023-04-15 15:52:10 - r - INFO: - Episode: 836/2000, Reward: 67.602, Step: 500 +2023-04-15 15:52:14 - r - INFO: - Episode: 837/2000, Reward: 73.431, Step: 500 +2023-04-15 15:52:17 - r - INFO: - Episode: 838/2000, Reward: -72.631, Step: 298 +2023-04-15 15:52:21 - r - INFO: - Episode: 839/2000, Reward: 56.665, Step: 500 +2023-04-15 15:52:23 - r - INFO: - Episode: 840/2000, Reward: -79.722, Step: 237 +2023-04-15 15:52:34 - r - INFO: - Episode: 841/2000, Reward: 88.766, Step: 500 +2023-04-15 15:52:38 - r - INFO: - Episode: 842/2000, Reward: 75.741, Step: 500 +2023-04-15 15:52:41 - r - INFO: - Episode: 843/2000, Reward: -42.834, Step: 303 +2023-04-15 15:52:44 - r - INFO: - Episode: 844/2000, Reward: -37.962, Step: 424 +2023-04-15 15:52:49 - r - INFO: - Episode: 845/2000, Reward: 81.303, Step: 500 +2023-04-15 15:53:00 - r - INFO: - Episode: 846/2000, Reward: 66.565, Step: 500 +2023-04-15 15:53:49 - r - INFO: - Episode: 847/2000, Reward: 85.903, Step: 500 +2023-04-15 15:53:53 - r - INFO: - Episode: 848/2000, Reward: 67.222, Step: 500 +2023-04-15 15:53:54 - r - INFO: - Episode: 849/2000, Reward: -109.538, Step: 66 +2023-04-15 15:53:55 - r - INFO: - Episode: 850/2000, Reward: -105.241, Step: 142 +2023-04-15 15:54:04 - r - INFO: - Episode: 851/2000, Reward: -55.160, Step: 477 +2023-04-15 15:54:06 - r - INFO: - Episode: 852/2000, Reward: -68.252, Step: 278 +2023-04-15 15:54:10 - r - INFO: - Episode: 853/2000, Reward: 77.322, Step: 500 +2023-04-15 15:54:14 - r - INFO: - Episode: 854/2000, Reward: -21.879, Step: 471 +2023-04-15 15:54:15 - r - INFO: - Episode: 855/2000, Reward: -115.687, Step: 110 +2023-04-15 15:54:22 - r - INFO: - Current episode 855 has the best eval reward: 89.017 +2023-04-15 15:54:26 - r - INFO: - Episode: 856/2000, Reward: 61.551, Step: 500 +2023-04-15 15:54:31 - r - INFO: - Episode: 857/2000, Reward: 80.643, Step: 500 +2023-04-15 15:54:35 - r - INFO: - Episode: 858/2000, Reward: 64.427, Step: 500 +2023-04-15 15:54:39 - r - INFO: - Episode: 859/2000, Reward: 59.771, Step: 500 +2023-04-15 15:54:44 - r - INFO: - Episode: 860/2000, Reward: 62.897, Step: 500 +2023-04-15 15:54:55 - r - INFO: - Episode: 861/2000, Reward: 77.588, Step: 500 +2023-04-15 15:54:57 - r - INFO: - Episode: 862/2000, Reward: -80.815, Step: 231 +2023-04-15 15:55:00 - r - INFO: - Episode: 863/2000, Reward: -52.242, Step: 383 +2023-04-15 15:55:04 - r - INFO: - Episode: 864/2000, Reward: 79.754, Step: 500 +2023-04-15 15:55:07 - r - INFO: - Episode: 865/2000, Reward: -88.709, Step: 244 +2023-04-15 15:55:14 - r - INFO: - Episode: 866/2000, Reward: -85.195, Step: 250 +2023-04-15 15:55:16 - r - INFO: - Episode: 867/2000, Reward: -81.564, Step: 246 +2023-04-15 15:55:17 - r - INFO: - Episode: 868/2000, Reward: -88.821, Step: 144 +2023-04-15 15:55:18 - r - INFO: - Episode: 869/2000, Reward: -86.757, Step: 168 +2023-04-15 15:55:20 - r - INFO: - Episode: 870/2000, Reward: -95.141, Step: 156 +2023-04-15 15:55:25 - r - INFO: - Episode: 871/2000, Reward: -81.943, Step: 290 +2023-04-15 15:57:53 - r - INFO: - Episode: 872/2000, Reward: -69.356, Step: 236 +2023-04-15 15:57:55 - r - INFO: - Episode: 873/2000, Reward: -85.056, Step: 241 +2023-04-15 15:57:56 - r - INFO: - Episode: 874/2000, Reward: -74.997, Step: 201 +2023-04-15 15:58:01 - r - INFO: - Episode: 875/2000, Reward: 65.163, Step: 500 +2023-04-15 15:58:12 - r - INFO: - Episode: 876/2000, Reward: 48.866, Step: 500 +2023-04-15 15:58:16 - r - INFO: - Episode: 877/2000, Reward: 72.887, Step: 500 +2023-04-15 15:58:20 - r - INFO: - Episode: 878/2000, Reward: 76.589, Step: 500 +2023-04-15 15:58:24 - r - INFO: - Episode: 879/2000, Reward: 78.974, Step: 500 +2023-04-15 15:58:29 - r - INFO: - Episode: 880/2000, Reward: 71.132, Step: 500 +2023-04-15 15:58:40 - r - INFO: - Episode: 881/2000, Reward: 80.263, Step: 500 +2023-04-15 15:58:44 - r - INFO: - Episode: 882/2000, Reward: 75.268, Step: 500 +2023-04-15 15:58:48 - r - INFO: - Episode: 883/2000, Reward: 92.223, Step: 500 +2023-04-15 15:58:52 - r - INFO: - Episode: 884/2000, Reward: 77.370, Step: 500 +2023-04-15 15:58:56 - r - INFO: - Episode: 885/2000, Reward: 74.276, Step: 500 +2023-04-15 15:59:08 - r - INFO: - Episode: 886/2000, Reward: 78.193, Step: 500 +2023-04-15 15:59:12 - r - INFO: - Episode: 887/2000, Reward: 79.882, Step: 500 +2023-04-15 15:59:16 - r - INFO: - Episode: 888/2000, Reward: 82.422, Step: 500 +2023-04-15 15:59:20 - r - INFO: - Episode: 889/2000, Reward: 56.836, Step: 500 +2023-04-15 15:59:24 - r - INFO: - Episode: 890/2000, Reward: -51.573, Step: 462 +2023-04-15 15:59:35 - r - INFO: - Episode: 891/2000, Reward: 74.436, Step: 500 +2023-04-15 15:59:40 - r - INFO: - Episode: 892/2000, Reward: 69.013, Step: 500 +2023-04-15 15:59:44 - r - INFO: - Episode: 893/2000, Reward: 80.548, Step: 500 +2023-04-15 15:59:48 - r - INFO: - Episode: 894/2000, Reward: 13.244, Step: 500 +2023-04-15 15:59:51 - r - INFO: - Episode: 895/2000, Reward: -67.332, Step: 383 +2023-04-15 15:59:57 - r - INFO: - Episode: 896/2000, Reward: -82.271, Step: 276 +2023-04-15 16:00:01 - r - INFO: - Episode: 897/2000, Reward: 61.972, Step: 500 +2023-04-15 16:00:06 - r - INFO: - Episode: 898/2000, Reward: 78.081, Step: 500 +2023-04-15 16:00:10 - r - INFO: - Episode: 899/2000, Reward: 93.652, Step: 500 +2023-04-15 16:00:11 - r - INFO: - Episode: 900/2000, Reward: -88.540, Step: 161 +2023-04-15 16:00:22 - r - INFO: - Episode: 901/2000, Reward: -52.771, Step: 432 +2023-04-15 16:00:26 - r - INFO: - Episode: 902/2000, Reward: 77.397, Step: 500 +2023-04-15 16:00:30 - r - INFO: - Episode: 903/2000, Reward: 58.317, Step: 500 +2023-04-15 16:00:35 - r - INFO: - Episode: 904/2000, Reward: 85.266, Step: 500 +2023-04-15 16:00:39 - r - INFO: - Episode: 905/2000, Reward: 79.294, Step: 500 +2023-04-15 16:00:50 - r - INFO: - Episode: 906/2000, Reward: 84.654, Step: 500 +2023-04-15 16:00:55 - r - INFO: - Episode: 907/2000, Reward: 91.012, Step: 500 +2023-04-15 16:00:59 - r - INFO: - Episode: 908/2000, Reward: 81.163, Step: 500 +2023-04-15 16:01:03 - r - INFO: - Episode: 909/2000, Reward: 68.462, Step: 500 +2023-04-15 16:01:07 - r - INFO: - Episode: 910/2000, Reward: 74.144, Step: 500 +2023-04-15 16:01:18 - r - INFO: - Episode: 911/2000, Reward: 60.909, Step: 500 +2023-04-15 16:01:23 - r - INFO: - Episode: 912/2000, Reward: 74.496, Step: 500 +2023-04-15 16:01:27 - r - INFO: - Episode: 913/2000, Reward: 56.114, Step: 500 +2023-04-15 16:01:31 - r - INFO: - Episode: 914/2000, Reward: 91.501, Step: 500 +2023-04-15 16:01:35 - r - INFO: - Episode: 915/2000, Reward: 73.408, Step: 500 +2023-04-15 16:01:42 - r - INFO: - Current episode 915 has the best eval reward: 97.911 +2023-04-15 16:01:47 - r - INFO: - Episode: 916/2000, Reward: 89.385, Step: 500 +2023-04-15 16:01:50 - r - INFO: - Episode: 917/2000, Reward: -61.257, Step: 358 +2023-04-15 16:01:52 - r - INFO: - Episode: 918/2000, Reward: -80.526, Step: 267 +2023-04-15 16:01:54 - r - INFO: - Episode: 919/2000, Reward: -87.274, Step: 192 +2023-04-15 16:01:57 - r - INFO: - Episode: 920/2000, Reward: -64.820, Step: 330 +2023-04-15 16:02:01 - r - INFO: - Episode: 921/2000, Reward: -81.848, Step: 177 +2023-04-15 16:02:03 - r - INFO: - Episode: 922/2000, Reward: -82.630, Step: 205 +2023-04-15 16:02:07 - r - INFO: - Episode: 923/2000, Reward: 73.468, Step: 500 +2023-04-15 16:02:11 - r - INFO: - Episode: 924/2000, Reward: 72.599, Step: 500 +2023-04-15 16:02:16 - r - INFO: - Episode: 925/2000, Reward: 79.581, Step: 500 +2023-04-15 16:02:27 - r - INFO: - Episode: 926/2000, Reward: 71.414, Step: 500 +2023-04-15 16:02:31 - r - INFO: - Episode: 927/2000, Reward: 80.186, Step: 500 +2023-04-15 16:02:36 - r - INFO: - Episode: 928/2000, Reward: 76.907, Step: 500 +2023-04-15 16:02:40 - r - INFO: - Episode: 929/2000, Reward: 89.474, Step: 500 +2023-04-15 16:02:44 - r - INFO: - Episode: 930/2000, Reward: 74.462, Step: 500 +2023-04-15 16:02:55 - r - INFO: - Episode: 931/2000, Reward: 50.643, Step: 500 +2023-04-15 16:03:00 - r - INFO: - Episode: 932/2000, Reward: 59.853, Step: 500 +2023-04-15 16:03:04 - r - INFO: - Episode: 933/2000, Reward: 93.702, Step: 500 +2023-04-15 16:03:08 - r - INFO: - Episode: 934/2000, Reward: 83.755, Step: 500 +2023-04-15 16:03:13 - r - INFO: - Episode: 935/2000, Reward: 94.468, Step: 500 +2023-04-15 16:03:20 - r - INFO: - Current episode 935 has the best eval reward: 99.891 +2023-04-15 16:03:24 - r - INFO: - Episode: 936/2000, Reward: 98.882, Step: 500 +2023-04-15 16:03:28 - r - INFO: - Episode: 937/2000, Reward: -30.513, Step: 451 +2023-04-15 16:03:32 - r - INFO: - Episode: 938/2000, Reward: 86.499, Step: 500 +2023-04-15 16:03:37 - r - INFO: - Episode: 939/2000, Reward: 94.563, Step: 500 +2023-04-15 16:03:41 - r - INFO: - Episode: 940/2000, Reward: 65.416, Step: 500 +2023-04-15 16:03:51 - r - INFO: - Episode: 941/2000, Reward: 95.841, Step: 500 +2023-04-15 16:03:55 - r - INFO: - Episode: 942/2000, Reward: 87.442, Step: 500 +2023-04-15 16:04:00 - r - INFO: - Episode: 943/2000, Reward: 99.057, Step: 500 +2023-04-15 16:04:04 - r - INFO: - Episode: 944/2000, Reward: 80.490, Step: 500 +2023-04-15 16:04:08 - r - INFO: - Episode: 945/2000, Reward: 90.520, Step: 500 +2023-04-15 16:04:19 - r - INFO: - Episode: 946/2000, Reward: 90.729, Step: 500 +2023-04-15 16:04:21 - r - INFO: - Episode: 947/2000, Reward: -84.831, Step: 243 +2023-04-15 16:04:26 - r - INFO: - Episode: 948/2000, Reward: 92.500, Step: 500 +2023-04-15 16:04:30 - r - INFO: - Episode: 949/2000, Reward: 87.331, Step: 500 +2023-04-15 16:04:32 - r - INFO: - Episode: 950/2000, Reward: -96.267, Step: 217 +2023-04-15 16:04:42 - r - INFO: - Episode: 951/2000, Reward: -41.952, Step: 498 +2023-04-15 16:04:46 - r - INFO: - Episode: 952/2000, Reward: 107.177, Step: 500 +2023-04-15 16:04:48 - r - INFO: - Episode: 953/2000, Reward: -112.698, Step: 237 +2023-04-15 16:04:49 - r - INFO: - Episode: 954/2000, Reward: -110.703, Step: 155 +2023-04-15 16:04:51 - r - INFO: - Episode: 955/2000, Reward: -108.181, Step: 169 +2023-04-15 16:04:55 - r - INFO: - Episode: 956/2000, Reward: -105.705, Step: 123 +2023-04-15 16:04:56 - r - INFO: - Episode: 957/2000, Reward: -106.489, Step: 155 +2023-04-15 16:05:01 - r - INFO: - Episode: 958/2000, Reward: -4.278, Step: 500 +2023-04-15 16:05:03 - r - INFO: - Episode: 959/2000, Reward: -113.427, Step: 327 +2023-04-15 16:05:08 - r - INFO: - Episode: 960/2000, Reward: -27.005, Step: 500 +2023-04-15 16:05:19 - r - INFO: - Episode: 961/2000, Reward: -31.999, Step: 500 +2023-04-15 16:05:20 - r - INFO: - Episode: 962/2000, Reward: -114.464, Step: 109 +2023-04-15 16:05:21 - r - INFO: - Episode: 963/2000, Reward: -110.080, Step: 121 +2023-04-15 16:05:25 - r - INFO: - Episode: 964/2000, Reward: -28.793, Step: 500 +2023-04-15 16:05:26 - r - INFO: - Episode: 965/2000, Reward: -110.512, Step: 103 +2023-04-15 16:05:29 - r - INFO: - Episode: 966/2000, Reward: -110.361, Step: 102 +2023-04-15 16:05:34 - r - INFO: - Episode: 967/2000, Reward: -23.887, Step: 500 +2023-04-15 16:05:38 - r - INFO: - Episode: 968/2000, Reward: -2.255, Step: 500 +2023-04-15 16:05:42 - r - INFO: - Episode: 969/2000, Reward: -13.776, Step: 500 +2023-04-15 16:05:47 - r - INFO: - Episode: 970/2000, Reward: -13.296, Step: 500 +2023-04-15 16:05:58 - r - INFO: - Episode: 971/2000, Reward: -23.954, Step: 500 +2023-04-15 16:06:03 - r - INFO: - Episode: 972/2000, Reward: -16.519, Step: 500 +2023-04-15 16:06:07 - r - INFO: - Episode: 973/2000, Reward: 1.669, Step: 500 +2023-04-15 16:06:11 - r - INFO: - Episode: 974/2000, Reward: 98.427, Step: 500 +2023-04-15 16:06:16 - r - INFO: - Episode: 975/2000, Reward: 25.365, Step: 500 +2023-04-15 16:06:23 - r - INFO: - Episode: 976/2000, Reward: -83.509, Step: 126 +2023-04-15 16:06:24 - r - INFO: - Episode: 977/2000, Reward: -107.193, Step: 145 +2023-04-15 16:06:29 - r - INFO: - Episode: 978/2000, Reward: 91.300, Step: 500 +2023-04-15 16:06:33 - r - INFO: - Episode: 979/2000, Reward: 78.528, Step: 500 +2023-04-15 16:06:37 - r - INFO: - Episode: 980/2000, Reward: 42.878, Step: 500 +2023-04-15 16:06:48 - r - INFO: - Episode: 981/2000, Reward: -72.003, Step: 369 +2023-04-15 16:06:52 - r - INFO: - Episode: 982/2000, Reward: 95.716, Step: 500 +2023-04-15 16:06:56 - r - INFO: - Episode: 983/2000, Reward: 98.654, Step: 500 +2023-04-15 16:07:01 - r - INFO: - Episode: 984/2000, Reward: 92.986, Step: 500 +2023-04-15 16:07:05 - r - INFO: - Episode: 985/2000, Reward: 87.218, Step: 500 +2023-04-15 16:07:12 - r - INFO: - Current episode 985 has the best eval reward: 114.978 +2023-04-15 16:07:16 - r - INFO: - Episode: 986/2000, Reward: 93.429, Step: 500 +2023-04-15 16:07:21 - r - INFO: - Episode: 987/2000, Reward: 95.591, Step: 500 +2023-04-15 16:07:25 - r - INFO: - Episode: 988/2000, Reward: 90.532, Step: 500 +2023-04-15 16:07:29 - r - INFO: - Episode: 989/2000, Reward: 96.223, Step: 500 +2023-04-15 16:07:34 - r - INFO: - Episode: 990/2000, Reward: 102.325, Step: 500 +2023-04-15 16:07:42 - r - INFO: - Episode: 991/2000, Reward: -92.504, Step: 119 +2023-04-15 16:07:46 - r - INFO: - Episode: 992/2000, Reward: 106.985, Step: 500 +2023-04-15 16:07:50 - r - INFO: - Episode: 993/2000, Reward: 91.128, Step: 500 +2023-04-15 16:07:55 - r - INFO: - Episode: 994/2000, Reward: 96.727, Step: 500 +2023-04-15 16:07:59 - r - INFO: - Episode: 995/2000, Reward: -10.043, Step: 500 +2023-04-15 16:08:09 - r - INFO: - Episode: 996/2000, Reward: -58.130, Step: 304 +2023-04-15 16:08:13 - r - INFO: - Episode: 997/2000, Reward: -38.904, Step: 379 +2023-04-15 16:08:15 - r - INFO: - Episode: 998/2000, Reward: -50.494, Step: 324 +2023-04-15 16:08:20 - r - INFO: - Episode: 999/2000, Reward: 80.029, Step: 500 +2023-04-15 16:08:24 - r - INFO: - Episode: 1000/2000, Reward: 86.055, Step: 500 +2023-04-15 16:08:34 - r - INFO: - Episode: 1001/2000, Reward: 78.492, Step: 500 +2023-04-15 16:08:38 - r - INFO: - Episode: 1002/2000, Reward: -45.317, Step: 384 +2023-04-15 16:08:42 - r - INFO: - Episode: 1003/2000, Reward: 2.446, Step: 500 +2023-04-15 16:08:46 - r - INFO: - Episode: 1004/2000, Reward: 35.510, Step: 500 +2023-04-15 16:08:51 - r - INFO: - Episode: 1005/2000, Reward: 59.063, Step: 500 +2023-04-15 16:08:58 - r - INFO: - Episode: 1006/2000, Reward: -84.574, Step: 147 +2023-04-15 16:09:02 - r - INFO: - Episode: 1007/2000, Reward: 115.185, Step: 500 +2023-04-15 16:09:06 - r - INFO: - Episode: 1008/2000, Reward: 85.989, Step: 500 +2023-04-15 16:09:10 - r - INFO: - Episode: 1009/2000, Reward: 97.671, Step: 500 +2023-04-15 16:09:15 - r - INFO: - Episode: 1010/2000, Reward: 25.630, Step: 500 +2023-04-15 16:09:24 - r - INFO: - Episode: 1011/2000, Reward: 92.674, Step: 500 +2023-04-15 16:09:28 - r - INFO: - Episode: 1012/2000, Reward: 38.561, Step: 500 +2023-04-15 16:09:33 - r - INFO: - Episode: 1013/2000, Reward: 81.952, Step: 500 +2023-04-15 16:09:37 - r - INFO: - Episode: 1014/2000, Reward: 36.425, Step: 500 +2023-04-15 16:09:41 - r - INFO: - Episode: 1015/2000, Reward: 118.020, Step: 500 +2023-04-15 16:09:48 - r - INFO: - Current episode 1015 has the best eval reward: 128.106 +2023-04-15 16:09:53 - r - INFO: - Episode: 1016/2000, Reward: 115.892, Step: 500 +2023-04-15 16:09:55 - r - INFO: - Episode: 1017/2000, Reward: -60.269, Step: 273 +2023-04-15 16:09:57 - r - INFO: - Episode: 1018/2000, Reward: -92.419, Step: 238 +2023-04-15 16:10:01 - r - INFO: - Episode: 1019/2000, Reward: 89.665, Step: 500 +2023-04-15 16:10:03 - r - INFO: - Episode: 1020/2000, Reward: -67.780, Step: 214 +2023-04-15 16:10:09 - r - INFO: - Episode: 1021/2000, Reward: -65.982, Step: 194 +2023-04-15 16:10:13 - r - INFO: - Episode: 1022/2000, Reward: -16.793, Step: 450 +2023-04-15 16:10:17 - r - INFO: - Episode: 1023/2000, Reward: 87.739, Step: 500 +2023-04-15 16:10:19 - r - INFO: - Episode: 1024/2000, Reward: -101.791, Step: 217 +2023-04-15 16:10:23 - r - INFO: - Episode: 1025/2000, Reward: 104.755, Step: 500 +2023-04-15 16:10:35 - r - INFO: - Episode: 1026/2000, Reward: 104.730, Step: 500 +2023-04-15 16:10:39 - r - INFO: - Episode: 1027/2000, Reward: -13.050, Step: 455 +2023-04-15 16:10:43 - r - INFO: - Episode: 1028/2000, Reward: 19.984, Step: 500 +2023-04-15 16:10:47 - r - INFO: - Episode: 1029/2000, Reward: 115.392, Step: 500 +2023-04-15 16:10:52 - r - INFO: - Episode: 1030/2000, Reward: 111.310, Step: 500 +2023-04-15 16:11:03 - r - INFO: - Episode: 1031/2000, Reward: 117.678, Step: 500 +2023-04-15 16:11:08 - r - INFO: - Episode: 1032/2000, Reward: 120.628, Step: 500 +2023-04-15 16:11:12 - r - INFO: - Episode: 1033/2000, Reward: 109.530, Step: 500 +2023-04-15 16:11:16 - r - INFO: - Episode: 1034/2000, Reward: 67.495, Step: 500 +2023-04-15 16:11:17 - r - INFO: - Episode: 1035/2000, Reward: -96.733, Step: 127 +2023-04-15 16:11:25 - r - INFO: - Episode: 1036/2000, Reward: -57.062, Step: 320 +2023-04-15 16:11:29 - r - INFO: - Episode: 1037/2000, Reward: 97.249, Step: 500 +2023-04-15 16:11:31 - r - INFO: - Episode: 1038/2000, Reward: -79.664, Step: 180 +2023-04-15 16:11:32 - r - INFO: - Episode: 1039/2000, Reward: -75.553, Step: 217 +2023-04-15 16:11:37 - r - INFO: - Episode: 1040/2000, Reward: 7.207, Step: 500 +2023-04-15 16:11:48 - r - INFO: - Episode: 1041/2000, Reward: 114.612, Step: 500 +2023-04-15 16:11:52 - r - INFO: - Episode: 1042/2000, Reward: 115.209, Step: 500 +2023-04-15 16:11:57 - r - INFO: - Episode: 1043/2000, Reward: 113.667, Step: 500 +2023-04-15 16:12:01 - r - INFO: - Episode: 1044/2000, Reward: 110.371, Step: 500 +2023-04-15 16:12:05 - r - INFO: - Episode: 1045/2000, Reward: 106.825, Step: 500 +2023-04-15 16:12:17 - r - INFO: - Episode: 1046/2000, Reward: 114.766, Step: 500 +2023-04-15 16:12:21 - r - INFO: - Episode: 1047/2000, Reward: 60.500, Step: 500 +2023-04-15 16:12:25 - r - INFO: - Episode: 1048/2000, Reward: 123.083, Step: 500 +2023-04-15 16:12:28 - r - INFO: - Episode: 1049/2000, Reward: -36.408, Step: 312 +2023-04-15 16:12:29 - r - INFO: - Episode: 1050/2000, Reward: -85.875, Step: 145 +2023-04-15 16:12:34 - r - INFO: - Episode: 1051/2000, Reward: -90.046, Step: 118 +2023-04-15 16:12:38 - r - INFO: - Episode: 1052/2000, Reward: 114.402, Step: 500 +2023-04-15 16:12:42 - r - INFO: - Episode: 1053/2000, Reward: 105.606, Step: 500 +2023-04-15 16:12:47 - r - INFO: - Episode: 1054/2000, Reward: 130.981, Step: 500 +2023-04-15 16:12:51 - r - INFO: - Episode: 1055/2000, Reward: 119.599, Step: 500 +2023-04-15 16:12:58 - r - INFO: - Current episode 1055 has the best eval reward: 130.869 +2023-04-15 16:13:03 - r - INFO: - Episode: 1056/2000, Reward: 114.507, Step: 500 +2023-04-15 16:13:07 - r - INFO: - Episode: 1057/2000, Reward: 116.602, Step: 500 +2023-04-15 16:13:11 - r - INFO: - Episode: 1058/2000, Reward: 104.173, Step: 500 +2023-04-15 16:13:16 - r - INFO: - Episode: 1059/2000, Reward: 113.015, Step: 500 +2023-04-15 16:13:20 - r - INFO: - Episode: 1060/2000, Reward: 118.092, Step: 500 +2023-04-15 16:13:27 - r - INFO: - Current episode 1060 has the best eval reward: 134.972 +2023-04-15 16:13:31 - r - INFO: - Episode: 1061/2000, Reward: 116.203, Step: 500 +2023-04-15 16:13:36 - r - INFO: - Episode: 1062/2000, Reward: 126.779, Step: 500 +2023-04-15 16:13:40 - r - INFO: - Episode: 1063/2000, Reward: 119.266, Step: 500 +2023-04-15 16:13:44 - r - INFO: - Episode: 1064/2000, Reward: 128.085, Step: 500 +2023-04-15 16:13:49 - r - INFO: - Episode: 1065/2000, Reward: 119.091, Step: 500 +2023-04-15 16:14:00 - r - INFO: - Episode: 1066/2000, Reward: 121.758, Step: 500 +2023-04-15 16:14:02 - r - INFO: - Episode: 1067/2000, Reward: -79.453, Step: 233 +2023-04-15 16:14:05 - r - INFO: - Episode: 1068/2000, Reward: -41.547, Step: 323 +2023-04-15 16:14:09 - r - INFO: - Episode: 1069/2000, Reward: 119.658, Step: 500 +2023-04-15 16:14:14 - r - INFO: - Episode: 1070/2000, Reward: 125.632, Step: 500 +2023-04-15 16:14:21 - r - INFO: - Current episode 1070 has the best eval reward: 140.951 +2023-04-15 16:14:25 - r - INFO: - Episode: 1071/2000, Reward: 121.948, Step: 500 +2023-04-15 16:14:29 - r - INFO: - Episode: 1072/2000, Reward: 118.592, Step: 500 +2023-04-15 16:14:34 - r - INFO: - Episode: 1073/2000, Reward: 130.150, Step: 500 +2023-04-15 16:14:38 - r - INFO: - Episode: 1074/2000, Reward: 130.027, Step: 500 +2023-04-15 16:14:42 - r - INFO: - Episode: 1075/2000, Reward: 128.036, Step: 500 +2023-04-15 16:14:54 - r - INFO: - Episode: 1076/2000, Reward: 117.349, Step: 500 +2023-04-15 16:14:58 - r - INFO: - Episode: 1077/2000, Reward: 124.144, Step: 500 +2023-04-15 16:15:03 - r - INFO: - Episode: 1078/2000, Reward: 123.809, Step: 500 +2023-04-15 16:15:07 - r - INFO: - Episode: 1079/2000, Reward: 129.816, Step: 500 +2023-04-15 16:15:11 - r - INFO: - Episode: 1080/2000, Reward: 126.589, Step: 500 +2023-04-15 16:15:23 - r - INFO: - Episode: 1081/2000, Reward: 124.001, Step: 500 +2023-04-15 16:15:27 - r - INFO: - Episode: 1082/2000, Reward: 119.228, Step: 500 +2023-04-15 16:15:32 - r - INFO: - Episode: 1083/2000, Reward: 125.376, Step: 500 +2023-04-15 16:15:36 - r - INFO: - Episode: 1084/2000, Reward: 121.389, Step: 500 +2023-04-15 16:15:40 - r - INFO: - Episode: 1085/2000, Reward: 125.326, Step: 500 +2023-04-15 16:15:52 - r - INFO: - Episode: 1086/2000, Reward: 130.340, Step: 500 +2023-04-15 16:15:56 - r - INFO: - Episode: 1087/2000, Reward: 131.424, Step: 500 +2023-04-15 16:16:00 - r - INFO: - Episode: 1088/2000, Reward: 115.953, Step: 500 +2023-04-15 16:16:05 - r - INFO: - Episode: 1089/2000, Reward: 118.519, Step: 500 +2023-04-15 16:16:09 - r - INFO: - Episode: 1090/2000, Reward: 126.423, Step: 500 +2023-04-15 16:16:16 - r - INFO: - Current episode 1090 has the best eval reward: 144.621 +2023-04-15 16:16:20 - r - INFO: - Episode: 1091/2000, Reward: 138.481, Step: 500 +2023-04-15 16:16:25 - r - INFO: - Episode: 1092/2000, Reward: 126.969, Step: 500 +2023-04-15 16:16:29 - r - INFO: - Episode: 1093/2000, Reward: 131.646, Step: 500 +2023-04-15 16:16:33 - r - INFO: - Episode: 1094/2000, Reward: 132.252, Step: 500 +2023-04-15 16:16:38 - r - INFO: - Episode: 1095/2000, Reward: 129.873, Step: 500 +2023-04-15 16:16:49 - r - INFO: - Episode: 1096/2000, Reward: 131.873, Step: 500 +2023-04-15 16:16:53 - r - INFO: - Episode: 1097/2000, Reward: 136.627, Step: 500 +2023-04-15 16:16:58 - r - INFO: - Episode: 1098/2000, Reward: 130.555, Step: 500 +2023-04-15 16:17:02 - r - INFO: - Episode: 1099/2000, Reward: 127.372, Step: 500 +2023-04-15 16:17:06 - r - INFO: - Episode: 1100/2000, Reward: 108.263, Step: 500 +2023-04-15 16:17:18 - r - INFO: - Episode: 1101/2000, Reward: 109.400, Step: 500 +2023-04-15 16:17:22 - r - INFO: - Episode: 1102/2000, Reward: 138.297, Step: 500 +2023-04-15 16:17:27 - r - INFO: - Episode: 1103/2000, Reward: 141.233, Step: 500 +2023-04-15 16:17:31 - r - INFO: - Episode: 1104/2000, Reward: 140.907, Step: 500 +2023-04-15 16:17:35 - r - INFO: - Episode: 1105/2000, Reward: 124.212, Step: 500 +2023-04-15 16:17:47 - r - INFO: - Episode: 1106/2000, Reward: 140.827, Step: 500 +2023-04-15 16:17:51 - r - INFO: - Episode: 1107/2000, Reward: 134.017, Step: 500 +2023-04-15 16:17:55 - r - INFO: - Episode: 1108/2000, Reward: 131.794, Step: 500 +2023-04-15 16:18:00 - r - INFO: - Episode: 1109/2000, Reward: 135.786, Step: 500 +2023-04-15 16:18:04 - r - INFO: - Episode: 1110/2000, Reward: 123.271, Step: 500 +2023-04-15 16:18:15 - r - INFO: - Episode: 1111/2000, Reward: 108.105, Step: 500 +2023-04-15 16:18:20 - r - INFO: - Episode: 1112/2000, Reward: 136.164, Step: 500 +2023-04-15 16:18:23 - r - INFO: - Episode: 1113/2000, Reward: -18.994, Step: 426 +2023-04-15 16:18:28 - r - INFO: - Episode: 1114/2000, Reward: 133.548, Step: 500 +2023-04-15 16:18:32 - r - INFO: - Episode: 1115/2000, Reward: 127.274, Step: 500 +2023-04-15 16:18:43 - r - INFO: - Episode: 1116/2000, Reward: 136.398, Step: 500 +2023-04-15 16:18:47 - r - INFO: - Episode: 1117/2000, Reward: 132.993, Step: 500 +2023-04-15 16:18:52 - r - INFO: - Episode: 1118/2000, Reward: 121.502, Step: 500 +2023-04-15 16:18:56 - r - INFO: - Episode: 1119/2000, Reward: 137.030, Step: 500 +2023-04-15 16:19:00 - r - INFO: - Episode: 1120/2000, Reward: 138.802, Step: 500 +2023-04-15 16:19:07 - r - INFO: - Current episode 1120 has the best eval reward: 146.546 +2023-04-15 16:19:12 - r - INFO: - Episode: 1121/2000, Reward: 142.043, Step: 500 +2023-04-15 16:19:16 - r - INFO: - Episode: 1122/2000, Reward: 126.733, Step: 500 +2023-04-15 16:19:21 - r - INFO: - Episode: 1123/2000, Reward: 144.484, Step: 500 +2023-04-15 16:19:25 - r - INFO: - Episode: 1124/2000, Reward: 144.649, Step: 500 +2023-04-15 16:19:29 - r - INFO: - Episode: 1125/2000, Reward: 131.096, Step: 500 +2023-04-15 16:19:41 - r - INFO: - Episode: 1126/2000, Reward: 135.308, Step: 500 +2023-04-15 16:19:45 - r - INFO: - Episode: 1127/2000, Reward: 136.258, Step: 500 +2023-04-15 16:19:49 - r - INFO: - Episode: 1128/2000, Reward: 142.941, Step: 500 +2023-04-15 16:19:54 - r - INFO: - Episode: 1129/2000, Reward: 140.961, Step: 500 +2023-04-15 16:19:58 - r - INFO: - Episode: 1130/2000, Reward: 135.486, Step: 500 +2023-04-15 16:20:05 - r - INFO: - Current episode 1130 has the best eval reward: 150.965 +2023-04-15 16:20:10 - r - INFO: - Episode: 1131/2000, Reward: 140.307, Step: 500 +2023-04-15 16:20:14 - r - INFO: - Episode: 1132/2000, Reward: 141.362, Step: 500 +2023-04-15 16:20:18 - r - INFO: - Episode: 1133/2000, Reward: 97.270, Step: 500 +2023-04-15 16:20:23 - r - INFO: - Episode: 1134/2000, Reward: 129.240, Step: 500 +2023-04-15 16:20:27 - r - INFO: - Episode: 1135/2000, Reward: 134.480, Step: 500 +2023-04-15 16:20:37 - r - INFO: - Episode: 1136/2000, Reward: -9.551, Step: 367 +2023-04-15 16:20:42 - r - INFO: - Episode: 1137/2000, Reward: 139.276, Step: 500 +2023-04-15 16:20:46 - r - INFO: - Episode: 1138/2000, Reward: 128.888, Step: 500 +2023-04-15 16:20:50 - r - INFO: - Episode: 1139/2000, Reward: 142.175, Step: 500 +2023-04-15 16:20:55 - r - INFO: - Episode: 1140/2000, Reward: 134.721, Step: 500 +2023-04-15 16:21:04 - r - INFO: - Episode: 1141/2000, Reward: 132.893, Step: 500 +2023-04-15 16:21:08 - r - INFO: - Episode: 1142/2000, Reward: 134.467, Step: 500 +2023-04-15 16:21:12 - r - INFO: - Episode: 1143/2000, Reward: 145.773, Step: 500 +2023-04-15 16:21:17 - r - INFO: - Episode: 1144/2000, Reward: 130.598, Step: 500 +2023-04-15 16:21:20 - r - INFO: - Episode: 1145/2000, Reward: -16.657, Step: 453 +2023-04-15 16:21:29 - r - INFO: - Episode: 1146/2000, Reward: -27.040, Step: 334 +2023-04-15 16:21:33 - r - INFO: - Episode: 1147/2000, Reward: 17.180, Step: 474 +2023-04-15 16:21:38 - r - INFO: - Episode: 1148/2000, Reward: 147.406, Step: 500 +2023-04-15 16:21:42 - r - INFO: - Episode: 1149/2000, Reward: 136.289, Step: 500 +2023-04-15 16:21:46 - r - INFO: - Episode: 1150/2000, Reward: 80.051, Step: 500 +2023-04-15 16:21:53 - r - INFO: - Current episode 1150 has the best eval reward: 153.639 +2023-04-15 16:21:58 - r - INFO: - Episode: 1151/2000, Reward: 148.850, Step: 500 +2023-04-15 16:22:02 - r - INFO: - Episode: 1152/2000, Reward: 149.354, Step: 500 +2023-04-15 16:22:06 - r - INFO: - Episode: 1153/2000, Reward: 145.779, Step: 500 +2023-04-15 16:22:11 - r - INFO: - Episode: 1154/2000, Reward: -5.455, Step: 477 +2023-04-15 16:22:15 - r - INFO: - Episode: 1155/2000, Reward: 147.438, Step: 500 +2023-04-15 16:22:22 - r - INFO: - Current episode 1155 has the best eval reward: 155.653 +2023-04-15 16:22:26 - r - INFO: - Episode: 1156/2000, Reward: 145.149, Step: 500 +2023-04-15 16:22:31 - r - INFO: - Episode: 1157/2000, Reward: 132.783, Step: 500 +2023-04-15 16:22:35 - r - INFO: - Episode: 1158/2000, Reward: 140.507, Step: 500 +2023-04-15 16:22:39 - r - INFO: - Episode: 1159/2000, Reward: 47.836, Step: 500 +2023-04-15 16:22:41 - r - INFO: - Episode: 1160/2000, Reward: -59.549, Step: 209 +2023-04-15 16:22:53 - r - INFO: - Episode: 1161/2000, Reward: 141.402, Step: 500 +2023-04-15 16:22:57 - r - INFO: - Episode: 1162/2000, Reward: 144.903, Step: 500 +2023-04-15 16:23:01 - r - INFO: - Episode: 1163/2000, Reward: 148.350, Step: 500 +2023-04-15 16:23:06 - r - INFO: - Episode: 1164/2000, Reward: 147.295, Step: 500 +2023-04-15 16:23:10 - r - INFO: - Episode: 1165/2000, Reward: 141.812, Step: 500 +2023-04-15 16:23:22 - r - INFO: - Episode: 1166/2000, Reward: 142.631, Step: 500 +2023-04-15 16:23:26 - r - INFO: - Episode: 1167/2000, Reward: 152.024, Step: 500 +2023-04-15 16:23:30 - r - INFO: - Episode: 1168/2000, Reward: 148.931, Step: 500 +2023-04-15 16:23:35 - r - INFO: - Episode: 1169/2000, Reward: 142.639, Step: 500 +2023-04-15 16:23:39 - r - INFO: - Episode: 1170/2000, Reward: 145.900, Step: 500 +2023-04-15 16:23:50 - r - INFO: - Episode: 1171/2000, Reward: 150.282, Step: 500 +2023-04-15 16:23:55 - r - INFO: - Episode: 1172/2000, Reward: 134.547, Step: 500 +2023-04-15 16:23:59 - r - INFO: - Episode: 1173/2000, Reward: 138.556, Step: 500 +2023-04-15 16:24:03 - r - INFO: - Episode: 1174/2000, Reward: 148.195, Step: 500 +2023-04-15 16:24:08 - r - INFO: - Episode: 1175/2000, Reward: 142.402, Step: 500 +2023-04-15 16:24:15 - r - INFO: - Current episode 1175 has the best eval reward: 155.856 +2023-04-15 16:24:19 - r - INFO: - Episode: 1176/2000, Reward: 152.851, Step: 500 +2023-04-15 16:24:23 - r - INFO: - Episode: 1177/2000, Reward: 143.433, Step: 500 +2023-04-15 16:24:28 - r - INFO: - Episode: 1178/2000, Reward: 151.907, Step: 500 +2023-04-15 16:24:32 - r - INFO: - Episode: 1179/2000, Reward: 155.131, Step: 500 +2023-04-15 16:24:36 - r - INFO: - Episode: 1180/2000, Reward: 148.064, Step: 500 +2023-04-15 16:24:48 - r - INFO: - Episode: 1181/2000, Reward: 140.459, Step: 500 +2023-04-15 16:24:52 - r - INFO: - Episode: 1182/2000, Reward: 139.441, Step: 500 +2023-04-15 16:24:57 - r - INFO: - Episode: 1183/2000, Reward: 66.977, Step: 500 +2023-04-15 16:24:58 - r - INFO: - Episode: 1184/2000, Reward: -69.871, Step: 191 +2023-04-15 16:25:03 - r - INFO: - Episode: 1185/2000, Reward: 145.510, Step: 500 +2023-04-15 16:25:10 - r - INFO: - Current episode 1185 has the best eval reward: 157.433 +2023-04-15 16:25:14 - r - INFO: - Episode: 1186/2000, Reward: 152.706, Step: 500 +2023-04-15 16:25:19 - r - INFO: - Episode: 1187/2000, Reward: 154.865, Step: 500 +2023-04-15 16:25:23 - r - INFO: - Episode: 1188/2000, Reward: 147.823, Step: 500 +2023-04-15 16:25:27 - r - INFO: - Episode: 1189/2000, Reward: 147.652, Step: 500 +2023-04-15 16:25:32 - r - INFO: - Episode: 1190/2000, Reward: 152.725, Step: 500 +2023-04-15 16:25:43 - r - INFO: - Episode: 1191/2000, Reward: 151.064, Step: 500 +2023-04-15 16:25:47 - r - INFO: - Episode: 1192/2000, Reward: 146.334, Step: 500 +2023-04-15 16:25:52 - r - INFO: - Episode: 1193/2000, Reward: 147.037, Step: 500 +2023-04-15 16:25:56 - r - INFO: - Episode: 1194/2000, Reward: 148.806, Step: 500 +2023-04-15 16:26:00 - r - INFO: - Episode: 1195/2000, Reward: 158.017, Step: 500 +2023-04-15 16:26:08 - r - INFO: - Current episode 1195 has the best eval reward: 163.561 +2023-04-15 16:26:12 - r - INFO: - Episode: 1196/2000, Reward: 149.401, Step: 500 +2023-04-15 16:26:16 - r - INFO: - Episode: 1197/2000, Reward: 162.002, Step: 500 +2023-04-15 16:26:21 - r - INFO: - Episode: 1198/2000, Reward: 150.712, Step: 500 +2023-04-15 16:26:25 - r - INFO: - Episode: 1199/2000, Reward: 141.444, Step: 500 +2023-04-15 16:26:29 - r - INFO: - Episode: 1200/2000, Reward: 142.955, Step: 500 +2023-04-15 16:26:41 - r - INFO: - Episode: 1201/2000, Reward: 138.875, Step: 500 +2023-04-15 16:29:03 - r - INFO: - Episode: 1202/2000, Reward: 136.920, Step: 500 +2023-04-15 16:29:07 - r - INFO: - Episode: 1203/2000, Reward: 142.703, Step: 500 +2023-04-15 16:29:12 - r - INFO: - Episode: 1204/2000, Reward: 152.001, Step: 500 +2023-04-15 16:29:16 - r - INFO: - Episode: 1205/2000, Reward: 147.088, Step: 500 +2023-04-15 16:29:24 - r - INFO: - Episode: 1206/2000, Reward: -56.167, Step: 219 +2023-04-15 16:29:28 - r - INFO: - Episode: 1207/2000, Reward: 151.515, Step: 500 +2023-04-15 16:29:30 - r - INFO: - Episode: 1208/2000, Reward: -57.827, Step: 228 +2023-04-15 16:29:34 - r - INFO: - Episode: 1209/2000, Reward: 152.895, Step: 500 +2023-04-15 16:29:38 - r - INFO: - Episode: 1210/2000, Reward: 151.669, Step: 500 +2023-04-15 16:29:49 - r - INFO: - Episode: 1211/2000, Reward: 149.996, Step: 500 +2023-04-15 16:29:54 - r - INFO: - Episode: 1212/2000, Reward: 152.821, Step: 500 +2023-04-15 16:29:58 - r - INFO: - Episode: 1213/2000, Reward: 153.276, Step: 500 +2023-04-15 16:30:02 - r - INFO: - Episode: 1214/2000, Reward: 147.853, Step: 500 +2023-04-15 16:30:06 - r - INFO: - Episode: 1215/2000, Reward: 144.806, Step: 500 +2023-04-15 16:30:17 - r - INFO: - Episode: 1216/2000, Reward: 150.216, Step: 500 +2023-04-15 16:30:21 - r - INFO: - Episode: 1217/2000, Reward: 143.971, Step: 500 +2023-04-15 16:30:26 - r - INFO: - Episode: 1218/2000, Reward: 154.949, Step: 500 +2023-04-15 16:30:30 - r - INFO: - Episode: 1219/2000, Reward: 154.383, Step: 500 +2023-04-15 16:30:34 - r - INFO: - Episode: 1220/2000, Reward: 145.632, Step: 500 +2023-04-15 16:30:41 - r - INFO: - Current episode 1220 has the best eval reward: 167.707 +2023-04-15 16:30:45 - r - INFO: - Episode: 1221/2000, Reward: 161.370, Step: 500 +2023-04-15 16:30:50 - r - INFO: - Episode: 1222/2000, Reward: 160.930, Step: 500 +2023-04-15 16:30:54 - r - INFO: - Episode: 1223/2000, Reward: 160.374, Step: 500 +2023-04-15 16:30:58 - r - INFO: - Episode: 1224/2000, Reward: 145.059, Step: 500 +2023-04-15 16:31:02 - r - INFO: - Episode: 1225/2000, Reward: 156.269, Step: 500 +2023-04-15 16:31:14 - r - INFO: - Episode: 1226/2000, Reward: 152.381, Step: 500 +2023-04-15 16:31:18 - r - INFO: - Episode: 1227/2000, Reward: 156.718, Step: 500 +2023-04-15 16:31:22 - r - INFO: - Episode: 1228/2000, Reward: 152.290, Step: 500 +2023-04-15 16:31:26 - r - INFO: - Episode: 1229/2000, Reward: 156.978, Step: 500 +2023-04-15 16:31:31 - r - INFO: - Episode: 1230/2000, Reward: 155.554, Step: 500 +2023-04-15 16:31:42 - r - INFO: - Episode: 1231/2000, Reward: 166.148, Step: 500 +2023-04-15 16:31:46 - r - INFO: - Episode: 1232/2000, Reward: 152.706, Step: 500 +2023-04-15 16:31:50 - r - INFO: - Episode: 1233/2000, Reward: 151.922, Step: 500 +2023-04-15 16:31:54 - r - INFO: - Episode: 1234/2000, Reward: 153.177, Step: 500 +2023-04-15 16:31:59 - r - INFO: - Episode: 1235/2000, Reward: 148.754, Step: 500 +2023-04-15 16:32:10 - r - INFO: - Episode: 1236/2000, Reward: 151.378, Step: 500 +2023-04-15 16:32:14 - r - INFO: - Episode: 1237/2000, Reward: 154.427, Step: 500 +2023-04-15 16:32:19 - r - INFO: - Episode: 1238/2000, Reward: 148.701, Step: 500 +2023-04-15 16:32:23 - r - INFO: - Episode: 1239/2000, Reward: 158.953, Step: 500 +2023-04-15 16:32:27 - r - INFO: - Episode: 1240/2000, Reward: 153.100, Step: 500 +2023-04-15 16:32:39 - r - INFO: - Episode: 1241/2000, Reward: 156.957, Step: 500 +2023-04-15 16:32:43 - r - INFO: - Episode: 1242/2000, Reward: 155.746, Step: 500 +2023-04-15 16:32:47 - r - INFO: - Episode: 1243/2000, Reward: 160.143, Step: 500 +2023-04-15 16:32:51 - r - INFO: - Episode: 1244/2000, Reward: 158.063, Step: 500 +2023-04-15 16:32:56 - r - INFO: - Episode: 1245/2000, Reward: 157.372, Step: 500 +2023-04-15 16:33:07 - r - INFO: - Episode: 1246/2000, Reward: 156.585, Step: 500 +2023-04-15 16:33:11 - r - INFO: - Episode: 1247/2000, Reward: 145.510, Step: 500 +2023-04-15 16:33:16 - r - INFO: - Episode: 1248/2000, Reward: 150.335, Step: 500 +2023-04-15 16:33:20 - r - INFO: - Episode: 1249/2000, Reward: 151.606, Step: 500 +2023-04-15 16:33:24 - r - INFO: - Episode: 1250/2000, Reward: 141.600, Step: 500 +2023-04-15 16:33:35 - r - INFO: - Episode: 1251/2000, Reward: 147.371, Step: 500 +2023-04-15 16:33:40 - r - INFO: - Episode: 1252/2000, Reward: 145.156, Step: 500 +2023-04-15 16:33:44 - r - INFO: - Episode: 1253/2000, Reward: 145.016, Step: 500 +2023-04-15 16:33:48 - r - INFO: - Episode: 1254/2000, Reward: 148.138, Step: 500 +2023-04-15 16:33:52 - r - INFO: - Episode: 1255/2000, Reward: 143.393, Step: 500 +2023-04-15 16:34:04 - r - INFO: - Episode: 1256/2000, Reward: 146.141, Step: 500 +2023-04-15 16:34:08 - r - INFO: - Episode: 1257/2000, Reward: 140.549, Step: 500 +2023-04-15 16:34:12 - r - INFO: - Episode: 1258/2000, Reward: 155.594, Step: 500 +2023-04-15 16:34:17 - r - INFO: - Episode: 1259/2000, Reward: 152.093, Step: 500 +2023-04-15 16:34:21 - r - INFO: - Episode: 1260/2000, Reward: 148.801, Step: 500 +2023-04-15 16:34:32 - r - INFO: - Episode: 1261/2000, Reward: 146.875, Step: 500 +2023-04-15 16:34:37 - r - INFO: - Episode: 1262/2000, Reward: 152.660, Step: 500 +2023-04-15 16:34:41 - r - INFO: - Episode: 1263/2000, Reward: 162.444, Step: 500 +2023-04-15 16:34:45 - r - INFO: - Episode: 1264/2000, Reward: 148.869, Step: 500 +2023-04-15 16:34:50 - r - INFO: - Episode: 1265/2000, Reward: 167.433, Step: 500 +2023-04-15 16:34:57 - r - INFO: - Current episode 1265 has the best eval reward: 168.807 +2023-04-15 16:35:01 - r - INFO: - Episode: 1266/2000, Reward: 158.619, Step: 500 +2023-04-15 16:35:05 - r - INFO: - Episode: 1267/2000, Reward: 162.576, Step: 500 +2023-04-15 16:35:10 - r - INFO: - Episode: 1268/2000, Reward: 163.835, Step: 500 +2023-04-15 16:35:14 - r - INFO: - Episode: 1269/2000, Reward: 151.696, Step: 500 +2023-04-15 16:35:18 - r - INFO: - Episode: 1270/2000, Reward: 160.253, Step: 500 +2023-04-15 16:35:30 - r - INFO: - Episode: 1271/2000, Reward: 153.865, Step: 500 +2023-04-15 16:35:34 - r - INFO: - Episode: 1272/2000, Reward: 158.959, Step: 500 +2023-04-15 16:35:38 - r - INFO: - Episode: 1273/2000, Reward: 160.767, Step: 500 +2023-04-15 16:35:42 - r - INFO: - Episode: 1274/2000, Reward: 162.615, Step: 500 +2023-04-15 16:35:47 - r - INFO: - Episode: 1275/2000, Reward: 161.869, Step: 500 +2023-04-15 16:35:58 - r - INFO: - Episode: 1276/2000, Reward: 167.471, Step: 500 +2023-04-15 16:36:03 - r - INFO: - Episode: 1277/2000, Reward: 157.087, Step: 500 +2023-04-15 16:36:07 - r - INFO: - Episode: 1278/2000, Reward: 156.381, Step: 500 +2023-04-15 16:36:11 - r - INFO: - Episode: 1279/2000, Reward: 159.448, Step: 500 +2023-04-15 16:36:15 - r - INFO: - Episode: 1280/2000, Reward: 158.229, Step: 500 +2023-04-15 16:36:22 - r - INFO: - Current episode 1280 has the best eval reward: 171.148 +2023-04-15 16:36:27 - r - INFO: - Episode: 1281/2000, Reward: 153.336, Step: 500 +2023-04-15 16:36:31 - r - INFO: - Episode: 1282/2000, Reward: 165.396, Step: 500 +2023-04-15 16:36:36 - r - INFO: - Episode: 1283/2000, Reward: 163.198, Step: 500 +2023-04-15 16:36:40 - r - INFO: - Episode: 1284/2000, Reward: 166.153, Step: 500 +2023-04-15 16:36:44 - r - INFO: - Episode: 1285/2000, Reward: 163.957, Step: 500 +2023-04-15 16:36:56 - r - INFO: - Episode: 1286/2000, Reward: 156.733, Step: 500 +2023-04-15 16:37:00 - r - INFO: - Episode: 1287/2000, Reward: 157.278, Step: 500 +2023-04-15 16:37:04 - r - INFO: - Episode: 1288/2000, Reward: 153.631, Step: 500 +2023-04-15 16:37:09 - r - INFO: - Episode: 1289/2000, Reward: 156.828, Step: 500 +2023-04-15 16:37:13 - r - INFO: - Episode: 1290/2000, Reward: 162.339, Step: 500 +2023-04-15 16:37:24 - r - INFO: - Episode: 1291/2000, Reward: 161.763, Step: 500 +2023-04-15 16:37:29 - r - INFO: - Episode: 1292/2000, Reward: 162.990, Step: 500 +2023-04-15 16:37:33 - r - INFO: - Episode: 1293/2000, Reward: 168.733, Step: 500 +2023-04-15 16:37:37 - r - INFO: - Episode: 1294/2000, Reward: 155.742, Step: 500 +2023-04-15 16:37:42 - r - INFO: - Episode: 1295/2000, Reward: 166.084, Step: 500 +2023-04-15 16:37:53 - r - INFO: - Episode: 1296/2000, Reward: 170.388, Step: 500 +2023-04-15 16:37:57 - r - INFO: - Episode: 1297/2000, Reward: 167.510, Step: 500 +2023-04-15 16:38:02 - r - INFO: - Episode: 1298/2000, Reward: 160.662, Step: 500 +2023-04-15 16:38:06 - r - INFO: - Episode: 1299/2000, Reward: 161.371, Step: 500 +2023-04-15 16:38:10 - r - INFO: - Episode: 1300/2000, Reward: 162.083, Step: 500 +2023-04-15 16:38:17 - r - INFO: - Current episode 1300 has the best eval reward: 171.903 +2023-04-15 16:38:22 - r - INFO: - Episode: 1301/2000, Reward: 155.715, Step: 500 +2023-04-15 16:38:26 - r - INFO: - Episode: 1302/2000, Reward: 155.814, Step: 500 +2023-04-15 16:38:30 - r - INFO: - Episode: 1303/2000, Reward: 155.193, Step: 500 +2023-04-15 16:38:32 - r - INFO: - Episode: 1304/2000, Reward: -44.185, Step: 263 +2023-04-15 16:38:37 - r - INFO: - Episode: 1305/2000, Reward: 141.253, Step: 500 +2023-04-15 16:38:48 - r - INFO: - Episode: 1306/2000, Reward: 150.348, Step: 500 +2023-04-15 16:38:52 - r - INFO: - Episode: 1307/2000, Reward: 163.094, Step: 500 +2023-04-15 16:38:57 - r - INFO: - Episode: 1308/2000, Reward: 171.130, Step: 500 +2023-04-15 16:39:01 - r - INFO: - Episode: 1309/2000, Reward: 156.783, Step: 500 +2023-04-15 16:39:05 - r - INFO: - Episode: 1310/2000, Reward: 165.278, Step: 500 +2023-04-15 16:39:12 - r - INFO: - Current episode 1310 has the best eval reward: 178.250 +2023-04-15 16:39:17 - r - INFO: - Episode: 1311/2000, Reward: 169.081, Step: 500 +2023-04-15 16:39:21 - r - INFO: - Episode: 1312/2000, Reward: 164.561, Step: 500 +2023-04-15 16:39:25 - r - INFO: - Episode: 1313/2000, Reward: 169.995, Step: 500 +2023-04-15 16:39:30 - r - INFO: - Episode: 1314/2000, Reward: 160.336, Step: 500 +2023-04-15 16:39:34 - r - INFO: - Episode: 1315/2000, Reward: 172.370, Step: 500 +2023-04-15 16:39:41 - r - INFO: - Current episode 1315 has the best eval reward: 181.181 +2023-04-15 16:39:46 - r - INFO: - Episode: 1316/2000, Reward: 165.336, Step: 500 +2023-04-15 16:39:50 - r - INFO: - Episode: 1317/2000, Reward: 156.946, Step: 500 +2023-04-15 16:39:54 - r - INFO: - Episode: 1318/2000, Reward: 162.699, Step: 500 +2023-04-15 16:39:58 - r - INFO: - Episode: 1319/2000, Reward: 157.845, Step: 500 +2023-04-15 16:40:03 - r - INFO: - Episode: 1320/2000, Reward: 152.657, Step: 500 +2023-04-15 16:40:14 - r - INFO: - Episode: 1321/2000, Reward: 173.483, Step: 500 +2023-04-15 16:40:19 - r - INFO: - Episode: 1322/2000, Reward: 163.987, Step: 500 +2023-04-15 16:40:23 - r - INFO: - Episode: 1323/2000, Reward: 151.327, Step: 500 +2023-04-15 16:40:27 - r - INFO: - Episode: 1324/2000, Reward: 158.784, Step: 500 +2023-04-15 16:40:32 - r - INFO: - Episode: 1325/2000, Reward: 173.653, Step: 500 +2023-04-15 16:40:43 - r - INFO: - Episode: 1326/2000, Reward: 162.407, Step: 500 +2023-04-15 16:40:48 - r - INFO: - Episode: 1327/2000, Reward: 162.595, Step: 500 +2023-04-15 16:40:52 - r - INFO: - Episode: 1328/2000, Reward: 153.828, Step: 500 +2023-04-15 16:40:56 - r - INFO: - Episode: 1329/2000, Reward: 163.516, Step: 500 +2023-04-15 16:41:01 - r - INFO: - Episode: 1330/2000, Reward: 164.987, Step: 500 +2023-04-15 16:41:12 - r - INFO: - Episode: 1331/2000, Reward: 155.331, Step: 500 +2023-04-15 16:41:15 - r - INFO: - Episode: 1332/2000, Reward: -32.332, Step: 290 +2023-04-15 16:41:19 - r - INFO: - Episode: 1333/2000, Reward: 159.481, Step: 500 +2023-04-15 16:41:23 - r - INFO: - Episode: 1334/2000, Reward: 161.389, Step: 500 +2023-04-15 16:41:28 - r - INFO: - Episode: 1335/2000, Reward: 158.215, Step: 500 +2023-04-15 16:41:39 - r - INFO: - Episode: 1336/2000, Reward: 166.264, Step: 500 +2023-04-15 16:41:43 - r - INFO: - Episode: 1337/2000, Reward: 162.033, Step: 500 +2023-04-15 16:41:48 - r - INFO: - Episode: 1338/2000, Reward: 160.443, Step: 500 +2023-04-15 16:41:52 - r - INFO: - Episode: 1339/2000, Reward: 165.857, Step: 500 +2023-04-15 16:41:56 - r - INFO: - Episode: 1340/2000, Reward: 163.221, Step: 500 +2023-04-15 16:42:07 - r - INFO: - Episode: 1341/2000, Reward: 175.147, Step: 500 +2023-04-15 16:42:12 - r - INFO: - Episode: 1342/2000, Reward: 162.508, Step: 500 +2023-04-15 16:42:16 - r - INFO: - Episode: 1343/2000, Reward: 167.007, Step: 500 +2023-04-15 16:42:20 - r - INFO: - Episode: 1344/2000, Reward: 170.826, Step: 500 +2023-04-15 16:42:25 - r - INFO: - Episode: 1345/2000, Reward: 174.644, Step: 500 +2023-04-15 16:42:36 - r - INFO: - Episode: 1346/2000, Reward: 172.169, Step: 500 +2023-04-15 16:42:40 - r - INFO: - Episode: 1347/2000, Reward: 163.649, Step: 500 +2023-04-15 16:42:43 - r - INFO: - Episode: 1348/2000, Reward: -15.429, Step: 361 +2023-04-15 16:42:45 - r - INFO: - Episode: 1349/2000, Reward: -87.373, Step: 129 +2023-04-15 16:42:49 - r - INFO: - Episode: 1350/2000, Reward: 173.693, Step: 500 +2023-04-15 16:43:00 - r - INFO: - Episode: 1351/2000, Reward: 162.558, Step: 500 +2023-04-15 16:43:05 - r - INFO: - Episode: 1352/2000, Reward: 164.846, Step: 500 +2023-04-15 16:43:09 - r - INFO: - Episode: 1353/2000, Reward: 169.086, Step: 500 +2023-04-15 16:43:13 - r - INFO: - Episode: 1354/2000, Reward: 163.968, Step: 500 +2023-04-15 16:43:18 - r - INFO: - Episode: 1355/2000, Reward: 164.894, Step: 500 +2023-04-15 16:43:29 - r - INFO: - Episode: 1356/2000, Reward: 160.069, Step: 500 +2023-04-15 16:43:33 - r - INFO: - Episode: 1357/2000, Reward: 163.126, Step: 500 +2023-04-15 16:43:38 - r - INFO: - Episode: 1358/2000, Reward: 168.802, Step: 500 +2023-04-15 16:43:42 - r - INFO: - Episode: 1359/2000, Reward: 170.020, Step: 500 +2023-04-15 16:43:46 - r - INFO: - Episode: 1360/2000, Reward: 166.069, Step: 500 +2023-04-15 16:43:53 - r - INFO: - Current episode 1360 has the best eval reward: 186.506 +2023-04-15 16:43:58 - r - INFO: - Episode: 1361/2000, Reward: 174.522, Step: 500 +2023-04-15 16:44:02 - r - INFO: - Episode: 1362/2000, Reward: 163.662, Step: 500 +2023-04-15 16:44:06 - r - INFO: - Episode: 1363/2000, Reward: 165.396, Step: 500 +2023-04-15 16:44:11 - r - INFO: - Episode: 1364/2000, Reward: 164.362, Step: 500 +2023-04-15 16:44:15 - r - INFO: - Episode: 1365/2000, Reward: 178.168, Step: 500 +2023-04-15 16:44:26 - r - INFO: - Episode: 1366/2000, Reward: 178.955, Step: 500 +2023-04-15 16:44:31 - r - INFO: - Episode: 1367/2000, Reward: 162.819, Step: 500 +2023-04-15 16:44:35 - r - INFO: - Episode: 1368/2000, Reward: 167.089, Step: 500 +2023-04-15 16:44:39 - r - INFO: - Episode: 1369/2000, Reward: 162.402, Step: 500 +2023-04-15 16:44:44 - r - INFO: - Episode: 1370/2000, Reward: 156.829, Step: 500 +2023-04-15 16:44:55 - r - INFO: - Episode: 1371/2000, Reward: 162.725, Step: 500 +2023-04-15 16:45:00 - r - INFO: - Episode: 1372/2000, Reward: 156.196, Step: 500 +2023-04-15 16:45:04 - r - INFO: - Episode: 1373/2000, Reward: 161.457, Step: 500 +2023-04-15 16:45:08 - r - INFO: - Episode: 1374/2000, Reward: 162.017, Step: 500 +2023-04-15 16:45:12 - r - INFO: - Episode: 1375/2000, Reward: 23.152, Step: 427 +2023-04-15 16:45:23 - r - INFO: - Episode: 1376/2000, Reward: 160.384, Step: 500 +2023-04-15 16:45:28 - r - INFO: - Episode: 1377/2000, Reward: 171.648, Step: 500 +2023-04-15 16:45:32 - r - INFO: - Episode: 1378/2000, Reward: 165.825, Step: 500 +2023-04-15 16:45:36 - r - INFO: - Episode: 1379/2000, Reward: 167.297, Step: 500 +2023-04-15 16:45:41 - r - INFO: - Episode: 1380/2000, Reward: 161.709, Step: 500 +2023-04-15 16:45:52 - r - INFO: - Episode: 1381/2000, Reward: 161.726, Step: 500 +2023-04-15 16:45:54 - r - INFO: - Episode: 1382/2000, Reward: -67.520, Step: 202 +2023-04-15 16:45:58 - r - INFO: - Episode: 1383/2000, Reward: 162.681, Step: 500 +2023-04-15 16:46:03 - r - INFO: - Episode: 1384/2000, Reward: 154.184, Step: 500 +2023-04-15 16:46:07 - r - INFO: - Episode: 1385/2000, Reward: 166.627, Step: 500 +2023-04-15 16:46:18 - r - INFO: - Episode: 1386/2000, Reward: 160.835, Step: 500 +2023-04-15 16:46:23 - r - INFO: - Episode: 1387/2000, Reward: 160.511, Step: 500 +2023-04-15 16:46:27 - r - INFO: - Episode: 1388/2000, Reward: 163.687, Step: 500 +2023-04-15 16:46:31 - r - INFO: - Episode: 1389/2000, Reward: 159.495, Step: 500 +2023-04-15 16:46:36 - r - INFO: - Episode: 1390/2000, Reward: 172.267, Step: 500 +2023-04-15 16:46:44 - r - INFO: - Episode: 1391/2000, Reward: -46.432, Step: 240 +2023-04-15 16:46:49 - r - INFO: - Episode: 1392/2000, Reward: 152.820, Step: 500 +2023-04-15 16:46:53 - r - INFO: - Episode: 1393/2000, Reward: 162.929, Step: 500 +2023-04-15 16:46:57 - r - INFO: - Episode: 1394/2000, Reward: 160.069, Step: 500 +2023-04-15 16:47:01 - r - INFO: - Episode: 1395/2000, Reward: 163.715, Step: 500 +2023-04-15 16:47:13 - r - INFO: - Episode: 1396/2000, Reward: 169.393, Step: 500 +2023-04-15 16:47:17 - r - INFO: - Episode: 1397/2000, Reward: 169.355, Step: 500 +2023-04-15 16:47:22 - r - INFO: - Episode: 1398/2000, Reward: 168.400, Step: 500 +2023-04-15 16:47:26 - r - INFO: - Episode: 1399/2000, Reward: 166.737, Step: 500 +2023-04-15 16:47:30 - r - INFO: - Episode: 1400/2000, Reward: 172.890, Step: 500 +2023-04-15 16:47:42 - r - INFO: - Episode: 1401/2000, Reward: 162.495, Step: 500 +2023-04-15 16:47:45 - r - INFO: - Episode: 1402/2000, Reward: 32.376, Step: 439 +2023-04-15 16:47:50 - r - INFO: - Episode: 1403/2000, Reward: 174.499, Step: 500 +2023-04-15 16:47:54 - r - INFO: - Episode: 1404/2000, Reward: 167.130, Step: 500 +2023-04-15 16:47:58 - r - INFO: - Episode: 1405/2000, Reward: 157.705, Step: 500 +2023-04-15 16:48:10 - r - INFO: - Episode: 1406/2000, Reward: 168.087, Step: 500 +2023-04-15 16:48:14 - r - INFO: - Episode: 1407/2000, Reward: 170.300, Step: 500 +2023-04-15 16:48:19 - r - INFO: - Episode: 1408/2000, Reward: 122.708, Step: 500 +2023-04-15 16:48:23 - r - INFO: - Episode: 1409/2000, Reward: 170.469, Step: 500 +2023-04-15 16:48:27 - r - INFO: - Episode: 1410/2000, Reward: 166.419, Step: 500 +2023-04-15 16:48:39 - r - INFO: - Episode: 1411/2000, Reward: 159.550, Step: 500 +2023-04-15 16:48:43 - r - INFO: - Episode: 1412/2000, Reward: 172.454, Step: 500 +2023-04-15 16:48:47 - r - INFO: - Episode: 1413/2000, Reward: 172.727, Step: 500 +2023-04-15 16:48:52 - r - INFO: - Episode: 1414/2000, Reward: 169.552, Step: 500 +2023-04-15 16:48:56 - r - INFO: - Episode: 1415/2000, Reward: 159.597, Step: 500 +2023-04-15 16:49:07 - r - INFO: - Episode: 1416/2000, Reward: 166.480, Step: 500 +2023-04-15 16:49:12 - r - INFO: - Episode: 1417/2000, Reward: 165.911, Step: 500 +2023-04-15 16:49:16 - r - INFO: - Episode: 1418/2000, Reward: 57.484, Step: 500 +2023-04-15 16:49:20 - r - INFO: - Episode: 1419/2000, Reward: 175.447, Step: 500 +2023-04-15 16:49:25 - r - INFO: - Episode: 1420/2000, Reward: 176.048, Step: 500 +2023-04-15 16:49:32 - r - INFO: - Current episode 1420 has the best eval reward: 192.696 +2023-04-15 16:49:36 - r - INFO: - Episode: 1421/2000, Reward: 185.095, Step: 500 +2023-04-15 16:49:40 - r - INFO: - Episode: 1422/2000, Reward: 174.469, Step: 500 +2023-04-15 16:49:45 - r - INFO: - Episode: 1423/2000, Reward: 166.095, Step: 500 +2023-04-15 16:49:49 - r - INFO: - Episode: 1424/2000, Reward: 168.499, Step: 500 +2023-04-15 16:49:53 - r - INFO: - Episode: 1425/2000, Reward: 169.456, Step: 500 +2023-04-15 16:50:05 - r - INFO: - Episode: 1426/2000, Reward: 158.374, Step: 500 +2023-04-15 16:50:09 - r - INFO: - Episode: 1427/2000, Reward: 172.851, Step: 500 +2023-04-15 16:50:13 - r - INFO: - Episode: 1428/2000, Reward: 26.894, Step: 500 +2023-04-15 16:50:17 - r - INFO: - Episode: 1429/2000, Reward: 166.808, Step: 500 +2023-04-15 16:50:22 - r - INFO: - Episode: 1430/2000, Reward: 168.746, Step: 500 +2023-04-15 16:50:33 - r - INFO: - Episode: 1431/2000, Reward: 162.027, Step: 500 +2023-04-15 16:50:38 - r - INFO: - Episode: 1432/2000, Reward: 160.192, Step: 500 +2023-04-15 16:50:42 - r - INFO: - Episode: 1433/2000, Reward: 160.721, Step: 500 +2023-04-15 16:50:46 - r - INFO: - Episode: 1434/2000, Reward: 77.525, Step: 500 +2023-04-15 16:50:51 - r - INFO: - Episode: 1435/2000, Reward: 153.827, Step: 500 +2023-04-15 16:51:02 - r - INFO: - Episode: 1436/2000, Reward: 164.276, Step: 500 +2023-04-15 16:51:06 - r - INFO: - Episode: 1437/2000, Reward: 170.150, Step: 500 +2023-04-15 16:51:11 - r - INFO: - Episode: 1438/2000, Reward: 165.980, Step: 500 +2023-04-15 16:51:15 - r - INFO: - Episode: 1439/2000, Reward: 158.667, Step: 500 +2023-04-15 16:51:19 - r - INFO: - Episode: 1440/2000, Reward: 162.818, Step: 500 +2023-04-15 16:51:31 - r - INFO: - Episode: 1441/2000, Reward: 167.953, Step: 500 +2023-04-15 16:51:35 - r - INFO: - Episode: 1442/2000, Reward: 167.181, Step: 500 +2023-04-15 16:51:39 - r - INFO: - Episode: 1443/2000, Reward: 169.182, Step: 500 +2023-04-15 16:51:42 - r - INFO: - Episode: 1444/2000, Reward: -60.369, Step: 310 +2023-04-15 16:51:46 - r - INFO: - Episode: 1445/2000, Reward: 171.494, Step: 500 +2023-04-15 16:51:58 - r - INFO: - Episode: 1446/2000, Reward: 162.708, Step: 500 +2023-04-15 16:52:02 - r - INFO: - Episode: 1447/2000, Reward: 169.706, Step: 500 +2023-04-15 16:52:06 - r - INFO: - Episode: 1448/2000, Reward: 172.995, Step: 500 +2023-04-15 16:52:11 - r - INFO: - Episode: 1449/2000, Reward: 163.999, Step: 500 +2023-04-15 16:52:15 - r - INFO: - Episode: 1450/2000, Reward: 166.603, Step: 500 +2023-04-15 16:52:27 - r - INFO: - Episode: 1451/2000, Reward: 162.601, Step: 500 +2023-04-15 16:52:31 - r - INFO: - Episode: 1452/2000, Reward: 162.529, Step: 500 +2023-04-15 16:52:35 - r - INFO: - Episode: 1453/2000, Reward: 158.385, Step: 500 +2023-04-15 16:52:39 - r - INFO: - Episode: 1454/2000, Reward: 163.427, Step: 500 +2023-04-15 16:52:44 - r - INFO: - Episode: 1455/2000, Reward: 164.198, Step: 500 +2023-04-15 16:52:55 - r - INFO: - Episode: 1456/2000, Reward: 172.664, Step: 500 +2023-04-15 16:52:59 - r - INFO: - Episode: 1457/2000, Reward: 164.267, Step: 500 +2023-04-15 16:53:04 - r - INFO: - Episode: 1458/2000, Reward: 168.525, Step: 500 +2023-04-15 16:53:08 - r - INFO: - Episode: 1459/2000, Reward: 171.524, Step: 500 +2023-04-15 16:53:12 - r - INFO: - Episode: 1460/2000, Reward: 163.281, Step: 500 +2023-04-15 16:53:24 - r - INFO: - Episode: 1461/2000, Reward: 168.903, Step: 500 +2023-04-15 16:53:28 - r - INFO: - Episode: 1462/2000, Reward: 169.152, Step: 500 +2023-04-15 16:53:33 - r - INFO: - Episode: 1463/2000, Reward: 166.361, Step: 500 +2023-04-15 16:53:37 - r - INFO: - Episode: 1464/2000, Reward: 168.562, Step: 500 +2023-04-15 16:53:41 - r - INFO: - Episode: 1465/2000, Reward: 169.910, Step: 500 +2023-04-15 16:53:53 - r - INFO: - Episode: 1466/2000, Reward: 168.345, Step: 500 +2023-04-15 16:53:57 - r - INFO: - Episode: 1467/2000, Reward: 163.331, Step: 500 +2023-04-15 16:54:01 - r - INFO: - Episode: 1468/2000, Reward: 163.870, Step: 500 +2023-04-15 16:54:06 - r - INFO: - Episode: 1469/2000, Reward: 168.885, Step: 500 +2023-04-15 16:54:10 - r - INFO: - Episode: 1470/2000, Reward: 162.591, Step: 500 +2023-04-15 16:54:21 - r - INFO: - Episode: 1471/2000, Reward: 165.763, Step: 500 +2023-04-15 16:54:26 - r - INFO: - Episode: 1472/2000, Reward: 156.391, Step: 500 +2023-04-15 16:54:30 - r - INFO: - Episode: 1473/2000, Reward: 159.242, Step: 500 +2023-04-15 16:54:34 - r - INFO: - Episode: 1474/2000, Reward: 168.022, Step: 500 +2023-04-15 16:54:39 - r - INFO: - Episode: 1475/2000, Reward: 166.664, Step: 500 +2023-04-15 16:54:50 - r - INFO: - Episode: 1476/2000, Reward: 168.250, Step: 500 +2023-04-15 16:54:54 - r - INFO: - Episode: 1477/2000, Reward: 172.336, Step: 500 +2023-04-15 16:54:59 - r - INFO: - Episode: 1478/2000, Reward: 164.654, Step: 500 +2023-04-15 16:55:03 - r - INFO: - Episode: 1479/2000, Reward: 172.167, Step: 500 +2023-04-15 16:55:07 - r - INFO: - Episode: 1480/2000, Reward: 174.926, Step: 500 +2023-04-15 16:55:19 - r - INFO: - Episode: 1481/2000, Reward: 167.485, Step: 500 +2023-04-15 16:55:23 - r - INFO: - Episode: 1482/2000, Reward: 170.358, Step: 500 +2023-04-15 16:56:44 - r - INFO: - Episode: 1483/2000, Reward: 173.460, Step: 500 +2023-04-15 16:56:48 - r - INFO: - Episode: 1484/2000, Reward: 162.102, Step: 500 +2023-04-15 16:56:52 - r - INFO: - Episode: 1485/2000, Reward: 179.137, Step: 500 +2023-04-15 16:57:03 - r - INFO: - Episode: 1486/2000, Reward: 173.043, Step: 500 +2023-04-15 16:57:08 - r - INFO: - Episode: 1487/2000, Reward: 171.089, Step: 500 +2023-04-15 16:57:12 - r - INFO: - Episode: 1488/2000, Reward: 169.241, Step: 500 +2023-04-15 16:57:16 - r - INFO: - Episode: 1489/2000, Reward: 160.976, Step: 500 +2023-04-15 16:57:20 - r - INFO: - Episode: 1490/2000, Reward: 148.451, Step: 500 +2023-04-15 16:57:31 - r - INFO: - Episode: 1491/2000, Reward: 167.487, Step: 500 +2023-04-15 16:57:35 - r - INFO: - Episode: 1492/2000, Reward: 161.800, Step: 500 +2023-04-15 16:57:40 - r - INFO: - Episode: 1493/2000, Reward: 135.415, Step: 500 +2023-04-15 16:57:44 - r - INFO: - Episode: 1494/2000, Reward: 164.823, Step: 500 +2023-04-15 16:57:48 - r - INFO: - Episode: 1495/2000, Reward: 167.508, Step: 500 +2023-04-15 16:57:59 - r - INFO: - Episode: 1496/2000, Reward: 164.849, Step: 500 +2023-04-15 16:58:04 - r - INFO: - Episode: 1497/2000, Reward: 167.353, Step: 500 +2023-04-15 16:58:08 - r - INFO: - Episode: 1498/2000, Reward: 171.661, Step: 500 +2023-04-15 16:58:12 - r - INFO: - Episode: 1499/2000, Reward: 162.653, Step: 500 +2023-04-15 16:58:16 - r - INFO: - Episode: 1500/2000, Reward: 151.088, Step: 500 +2023-04-15 16:58:28 - r - INFO: - Episode: 1501/2000, Reward: 164.161, Step: 500 +2023-04-15 16:58:32 - r - INFO: - Episode: 1502/2000, Reward: 167.301, Step: 500 +2023-04-15 16:58:36 - r - INFO: - Episode: 1503/2000, Reward: 161.686, Step: 500 +2023-04-15 16:58:40 - r - INFO: - Episode: 1504/2000, Reward: 170.290, Step: 500 +2023-04-15 16:58:45 - r - INFO: - Episode: 1505/2000, Reward: 170.683, Step: 500 +2023-04-15 16:58:56 - r - INFO: - Episode: 1506/2000, Reward: 167.892, Step: 500 +2023-04-15 16:59:00 - r - INFO: - Episode: 1507/2000, Reward: 156.766, Step: 500 +2023-04-15 16:59:05 - r - INFO: - Episode: 1508/2000, Reward: 163.372, Step: 500 +2023-04-15 16:59:09 - r - INFO: - Episode: 1509/2000, Reward: 163.700, Step: 500 +2023-04-15 16:59:13 - r - INFO: - Episode: 1510/2000, Reward: 167.330, Step: 500 +2023-04-15 16:59:24 - r - INFO: - Episode: 1511/2000, Reward: 166.159, Step: 500 +2023-04-15 16:59:29 - r - INFO: - Episode: 1512/2000, Reward: 169.929, Step: 500 +2023-04-15 16:59:33 - r - INFO: - Episode: 1513/2000, Reward: 167.378, Step: 500 +2023-04-15 16:59:37 - r - INFO: - Episode: 1514/2000, Reward: 176.540, Step: 500 +2023-04-15 16:59:41 - r - INFO: - Episode: 1515/2000, Reward: 167.902, Step: 500 +2023-04-15 16:59:53 - r - INFO: - Episode: 1516/2000, Reward: 169.052, Step: 500 +2023-04-15 16:59:57 - r - INFO: - Episode: 1517/2000, Reward: 175.824, Step: 500 +2023-04-15 17:00:01 - r - INFO: - Episode: 1518/2000, Reward: 157.596, Step: 500 +2023-04-15 17:00:05 - r - INFO: - Episode: 1519/2000, Reward: 158.857, Step: 500 +2023-04-15 17:00:10 - r - INFO: - Episode: 1520/2000, Reward: 165.900, Step: 500 +2023-04-15 17:00:21 - r - INFO: - Episode: 1521/2000, Reward: 170.725, Step: 500 +2023-04-15 17:00:25 - r - INFO: - Episode: 1522/2000, Reward: 166.554, Step: 500 +2023-04-15 17:00:30 - r - INFO: - Episode: 1523/2000, Reward: 162.531, Step: 500 +2023-04-15 17:00:34 - r - INFO: - Episode: 1524/2000, Reward: 154.925, Step: 500 +2023-04-15 17:00:38 - r - INFO: - Episode: 1525/2000, Reward: 156.979, Step: 500 +2023-04-15 17:00:49 - r - INFO: - Episode: 1526/2000, Reward: 154.319, Step: 500 +2023-04-15 17:00:54 - r - INFO: - Episode: 1527/2000, Reward: 161.431, Step: 500 +2023-04-15 17:00:58 - r - INFO: - Episode: 1528/2000, Reward: 163.032, Step: 500 +2023-04-15 17:01:02 - r - INFO: - Episode: 1529/2000, Reward: 155.748, Step: 500 +2023-04-15 17:01:06 - r - INFO: - Episode: 1530/2000, Reward: 163.791, Step: 500 +2023-04-15 17:01:18 - r - INFO: - Episode: 1531/2000, Reward: 158.047, Step: 500 +2023-04-15 17:01:22 - r - INFO: - Episode: 1532/2000, Reward: 166.720, Step: 500 +2023-04-15 17:01:27 - r - INFO: - Episode: 1533/2000, Reward: 172.160, Step: 500 +2023-04-15 17:01:31 - r - INFO: - Episode: 1534/2000, Reward: 169.809, Step: 500 +2023-04-15 17:01:35 - r - INFO: - Episode: 1535/2000, Reward: 163.334, Step: 500 +2023-04-15 17:01:47 - r - INFO: - Episode: 1536/2000, Reward: 171.252, Step: 500 +2023-04-15 17:01:51 - r - INFO: - Episode: 1537/2000, Reward: 170.419, Step: 500 +2023-04-15 17:01:55 - r - INFO: - Episode: 1538/2000, Reward: 167.047, Step: 500 +2023-04-15 17:02:00 - r - INFO: - Episode: 1539/2000, Reward: 169.425, Step: 500 +2023-04-15 17:02:04 - r - INFO: - Episode: 1540/2000, Reward: 168.026, Step: 500 +2023-04-15 17:02:16 - r - INFO: - Episode: 1541/2000, Reward: 173.243, Step: 500 +2023-04-15 17:02:20 - r - INFO: - Episode: 1542/2000, Reward: 157.604, Step: 500 +2023-04-15 17:02:24 - r - INFO: - Episode: 1543/2000, Reward: 159.268, Step: 500 +2023-04-15 17:02:28 - r - INFO: - Episode: 1544/2000, Reward: 167.514, Step: 500 +2023-04-15 17:02:33 - r - INFO: - Episode: 1545/2000, Reward: 170.739, Step: 500 +2023-04-15 17:02:44 - r - INFO: - Episode: 1546/2000, Reward: 172.580, Step: 500 +2023-04-15 17:02:48 - r - INFO: - Episode: 1547/2000, Reward: 175.986, Step: 500 +2023-04-15 17:02:52 - r - INFO: - Episode: 1548/2000, Reward: 169.381, Step: 500 +2023-04-15 17:02:57 - r - INFO: - Episode: 1549/2000, Reward: 172.752, Step: 500 +2023-04-15 17:03:01 - r - INFO: - Episode: 1550/2000, Reward: 163.782, Step: 500 +2023-04-15 17:03:13 - r - INFO: - Episode: 1551/2000, Reward: 165.366, Step: 500 +2023-04-15 17:03:17 - r - INFO: - Episode: 1552/2000, Reward: 162.509, Step: 500 +2023-04-15 17:03:21 - r - INFO: - Episode: 1553/2000, Reward: 167.205, Step: 500 +2023-04-15 17:03:25 - r - INFO: - Episode: 1554/2000, Reward: 163.343, Step: 500 +2023-04-15 17:03:30 - r - INFO: - Episode: 1555/2000, Reward: 172.057, Step: 500 +2023-04-15 17:03:41 - r - INFO: - Episode: 1556/2000, Reward: 165.639, Step: 500 +2023-04-15 17:03:45 - r - INFO: - Episode: 1557/2000, Reward: 169.233, Step: 500 +2023-04-15 17:03:50 - r - INFO: - Episode: 1558/2000, Reward: 134.785, Step: 500 +2023-04-15 17:03:54 - r - INFO: - Episode: 1559/2000, Reward: 187.292, Step: 500 +2023-04-15 17:03:58 - r - INFO: - Episode: 1560/2000, Reward: 175.392, Step: 500 +2023-04-15 17:04:10 - r - INFO: - Episode: 1561/2000, Reward: 171.549, Step: 500 +2023-04-15 17:04:14 - r - INFO: - Episode: 1562/2000, Reward: 176.708, Step: 500 +2023-04-15 17:04:18 - r - INFO: - Episode: 1563/2000, Reward: 170.675, Step: 500 +2023-04-15 17:04:23 - r - INFO: - Episode: 1564/2000, Reward: 170.385, Step: 500 +2023-04-15 17:04:27 - r - INFO: - Episode: 1565/2000, Reward: 171.957, Step: 500 +2023-04-15 17:04:38 - r - INFO: - Episode: 1566/2000, Reward: 170.395, Step: 500 +2023-04-15 17:04:41 - r - INFO: - Episode: 1567/2000, Reward: -19.116, Step: 299 +2023-04-15 17:04:45 - r - INFO: - Episode: 1568/2000, Reward: 164.515, Step: 500 +2023-04-15 17:04:50 - r - INFO: - Episode: 1569/2000, Reward: 160.290, Step: 500 +2023-04-15 17:04:54 - r - INFO: - Episode: 1570/2000, Reward: 166.077, Step: 500 +2023-04-15 17:05:05 - r - INFO: - Episode: 1571/2000, Reward: 180.818, Step: 500 +2023-04-15 17:05:10 - r - INFO: - Episode: 1572/2000, Reward: 167.499, Step: 500 +2023-04-15 17:05:14 - r - INFO: - Episode: 1573/2000, Reward: 162.800, Step: 500 +2023-04-15 17:05:18 - r - INFO: - Episode: 1574/2000, Reward: 170.635, Step: 500 +2023-04-15 17:05:23 - r - INFO: - Episode: 1575/2000, Reward: 166.846, Step: 500 +2023-04-15 17:05:34 - r - INFO: - Episode: 1576/2000, Reward: 176.942, Step: 500 +2023-04-15 17:05:38 - r - INFO: - Episode: 1577/2000, Reward: 176.993, Step: 500 +2023-04-15 17:05:42 - r - INFO: - Episode: 1578/2000, Reward: 166.038, Step: 500 +2023-04-15 17:05:47 - r - INFO: - Episode: 1579/2000, Reward: 164.368, Step: 500 +2023-04-15 17:05:51 - r - INFO: - Episode: 1580/2000, Reward: 169.332, Step: 500 +2023-04-15 17:06:02 - r - INFO: - Episode: 1581/2000, Reward: 172.098, Step: 500 +2023-04-15 17:06:07 - r - INFO: - Episode: 1582/2000, Reward: 166.560, Step: 500 +2023-04-15 17:06:11 - r - INFO: - Episode: 1583/2000, Reward: 168.031, Step: 500 +2023-04-15 17:06:15 - r - INFO: - Episode: 1584/2000, Reward: 160.163, Step: 500 +2023-04-15 17:06:20 - r - INFO: - Episode: 1585/2000, Reward: 178.816, Step: 500 +2023-04-15 17:06:31 - r - INFO: - Episode: 1586/2000, Reward: 171.628, Step: 500 +2023-04-15 17:06:35 - r - INFO: - Episode: 1587/2000, Reward: 165.065, Step: 500 +2023-04-15 17:06:40 - r - INFO: - Episode: 1588/2000, Reward: 168.568, Step: 500 +2023-04-15 17:06:44 - r - INFO: - Episode: 1589/2000, Reward: 169.493, Step: 500 +2023-04-15 17:06:48 - r - INFO: - Episode: 1590/2000, Reward: 176.106, Step: 500 +2023-04-15 17:07:00 - r - INFO: - Episode: 1591/2000, Reward: 164.056, Step: 500 +2023-04-15 17:07:04 - r - INFO: - Episode: 1592/2000, Reward: 171.978, Step: 500 +2023-04-15 17:07:08 - r - INFO: - Episode: 1593/2000, Reward: 174.524, Step: 500 +2023-04-15 17:07:13 - r - INFO: - Episode: 1594/2000, Reward: 167.584, Step: 500 +2023-04-15 17:07:17 - r - INFO: - Episode: 1595/2000, Reward: 172.107, Step: 500 +2023-04-15 17:07:28 - r - INFO: - Episode: 1596/2000, Reward: 173.348, Step: 500 +2023-04-15 17:07:33 - r - INFO: - Episode: 1597/2000, Reward: 172.077, Step: 500 +2023-04-15 17:07:37 - r - INFO: - Episode: 1598/2000, Reward: 162.065, Step: 500 +2023-04-15 17:07:41 - r - INFO: - Episode: 1599/2000, Reward: 163.751, Step: 500 +2023-04-15 17:07:46 - r - INFO: - Episode: 1600/2000, Reward: 160.412, Step: 500 +2023-04-15 17:07:56 - r - INFO: - Episode: 1601/2000, Reward: 34.110, Step: 445 +2023-04-15 17:08:01 - r - INFO: - Episode: 1602/2000, Reward: 167.423, Step: 500 +2023-04-15 17:08:05 - r - INFO: - Episode: 1603/2000, Reward: 171.864, Step: 500 +2023-04-15 17:08:09 - r - INFO: - Episode: 1604/2000, Reward: 175.129, Step: 500 +2023-04-15 17:08:14 - r - INFO: - Episode: 1605/2000, Reward: 166.500, Step: 500 +2023-04-15 17:08:25 - r - INFO: - Episode: 1606/2000, Reward: 177.378, Step: 500 +2023-04-15 17:08:29 - r - INFO: - Episode: 1607/2000, Reward: 168.065, Step: 500 +2023-04-15 17:08:34 - r - INFO: - Episode: 1608/2000, Reward: 172.448, Step: 500 +2023-04-15 17:08:38 - r - INFO: - Episode: 1609/2000, Reward: 176.147, Step: 500 +2023-04-15 17:08:42 - r - INFO: - Episode: 1610/2000, Reward: 166.194, Step: 500 +2023-04-15 17:08:54 - r - INFO: - Episode: 1611/2000, Reward: 173.803, Step: 500 +2023-04-15 17:10:41 - r - INFO: - Episode: 1612/2000, Reward: 169.478, Step: 500 +2023-04-15 17:10:45 - r - INFO: - Episode: 1613/2000, Reward: 168.722, Step: 500 +2023-04-15 17:10:49 - r - INFO: - Episode: 1614/2000, Reward: 168.291, Step: 500 +2023-04-15 17:10:53 - r - INFO: - Episode: 1615/2000, Reward: 166.663, Step: 500 +2023-04-15 17:11:05 - r - INFO: - Episode: 1616/2000, Reward: 175.879, Step: 500 +2023-04-15 17:11:09 - r - INFO: - Episode: 1617/2000, Reward: 174.496, Step: 500 +2023-04-15 17:11:13 - r - INFO: - Episode: 1618/2000, Reward: 178.576, Step: 500 +2023-04-15 17:11:17 - r - INFO: - Episode: 1619/2000, Reward: 175.074, Step: 500 +2023-04-15 17:11:21 - r - INFO: - Episode: 1620/2000, Reward: 183.019, Step: 500 +2023-04-15 17:11:33 - r - INFO: - Episode: 1621/2000, Reward: 171.073, Step: 500 +2023-04-15 17:11:37 - r - INFO: - Episode: 1622/2000, Reward: 161.003, Step: 500 +2023-04-15 17:11:41 - r - INFO: - Episode: 1623/2000, Reward: 169.812, Step: 500 +2023-04-15 17:11:45 - r - INFO: - Episode: 1624/2000, Reward: 173.422, Step: 500 +2023-04-15 17:11:49 - r - INFO: - Episode: 1625/2000, Reward: 169.453, Step: 500 +2023-04-15 17:12:01 - r - INFO: - Episode: 1626/2000, Reward: 169.603, Step: 500 +2023-04-15 17:12:05 - r - INFO: - Episode: 1627/2000, Reward: 169.634, Step: 500 +2023-04-15 17:12:09 - r - INFO: - Episode: 1628/2000, Reward: 173.967, Step: 500 +2023-04-15 17:12:13 - r - INFO: - Episode: 1629/2000, Reward: 176.441, Step: 500 +2023-04-15 17:12:18 - r - INFO: - Episode: 1630/2000, Reward: 168.544, Step: 500 +2023-04-15 17:12:29 - r - INFO: - Episode: 1631/2000, Reward: 164.393, Step: 500 +2023-04-15 17:12:33 - r - INFO: - Episode: 1632/2000, Reward: 169.804, Step: 500 +2023-04-15 17:12:37 - r - INFO: - Episode: 1633/2000, Reward: 168.495, Step: 500 +2023-04-15 17:12:42 - r - INFO: - Episode: 1634/2000, Reward: 172.145, Step: 500 +2023-04-15 17:12:46 - r - INFO: - Episode: 1635/2000, Reward: 180.428, Step: 500 +2023-04-15 17:12:57 - r - INFO: - Episode: 1636/2000, Reward: 167.109, Step: 500 +2023-04-15 17:13:01 - r - INFO: - Episode: 1637/2000, Reward: 172.311, Step: 500 +2023-04-15 17:13:06 - r - INFO: - Episode: 1638/2000, Reward: 179.647, Step: 500 +2023-04-15 17:13:10 - r - INFO: - Episode: 1639/2000, Reward: 181.251, Step: 500 +2023-04-15 17:13:14 - r - INFO: - Episode: 1640/2000, Reward: 170.863, Step: 500 +2023-04-15 17:13:26 - r - INFO: - Episode: 1641/2000, Reward: 176.744, Step: 500 +2023-04-15 17:13:30 - r - INFO: - Episode: 1642/2000, Reward: 179.735, Step: 500 +2023-04-15 17:13:34 - r - INFO: - Episode: 1643/2000, Reward: 178.655, Step: 500 +2023-04-15 17:13:38 - r - INFO: - Episode: 1644/2000, Reward: 178.962, Step: 500 +2023-04-15 17:13:43 - r - INFO: - Episode: 1645/2000, Reward: 174.075, Step: 500 +2023-04-15 17:13:54 - r - INFO: - Episode: 1646/2000, Reward: 42.560, Step: 474 +2023-04-15 17:13:58 - r - INFO: - Episode: 1647/2000, Reward: 180.142, Step: 500 +2023-04-15 17:14:02 - r - INFO: - Episode: 1648/2000, Reward: 179.251, Step: 500 +2023-04-15 17:14:07 - r - INFO: - Episode: 1649/2000, Reward: 176.717, Step: 500 +2023-04-15 17:14:11 - r - INFO: - Episode: 1650/2000, Reward: 168.455, Step: 500 +2023-04-15 17:14:22 - r - INFO: - Episode: 1651/2000, Reward: 179.935, Step: 500 +2023-04-15 17:14:26 - r - INFO: - Episode: 1652/2000, Reward: 176.907, Step: 500 +2023-04-15 17:14:31 - r - INFO: - Episode: 1653/2000, Reward: 183.348, Step: 500 +2023-04-15 17:14:35 - r - INFO: - Episode: 1654/2000, Reward: 178.977, Step: 500 +2023-04-15 17:14:39 - r - INFO: - Episode: 1655/2000, Reward: 178.516, Step: 500 +2023-04-15 17:14:51 - r - INFO: - Episode: 1656/2000, Reward: 174.131, Step: 500 +2023-04-15 17:14:55 - r - INFO: - Episode: 1657/2000, Reward: 185.195, Step: 500 +2023-04-15 17:14:59 - r - INFO: - Episode: 1658/2000, Reward: 181.087, Step: 500 +2023-04-15 17:15:04 - r - INFO: - Episode: 1659/2000, Reward: 174.128, Step: 500 +2023-04-15 17:15:08 - r - INFO: - Episode: 1660/2000, Reward: 33.652, Step: 453 +2023-04-15 17:15:19 - r - INFO: - Episode: 1661/2000, Reward: 178.465, Step: 500 +2023-04-15 17:15:23 - r - INFO: - Episode: 1662/2000, Reward: 179.992, Step: 500 +2023-04-15 17:15:27 - r - INFO: - Episode: 1663/2000, Reward: 182.272, Step: 500 +2023-04-15 17:15:32 - r - INFO: - Episode: 1664/2000, Reward: 177.841, Step: 500 +2023-04-15 17:15:36 - r - INFO: - Episode: 1665/2000, Reward: 182.752, Step: 500 +2023-04-15 17:15:47 - r - INFO: - Episode: 1666/2000, Reward: 176.931, Step: 500 +2023-04-15 17:15:52 - r - INFO: - Episode: 1667/2000, Reward: 178.196, Step: 500 +2023-04-15 17:15:56 - r - INFO: - Episode: 1668/2000, Reward: 178.505, Step: 500 +2023-04-15 17:16:00 - r - INFO: - Episode: 1669/2000, Reward: 164.524, Step: 500 +2023-04-15 17:16:04 - r - INFO: - Episode: 1670/2000, Reward: 171.471, Step: 500 +2023-04-15 17:16:16 - r - INFO: - Episode: 1671/2000, Reward: 173.352, Step: 500 +2023-04-15 17:16:20 - r - INFO: - Episode: 1672/2000, Reward: 181.510, Step: 500 +2023-04-15 17:16:24 - r - INFO: - Episode: 1673/2000, Reward: 179.264, Step: 500 +2023-04-15 17:16:29 - r - INFO: - Episode: 1674/2000, Reward: 187.119, Step: 500 +2023-04-15 17:16:33 - r - INFO: - Episode: 1675/2000, Reward: 178.467, Step: 500 +2023-04-15 17:16:44 - r - INFO: - Episode: 1676/2000, Reward: 178.918, Step: 500 +2023-04-15 17:16:49 - r - INFO: - Episode: 1677/2000, Reward: 179.194, Step: 500 +2023-04-15 17:16:53 - r - INFO: - Episode: 1678/2000, Reward: 179.367, Step: 500 +2023-04-15 17:16:57 - r - INFO: - Episode: 1679/2000, Reward: 181.355, Step: 500 +2023-04-15 17:17:01 - r - INFO: - Episode: 1680/2000, Reward: 174.981, Step: 500 +2023-04-15 17:17:13 - r - INFO: - Episode: 1681/2000, Reward: 177.879, Step: 500 +2023-04-15 17:17:17 - r - INFO: - Episode: 1682/2000, Reward: 177.223, Step: 500 +2023-04-15 17:17:21 - r - INFO: - Episode: 1683/2000, Reward: 171.136, Step: 500 +2023-04-15 17:17:26 - r - INFO: - Episode: 1684/2000, Reward: 166.853, Step: 500 +2023-04-15 17:17:30 - r - INFO: - Episode: 1685/2000, Reward: 175.560, Step: 500 +2023-04-15 17:17:41 - r - INFO: - Episode: 1686/2000, Reward: 163.482, Step: 500 +2023-04-15 17:17:46 - r - INFO: - Episode: 1687/2000, Reward: 183.702, Step: 500 +2023-04-15 17:17:50 - r - INFO: - Episode: 1688/2000, Reward: 180.518, Step: 500 +2023-04-15 17:17:54 - r - INFO: - Episode: 1689/2000, Reward: 168.532, Step: 500 +2023-04-15 17:17:59 - r - INFO: - Episode: 1690/2000, Reward: 169.640, Step: 500 +2023-04-15 17:18:10 - r - INFO: - Episode: 1691/2000, Reward: 181.464, Step: 500 +2023-04-15 17:18:15 - r - INFO: - Episode: 1692/2000, Reward: 173.503, Step: 500 +2023-04-15 17:18:19 - r - INFO: - Episode: 1693/2000, Reward: 182.852, Step: 500 +2023-04-15 17:18:23 - r - INFO: - Episode: 1694/2000, Reward: 162.777, Step: 500 +2023-04-15 17:18:27 - r - INFO: - Episode: 1695/2000, Reward: 175.194, Step: 500 +2023-04-15 17:18:39 - r - INFO: - Episode: 1696/2000, Reward: 174.928, Step: 500 +2023-04-15 17:18:43 - r - INFO: - Episode: 1697/2000, Reward: 181.179, Step: 500 +2023-04-15 17:18:47 - r - INFO: - Episode: 1698/2000, Reward: 172.347, Step: 500 +2023-04-15 17:18:52 - r - INFO: - Episode: 1699/2000, Reward: 163.380, Step: 500 +2023-04-15 17:18:56 - r - INFO: - Episode: 1700/2000, Reward: 185.020, Step: 500 +2023-04-15 17:22:12 - r - INFO: - Current episode 1700 has the best eval reward: 195.819 +2023-04-15 17:22:16 - r - INFO: - Episode: 1701/2000, Reward: 162.964, Step: 500 +2023-04-15 17:22:21 - r - INFO: - Episode: 1702/2000, Reward: 173.283, Step: 500 +2023-04-15 17:22:25 - r - INFO: - Episode: 1703/2000, Reward: 179.235, Step: 500 +2023-04-15 17:22:29 - r - INFO: - Episode: 1704/2000, Reward: 165.099, Step: 500 +2023-04-15 17:22:33 - r - INFO: - Episode: 1705/2000, Reward: 180.331, Step: 500 +2023-04-15 17:22:45 - r - INFO: - Episode: 1706/2000, Reward: 168.017, Step: 500 +2023-04-15 17:22:49 - r - INFO: - Episode: 1707/2000, Reward: 174.746, Step: 500 +2023-04-15 17:22:53 - r - INFO: - Episode: 1708/2000, Reward: 176.093, Step: 500 +2023-04-15 17:22:57 - r - INFO: - Episode: 1709/2000, Reward: 181.259, Step: 500 +2023-04-15 17:23:01 - r - INFO: - Episode: 1710/2000, Reward: 178.177, Step: 500 +2023-04-15 17:23:13 - r - INFO: - Episode: 1711/2000, Reward: 178.861, Step: 500 +2023-04-15 17:23:17 - r - INFO: - Episode: 1712/2000, Reward: 174.321, Step: 500 +2023-04-15 17:23:21 - r - INFO: - Episode: 1713/2000, Reward: 178.517, Step: 500 +2023-04-15 17:23:25 - r - INFO: - Episode: 1714/2000, Reward: 172.991, Step: 500 +2023-04-15 17:23:30 - r - INFO: - Episode: 1715/2000, Reward: 173.234, Step: 500 +2023-04-15 17:23:41 - r - INFO: - Episode: 1716/2000, Reward: 177.090, Step: 500 +2023-04-15 17:23:45 - r - INFO: - Episode: 1717/2000, Reward: 179.384, Step: 500 +2023-04-15 17:23:49 - r - INFO: - Episode: 1718/2000, Reward: 182.963, Step: 500 +2023-04-15 17:23:54 - r - INFO: - Episode: 1719/2000, Reward: 180.288, Step: 500 +2023-04-15 17:23:58 - r - INFO: - Episode: 1720/2000, Reward: 177.562, Step: 500 +2023-04-15 17:24:09 - r - INFO: - Episode: 1721/2000, Reward: 175.510, Step: 500 +2023-04-15 17:24:13 - r - INFO: - Episode: 1722/2000, Reward: 183.487, Step: 500 +2023-04-15 17:24:18 - r - INFO: - Episode: 1723/2000, Reward: 176.436, Step: 500 +2023-04-15 17:24:22 - r - INFO: - Episode: 1724/2000, Reward: 171.949, Step: 500 +2023-04-15 17:24:26 - r - INFO: - Episode: 1725/2000, Reward: 181.924, Step: 500 +2023-04-15 17:24:38 - r - INFO: - Episode: 1726/2000, Reward: 177.366, Step: 500 +2023-04-15 17:24:42 - r - INFO: - Episode: 1727/2000, Reward: 181.957, Step: 500 +2023-04-15 17:24:46 - r - INFO: - Episode: 1728/2000, Reward: 185.237, Step: 500 +2023-04-15 17:24:50 - r - INFO: - Episode: 1729/2000, Reward: 181.250, Step: 500 +2023-04-15 17:24:55 - r - INFO: - Episode: 1730/2000, Reward: 177.610, Step: 500 +2023-04-15 17:25:06 - r - INFO: - Episode: 1731/2000, Reward: 182.223, Step: 500 +2023-04-15 17:25:10 - r - INFO: - Episode: 1732/2000, Reward: 171.946, Step: 500 +2023-04-15 17:25:14 - r - INFO: - Episode: 1733/2000, Reward: 172.403, Step: 500 +2023-04-15 17:25:19 - r - INFO: - Episode: 1734/2000, Reward: 171.604, Step: 500 +2023-04-15 17:25:23 - r - INFO: - Episode: 1735/2000, Reward: 169.863, Step: 500 +2023-04-15 17:25:34 - r - INFO: - Episode: 1736/2000, Reward: 173.186, Step: 500 +2023-04-15 17:25:39 - r - INFO: - Episode: 1737/2000, Reward: 179.608, Step: 500 +2023-04-15 17:25:43 - r - INFO: - Episode: 1738/2000, Reward: 174.642, Step: 500 +2023-04-15 17:25:46 - r - INFO: - Episode: 1739/2000, Reward: -13.331, Step: 344 +2023-04-15 17:25:50 - r - INFO: - Episode: 1740/2000, Reward: 175.932, Step: 500 +2023-04-15 17:26:02 - r - INFO: - Episode: 1741/2000, Reward: 165.825, Step: 500 +2023-04-15 17:26:06 - r - INFO: - Episode: 1742/2000, Reward: 174.859, Step: 500 +2023-04-15 17:26:10 - r - INFO: - Episode: 1743/2000, Reward: 173.641, Step: 500 +2023-04-15 17:26:14 - r - INFO: - Episode: 1744/2000, Reward: 183.587, Step: 500 +2023-04-15 17:26:19 - r - INFO: - Episode: 1745/2000, Reward: 180.676, Step: 500 +2023-04-15 17:26:30 - r - INFO: - Episode: 1746/2000, Reward: 180.643, Step: 500 +2023-04-15 17:26:35 - r - INFO: - Episode: 1747/2000, Reward: 176.961, Step: 500 +2023-04-15 17:26:39 - r - INFO: - Episode: 1748/2000, Reward: 186.130, Step: 500 +2023-04-15 17:26:43 - r - INFO: - Episode: 1749/2000, Reward: 180.409, Step: 500 +2023-04-15 17:26:47 - r - INFO: - Episode: 1750/2000, Reward: 171.934, Step: 500 +2023-04-15 17:26:59 - r - INFO: - Episode: 1751/2000, Reward: 168.204, Step: 500 +2023-04-15 17:27:03 - r - INFO: - Episode: 1752/2000, Reward: 179.669, Step: 500 +2023-04-15 17:27:07 - r - INFO: - Episode: 1753/2000, Reward: 177.090, Step: 500 +2023-04-15 17:27:12 - r - INFO: - Episode: 1754/2000, Reward: 170.887, Step: 500 +2023-04-15 17:27:16 - r - INFO: - Episode: 1755/2000, Reward: 180.659, Step: 500 +2023-04-15 17:27:27 - r - INFO: - Episode: 1756/2000, Reward: 173.893, Step: 500 +2023-04-15 17:27:32 - r - INFO: - Episode: 1757/2000, Reward: 167.864, Step: 500 +2023-04-15 17:27:36 - r - INFO: - Episode: 1758/2000, Reward: 177.626, Step: 500 +2023-04-15 17:27:40 - r - INFO: - Episode: 1759/2000, Reward: 176.625, Step: 500 +2023-04-15 17:27:44 - r - INFO: - Episode: 1760/2000, Reward: 177.207, Step: 500 +2023-04-15 17:27:56 - r - INFO: - Episode: 1761/2000, Reward: 182.239, Step: 500 +2023-04-15 17:28:00 - r - INFO: - Episode: 1762/2000, Reward: 180.671, Step: 500 +2023-04-15 17:28:04 - r - INFO: - Episode: 1763/2000, Reward: 171.815, Step: 500 +2023-04-15 17:28:09 - r - INFO: - Episode: 1764/2000, Reward: 173.669, Step: 500 +2023-04-15 17:28:13 - r - INFO: - Episode: 1765/2000, Reward: 169.765, Step: 500 +2023-04-15 17:28:24 - r - INFO: - Episode: 1766/2000, Reward: 168.729, Step: 500 +2023-04-15 17:28:29 - r - INFO: - Episode: 1767/2000, Reward: 168.870, Step: 500 +2023-04-15 17:28:33 - r - INFO: - Episode: 1768/2000, Reward: 174.265, Step: 500 +2023-04-15 17:28:37 - r - INFO: - Episode: 1769/2000, Reward: 175.286, Step: 500 +2023-04-15 17:28:41 - r - INFO: - Episode: 1770/2000, Reward: 179.886, Step: 500 +2023-04-15 17:28:53 - r - INFO: - Episode: 1771/2000, Reward: 171.018, Step: 500 +2023-04-15 17:28:57 - r - INFO: - Episode: 1772/2000, Reward: 176.802, Step: 500 +2023-04-15 17:30:15 - r - INFO: - Episode: 1773/2000, Reward: 176.991, Step: 500 +2023-04-15 17:30:20 - r - INFO: - Episode: 1774/2000, Reward: 174.630, Step: 500 +2023-04-15 17:30:24 - r - INFO: - Episode: 1775/2000, Reward: 178.769, Step: 500 +2023-04-15 17:30:35 - r - INFO: - Episode: 1776/2000, Reward: 173.925, Step: 500 +2023-04-15 17:30:39 - r - INFO: - Episode: 1777/2000, Reward: 173.252, Step: 500 +2023-04-15 17:30:43 - r - INFO: - Episode: 1778/2000, Reward: 168.836, Step: 500 +2023-04-15 17:30:47 - r - INFO: - Episode: 1779/2000, Reward: 184.162, Step: 500 +2023-04-15 17:30:52 - r - INFO: - Episode: 1780/2000, Reward: 174.730, Step: 500 +2023-04-15 17:31:03 - r - INFO: - Episode: 1781/2000, Reward: 170.494, Step: 500 +2023-04-15 17:31:07 - r - INFO: - Episode: 1782/2000, Reward: 172.504, Step: 500 +2023-04-15 17:31:11 - r - INFO: - Episode: 1783/2000, Reward: 174.788, Step: 500 +2023-04-15 17:31:16 - r - INFO: - Episode: 1784/2000, Reward: 174.578, Step: 500 +2023-04-15 17:31:20 - r - INFO: - Episode: 1785/2000, Reward: 168.671, Step: 500 +2023-04-15 17:31:31 - r - INFO: - Episode: 1786/2000, Reward: 176.506, Step: 500 +2023-04-15 17:31:35 - r - INFO: - Episode: 1787/2000, Reward: 181.402, Step: 500 +2023-04-15 17:31:40 - r - INFO: - Episode: 1788/2000, Reward: 174.526, Step: 500 +2023-04-15 17:31:44 - r - INFO: - Episode: 1789/2000, Reward: 177.149, Step: 500 +2023-04-15 17:31:48 - r - INFO: - Episode: 1790/2000, Reward: 168.272, Step: 500 +2023-04-15 17:31:59 - r - INFO: - Episode: 1791/2000, Reward: 174.638, Step: 500 +2023-04-15 17:32:04 - r - INFO: - Episode: 1792/2000, Reward: 177.967, Step: 500 +2023-04-15 17:32:08 - r - INFO: - Episode: 1793/2000, Reward: 169.771, Step: 500 +2023-04-15 17:32:12 - r - INFO: - Episode: 1794/2000, Reward: 175.805, Step: 500 +2023-04-15 17:32:16 - r - INFO: - Episode: 1795/2000, Reward: 176.256, Step: 500 +2023-04-15 17:32:28 - r - INFO: - Episode: 1796/2000, Reward: 179.647, Step: 500 +2023-04-15 17:32:32 - r - INFO: - Episode: 1797/2000, Reward: 182.419, Step: 500 +2023-04-15 17:32:36 - r - INFO: - Episode: 1798/2000, Reward: 167.527, Step: 500 +2023-04-15 17:32:41 - r - INFO: - Episode: 1799/2000, Reward: 168.132, Step: 500 +2023-04-15 17:32:45 - r - INFO: - Episode: 1800/2000, Reward: 179.810, Step: 500 +2023-04-15 17:32:56 - r - INFO: - Episode: 1801/2000, Reward: 176.918, Step: 500 +2023-04-15 17:33:01 - r - INFO: - Episode: 1802/2000, Reward: 178.087, Step: 500 +2023-04-15 17:33:06 - r - INFO: - Episode: 1803/2000, Reward: 175.516, Step: 500 +2023-04-15 17:33:10 - r - INFO: - Episode: 1804/2000, Reward: 170.837, Step: 500 +2023-04-15 17:33:14 - r - INFO: - Episode: 1805/2000, Reward: 184.293, Step: 500 +2023-04-15 17:33:25 - r - INFO: - Episode: 1806/2000, Reward: 183.139, Step: 500 +2023-04-15 17:33:30 - r - INFO: - Episode: 1807/2000, Reward: 185.139, Step: 500 +2023-04-15 17:33:34 - r - INFO: - Episode: 1808/2000, Reward: 184.552, Step: 500 +2023-04-15 17:33:38 - r - INFO: - Episode: 1809/2000, Reward: 183.223, Step: 500 +2023-04-15 17:33:43 - r - INFO: - Episode: 1810/2000, Reward: 185.231, Step: 500 +2023-04-15 17:33:54 - r - INFO: - Episode: 1811/2000, Reward: 182.895, Step: 500 +2023-04-15 17:33:58 - r - INFO: - Episode: 1812/2000, Reward: 196.688, Step: 500 +2023-04-15 17:34:03 - r - INFO: - Episode: 1813/2000, Reward: 175.804, Step: 500 +2023-04-15 17:34:07 - r - INFO: - Episode: 1814/2000, Reward: 178.941, Step: 500 +2023-04-15 17:34:11 - r - INFO: - Episode: 1815/2000, Reward: 180.348, Step: 500 +2023-04-15 17:34:22 - r - INFO: - Episode: 1816/2000, Reward: 185.567, Step: 500 +2023-04-15 17:34:26 - r - INFO: - Episode: 1817/2000, Reward: 181.703, Step: 500 +2023-04-15 17:34:30 - r - INFO: - Episode: 1818/2000, Reward: 181.120, Step: 500 +2023-04-15 17:34:35 - r - INFO: - Episode: 1819/2000, Reward: 183.482, Step: 500 +2023-04-15 17:34:39 - r - INFO: - Episode: 1820/2000, Reward: 164.342, Step: 500 +2023-04-15 17:34:51 - r - INFO: - Episode: 1821/2000, Reward: 187.423, Step: 500 +2023-04-15 17:34:55 - r - INFO: - Episode: 1822/2000, Reward: 61.888, Step: 483 +2023-04-15 17:34:59 - r - INFO: - Episode: 1823/2000, Reward: 182.613, Step: 500 +2023-04-15 17:35:03 - r - INFO: - Episode: 1824/2000, Reward: 196.402, Step: 500 +2023-04-15 17:35:08 - r - INFO: - Episode: 1825/2000, Reward: 186.813, Step: 500 +2023-04-15 17:35:19 - r - INFO: - Episode: 1826/2000, Reward: 183.799, Step: 500 +2023-04-15 17:35:24 - r - INFO: - Episode: 1827/2000, Reward: 182.511, Step: 500 +2023-04-15 17:35:28 - r - INFO: - Episode: 1828/2000, Reward: 179.894, Step: 500 +2023-04-15 17:35:32 - r - INFO: - Episode: 1829/2000, Reward: 176.384, Step: 500 +2023-04-15 17:35:37 - r - INFO: - Episode: 1830/2000, Reward: 176.979, Step: 500 +2023-04-15 17:35:48 - r - INFO: - Episode: 1831/2000, Reward: 183.862, Step: 500 +2023-04-15 17:35:52 - r - INFO: - Episode: 1832/2000, Reward: 182.122, Step: 500 +2023-04-15 17:35:57 - r - INFO: - Episode: 1833/2000, Reward: 183.581, Step: 500 +2023-04-15 17:36:01 - r - INFO: - Episode: 1834/2000, Reward: 176.754, Step: 500 +2023-04-15 17:36:05 - r - INFO: - Episode: 1835/2000, Reward: 191.092, Step: 500 +2023-04-15 17:36:17 - r - INFO: - Episode: 1836/2000, Reward: 183.811, Step: 500 +2023-04-15 17:36:21 - r - INFO: - Episode: 1837/2000, Reward: 185.218, Step: 500 +2023-04-15 17:36:25 - r - INFO: - Episode: 1838/2000, Reward: 177.169, Step: 500 +2023-04-15 17:36:30 - r - INFO: - Episode: 1839/2000, Reward: 168.139, Step: 500 +2023-04-15 17:36:34 - r - INFO: - Episode: 1840/2000, Reward: 175.926, Step: 500 +2023-04-15 17:36:46 - r - INFO: - Episode: 1841/2000, Reward: 183.160, Step: 500 +2023-04-15 17:36:50 - r - INFO: - Episode: 1842/2000, Reward: 178.835, Step: 500 +2023-04-15 17:36:54 - r - INFO: - Episode: 1843/2000, Reward: 184.955, Step: 500 +2023-04-15 17:36:58 - r - INFO: - Episode: 1844/2000, Reward: 177.118, Step: 500 +2023-04-15 17:37:03 - r - INFO: - Episode: 1845/2000, Reward: 183.304, Step: 500 +2023-04-15 17:37:14 - r - INFO: - Episode: 1846/2000, Reward: 187.948, Step: 500 +2023-04-15 17:37:19 - r - INFO: - Episode: 1847/2000, Reward: 182.864, Step: 500 +2023-04-15 17:37:23 - r - INFO: - Episode: 1848/2000, Reward: 174.390, Step: 500 +2023-04-15 17:37:27 - r - INFO: - Episode: 1849/2000, Reward: 170.499, Step: 500 +2023-04-15 17:37:31 - r - INFO: - Episode: 1850/2000, Reward: 186.822, Step: 500 +2023-04-15 17:37:43 - r - INFO: - Episode: 1851/2000, Reward: 175.256, Step: 500 +2023-04-15 17:37:47 - r - INFO: - Episode: 1852/2000, Reward: 179.223, Step: 500 +2023-04-15 17:37:51 - r - INFO: - Episode: 1853/2000, Reward: 183.485, Step: 500 +2023-04-15 17:37:56 - r - INFO: - Episode: 1854/2000, Reward: 175.090, Step: 500 +2023-04-15 17:38:00 - r - INFO: - Episode: 1855/2000, Reward: 173.608, Step: 500 +2023-04-15 17:38:12 - r - INFO: - Episode: 1856/2000, Reward: 170.073, Step: 500 +2023-04-15 17:38:16 - r - INFO: - Episode: 1857/2000, Reward: 174.501, Step: 500 +2023-04-15 17:38:20 - r - INFO: - Episode: 1858/2000, Reward: 176.623, Step: 500 +2023-04-15 17:38:25 - r - INFO: - Episode: 1859/2000, Reward: 178.123, Step: 500 +2023-04-15 17:38:29 - r - INFO: - Episode: 1860/2000, Reward: 171.879, Step: 500 +2023-04-15 17:38:40 - r - INFO: - Episode: 1861/2000, Reward: 182.821, Step: 500 +2023-04-15 17:38:45 - r - INFO: - Episode: 1862/2000, Reward: 185.760, Step: 500 +2023-04-15 17:38:49 - r - INFO: - Episode: 1863/2000, Reward: 184.854, Step: 500 +2023-04-15 17:38:53 - r - INFO: - Episode: 1864/2000, Reward: 177.490, Step: 500 +2023-04-15 17:38:58 - r - INFO: - Episode: 1865/2000, Reward: 173.750, Step: 500 +2023-04-15 17:39:10 - r - INFO: - Episode: 1866/2000, Reward: 171.524, Step: 500 +2023-04-15 17:39:14 - r - INFO: - Episode: 1867/2000, Reward: 175.440, Step: 500 +2023-04-15 17:39:18 - r - INFO: - Episode: 1868/2000, Reward: 181.501, Step: 500 +2023-04-15 17:39:23 - r - INFO: - Episode: 1869/2000, Reward: 174.766, Step: 500 +2023-04-15 17:39:27 - r - INFO: - Episode: 1870/2000, Reward: 188.920, Step: 500 +2023-04-15 17:39:38 - r - INFO: - Episode: 1871/2000, Reward: 184.018, Step: 500 +2023-04-15 17:39:43 - r - INFO: - Episode: 1872/2000, Reward: 178.308, Step: 500 +2023-04-15 17:39:47 - r - INFO: - Episode: 1873/2000, Reward: 183.189, Step: 500 +2023-04-15 17:39:51 - r - INFO: - Episode: 1874/2000, Reward: 178.231, Step: 500 +2023-04-15 17:39:56 - r - INFO: - Episode: 1875/2000, Reward: 172.420, Step: 500 +2023-04-15 17:40:07 - r - INFO: - Episode: 1876/2000, Reward: 176.544, Step: 500 +2023-04-15 17:40:11 - r - INFO: - Episode: 1877/2000, Reward: 175.672, Step: 500 +2023-04-15 17:40:16 - r - INFO: - Episode: 1878/2000, Reward: 183.388, Step: 500 +2023-04-15 17:40:20 - r - INFO: - Episode: 1879/2000, Reward: 177.401, Step: 500 +2023-04-15 17:40:24 - r - INFO: - Episode: 1880/2000, Reward: 181.090, Step: 500 +2023-04-15 17:40:36 - r - INFO: - Episode: 1881/2000, Reward: 176.277, Step: 500 +2023-04-15 17:40:40 - r - INFO: - Episode: 1882/2000, Reward: 184.641, Step: 500 +2023-04-15 17:40:45 - r - INFO: - Episode: 1883/2000, Reward: 173.254, Step: 500 +2023-04-15 17:40:49 - r - INFO: - Episode: 1884/2000, Reward: 179.799, Step: 500 +2023-04-15 17:40:53 - r - INFO: - Episode: 1885/2000, Reward: 185.067, Step: 500 +2023-04-15 17:41:05 - r - INFO: - Episode: 1886/2000, Reward: 184.024, Step: 500 +2023-04-15 17:41:09 - r - INFO: - Episode: 1887/2000, Reward: 178.924, Step: 500 +2023-04-15 17:41:13 - r - INFO: - Episode: 1888/2000, Reward: 171.174, Step: 500 +2023-04-15 17:41:18 - r - INFO: - Episode: 1889/2000, Reward: 180.529, Step: 500 +2023-04-15 17:41:22 - r - INFO: - Episode: 1890/2000, Reward: 181.912, Step: 500 +2023-04-15 17:41:34 - r - INFO: - Episode: 1891/2000, Reward: 181.834, Step: 500 +2023-04-15 17:41:38 - r - INFO: - Episode: 1892/2000, Reward: 177.927, Step: 500 +2023-04-15 17:41:42 - r - INFO: - Episode: 1893/2000, Reward: 176.801, Step: 500 +2023-04-15 17:41:47 - r - INFO: - Episode: 1894/2000, Reward: 183.159, Step: 500 +2023-04-15 17:41:51 - r - INFO: - Episode: 1895/2000, Reward: 179.378, Step: 500 +2023-04-15 17:42:02 - r - INFO: - Episode: 1896/2000, Reward: 169.861, Step: 500 +2023-04-15 17:42:07 - r - INFO: - Episode: 1897/2000, Reward: 180.793, Step: 500 +2023-04-15 17:42:11 - r - INFO: - Episode: 1898/2000, Reward: 179.137, Step: 500 +2023-04-15 17:42:15 - r - INFO: - Episode: 1899/2000, Reward: 183.763, Step: 500 +2023-04-15 17:42:20 - r - INFO: - Episode: 1900/2000, Reward: 175.809, Step: 500 +2023-04-15 17:42:31 - r - INFO: - Episode: 1901/2000, Reward: 188.543, Step: 500 +2023-04-15 17:42:36 - r - INFO: - Episode: 1902/2000, Reward: 177.117, Step: 500 +2023-04-15 17:42:40 - r - INFO: - Episode: 1903/2000, Reward: 178.438, Step: 500 +2023-04-15 17:42:44 - r - INFO: - Episode: 1904/2000, Reward: 179.330, Step: 500 +2023-04-15 17:42:49 - r - INFO: - Episode: 1905/2000, Reward: 182.939, Step: 500 +2023-04-15 17:43:00 - r - INFO: - Episode: 1906/2000, Reward: 184.640, Step: 500 +2023-04-15 17:43:04 - r - INFO: - Episode: 1907/2000, Reward: 183.449, Step: 500 +2023-04-15 17:43:09 - r - INFO: - Episode: 1908/2000, Reward: 183.414, Step: 500 +2023-04-15 17:43:13 - r - INFO: - Episode: 1909/2000, Reward: 175.099, Step: 500 +2023-04-15 17:43:17 - r - INFO: - Episode: 1910/2000, Reward: 181.543, Step: 500 +2023-04-15 17:43:29 - r - INFO: - Episode: 1911/2000, Reward: 175.130, Step: 500 +2023-04-15 17:43:33 - r - INFO: - Episode: 1912/2000, Reward: 176.879, Step: 500 +2023-04-15 17:43:38 - r - INFO: - Episode: 1913/2000, Reward: 171.241, Step: 500 +2023-04-15 17:43:42 - r - INFO: - Episode: 1914/2000, Reward: 171.384, Step: 500 +2023-04-15 17:43:46 - r - INFO: - Episode: 1915/2000, Reward: 178.384, Step: 500 +2023-04-15 17:43:58 - r - INFO: - Episode: 1916/2000, Reward: 176.446, Step: 500 +2023-04-15 17:44:02 - r - INFO: - Episode: 1917/2000, Reward: 177.310, Step: 500 +2023-04-15 17:44:06 - r - INFO: - Episode: 1918/2000, Reward: 189.884, Step: 500 +2023-04-15 17:44:11 - r - INFO: - Episode: 1919/2000, Reward: 181.427, Step: 500 +2023-04-15 17:44:15 - r - INFO: - Episode: 1920/2000, Reward: 182.405, Step: 500 +2023-04-15 17:44:27 - r - INFO: - Episode: 1921/2000, Reward: 177.435, Step: 500 +2023-04-15 17:44:31 - r - INFO: - Episode: 1922/2000, Reward: 177.886, Step: 500 +2023-04-15 17:44:35 - r - INFO: - Episode: 1923/2000, Reward: 188.396, Step: 500 +2023-04-15 17:44:40 - r - INFO: - Episode: 1924/2000, Reward: 177.858, Step: 500 +2023-04-15 17:44:44 - r - INFO: - Episode: 1925/2000, Reward: 185.079, Step: 500 +2023-04-15 17:44:55 - r - INFO: - Episode: 1926/2000, Reward: 185.063, Step: 500 +2023-04-15 17:45:00 - r - INFO: - Episode: 1927/2000, Reward: 191.834, Step: 500 +2023-04-15 17:45:04 - r - INFO: - Episode: 1928/2000, Reward: 184.083, Step: 500 +2023-04-15 17:45:08 - r - INFO: - Episode: 1929/2000, Reward: 190.067, Step: 500 +2023-04-15 17:45:13 - r - INFO: - Episode: 1930/2000, Reward: 193.351, Step: 500 +2023-04-15 17:45:24 - r - INFO: - Episode: 1931/2000, Reward: 181.314, Step: 500 +2023-04-15 17:45:29 - r - INFO: - Episode: 1932/2000, Reward: 186.995, Step: 500 +2023-04-15 17:45:33 - r - INFO: - Episode: 1933/2000, Reward: 176.408, Step: 500 +2023-04-15 17:45:37 - r - INFO: - Episode: 1934/2000, Reward: 179.614, Step: 500 +2023-04-15 17:45:42 - r - INFO: - Episode: 1935/2000, Reward: 179.372, Step: 500 +2023-04-15 17:45:49 - r - INFO: - Current episode 1935 has the best eval reward: 204.068 +2023-04-15 17:45:53 - r - INFO: - Episode: 1936/2000, Reward: 178.036, Step: 500 +2023-04-15 17:45:57 - r - INFO: - Episode: 1937/2000, Reward: 185.358, Step: 500 +2023-04-15 17:46:02 - r - INFO: - Episode: 1938/2000, Reward: 186.404, Step: 500 +2023-04-15 17:46:06 - r - INFO: - Episode: 1939/2000, Reward: 179.210, Step: 500 +2023-04-15 17:46:10 - r - INFO: - Episode: 1940/2000, Reward: 177.714, Step: 500 +2023-04-15 17:46:22 - r - INFO: - Episode: 1941/2000, Reward: 186.607, Step: 500 +2023-04-15 17:46:26 - r - INFO: - Episode: 1942/2000, Reward: 184.224, Step: 500 +2023-04-15 17:46:30 - r - INFO: - Episode: 1943/2000, Reward: 185.085, Step: 500 +2023-04-15 17:46:35 - r - INFO: - Episode: 1944/2000, Reward: 188.543, Step: 500 +2023-04-15 17:46:39 - r - INFO: - Episode: 1945/2000, Reward: 180.505, Step: 500 +2023-04-15 17:46:51 - r - INFO: - Episode: 1946/2000, Reward: 176.329, Step: 500 +2023-04-15 17:46:55 - r - INFO: - Episode: 1947/2000, Reward: 174.286, Step: 500 +2023-04-15 17:46:59 - r - INFO: - Episode: 1948/2000, Reward: 178.653, Step: 500 +2023-04-15 17:47:03 - r - INFO: - Episode: 1949/2000, Reward: 189.436, Step: 500 +2023-04-15 17:47:08 - r - INFO: - Episode: 1950/2000, Reward: 195.411, Step: 500 +2023-04-15 17:47:19 - r - INFO: - Episode: 1951/2000, Reward: 188.869, Step: 500 +2023-04-15 17:47:24 - r - INFO: - Episode: 1952/2000, Reward: 187.426, Step: 500 +2023-04-15 17:47:28 - r - INFO: - Episode: 1953/2000, Reward: 187.143, Step: 500 +2023-04-15 17:47:32 - r - INFO: - Episode: 1954/2000, Reward: 187.463, Step: 500 +2023-04-15 17:47:37 - r - INFO: - Episode: 1955/2000, Reward: 179.338, Step: 500 +2023-04-15 17:47:48 - r - INFO: - Episode: 1956/2000, Reward: 170.162, Step: 500 +2023-04-15 17:47:52 - r - INFO: - Episode: 1957/2000, Reward: 186.970, Step: 500 +2023-04-15 17:47:57 - r - INFO: - Episode: 1958/2000, Reward: 189.166, Step: 500 +2023-04-15 17:48:01 - r - INFO: - Episode: 1959/2000, Reward: 180.887, Step: 500 +2023-04-15 17:48:05 - r - INFO: - Episode: 1960/2000, Reward: 169.419, Step: 500 +2023-04-15 17:48:17 - r - INFO: - Episode: 1961/2000, Reward: 187.653, Step: 500 +2023-04-15 17:48:21 - r - INFO: - Episode: 1962/2000, Reward: 187.440, Step: 500 +2023-04-15 17:48:25 - r - INFO: - Episode: 1963/2000, Reward: 179.335, Step: 500 +2023-04-15 17:48:30 - r - INFO: - Episode: 1964/2000, Reward: 181.468, Step: 500 +2023-04-15 17:48:34 - r - INFO: - Episode: 1965/2000, Reward: 180.077, Step: 500 +2023-04-15 17:48:46 - r - INFO: - Episode: 1966/2000, Reward: 176.276, Step: 500 +2023-04-15 17:48:50 - r - INFO: - Episode: 1967/2000, Reward: 176.152, Step: 500 +2023-04-15 17:48:54 - r - INFO: - Episode: 1968/2000, Reward: 178.413, Step: 500 +2023-04-15 17:48:58 - r - INFO: - Episode: 1969/2000, Reward: 183.751, Step: 500 +2023-04-15 17:49:03 - r - INFO: - Episode: 1970/2000, Reward: 175.015, Step: 500 +2023-04-15 17:49:14 - r - INFO: - Episode: 1971/2000, Reward: 171.655, Step: 500 +2023-04-15 17:49:19 - r - INFO: - Episode: 1972/2000, Reward: 186.939, Step: 500 +2023-04-15 17:49:23 - r - INFO: - Episode: 1973/2000, Reward: 182.054, Step: 500 +2023-04-15 17:49:27 - r - INFO: - Episode: 1974/2000, Reward: 182.490, Step: 500 +2023-04-15 17:49:32 - r - INFO: - Episode: 1975/2000, Reward: 184.478, Step: 500 +2023-04-15 17:49:43 - r - INFO: - Episode: 1976/2000, Reward: 179.534, Step: 500 +2023-04-15 17:49:47 - r - INFO: - Episode: 1977/2000, Reward: 180.326, Step: 500 +2023-04-15 17:49:52 - r - INFO: - Episode: 1978/2000, Reward: 178.568, Step: 500 +2023-04-15 17:49:56 - r - INFO: - Episode: 1979/2000, Reward: 180.807, Step: 500 +2023-04-15 17:50:00 - r - INFO: - Episode: 1980/2000, Reward: 174.378, Step: 500 +2023-04-15 17:50:12 - r - INFO: - Episode: 1981/2000, Reward: 183.118, Step: 500 +2023-04-15 17:50:16 - r - INFO: - Episode: 1982/2000, Reward: 182.554, Step: 500 +2023-04-15 17:50:20 - r - INFO: - Episode: 1983/2000, Reward: 176.973, Step: 500 +2023-04-15 17:50:24 - r - INFO: - Episode: 1984/2000, Reward: 185.415, Step: 500 +2023-04-15 17:50:29 - r - INFO: - Episode: 1985/2000, Reward: 199.738, Step: 500 +2023-04-15 17:50:40 - r - INFO: - Episode: 1986/2000, Reward: 191.124, Step: 500 +2023-04-15 17:50:45 - r - INFO: - Episode: 1987/2000, Reward: 189.937, Step: 500 +2023-04-15 17:50:49 - r - INFO: - Episode: 1988/2000, Reward: 195.043, Step: 500 +2023-04-15 17:50:53 - r - INFO: - Episode: 1989/2000, Reward: 184.873, Step: 500 +2023-04-15 17:50:58 - r - INFO: - Episode: 1990/2000, Reward: 181.201, Step: 500 +2023-04-15 17:51:09 - r - INFO: - Episode: 1991/2000, Reward: 180.935, Step: 500 +2023-04-15 17:51:13 - r - INFO: - Episode: 1992/2000, Reward: 187.284, Step: 500 +2023-04-15 17:51:18 - r - INFO: - Episode: 1993/2000, Reward: 187.060, Step: 500 +2023-04-15 17:51:22 - r - INFO: - Episode: 1994/2000, Reward: 181.119, Step: 500 +2023-04-15 17:51:26 - r - INFO: - Episode: 1995/2000, Reward: 177.719, Step: 500 +2023-04-15 17:51:38 - r - INFO: - Episode: 1996/2000, Reward: 180.708, Step: 500 +2023-04-15 17:51:42 - r - INFO: - Episode: 1997/2000, Reward: 180.392, Step: 500 +2023-04-15 17:51:47 - r - INFO: - Episode: 1998/2000, Reward: 174.726, Step: 500 +2023-04-15 17:51:51 - r - INFO: - Episode: 1999/2000, Reward: 181.611, Step: 500 +2023-04-15 17:51:55 - r - INFO: - Episode: 2000/2000, Reward: 183.335, Step: 500 +2023-04-15 17:52:02 - r - INFO: - Finish training! diff --git a/BipedalWalker-v3/Train_gym_SAC_20230415-140928/models/checkpoint.pt b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..868e3ef211f6ada934b8b423dfd664f3102c8ffe --- /dev/null +++ b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e57f7720e0d181816c759181e068984e6533e701e78fe8355947e539090488d2 +size 3263885 diff --git a/BipedalWalker-v3/Train_gym_SAC_20230415-140928/results/learning_curve.png b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..0a87c13b75360e89a040145f4d8ae5cdf56eb03d Binary files /dev/null and b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/results/learning_curve.png differ diff --git a/BipedalWalker-v3/Train_gym_SAC_20230415-140928/results/res.csv b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..38bbcfc55bf01fc6e98311f1b66a4fc7f29bce37 --- /dev/null +++ b/BipedalWalker-v3/Train_gym_SAC_20230415-140928/results/res.csv @@ -0,0 +1,2001 @@ +episodes,rewards,steps +0,-109.17609278473165,75 +1,-100.01576877694329,93 +2,-101.03980851535499,91 +3,-108.70971189774015,141 +4,-27.625621148421125,500 +5,-103.04416789395238,71 +6,-27.145343235799682,500 +7,-107.5987491886119,47 +8,-99.18924822770715,98 +9,-26.911283808039,500 +10,-114.28422482655135,66 +11,-35.445630212150924,500 +12,-25.798874869215215,500 +13,-114.29628114350265,89 +14,-97.41333986382396,84 +15,-35.19394143203207,500 +16,-26.21470441150061,500 +17,-98.7024272034457,80 +18,-106.98310814924211,57 +19,-26.97144223131296,500 +20,-109.31461922142375,149 +21,-100.05216317099364,86 +22,-109.19703117621907,41 +23,-24.714292216202782,500 +24,-25.42934448404529,500 +25,-106.01485414230885,62 +26,-101.94488953041922,76 +27,-101.86377554367192,106 +28,-116.62963199574035,70 +29,-110.27211371342139,46 +30,-98.83221751113919,110 +31,-107.2607261092638,93 +32,-115.47958575566273,90 +33,-101.45122479578015,85 +34,-122.46549345041129,80 +35,-99.98559200359198,58 +36,-100.19421544374067,80 +37,-118.43955320563911,76 +38,-109.70753376998597,42 +39,-108.76314512012185,42 +40,-99.49458336702197,81 +41,-113.15555524995722,81 +42,-23.712108169979313,500 +43,-25.02513106135896,500 +44,-97.8360496230799,70 +45,-21.99814765671224,500 +46,-110.37325008089095,66 +47,-22.773966961617997,500 +48,-99.66569493129508,65 +49,-36.20901389802536,500 +50,-111.62684777663229,248 +51,-23.537034781381223,500 +52,-124.57002624657315,80 +53,-98.71153948731903,81 +54,-125.50173854186386,83 +55,-28.697982124020594,500 +56,-33.80497307319849,500 +57,-107.62331898212278,72 +58,-102.41168292381599,109 +59,-104.6902384254147,124 +60,-121.3198278723387,99 +61,-105.05686165610499,78 +62,-31.16932013093945,500 +63,-150.0230899610769,466 +64,-22.506043827727694,500 +65,-8.33709733654741,500 +66,-17.253484087761016,500 +67,-111.36003468335637,182 +68,-24.281115027233405,500 +69,-98.75801705496696,77 +70,-20.299350614704945,500 +71,-15.941391988116473,500 +72,-14.262358797675,500 +73,-32.540663543485444,500 +74,-26.717654138056385,500 +75,-32.42343197355879,500 +76,-28.473747462403544,500 +77,-24.27863385875523,500 +78,-19.190654872622144,500 +79,-23.84401850122678,500 +80,-26.297334590452326,500 +81,-21.796243168549243,500 +82,-25.648329205215447,500 +83,-25.68794259615678,500 +84,-97.20575065001101,235 +85,-15.132652338628418,500 +86,-36.64093673749493,500 +87,-38.777925614131156,500 +88,-40.07954568159687,500 +89,-28.344545627744385,500 +90,-117.49736551042429,75 +91,-28.844239608195306,500 +92,-26.97807828315116,500 +93,-32.589316577893086,500 +94,-27.701944537502406,500 +95,-17.24624489459286,500 +96,-16.413278849084165,500 +97,-23.937339475088976,500 +98,-18.235752973055583,500 +99,-20.87217298721726,500 +100,-21.801843160298688,500 +101,-26.1847651416876,500 +102,-26.586543188761127,500 +103,-27.465877663005728,500 +104,-24.47458545748045,500 +105,-23.65532561863291,500 +106,-21.923778311885197,500 +107,-26.563432240537974,500 +108,-97.67063829658522,86 +109,-113.1259621029868,134 +110,-99.38368117123827,152 +111,-15.168569289154325,500 +112,-17.825059547444187,500 +113,-20.76757011780515,500 +114,-14.060824182467579,500 +115,-15.810821868476213,500 +116,-27.770175577766818,500 +117,-26.5715778592682,500 +118,-25.633209304738937,500 +119,-26.570085403126978,500 +120,-21.23777115242916,500 +121,-10.052395461085958,500 +122,-121.5446874761662,74 +123,-23.98322423279163,500 +124,-19.3049044806921,500 +125,-134.3141245272234,163 +126,-27.830836003268296,500 +127,-32.146149072539984,500 +128,-31.763206672907792,500 +129,-29.210034371686053,500 +130,-20.699486348037603,500 +131,-32.027149691623634,500 +132,-26.394613801337798,500 +133,-32.67467935969376,500 +134,-36.649358443900915,500 +135,-15.825760523567348,500 +136,-15.980268085766163,500 +137,-12.32125515477778,500 +138,-4.983826762093624,500 +139,-11.358654094151495,500 +140,-15.740982237690897,500 +141,-5.928600838592045,500 +142,-11.563864501850441,500 +143,-15.421263126829027,500 +144,-20.92441564763817,500 +145,-14.13673565846833,500 +146,-21.477853857727414,500 +147,-16.3858993775666,500 +148,-17.58033092053942,500 +149,-7.454604173679217,500 +150,-10.209002989720238,500 +151,-41.64049186342532,500 +152,-26.293526682769045,500 +153,-19.839427874592054,500 +154,-30.129201239252897,500 +155,-27.87746808686093,500 +156,-28.418623118800756,500 +157,-26.45029904139105,500 +158,-29.323353688921422,500 +159,-22.72341163935282,500 +160,-12.633734159285286,500 +161,-20.11361768613562,500 +162,-14.455103045891812,500 +163,-19.218593745022392,500 +164,-17.959568911774088,500 +165,-22.410591931713405,500 +166,-24.840146158889173,500 +167,-28.98412185185619,500 +168,-114.68173782167584,46 +169,-18.947463610934395,500 +170,-18.739492059039723,500 +171,-14.313702580303737,500 +172,-19.59240550405077,500 +173,-33.01355039693825,500 +174,-20.05853678875212,500 +175,-15.57136422605867,500 +176,-10.026313637741794,500 +177,-6.414279457498342,500 +178,-23.35225791922212,500 +179,-27.633837278842243,500 +180,-104.61052898384445,158 +181,-17.79895505444797,500 +182,-16.624789455479284,500 +183,-12.714431568488477,500 +184,-8.523970718214434,500 +185,-23.866391787606833,500 +186,-20.090610758162146,500 +187,-23.36440153064766,500 +188,-1.8684518259974814,500 +189,-15.353624580008367,500 +190,-116.03525574134446,84 +191,-15.563315623205234,500 +192,-18.837776913328707,500 +193,-11.03732222112917,500 +194,-17.94612867950884,500 +195,-16.63067639720006,500 +196,-10.617591375686303,500 +197,-17.572780910467117,500 +198,-15.369273402707343,500 +199,-11.963311328298333,500 +200,-27.45751172857041,500 +201,-16.49366653119617,500 +202,-9.79316800123453,500 +203,-8.005437258440375,500 +204,-21.846067463663616,500 +205,-17.837632805886486,500 +206,-17.027138914305812,500 +207,-17.97568199660744,500 +208,-13.78454522954658,500 +209,-9.318651637981034,500 +210,-14.769782968034585,500 +211,-13.772556779742212,500 +212,-17.483784077587835,500 +213,-26.864351156370702,500 +214,-12.291301843276635,500 +215,-5.674247383089307,500 +216,-5.6261961534894045,500 +217,-10.13338548533499,500 +218,-16.10760257774157,500 +219,-14.999823549841878,500 +220,-18.99372235487873,500 +221,-14.804086181133016,500 +222,-16.160007543544687,500 +223,-106.23755861554652,112 +224,-104.90377705955893,108 +225,-8.520471366388842,500 +226,0.1324688811219894,500 +227,-11.455512988016237,500 +228,1.3463019317966,500 +229,4.278490338790777,500 +230,-19.49892064204626,500 +231,-6.462880689967423,500 +232,-107.03329060953297,66 +233,-117.2875547520214,81 +234,-124.23528260911101,103 +235,-119.85969594066637,148 +236,6.657376971747192,500 +237,-3.2626132160924963,500 +238,-17.618563797868408,500 +239,-5.57368512780914,500 +240,-4.528674751740879,500 +241,-15.192114733418611,500 +242,-15.512732439180331,500 +243,-119.44819387236424,53 +244,-13.981009427349546,500 +245,-13.005048514353412,500 +246,-18.20201768499285,500 +247,-21.412200914782787,500 +248,-16.893660635617252,500 +249,-18.05776560367139,500 +250,-11.73265324894525,500 +251,-104.11548207070337,92 +252,-12.756323756295343,500 +253,-12.749340501399018,500 +254,-17.66217726077333,500 +255,-10.871114640019057,500 +256,-6.162297410011449,500 +257,-15.871487066361734,500 +258,-9.629397479939733,500 +259,-9.856342458098046,500 +260,-10.79936415619528,500 +261,-14.232795652693126,500 +262,-117.47587130416805,74 +263,-9.24509855282415,500 +264,-6.390627230540343,500 +265,-5.09919112430357,500 +266,-12.592435762643495,500 +267,-11.617799980118965,500 +268,-4.880749620210691,500 +269,-10.68767346659124,500 +270,3.810317677924917,500 +271,-8.231311832249162,500 +272,-11.89010861470403,500 +273,-11.666484361802771,500 +274,-9.38550597429307,500 +275,-7.015584525847503,500 +276,-12.341359367132586,500 +277,-8.582267175173437,500 +278,-12.315944226410092,500 +279,-8.332835795564101,500 +280,-13.786859653577425,500 +281,-12.310882907933719,500 +282,-9.030160235182043,500 +283,-2.3414156077384236,500 +284,-20.645521879226905,500 +285,-11.439924243746638,500 +286,-9.506988769728842,500 +287,-7.310688313497503,500 +288,-8.184347754628359,500 +289,-11.065696785317419,500 +290,-8.23661400683581,500 +291,-12.0216417458416,500 +292,-103.54918505178027,223 +293,-11.441160574491592,500 +294,-9.513760830690364,500 +295,-14.983920383371792,500 +296,-12.713983412777543,500 +297,-10.47947357335562,500 +298,-14.815660464887353,500 +299,-13.523749117067247,500 +300,-8.391254885019,500 +301,-7.577403942800592,500 +302,-11.211862320194156,500 +303,-8.34842136444698,500 +304,-7.505917262443421,500 +305,-10.062304676873072,500 +306,-14.074796663775887,500 +307,-10.751709856437524,500 +308,-11.910069308734892,500 +309,-10.6596813962358,500 +310,-8.368757822619145,500 +311,-6.3899095061665925,500 +312,-9.036923710896323,500 +313,-9.121300188158555,500 +314,-7.9411584856728235,500 +315,-11.645479144605813,500 +316,-11.018596915678541,500 +317,-6.295783146344752,500 +318,-8.145500876407683,500 +319,-11.197891736496077,500 +320,-5.290291848567314,500 +321,-112.79306385655732,58 +322,-6.790185864205122,500 +323,-5.742412984187476,500 +324,-11.808231785094822,500 +325,-12.122019173632644,500 +326,-15.91748850490432,500 +327,-9.735758102465464,500 +328,-5.094848082678231,500 +329,-15.520794122382531,500 +330,-9.908185955589092,500 +331,-6.517741585234021,500 +332,-23.382139863160738,500 +333,-13.617767366679027,500 +334,-9.759011448935105,500 +335,-13.49479712272588,500 +336,-12.344397738910518,500 +337,-8.386104089588397,500 +338,-12.025444658870253,500 +339,-12.333300898999713,500 +340,-9.670071803379182,500 +341,-10.5342483992156,500 +342,-8.564983200680846,500 +343,-9.201983268592503,500 +344,-5.239818920347529,500 +345,-1.9219283669701177,500 +346,-7.598414349084717,500 +347,-9.927594350518273,500 +348,-11.22044088974933,500 +349,-1.991740197298924,500 +350,-7.595709698355918,500 +351,-12.467923390687801,500 +352,-8.078479860353047,500 +353,-7.660905231708199,500 +354,-6.740950190616308,500 +355,-9.661052107900145,500 +356,-7.811708279644333,500 +357,-10.593286246792957,500 +358,-12.706031130217788,500 +359,-12.909091503071291,500 +360,-11.771769595153007,500 +361,-11.704676223095971,500 +362,-12.034159382919015,500 +363,-10.023289463894477,500 +364,-13.374729972983008,500 +365,-7.6349228060158145,500 +366,-5.3588005525411555,500 +367,-4.909533750040692,500 +368,-2.8541494549982414,500 +369,-3.1512967460514476,500 +370,-2.779108680910352,500 +371,-0.5430016601767486,500 +372,-4.335177588598147,500 +373,-3.4003215138999923,500 +374,-3.607963110379329,500 +375,-129.06830404279552,138 +376,-116.29267242368745,63 +377,-6.746209618429965,500 +378,-26.47406104579999,500 +379,-6.4860818663404025,500 +380,-30.548034565133964,500 +381,-8.92098297559734,500 +382,-7.344787808378383,500 +383,-3.582528477002201,500 +384,-6.887441131907284,500 +385,-2.708788010352296,500 +386,-3.9782807498916544,500 +387,-5.675259285485641,500 +388,-5.4245458164840645,500 +389,-14.963778594825852,500 +390,1.5666639536172127,500 +391,-18.502876276782636,500 +392,0.12777505999806907,500 +393,-6.232919943239066,500 +394,-22.64486936277975,500 +395,-7.545151793090315,500 +396,-2.9513650358041454,500 +397,-118.12314813779368,294 +398,-3.1121339658265894,500 +399,-27.356492444949055,500 +400,-8.458856429015668,500 +401,-12.447788500498364,500 +402,-8.39379992993235,500 +403,-6.495279042056523,500 +404,-5.858412087803316,500 +405,-7.144104910775714,500 +406,-6.441302386879879,500 +407,-12.860898495240606,500 +408,-14.663192050075004,500 +409,-15.694480230024205,500 +410,-15.418308262149022,500 +411,-11.271817358033195,500 +412,-8.752080060586922,500 +413,-9.359176532478616,500 +414,-10.376737667025337,500 +415,-15.091207139044993,500 +416,-19.41527887865358,500 +417,-1.7916046327130535,500 +418,-4.054900301909033,500 +419,-2.523391218977828,500 +420,-7.988211117619461,500 +421,-0.49032922224009673,500 +422,-2.9225939212793426,500 +423,-131.18476687792938,137 +424,-11.488952403927069,500 +425,-7.105277289378099,500 +426,-18.1046048370214,500 +427,-2.8791172609564395,500 +428,-8.960327263766912,500 +429,1.8547178288147539,500 +430,4.996149140821256,500 +431,1.6706571309759017,500 +432,-107.07267892338956,46 +433,-110.94117231205168,41 +434,-4.009793555141503,500 +435,6.711054493037127,500 +436,-2.6839362089855903,500 +437,1.4946020327797573,500 +438,3.3689974781439713,500 +439,2.5735691369331137,500 +440,2.7829998149176443,500 +441,-0.8089168395164922,500 +442,-2.5362608309790167,500 +443,-0.4511310177308088,500 +444,-1.1695360410305053,500 +445,-1.1961825012716742,500 +446,-102.12536722506688,165 +447,7.79549692836419,500 +448,5.972884385219153,500 +449,8.617393756201405,500 +450,4.560785811045668,500 +451,-0.8064916177421132,500 +452,-8.21984682595039,500 +453,-0.42567827311243434,500 +454,-2.2457308576745403,500 +455,-103.83902519178716,170 +456,0.9221928376409527,500 +457,8.420847466749844,500 +458,0.35448842696822225,500 +459,3.6124107006969823,500 +460,6.723719279065574,500 +461,4.490934012617379,500 +462,3.4982277976611686,500 +463,-2.396418543842157,500 +464,-6.79736823547689,500 +465,3.2250370816401386,500 +466,-6.996565163639064,500 +467,-2.784318264565392,500 +468,-12.632123089817155,500 +469,-10.808304516839488,500 +470,-12.80923718445523,500 +471,-2.3338115794243857,500 +472,-4.626570004880192,500 +473,-11.21555913435974,500 +474,-3.4395237910965575,500 +475,-2.8783785888389297,500 +476,-6.8295808341526865,500 +477,-1.277528931456461,500 +478,-3.5082222505044625,500 +479,-2.849908936189645,500 +480,-1.147960060712878,500 +481,7.964679905891426,500 +482,-11.033158084716803,500 +483,-1.2771137183823964,500 +484,1.4132662304736074,500 +485,-0.7775499223604192,500 +486,5.6156620519105305,500 +487,2.638185338398055,500 +488,-101.53367756028473,448 +489,-12.62639200559786,500 +490,-12.3748012474288,500 +491,-11.26506054131097,500 +492,-0.004484325020923052,500 +493,2.8476811635439114,500 +494,5.061123374340979,500 +495,5.54475996242316,500 +496,0.7236001486674638,500 +497,-0.548463300120086,500 +498,3.5745168267083347,500 +499,2.160975593878808,500 +500,6.460362367704967,500 +501,-2.084758211562214,500 +502,8.528493500190711,500 +503,11.349339371395958,500 +504,-0.19036179262648206,500 +505,2.364141178916596,500 +506,-13.311985564134147,500 +507,-14.232546462810099,500 +508,-12.695621376047532,500 +509,6.034154003808106,500 +510,-14.896999897315624,500 +511,-3.0319987370036516,500 +512,5.358759577461515,500 +513,-2.764296208722752,500 +514,-0.38324377272631444,500 +515,-120.61334458200137,88 +516,3.263165105455554,500 +517,6.690919179319005,500 +518,11.97813651802775,500 +519,5.178713264888773,500 +520,1.582633908464342,500 +521,10.07619314995027,500 +522,5.093791704671192,500 +523,4.144072405865576,500 +524,8.6239251621546,500 +525,6.502252235396798,500 +526,-26.375216406455728,500 +527,-15.479832699456404,500 +528,-8.949197428207457,500 +529,-13.200251873628773,500 +530,-12.305499470745069,500 +531,-14.720368358436275,500 +532,-15.948733353153022,500 +533,-10.006946420553126,500 +534,-14.725836728812276,500 +535,-22.916753050412115,500 +536,9.52923279556437,500 +537,12.094231683174382,500 +538,5.354176095313157,500 +539,9.35293850750848,500 +540,6.630364113534801,500 +541,15.757943805859588,500 +542,15.400903079612787,500 +543,9.888092846389904,500 +544,5.234233287232384,500 +545,11.365180615692468,500 +546,8.067888096243689,500 +547,6.83511492306402,500 +548,9.4458209373757,500 +549,11.187545848633544,500 +550,4.881641875913442,500 +551,-114.72032207299706,75 +552,10.180547540086625,500 +553,14.422699068648589,500 +554,5.541105061631264,500 +555,9.004555801905102,500 +556,10.035137151005918,500 +557,9.20952155236838,500 +558,7.382816089975331,500 +559,11.616518165474965,500 +560,11.469531576010539,500 +561,7.641672140034807,500 +562,9.005596295421627,500 +563,10.710725792432115,500 +564,8.97484646446634,500 +565,19.73367820862301,500 +566,19.417691411687752,500 +567,11.406407873717496,500 +568,14.069964649504822,500 +569,10.249805421664451,500 +570,10.881417703973446,500 +571,11.689086955377842,500 +572,15.493820571743962,500 +573,15.167633613371608,500 +574,16.677028370957288,500 +575,7.420700036594182,500 +576,11.72524281267022,500 +577,-24.252380453916043,500 +578,-20.685589064936075,500 +579,-18.20632714579945,500 +580,-19.99486445496697,500 +581,-18.656386600391873,500 +582,-15.28516654644797,500 +583,-109.62068994610664,185 +584,12.77894256763804,500 +585,-12.09206040106699,500 +586,20.771977106881007,500 +587,14.499273687210216,500 +588,6.674178813352172,500 +589,6.450069236937748,500 +590,12.601394552527143,500 +591,10.867924383707892,500 +592,10.012911562572858,500 +593,3.671972671314618,500 +594,6.986753932441975,500 +595,16.27428199923195,500 +596,11.26535803961685,500 +597,10.18944457907311,500 +598,10.319473832387207,500 +599,18.887791428846047,500 +600,-2.0533743383334264,500 +601,6.030369236978927,500 +602,10.703035225957093,500 +603,9.976240285780388,500 +604,-115.95553307821338,108 +605,-21.729937584535612,500 +606,16.4523303606646,500 +607,11.82960050554418,500 +608,3.7327751795072115,500 +609,9.609501151556387,500 +610,12.698116468628976,500 +611,11.53882658569709,500 +612,13.384397966898602,500 +613,11.95099329576428,500 +614,11.51066363144899,500 +615,7.632554669059387,500 +616,7.308013302402656,500 +617,14.279456634793934,500 +618,12.519323195653193,500 +619,11.558295891176531,500 +620,16.55345751981316,500 +621,13.817802401075069,500 +622,12.585186065630552,500 +623,15.298032444584724,500 +624,9.067266378613805,500 +625,8.33483962159364,500 +626,12.433028170188464,500 +627,-7.862932432922574,500 +628,13.251497003777112,500 +629,7.224888588953521,500 +630,26.817751401731208,500 +631,16.06626908480959,500 +632,16.948330577698144,500 +633,20.84065639583816,500 +634,13.545343717707674,500 +635,18.802280443946433,500 +636,18.330429724462405,500 +637,23.574325997990382,500 +638,13.446455065642018,500 +639,11.189260655121391,500 +640,18.787953710247344,500 +641,12.257581834580373,500 +642,20.47632658820613,500 +643,-10.046922593379467,500 +644,-8.868652358889506,500 +645,9.15682660187257,500 +646,18.843920741258287,500 +647,15.014553335776075,500 +648,14.49524410885952,500 +649,12.12613737027335,500 +650,11.521173748279768,500 +651,15.619192576930557,500 +652,17.102159841585927,500 +653,1.0248426896402727,500 +654,11.629183900752878,500 +655,20.304872165680123,500 +656,17.682539033444876,500 +657,26.588120643800732,500 +658,16.231540495802122,500 +659,22.258248086875007,500 +660,11.31129569342144,500 +661,-87.91041263699324,497 +662,16.31164860201471,500 +663,23.847438884013272,500 +664,14.506103404789064,500 +665,4.5323489630758695,500 +666,17.959595148985922,500 +667,22.43184095810721,500 +668,21.41626332498558,500 +669,-1.83307602948827,500 +670,14.51722303790848,500 +671,16.57681812165673,500 +672,12.015200327904283,500 +673,17.430397885365018,500 +674,23.087802188706586,500 +675,19.49735820891537,500 +676,24.453649611498882,500 +677,22.332513863709003,500 +678,15.555191690370721,500 +679,20.126276884704243,500 +680,-97.0388368273412,395 +681,22.007290426794423,500 +682,20.58646548504321,500 +683,11.817964053565321,500 +684,15.060148201288897,500 +685,16.214967147518575,500 +686,8.026628691868858,500 +687,24.20362880993883,500 +688,-100.58607312951392,461 +689,24.722163652058374,500 +690,20.824565221579178,500 +691,29.185628835842927,500 +692,21.66275804363246,500 +693,21.200956699618846,500 +694,19.88779502720588,500 +695,8.29603671426512,500 +696,13.200917047774727,500 +697,23.525094798041422,500 +698,14.085544321329566,500 +699,19.184936974911015,500 +700,13.815397483234456,500 +701,18.29727764698249,500 +702,14.80332711194556,500 +703,14.726074779815017,500 +704,21.330808947690535,500 +705,23.327794235506296,500 +706,20.352910817712118,500 +707,26.935770180079118,500 +708,26.8828055639065,500 +709,18.027323342923857,500 +710,20.142742951917434,500 +711,21.35974424325327,500 +712,21.010668864734807,500 +713,10.665116556921978,500 +714,22.60167426442991,500 +715,27.262492316217443,500 +716,33.10608928545517,500 +717,22.352313160544888,500 +718,34.13881280947051,500 +719,35.637536027912475,500 +720,29.88324799453206,500 +721,36.864844741773304,500 +722,34.763076334034956,500 +723,33.08725303585873,500 +724,38.52470120316778,500 +725,39.8045888335618,500 +726,27.953245576733426,500 +727,36.57984264603601,500 +728,35.15059548465662,500 +729,18.732849157213,500 +730,35.84416797240623,500 +731,29.151128787662678,500 +732,31.56666885970194,500 +733,23.992674204280945,500 +734,28.34332440088829,500 +735,34.66292099632545,500 +736,40.770672240913356,500 +737,29.827403933679733,500 +738,42.81268282021116,500 +739,38.556013786416585,500 +740,27.16467273736277,500 +741,40.703482822866526,500 +742,32.477992908664746,500 +743,27.483899306415857,500 +744,41.9101328097224,500 +745,46.42153208935749,500 +746,40.82918511432665,500 +747,34.300970606108926,500 +748,29.764703915025454,500 +749,29.565524356725597,500 +750,37.68346644641278,500 +751,28.498470476053953,500 +752,47.23717713562029,500 +753,43.98956072252835,500 +754,20.420422953457233,500 +755,53.71345088666509,500 +756,-95.59851018170818,471 +757,28.403969929929556,500 +758,44.021023505618395,500 +759,47.92074102081969,500 +760,43.74147198143664,500 +761,40.39069914485316,500 +762,38.22442982773546,500 +763,32.84299142440754,500 +764,29.753071116088787,500 +765,25.84588061181739,500 +766,52.90150685373464,500 +767,29.693844021704756,500 +768,42.85220754640487,500 +769,56.68857325835739,500 +770,36.62208118749911,500 +771,39.374377460132834,500 +772,31.738078235475136,500 +773,35.430921959795654,500 +774,52.73584193390459,500 +775,39.405217637838746,500 +776,35.16939426394787,500 +777,42.58599553615876,500 +778,49.04198784553292,500 +779,55.691785463191266,500 +780,-87.27169023902319,380 +781,54.921386686127455,500 +782,59.66992104058884,500 +783,46.41123266721601,500 +784,46.253032845814026,500 +785,60.769108130351846,500 +786,58.92563633686275,500 +787,62.134807322880846,500 +788,55.03071972193993,500 +789,62.24777145792912,500 +790,51.46072934961999,500 +791,-76.64571959019085,195 +792,66.50712670053215,500 +793,47.660282414845526,500 +794,49.43114453381568,500 +795,50.89252175160168,500 +796,-106.54975693292822,75 +797,63.72010685963161,500 +798,52.73780543957279,500 +799,44.01487029192235,500 +800,49.73703725261307,500 +801,38.07338729014226,500 +802,50.792401156438025,500 +803,49.64230238776828,500 +804,64.16920705245599,500 +805,50.46005601781329,500 +806,36.26510709607415,500 +807,45.74053117041633,500 +808,42.38206203648576,500 +809,60.78989514428125,500 +810,71.12073858731968,500 +811,73.18262984875192,500 +812,65.24102353333888,500 +813,-70.45403976017266,269 +814,51.75783865267681,500 +815,63.90072431564511,500 +816,63.709527333866774,500 +817,73.57842418947102,500 +818,70.3375997420539,500 +819,52.38311723592233,500 +820,66.9521288324241,500 +821,60.30107532252433,500 +822,56.45205693838031,500 +823,48.26893123960936,500 +824,77.62264775519621,500 +825,66.08563146848043,500 +826,59.02478190733736,500 +827,78.8406548206397,500 +828,80.89794990976267,500 +829,75.05798906701204,500 +830,82.08881076868377,500 +831,82.58601496638065,500 +832,-80.46650113721627,169 +833,77.16731578089977,500 +834,57.07407965996011,500 +835,67.60240007885716,500 +836,73.43129011728551,500 +837,-72.63099134664921,298 +838,56.66516364890388,500 +839,-79.72237766213665,237 +840,88.76564060831856,500 +841,75.74114030217275,500 +842,-42.83409756864133,303 +843,-37.96154248766178,424 +844,81.30314080365686,500 +845,66.56521848621954,500 +846,85.90271056720985,500 +847,67.2216417710174,500 +848,-109.53797841737791,66 +849,-105.24079754737097,142 +850,-55.16001509975214,477 +851,-68.25193655632668,278 +852,77.3222615750044,500 +853,-21.878733436570684,471 +854,-115.6871884259653,110 +855,61.55111892150965,500 +856,80.64349741363186,500 +857,64.42697246974626,500 +858,59.77102597657686,500 +859,62.897385889494785,500 +860,77.58762718553677,500 +861,-80.8149533878599,231 +862,-52.24184975535196,383 +863,79.75428722587975,500 +864,-88.7087359209039,244 +865,-85.19510108049012,250 +866,-81.56403239015071,246 +867,-88.82107105690349,144 +868,-86.75710393597869,168 +869,-95.14082786774573,156 +870,-81.94266784989814,290 +871,-69.3555268339791,236 +872,-85.0560312883217,241 +873,-74.99739695577107,201 +874,65.16268295445262,500 +875,48.86581182785706,500 +876,72.88720208418509,500 +877,76.58935371387204,500 +878,78.97437217897938,500 +879,71.13150726205807,500 +880,80.26304509475182,500 +881,75.26824439742117,500 +882,92.22332700641401,500 +883,77.37020817963683,500 +884,74.2755089783535,500 +885,78.19283132381874,500 +886,79.88152925941196,500 +887,82.4223710339667,500 +888,56.835835586983634,500 +889,-51.57302463614724,462 +890,74.43556314218729,500 +891,69.0129955177623,500 +892,80.54839049513369,500 +893,13.243728873663065,500 +894,-67.33239281181304,383 +895,-82.27085351881499,276 +896,61.97239984572718,500 +897,78.08108202635015,500 +898,93.65155203074684,500 +899,-88.54013517620182,161 +900,-52.77052764333269,432 +901,77.39659654400886,500 +902,58.3168718155742,500 +903,85.26615247061315,500 +904,79.29388109090146,500 +905,84.65419838316441,500 +906,91.011571839071,500 +907,81.16289687821083,500 +908,68.46221849108562,500 +909,74.144167779211,500 +910,60.90894868274479,500 +911,74.49587043712918,500 +912,56.113899484780475,500 +913,91.50095863340314,500 +914,73.40818575477769,500 +915,89.38512318151797,500 +916,-61.257335973589996,358 +917,-80.52591613007152,267 +918,-87.27419511079074,192 +919,-64.82030517101025,330 +920,-81.84811721883614,177 +921,-82.63004198328964,205 +922,73.46763242061738,500 +923,72.59856927336403,500 +924,79.58078833361319,500 +925,71.41379101718302,500 +926,80.18597795208368,500 +927,76.90660456561443,500 +928,89.47381523961266,500 +929,74.46212390748317,500 +930,50.642724148396155,500 +931,59.85346718172013,500 +932,93.70235599999029,500 +933,83.75479557532041,500 +934,94.46752420502476,500 +935,98.88156627011152,500 +936,-30.513040128551594,451 +937,86.49941349254651,500 +938,94.56340347722613,500 +939,65.41592454985349,500 +940,95.84107971678966,500 +941,87.4424187698841,500 +942,99.05700261838531,500 +943,80.48956699651339,500 +944,90.5203823039678,500 +945,90.72939200152057,500 +946,-84.83103070827836,243 +947,92.50016198696868,500 +948,87.33131064203928,500 +949,-96.26684800596954,217 +950,-41.95194077649648,498 +951,107.17710729499046,500 +952,-112.69774347290506,237 +953,-110.70280705112032,155 +954,-108.18079824852718,169 +955,-105.70520854232119,123 +956,-106.48936606714503,155 +957,-4.278315033761939,500 +958,-113.42724204043121,327 +959,-27.005477218952024,500 +960,-31.998577014093755,500 +961,-114.46421445042353,109 +962,-110.07988365491387,121 +963,-28.79290085798116,500 +964,-110.51222966562503,103 +965,-110.36077649964454,102 +966,-23.88721210865653,500 +967,-2.2550229402000923,500 +968,-13.776350802127922,500 +969,-13.296342232728296,500 +970,-23.954182518954596,500 +971,-16.518903760764974,500 +972,1.6685410395348392,500 +973,98.42737078204175,500 +974,25.36535594553344,500 +975,-83.50938330077167,126 +976,-107.19325089070759,145 +977,91.30015228873187,500 +978,78.52797969708229,500 +979,42.878117859564725,500 +980,-72.00267938428124,369 +981,95.71643827531629,500 +982,98.65412309393513,500 +983,92.9855509975578,500 +984,87.21774035543461,500 +985,93.42920862539192,500 +986,95.59092868784289,500 +987,90.53236173914712,500 +988,96.22338647327453,500 +989,102.32484246560556,500 +990,-92.50424581945859,119 +991,106.98510458206825,500 +992,91.12847344673153,500 +993,96.72672110375946,500 +994,-10.042995043643327,500 +995,-58.12963592581041,304 +996,-38.90381335117778,379 +997,-50.49386171316885,324 +998,80.02852990215544,500 +999,86.054811019572,500 +1000,78.49161301437428,500 +1001,-45.316845159452825,384 +1002,2.4460031809938663,500 +1003,35.51031613296992,500 +1004,59.06330313483632,500 +1005,-84.57402417084731,147 +1006,115.1851505533332,500 +1007,85.98867158319293,500 +1008,97.67133185725547,500 +1009,25.6298875434555,500 +1010,92.6739626635252,500 +1011,38.560815769909304,500 +1012,81.95219605288867,500 +1013,36.42461901234213,500 +1014,118.0203948155477,500 +1015,115.89234567293539,500 +1016,-60.26913019368083,273 +1017,-92.4191168321728,238 +1018,89.66495049830567,500 +1019,-67.78037694000123,214 +1020,-65.98195062112964,194 +1021,-16.792999351735403,450 +1022,87.73925040496442,500 +1023,-101.7909454267942,217 +1024,104.75453014496715,500 +1025,104.72998116147372,500 +1026,-13.050393336426296,455 +1027,19.984322344733176,500 +1028,115.39151494111385,500 +1029,111.30993101980738,500 +1030,117.6778737549032,500 +1031,120.6281841703761,500 +1032,109.52953201405728,500 +1033,67.49488329027841,500 +1034,-96.73273923124839,127 +1035,-57.06173505996461,320 +1036,97.2485129451277,500 +1037,-79.66407872413205,180 +1038,-75.55276981343933,217 +1039,7.207099404374024,500 +1040,114.61151659545689,500 +1041,115.20906298209513,500 +1042,113.6672514289759,500 +1043,110.37130737700204,500 +1044,106.82513278609164,500 +1045,114.76598764237718,500 +1046,60.49963842039211,500 +1047,123.08326319901502,500 +1048,-36.40786311464537,312 +1049,-85.87451000156346,145 +1050,-90.04627389562937,118 +1051,114.40247635903644,500 +1052,105.6060267993862,500 +1053,130.98054358537536,500 +1054,119.59858441319984,500 +1055,114.50667034866764,500 +1056,116.60248006767152,500 +1057,104.17256166907502,500 +1058,113.01481613270373,500 +1059,118.09191292067678,500 +1060,116.20321389624523,500 +1061,126.77919858484147,500 +1062,119.26615331647285,500 +1063,128.08510170918748,500 +1064,119.09068045006785,500 +1065,121.7583498660111,500 +1066,-79.45286903139291,233 +1067,-41.547441904678834,323 +1068,119.65767757249245,500 +1069,125.63156250600984,500 +1070,121.94812552907553,500 +1071,118.59236380165471,500 +1072,130.14966193383606,500 +1073,130.0268662385699,500 +1074,128.03628289114994,500 +1075,117.34927272011662,500 +1076,124.14379219513467,500 +1077,123.80926907795664,500 +1078,129.81619591367806,500 +1079,126.58896107379731,500 +1080,124.0005508116661,500 +1081,119.22762300159152,500 +1082,125.37560964435131,500 +1083,121.3888524678472,500 +1084,125.32578001511474,500 +1085,130.3397680592554,500 +1086,131.4235474940826,500 +1087,115.9532753905639,500 +1088,118.51890860693959,500 +1089,126.42284151154247,500 +1090,138.48084802011587,500 +1091,126.96913853087258,500 +1092,131.64622340018124,500 +1093,132.25192492299772,500 +1094,129.87309192826245,500 +1095,131.87321706438493,500 +1096,136.6274700858279,500 +1097,130.55534437419914,500 +1098,127.37189007280497,500 +1099,108.26308381499342,500 +1100,109.39989812103353,500 +1101,138.29691060860029,500 +1102,141.23282117208447,500 +1103,140.9068813861025,500 +1104,124.21169190282892,500 +1105,140.8274115431309,500 +1106,134.0171036064141,500 +1107,131.79353048875066,500 +1108,135.78631365560457,500 +1109,123.27112381108456,500 +1110,108.10486873785707,500 +1111,136.1639394808033,500 +1112,-18.99403883714787,426 +1113,133.54818779397237,500 +1114,127.27368853748676,500 +1115,136.39808055314808,500 +1116,132.9925451048509,500 +1117,121.50232910320446,500 +1118,137.0301905505558,500 +1119,138.80182062334012,500 +1120,142.04283155886503,500 +1121,126.73304047757526,500 +1122,144.48381775410016,500 +1123,144.64903915883312,500 +1124,131.09554361980162,500 +1125,135.30845871208845,500 +1126,136.2577266797198,500 +1127,142.94136358800392,500 +1128,140.96096203051775,500 +1129,135.48552980211338,500 +1130,140.30663766268412,500 +1131,141.36248661776577,500 +1132,97.27003872673922,500 +1133,129.23958466303324,500 +1134,134.47987907979987,500 +1135,-9.550673522327315,367 +1136,139.2757873796573,500 +1137,128.88837947607072,500 +1138,142.1751917608306,500 +1139,134.72105552651954,500 +1140,132.89286262907135,500 +1141,134.4670010982676,500 +1142,145.77257125002646,500 +1143,130.598488535041,500 +1144,-16.65719550088616,453 +1145,-27.04041924375693,334 +1146,17.179663419059636,474 +1147,147.40559843308276,500 +1148,136.28943692783068,500 +1149,80.05079580004337,500 +1150,148.84959894154957,500 +1151,149.35389849646043,500 +1152,145.77889181968,500 +1153,-5.454833927899003,477 +1154,147.43811383783304,500 +1155,145.14890243403755,500 +1156,132.78314072874528,500 +1157,140.50662283983235,500 +1158,47.83569108420782,500 +1159,-59.54948866231649,209 +1160,141.40220693398203,500 +1161,144.90306751523661,500 +1162,148.3497166937149,500 +1163,147.29463722346532,500 +1164,141.81240275537544,500 +1165,142.6309499239545,500 +1166,152.0235157580737,500 +1167,148.93141981129435,500 +1168,142.63865935741495,500 +1169,145.89957479194865,500 +1170,150.28157971684973,500 +1171,134.54740586810811,500 +1172,138.55646672677997,500 +1173,148.19510625555353,500 +1174,142.40215516239766,500 +1175,152.8513931758948,500 +1176,143.4325344271554,500 +1177,151.90672751194612,500 +1178,155.13117843368764,500 +1179,148.0641245062225,500 +1180,140.4593451205472,500 +1181,139.44124393084027,500 +1182,66.97652415471123,500 +1183,-69.87074704215301,191 +1184,145.5098077559184,500 +1185,152.70593820567598,500 +1186,154.86482850073244,500 +1187,147.82300949759232,500 +1188,147.65176624580434,500 +1189,152.72459412643983,500 +1190,151.0638138456796,500 +1191,146.33351745512905,500 +1192,147.03742116570396,500 +1193,148.80635459332316,500 +1194,158.01668868340738,500 +1195,149.40070493034415,500 +1196,162.00203856367693,500 +1197,150.71239677618354,500 +1198,141.44377076045254,500 +1199,142.95472144474166,500 +1200,138.87549377914502,500 +1201,136.92001036442335,500 +1202,142.70308187587585,500 +1203,152.00076308354028,500 +1204,147.08785449933984,500 +1205,-56.166655577073485,219 +1206,151.5148897649768,500 +1207,-57.82745943670492,228 +1208,152.89505396794758,500 +1209,151.66931428682824,500 +1210,149.99593803032988,500 +1211,152.82142853506483,500 +1212,153.2759967735706,500 +1213,147.85270945897366,500 +1214,144.80602849198118,500 +1215,150.21568290791873,500 +1216,143.97067388774298,500 +1217,154.94901238879226,500 +1218,154.38342020409476,500 +1219,145.63156125241778,500 +1220,161.3699274107668,500 +1221,160.93000540798178,500 +1222,160.3742190804247,500 +1223,145.05931956997708,500 +1224,156.26921562801198,500 +1225,152.38069645767305,500 +1226,156.71770927568377,500 +1227,152.29016740038185,500 +1228,156.97794981622172,500 +1229,155.553882233291,500 +1230,166.1480319905051,500 +1231,152.7061951492895,500 +1232,151.92241157713514,500 +1233,153.17673827103724,500 +1234,148.75353899488934,500 +1235,151.37837023506987,500 +1236,154.42714311023244,500 +1237,148.70063991255432,500 +1238,158.95307887117468,500 +1239,153.10032937706598,500 +1240,156.956793792626,500 +1241,155.74601757777555,500 +1242,160.14300072011932,500 +1243,158.063482767133,500 +1244,157.37151367277127,500 +1245,156.58530217068144,500 +1246,145.50967437141685,500 +1247,150.33525039379307,500 +1248,151.60590222635133,500 +1249,141.6002525096468,500 +1250,147.37087594779965,500 +1251,145.15570661901606,500 +1252,145.01573825195052,500 +1253,148.1384149829331,500 +1254,143.39297370220504,500 +1255,146.1413913904233,500 +1256,140.54903943146184,500 +1257,155.59438031296025,500 +1258,152.0925706933064,500 +1259,148.8013552570553,500 +1260,146.87497446171045,500 +1261,152.66006572797858,500 +1262,162.44378414733615,500 +1263,148.868942884486,500 +1264,167.43274096927487,500 +1265,158.6189422248241,500 +1266,162.57646003508663,500 +1267,163.83496999601095,500 +1268,151.69593256638095,500 +1269,160.25336814922986,500 +1270,153.86502368815337,500 +1271,158.9593415331138,500 +1272,160.76659480871697,500 +1273,162.61525337900574,500 +1274,161.86923053731908,500 +1275,167.47119134313309,500 +1276,157.0865516947842,500 +1277,156.38113076142687,500 +1278,159.44815340011743,500 +1279,158.2292493397023,500 +1280,153.33576545640452,500 +1281,165.39601317332415,500 +1282,163.19799566517145,500 +1283,166.1533229035437,500 +1284,163.95748961564806,500 +1285,156.7325061645853,500 +1286,157.27774796375,500 +1287,153.6312101128775,500 +1288,156.82759248237306,500 +1289,162.3388892513428,500 +1290,161.76290355020544,500 +1291,162.99017591587102,500 +1292,168.73273659033222,500 +1293,155.74159714910925,500 +1294,166.08397251056408,500 +1295,170.38773200004508,500 +1296,167.51032948901718,500 +1297,160.66245919192966,500 +1298,161.37103261022915,500 +1299,162.08296664280962,500 +1300,155.71477312047986,500 +1301,155.81439397743674,500 +1302,155.193406037296,500 +1303,-44.18511331543697,263 +1304,141.25312332391437,500 +1305,150.34844517515214,500 +1306,163.09384315500742,500 +1307,171.12998841344915,500 +1308,156.7832668137227,500 +1309,165.2779626258783,500 +1310,169.08146449885623,500 +1311,164.56139024364222,500 +1312,169.99534028830723,500 +1313,160.3357467698703,500 +1314,172.37046519591792,500 +1315,165.3358567749376,500 +1316,156.94649914300402,500 +1317,162.699381760307,500 +1318,157.84521756913358,500 +1319,152.65748604608183,500 +1320,173.48330891449513,500 +1321,163.98744567630013,500 +1322,151.3271166724632,500 +1323,158.78386421502958,500 +1324,173.6528840187649,500 +1325,162.40733376973554,500 +1326,162.59488026473457,500 +1327,153.8282998220873,500 +1328,163.5164180333525,500 +1329,164.98724256100974,500 +1330,155.33071178872703,500 +1331,-32.332193594137095,290 +1332,159.48138147303666,500 +1333,161.3893447078286,500 +1334,158.21541989321986,500 +1335,166.264337597724,500 +1336,162.03281367215055,500 +1337,160.44269814013322,500 +1338,165.85681210364908,500 +1339,163.22110758744054,500 +1340,175.14704398733642,500 +1341,162.50817297569878,500 +1342,167.00690777063588,500 +1343,170.82558097110396,500 +1344,174.64362441033734,500 +1345,172.168798563864,500 +1346,163.6491487034364,500 +1347,-15.429328442149881,361 +1348,-87.37317337652419,129 +1349,173.69345806919037,500 +1350,162.5579166192668,500 +1351,164.8459193806906,500 +1352,169.08612737370228,500 +1353,163.96844576118795,500 +1354,164.89390287335402,500 +1355,160.06928910701888,500 +1356,163.1257680556392,500 +1357,168.80198490010807,500 +1358,170.0202824097341,500 +1359,166.0689975865171,500 +1360,174.5222614232441,500 +1361,163.66221327721325,500 +1362,165.3957025912253,500 +1363,164.36181241074982,500 +1364,178.16804109208496,500 +1365,178.95456258731733,500 +1366,162.81853313106086,500 +1367,167.0885545347905,500 +1368,162.40240919801045,500 +1369,156.82941671359814,500 +1370,162.72509369228393,500 +1371,156.1964224557667,500 +1372,161.45680443299725,500 +1373,162.01720609744532,500 +1374,23.15164830722607,427 +1375,160.38394450705292,500 +1376,171.648422202141,500 +1377,165.82462084404443,500 +1378,167.29695971933324,500 +1379,161.70857596532332,500 +1380,161.7262235088023,500 +1381,-67.5204411577036,202 +1382,162.68107312514968,500 +1383,154.1837510460254,500 +1384,166.62729372811802,500 +1385,160.8346659358016,500 +1386,160.51095476916893,500 +1387,163.68678832764445,500 +1388,159.4954473329282,500 +1389,172.26680267512873,500 +1390,-46.43178198719862,240 +1391,152.82004565057488,500 +1392,162.92904032360195,500 +1393,160.06890054972692,500 +1394,163.7153479158755,500 +1395,169.39260828606496,500 +1396,169.35515620253798,500 +1397,168.39978781731918,500 +1398,166.7374224199439,500 +1399,172.89020994400053,500 +1400,162.49504000559176,500 +1401,32.376278389721534,439 +1402,174.49893492080864,500 +1403,167.13011225122506,500 +1404,157.70540048286398,500 +1405,168.08741428606294,500 +1406,170.30035373141138,500 +1407,122.70836476991249,500 +1408,170.4688961352569,500 +1409,166.41930797645165,500 +1410,159.5502857337031,500 +1411,172.45399898965695,500 +1412,172.72735531714363,500 +1413,169.55161232642607,500 +1414,159.5966917372201,500 +1415,166.4804188760932,500 +1416,165.91102447309603,500 +1417,57.483511242790044,500 +1418,175.44655734221453,500 +1419,176.047796429747,500 +1420,185.09489059794748,500 +1421,174.4692156694046,500 +1422,166.0945935851575,500 +1423,168.49949327168892,500 +1424,169.45616435430853,500 +1425,158.37423568567473,500 +1426,172.85095988682065,500 +1427,26.894152806224778,500 +1428,166.80767715862038,500 +1429,168.74561253557687,500 +1430,162.0267731385443,500 +1431,160.1924388040171,500 +1432,160.72074269636232,500 +1433,77.5252495069263,500 +1434,153.8270078447244,500 +1435,164.27611981509403,500 +1436,170.14969678134418,500 +1437,165.98038155276922,500 +1438,158.6665092497625,500 +1439,162.81773041348,500 +1440,167.9530792096407,500 +1441,167.18075580412398,500 +1442,169.1821340943638,500 +1443,-60.36883563404767,310 +1444,171.49441056509818,500 +1445,162.70766023795215,500 +1446,169.7063238211167,500 +1447,172.99478892717306,500 +1448,163.99898721521254,500 +1449,166.6025710703614,500 +1450,162.60143556472315,500 +1451,162.52871317436987,500 +1452,158.38483170109313,500 +1453,163.42670309841202,500 +1454,164.197833901979,500 +1455,172.66382394711354,500 +1456,164.266924774984,500 +1457,168.52450577542143,500 +1458,171.52429284908985,500 +1459,163.28112819055565,500 +1460,168.903458214473,500 +1461,169.15182282196741,500 +1462,166.36083527812886,500 +1463,168.5618434267544,500 +1464,169.91033611915472,500 +1465,168.34528436576394,500 +1466,163.33094134707375,500 +1467,163.87016320133657,500 +1468,168.88499725903662,500 +1469,162.59132952162471,500 +1470,165.76300438798103,500 +1471,156.39132130580552,500 +1472,159.24188082130317,500 +1473,168.02158054009726,500 +1474,166.6641538460541,500 +1475,168.24994658440005,500 +1476,172.33569114913274,500 +1477,164.65423761066572,500 +1478,172.1674327035378,500 +1479,174.92598268005545,500 +1480,167.48488648297175,500 +1481,170.35795495570022,500 +1482,173.45957003584923,500 +1483,162.10211163084554,500 +1484,179.13741036602676,500 +1485,173.04343215243165,500 +1486,171.08915996374665,500 +1487,169.24146749488864,500 +1488,160.97644186418628,500 +1489,148.4505588511382,500 +1490,167.48727952005032,500 +1491,161.80049273821734,500 +1492,135.41533115419188,500 +1493,164.8231309400302,500 +1494,167.5076835978919,500 +1495,164.8488639676275,500 +1496,167.35299884125544,500 +1497,171.6613350826359,500 +1498,162.65277943476568,500 +1499,151.08809933202355,500 +1500,164.16062725105277,500 +1501,167.30106584781012,500 +1502,161.68554515788261,500 +1503,170.29027511318608,500 +1504,170.6834100576194,500 +1505,167.89151331705816,500 +1506,156.76557994013683,500 +1507,163.3716452191131,500 +1508,163.70010499409102,500 +1509,167.3301713536758,500 +1510,166.1587122660503,500 +1511,169.92873075404637,500 +1512,167.37788542004196,500 +1513,176.5398925601673,500 +1514,167.9017824229203,500 +1515,169.05151911601934,500 +1516,175.82392681974358,500 +1517,157.5964813170656,500 +1518,158.85675330973984,500 +1519,165.8999423371939,500 +1520,170.72460127905237,500 +1521,166.55351575786258,500 +1522,162.53097621395452,500 +1523,154.92478786219976,500 +1524,156.97898234521648,500 +1525,154.3186567590825,500 +1526,161.43055586400104,500 +1527,163.03234709310883,500 +1528,155.74768171627164,500 +1529,163.7913245552905,500 +1530,158.04685355330042,500 +1531,166.71987252280775,500 +1532,172.15953548957233,500 +1533,169.80947851669006,500 +1534,163.3344997882229,500 +1535,171.2518197305245,500 +1536,170.4189929562283,500 +1537,167.0465816644215,500 +1538,169.42497345170105,500 +1539,168.02573842264687,500 +1540,173.24296778530723,500 +1541,157.6038563183277,500 +1542,159.26770609021105,500 +1543,167.51397148328252,500 +1544,170.73861712862887,500 +1545,172.57958415259884,500 +1546,175.9863754746224,500 +1547,169.3811964260908,500 +1548,172.7520376116906,500 +1549,163.78213506846316,500 +1550,165.3663238096978,500 +1551,162.50943807408137,500 +1552,167.20471815363945,500 +1553,163.34293245086005,500 +1554,172.05710392287037,500 +1555,165.6388513683318,500 +1556,169.2334086563492,500 +1557,134.78454005907162,500 +1558,187.29159587599582,500 +1559,175.39158514002338,500 +1560,171.54858332676753,500 +1561,176.70750163589042,500 +1562,170.67532738829073,500 +1563,170.3848223766428,500 +1564,171.9567342292431,500 +1565,170.3949243415546,500 +1566,-19.116369690695322,299 +1567,164.51450014559575,500 +1568,160.28980507610115,500 +1569,166.07689941270155,500 +1570,180.81768727576988,500 +1571,167.4991394925908,500 +1572,162.80028284493324,500 +1573,170.63496053634302,500 +1574,166.8462899865784,500 +1575,176.94218642310324,500 +1576,176.99344249316744,500 +1577,166.03806944901703,500 +1578,164.36842657356183,500 +1579,169.33156614619202,500 +1580,172.09838527506105,500 +1581,166.55989212018943,500 +1582,168.03110481993988,500 +1583,160.16259839722747,500 +1584,178.816132368039,500 +1585,171.62795331340618,500 +1586,165.06467561848032,500 +1587,168.56762370929528,500 +1588,169.49310949992534,500 +1589,176.1055570352436,500 +1590,164.05637904657792,500 +1591,171.97778107735672,500 +1592,174.52415305964377,500 +1593,167.5843863727339,500 +1594,172.10679243302846,500 +1595,173.34802962722154,500 +1596,172.07670826258675,500 +1597,162.06456548498636,500 +1598,163.75114126576324,500 +1599,160.41167457152875,500 +1600,34.110232511202355,445 +1601,167.42335349395174,500 +1602,171.86429774793186,500 +1603,175.1287015613739,500 +1604,166.4996381535579,500 +1605,177.37810211053068,500 +1606,168.065200857105,500 +1607,172.4477408043514,500 +1608,176.1472266249607,500 +1609,166.19448721978117,500 +1610,173.80322600050397,500 +1611,169.4776567838928,500 +1612,168.72150973567292,500 +1613,168.29131811744378,500 +1614,166.6625860646641,500 +1615,175.8786484220087,500 +1616,174.49559271483636,500 +1617,178.57577871409137,500 +1618,175.07367883485594,500 +1619,183.01897471062554,500 +1620,171.07257543237762,500 +1621,161.00277915538783,500 +1622,169.81189504531815,500 +1623,173.42233719159432,500 +1624,169.4525084160957,500 +1625,169.6027076863534,500 +1626,169.63350473164093,500 +1627,173.9673065521732,500 +1628,176.44125379916954,500 +1629,168.5440805286284,500 +1630,164.3928829490414,500 +1631,169.80368819457684,500 +1632,168.49468064946976,500 +1633,172.1448041794059,500 +1634,180.42842565222918,500 +1635,167.1089204998588,500 +1636,172.31130146539059,500 +1637,179.64668557524666,500 +1638,181.2505927433777,500 +1639,170.8627407227016,500 +1640,176.74401928690966,500 +1641,179.73548169231657,500 +1642,178.65504365566156,500 +1643,178.9623479311324,500 +1644,174.07461169018225,500 +1645,42.5597585502976,474 +1646,180.14169688651188,500 +1647,179.25117308280198,500 +1648,176.71686046427587,500 +1649,168.45492132003238,500 +1650,179.93518794543297,500 +1651,176.90722084176704,500 +1652,183.34805368314005,500 +1653,178.9769780278021,500 +1654,178.51634807395007,500 +1655,174.13064856806758,500 +1656,185.19535633897476,500 +1657,181.08734752081676,500 +1658,174.12751788594608,500 +1659,33.651682416177664,453 +1660,178.46483517363754,500 +1661,179.99225249187944,500 +1662,182.27202775896518,500 +1663,177.84062394415906,500 +1664,182.75184583989687,500 +1665,176.9308616126401,500 +1666,178.1958402614168,500 +1667,178.50479844354737,500 +1668,164.5244695174313,500 +1669,171.470947897506,500 +1670,173.35188480974662,500 +1671,181.51023043125633,500 +1672,179.26367162378676,500 +1673,187.1185767581692,500 +1674,178.4666683655735,500 +1675,178.91788266632497,500 +1676,179.19389190055315,500 +1677,179.36723129059365,500 +1678,181.35533603282053,500 +1679,174.98080355120845,500 +1680,177.87915148505243,500 +1681,177.22299230429883,500 +1682,171.13642523447945,500 +1683,166.85268785341623,500 +1684,175.56012878220056,500 +1685,163.4823377058719,500 +1686,183.7016543163649,500 +1687,180.51764734338713,500 +1688,168.53197871308316,500 +1689,169.64046249665864,500 +1690,181.46350451463056,500 +1691,173.5032064034548,500 +1692,182.85224160096027,500 +1693,162.77709401449007,500 +1694,175.19392322548595,500 +1695,174.9275155374901,500 +1696,181.17875293200086,500 +1697,172.34652049065764,500 +1698,163.3797056299127,500 +1699,185.02017628258795,500 +1700,162.96354523334034,500 +1701,173.2831833551,500 +1702,179.2351543039497,500 +1703,165.09870241757406,500 +1704,180.33109594742,500 +1705,168.01716961363772,500 +1706,174.74630083532304,500 +1707,176.09315200853803,500 +1708,181.25916009916227,500 +1709,178.1772327684256,500 +1710,178.860619484286,500 +1711,174.32133025795196,500 +1712,178.51662550437112,500 +1713,172.99083478468557,500 +1714,173.23413456008993,500 +1715,177.09011103955476,500 +1716,179.3843094598721,500 +1717,182.96258189644647,500 +1718,180.28757900296395,500 +1719,177.56189961244027,500 +1720,175.51039435053548,500 +1721,183.48662587930042,500 +1722,176.43559214355912,500 +1723,171.94869470813694,500 +1724,181.92360459040702,500 +1725,177.36595209982082,500 +1726,181.9568615196534,500 +1727,185.2370947566658,500 +1728,181.25008542870899,500 +1729,177.61021423225287,500 +1730,182.2234376979956,500 +1731,171.94591932789623,500 +1732,172.40299633139793,500 +1733,171.60409226911653,500 +1734,169.86330701508513,500 +1735,173.18640522378817,500 +1736,179.6083090843083,500 +1737,174.642346523776,500 +1738,-13.331293147338243,344 +1739,175.93215431316827,500 +1740,165.8249463338198,500 +1741,174.8586935941565,500 +1742,173.6412654490391,500 +1743,183.58726084592388,500 +1744,180.67580051138253,500 +1745,180.64291739477326,500 +1746,176.9605624801105,500 +1747,186.1303204157919,500 +1748,180.40913896701042,500 +1749,171.93421997969332,500 +1750,168.20411786657073,500 +1751,179.66891772510564,500 +1752,177.09045284203728,500 +1753,170.88652219685872,500 +1754,180.65880449208683,500 +1755,173.8927384081403,500 +1756,167.86382920224736,500 +1757,177.62609613306378,500 +1758,176.62484521596866,500 +1759,177.20710988432492,500 +1760,182.2387598909089,500 +1761,180.67127725522022,500 +1762,171.81453002883833,500 +1763,173.66865632719797,500 +1764,169.76482410152042,500 +1765,168.72875188416396,500 +1766,168.87021128886082,500 +1767,174.26471532277625,500 +1768,175.2862107211044,500 +1769,179.88628621014044,500 +1770,171.01803197372175,500 +1771,176.80217546799403,500 +1772,176.99071911796605,500 +1773,174.6299428856007,500 +1774,178.76850265562265,500 +1775,173.92546172440572,500 +1776,173.2523327911368,500 +1777,168.83649408124327,500 +1778,184.16167529454697,500 +1779,174.73029938447922,500 +1780,170.49427543405127,500 +1781,172.5042764006366,500 +1782,174.78797371555035,500 +1783,174.57766656067574,500 +1784,168.67110059690282,500 +1785,176.50591853409756,500 +1786,181.40184156001555,500 +1787,174.52562424944574,500 +1788,177.1486068606066,500 +1789,168.27194741205366,500 +1790,174.63823884077135,500 +1791,177.9670815141336,500 +1792,169.77104508358173,500 +1793,175.80504902871587,500 +1794,176.25584303518306,500 +1795,179.64718803522427,500 +1796,182.41881556850097,500 +1797,167.5267976218727,500 +1798,168.13233765790375,500 +1799,179.80998818628987,500 +1800,176.91771118527072,500 +1801,178.08727393521494,500 +1802,175.51634622199674,500 +1803,170.83707030375982,500 +1804,184.29326263640326,500 +1805,183.13873644420167,500 +1806,185.13888962937077,500 +1807,184.5519720071605,500 +1808,183.222507243015,500 +1809,185.23133274983886,500 +1810,182.89461779692846,500 +1811,196.68768152456354,500 +1812,175.8040021605857,500 +1813,178.94100962967485,500 +1814,180.34821762490253,500 +1815,185.5673814469456,500 +1816,181.7028302994973,500 +1817,181.1203816339489,500 +1818,183.4822790178548,500 +1819,164.34161434960507,500 +1820,187.4229373601789,500 +1821,61.88826724285329,483 +1822,182.61289157754192,500 +1823,196.40152095759242,500 +1824,186.81282994578436,500 +1825,183.7986988711558,500 +1826,182.5109548573848,500 +1827,179.89351956110067,500 +1828,176.38354831380767,500 +1829,176.9791635217182,500 +1830,183.86160595878064,500 +1831,182.12202440332268,500 +1832,183.58077292680602,500 +1833,176.7539751431995,500 +1834,191.09175679061966,500 +1835,183.81134949087482,500 +1836,185.21827729462566,500 +1837,177.16919225925304,500 +1838,168.13850355775415,500 +1839,175.9258633826931,500 +1840,183.1600524016265,500 +1841,178.83470545778863,500 +1842,184.9551481559026,500 +1843,177.11841091455008,500 +1844,183.3036159316416,500 +1845,187.9479548559729,500 +1846,182.8636227787621,500 +1847,174.39046452553833,500 +1848,170.499419190054,500 +1849,186.82206908092633,500 +1850,175.25599764942075,500 +1851,179.22348962188806,500 +1852,183.48521887651773,500 +1853,175.0903980345181,500 +1854,173.60847203167378,500 +1855,170.072974251404,500 +1856,174.50141860435562,500 +1857,176.62283785715354,500 +1858,178.12309582197645,500 +1859,171.87871381712438,500 +1860,182.82088959740364,500 +1861,185.76017900653582,500 +1862,184.85374336503887,500 +1863,177.48979983860738,500 +1864,173.74979608920168,500 +1865,171.52441189416703,500 +1866,175.43977043678217,500 +1867,181.50124062146187,500 +1868,174.76604978964997,500 +1869,188.92006466512746,500 +1870,184.01787733429123,500 +1871,178.30785378016452,500 +1872,183.18894035362248,500 +1873,178.23129063222788,500 +1874,172.4200875205565,500 +1875,176.54372599559082,500 +1876,175.67175660754373,500 +1877,183.38835460639896,500 +1878,177.40078734779888,500 +1879,181.08994514978502,500 +1880,176.27706919293485,500 +1881,184.64071601626796,500 +1882,173.25437492646574,500 +1883,179.79873156641014,500 +1884,185.0673496949901,500 +1885,184.02438781499632,500 +1886,178.92380275135284,500 +1887,171.1740209919592,500 +1888,180.52897447597215,500 +1889,181.91222049345913,500 +1890,181.83435323179214,500 +1891,177.92680812222417,500 +1892,176.8014763807251,500 +1893,183.15877815732273,500 +1894,179.37751961063313,500 +1895,169.8611538667412,500 +1896,180.7925855020993,500 +1897,179.13676323045738,500 +1898,183.7625760950682,500 +1899,175.80934241001148,500 +1900,188.5434139875575,500 +1901,177.11704261542727,500 +1902,178.43846555786882,500 +1903,179.3298205753156,500 +1904,182.9388005445676,500 +1905,184.64010923273608,500 +1906,183.4493622384663,500 +1907,183.4144591303478,500 +1908,175.09899721045073,500 +1909,181.5431351153174,500 +1910,175.12957622893265,500 +1911,176.87860960193055,500 +1912,171.24136315291275,500 +1913,171.3844792177297,500 +1914,178.38428740131263,500 +1915,176.44602347417813,500 +1916,177.30987592985278,500 +1917,189.8841102032437,500 +1918,181.42655217376603,500 +1919,182.4053164174304,500 +1920,177.43503246076034,500 +1921,177.8856681879835,500 +1922,188.3959354936482,500 +1923,177.8579605542309,500 +1924,185.07888550428515,500 +1925,185.06291993456202,500 +1926,191.83410892095722,500 +1927,184.0831545749442,500 +1928,190.06676508201687,500 +1929,193.35067073788363,500 +1930,181.31406829643495,500 +1931,186.99517454996922,500 +1932,176.40785164426555,500 +1933,179.61438785546503,500 +1934,179.37243249750512,500 +1935,178.03556259138512,500 +1936,185.35829120692568,500 +1937,186.40444727508276,500 +1938,179.20983490557748,500 +1939,177.713932288329,500 +1940,186.60698833912008,500 +1941,184.22365317114316,500 +1942,185.0848369990515,500 +1943,188.54279425074205,500 +1944,180.50549659828607,500 +1945,176.32945583691978,500 +1946,174.28589910819045,500 +1947,178.6532041125519,500 +1948,189.43617197568955,500 +1949,195.41149426779103,500 +1950,188.86886198192724,500 +1951,187.4264878254441,500 +1952,187.14268433633822,500 +1953,187.46250262505686,500 +1954,179.33789668669954,500 +1955,170.162229479365,500 +1956,186.97005805598033,500 +1957,189.16623715665304,500 +1958,180.88707486539386,500 +1959,169.4185878191503,500 +1960,187.65290404140643,500 +1961,187.43952284359673,500 +1962,179.3350861712521,500 +1963,181.4678713922971,500 +1964,180.07706254852525,500 +1965,176.27627842326496,500 +1966,176.1522835468194,500 +1967,178.41322792686145,500 +1968,183.7510812107608,500 +1969,175.01464046426705,500 +1970,171.6549489781948,500 +1971,186.93892098037045,500 +1972,182.05427742810292,500 +1973,182.49018846154934,500 +1974,184.47824584101625,500 +1975,179.5335736342317,500 +1976,180.32553089973965,500 +1977,178.56771589163733,500 +1978,180.80709490257794,500 +1979,174.3777889734563,500 +1980,183.11822817127526,500 +1981,182.55364984583505,500 +1982,176.97259432308172,500 +1983,185.41496081747326,500 +1984,199.73840843614647,500 +1985,191.12384515851195,500 +1986,189.9371167036408,500 +1987,195.04341171997478,500 +1988,184.87256885393208,500 +1989,181.2011597264032,500 +1990,180.93506806292206,500 +1991,187.28416622578425,500 +1992,187.06017476513895,500 +1993,181.11904804731492,500 +1994,177.71859439999517,500 +1995,180.70841464208257,500 +1996,180.39207737221432,500 +1997,174.72587838599398,500 +1998,181.61098502174107,500 +1999,183.3349530114202,500 diff --git a/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/config.yaml b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fae567911cf2157a73a875ab5cc0331cf6cee758 --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: DQN_CNN + device: cuda + env_name: Breakout-v4 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_Breakout-v4_DQN_CNN_20221207-214917 + max_steps: 1000 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 20 + train_eps: 200 + wrapper: envs.wrappers.MarioWrappers +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 diff --git a/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/logs/log.txt b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff4e6f4d4dea3ac6870911a598196c0a13afac7b --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/logs/log.txt @@ -0,0 +1,53 @@ +2022-12-08 10:15:45 - r - INFO: - Hyperparameters: +2022-12-08 10:15:45 - r - INFO: - ================================================================================ +2022-12-08 10:15:45 - r - INFO: - Name Value Type +2022-12-08 10:15:45 - r - INFO: - env_name Breakout-v4 +2022-12-08 10:15:45 - r - INFO: - new_step_api 1 +2022-12-08 10:15:45 - r - INFO: - wrapper envs.wrappers.MarioWrappers +2022-12-08 10:15:45 - r - INFO: - render 0 +2022-12-08 10:15:45 - r - INFO: - algo_name DQN_CNN +2022-12-08 10:15:45 - r - INFO: - mode test +2022-12-08 10:15:45 - r - INFO: - seed 1 +2022-12-08 10:15:45 - r - INFO: - device cuda +2022-12-08 10:15:45 - r - INFO: - train_eps 200 +2022-12-08 10:15:45 - r - INFO: - test_eps 20 +2022-12-08 10:15:45 - r - INFO: - eval_eps 10 +2022-12-08 10:15:45 - r - INFO: - eval_per_episode 5 +2022-12-08 10:15:45 - r - INFO: - max_steps 1000 +2022-12-08 10:15:45 - r - INFO: - load_checkpoint 1 +2022-12-08 10:15:45 - r - INFO: - load_path Train_Breakout-v4_DQN_CNN_20221207-214917 +2022-12-08 10:15:45 - r - INFO: - show_fig 0 +2022-12-08 10:15:45 - r - INFO: - save_fig 1 +2022-12-08 10:15:45 - r - INFO: - epsilon_start 0.95 +2022-12-08 10:15:45 - r - INFO: - epsilon_end 0.01 +2022-12-08 10:15:45 - r - INFO: - epsilon_decay 500 +2022-12-08 10:15:45 - r - INFO: - gamma 0.99 +2022-12-08 10:15:45 - r - INFO: - lr 0.0001 +2022-12-08 10:15:45 - r - INFO: - buffer_size 100000 +2022-12-08 10:15:45 - r - INFO: - batch_size 64 +2022-12-08 10:15:45 - r - INFO: - target_update 4 +2022-12-08 10:15:45 - r - INFO: - ================================================================================ +2022-12-08 10:15:46 - r - INFO: - n_states: 4, n_actions: 4 +2022-12-08 10:15:50 - r - INFO: - Start testing! +2022-12-08 10:15:50 - r - INFO: - Env: Breakout-v4, Algorithm: DQN_CNN, Device: cuda +2022-12-08 10:15:55 - r - INFO: - Episode: 1/20, Reward: 4.000, Step: 1000 +2022-12-08 10:15:58 - r - INFO: - Episode: 2/20, Reward: 5.000, Step: 1000 +2022-12-08 10:15:58 - r - INFO: - Episode: 3/20, Reward: 1.000, Step: 50 +2022-12-08 10:16:01 - r - INFO: - Episode: 4/20, Reward: 3.000, Step: 1000 +2022-12-08 10:16:01 - r - INFO: - Episode: 5/20, Reward: 2.000, Step: 63 +2022-12-08 10:16:01 - r - INFO: - Episode: 6/20, Reward: 5.000, Step: 102 +2022-12-08 10:16:04 - r - INFO: - Episode: 7/20, Reward: 5.000, Step: 1000 +2022-12-08 10:16:05 - r - INFO: - Episode: 8/20, Reward: 3.000, Step: 83 +2022-12-08 10:16:07 - r - INFO: - Episode: 9/20, Reward: 4.000, Step: 1000 +2022-12-08 10:16:10 - r - INFO: - Episode: 10/20, Reward: 5.000, Step: 1000 +2022-12-08 10:16:11 - r - INFO: - Episode: 11/20, Reward: 5.000, Step: 102 +2022-12-08 10:16:11 - r - INFO: - Episode: 12/20, Reward: 1.000, Step: 52 +2022-12-08 10:16:14 - r - INFO: - Episode: 13/20, Reward: 5.000, Step: 1000 +2022-12-08 10:16:14 - r - INFO: - Episode: 14/20, Reward: 3.000, Step: 71 +2022-12-08 10:16:18 - r - INFO: - Episode: 15/20, Reward: 5.000, Step: 1000 +2022-12-08 10:16:18 - r - INFO: - Episode: 16/20, Reward: 1.000, Step: 52 +2022-12-08 10:16:21 - r - INFO: - Episode: 17/20, Reward: 2.000, Step: 1000 +2022-12-08 10:16:21 - r - INFO: - Episode: 18/20, Reward: 5.000, Step: 96 +2022-12-08 10:16:21 - r - INFO: - Episode: 19/20, Reward: 2.000, Step: 61 +2022-12-08 10:16:24 - r - INFO: - Episode: 20/20, Reward: 2.000, Step: 1000 +2022-12-08 10:16:24 - r - INFO: - Finish testing! diff --git a/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/models/checkpoint.pt b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e85d1f7235c6cab0ef445d6390d3d050331b0dfd --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28265c6f02e3c5690ab9faac6ebc6cb72c05a24d23d21259c73e197d3eea863a +size 6747719 diff --git a/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/results/learning_curve.png b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..01d09341f3e9fb7324f80b4241cb459a05e44f35 Binary files /dev/null and b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/results/learning_curve.png differ diff --git a/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/results/res.csv b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..fc35ba881a92373480bc2b06c8c42182fd20d336 --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_DQN_CNN_20221208-101545/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,4.0,1000 +1,5.0,1000 +2,1.0,50 +3,3.0,1000 +4,2.0,63 +5,5.0,102 +6,5.0,1000 +7,3.0,83 +8,4.0,1000 +9,5.0,1000 +10,5.0,102 +11,1.0,52 +12,5.0,1000 +13,3.0,71 +14,5.0,1000 +15,1.0,52 +16,2.0,1000 +17,5.0,96 +18,2.0,61 +19,2.0,1000 diff --git a/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/config.yaml b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d3539f72ed2e10b31dfd8cff3fa1ebabfa6e9e9 --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/config.yaml @@ -0,0 +1,28 @@ +general_cfg: + algo_name: DQN_CNN + device: cpu + env_name: Breakout-v4 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_Breakout-v4_mp_DQN_CNN_20230320-084801 + max_steps: 500 + mode: test + n_workers: 1 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 500 + wrapper: envs.wrappers.MarioWrappers +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 diff --git a/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/logs/log.txt b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..21af5175a4876c85f6738e03e8591c2760fa41c5 --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/logs/log.txt @@ -0,0 +1,49 @@ +2023-03-20 13:47:30 - r - INFO: - Hyperparameters: +2023-03-20 13:47:30 - r - INFO: - ================================================================================ +2023-03-20 13:47:30 - r - INFO: - Name Value Type +2023-03-20 13:47:30 - r - INFO: - env_name Breakout-v4 +2023-03-20 13:47:30 - r - INFO: - new_step_api 1 +2023-03-20 13:47:30 - r - INFO: - wrapper envs.wrappers.MarioWrappers +2023-03-20 13:47:30 - r - INFO: - render 0 +2023-03-20 13:47:30 - r - INFO: - algo_name DQN_CNN +2023-03-20 13:47:30 - r - INFO: - mode test +2023-03-20 13:47:30 - r - INFO: - seed 1 +2023-03-20 13:47:30 - r - INFO: - device cpu +2023-03-20 13:47:30 - r - INFO: - train_eps 500 +2023-03-20 13:47:30 - r - INFO: - test_eps 10 +2023-03-20 13:47:30 - r - INFO: - eval_eps 10 +2023-03-20 13:47:30 - r - INFO: - eval_per_episode 5 +2023-03-20 13:47:30 - r - INFO: - max_steps 500 +2023-03-20 13:47:30 - r - INFO: - load_checkpoint 1 +2023-03-20 13:47:30 - r - INFO: - load_path Train_Breakout-v4_mp_DQN_CNN_20230320-084801 +2023-03-20 13:47:30 - r - INFO: - show_fig 0 +2023-03-20 13:47:30 - r - INFO: - save_fig 1 +2023-03-20 13:47:30 - r - INFO: - n_workers 1 +2023-03-20 13:47:30 - r - INFO: - epsilon_start 0.95 +2023-03-20 13:47:30 - r - INFO: - epsilon_end 0.01 +2023-03-20 13:47:30 - r - INFO: - epsilon_decay 500 +2023-03-20 13:47:30 - r - INFO: - gamma 0.99 +2023-03-20 13:47:30 - r - INFO: - lr 0.0001 +2023-03-20 13:47:30 - r - INFO: - buffer_size 100000 +2023-03-20 13:47:30 - r - INFO: - batch_size 64 +2023-03-20 13:47:30 - r - INFO: - target_update 4 +2023-03-20 13:47:30 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_Breakout-v4_DQN_CNN_20230320-134730 +2023-03-20 13:47:30 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_Breakout-v4_DQN_CNN_20230320-134730/results +2023-03-20 13:47:30 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_Breakout-v4_DQN_CNN_20230320-134730/logs +2023-03-20 13:47:30 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_Breakout-v4_DQN_CNN_20230320-134730/traj +2023-03-20 13:47:30 - r - INFO: - tb_dir /media/disk/gsc/joyrl-offline/tasks/Test_Breakout-v4_DQN_CNN_20230320-134730/tb_logs +2023-03-20 13:47:30 - r - INFO: - ================================================================================ +2023-03-20 13:47:32 - r - INFO: - n_states: 4, n_actions: 4 +2023-03-20 13:47:32 - r - INFO: - Start testing! +2023-03-20 13:47:32 - r - INFO: - Env: Breakout-v4, Algorithm: DQN_CNN, Device: cpu +2023-03-20 13:47:35 - r - INFO: - Episode: 1/10, Reward: 6.000, Step: 500 +2023-03-20 13:47:38 - r - INFO: - Episode: 2/10, Reward: 6.000, Step: 500 +2023-03-20 13:47:41 - r - INFO: - Episode: 3/10, Reward: 6.000, Step: 500 +2023-03-20 13:47:45 - r - INFO: - Episode: 4/10, Reward: 6.000, Step: 500 +2023-03-20 13:47:48 - r - INFO: - Episode: 5/10, Reward: 6.000, Step: 500 +2023-03-20 13:47:51 - r - INFO: - Episode: 6/10, Reward: 6.000, Step: 500 +2023-03-20 13:47:55 - r - INFO: - Episode: 7/10, Reward: 6.000, Step: 500 +2023-03-20 13:47:58 - r - INFO: - Episode: 8/10, Reward: 6.000, Step: 500 +2023-03-20 13:48:01 - r - INFO: - Episode: 9/10, Reward: 6.000, Step: 500 +2023-03-20 13:48:04 - r - INFO: - Episode: 10/10, Reward: 6.000, Step: 500 +2023-03-20 13:48:04 - r - INFO: - Finish testing! diff --git a/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/models/checkpoint.pt b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..0162aea1bfaacfd68912e0e9037e59f37994c172 --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec74cc055c2f33ef16473f9aee7b4ec9e1229fcffc4a1be7ba4668bf97e56ad +size 6747591 diff --git a/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/results/learning_curve.png b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..6fe193d38b071cf2bc3e508cc5e6c436529e3fc8 Binary files /dev/null and b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/results/learning_curve.png differ diff --git a/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/results/res.csv b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..fb577fb37a2ef87dff05354f500543f37515c80b --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,6.0,500 +1,6.0,500 +2,6.0,500 +3,6.0,500 +4,6.0,500 +5,6.0,500 +6,6.0,500 +7,6.0,500 +8,6.0,500 +9,6.0,500 diff --git a/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/tb_logs/events.out.tfevents.1679291250.ML3090.652699.0 b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/tb_logs/events.out.tfevents.1679291250.ML3090.652699.0 new file mode 100644 index 0000000000000000000000000000000000000000..f06455cedeef17a9a1b1a1cd6e24bf0fe33cc45d --- /dev/null +++ b/Breakout-v4/Test_Breakout-v4_mp_DQN_CNN_20230320-134730/tb_logs/events.out.tfevents.1679291250.ML3090.652699.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6585db129edd75bbffcc4cb2ee09451cceb4fab3e6ccb0c4d79115c70b63c002 +size 40 diff --git a/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/config.yaml b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73acfc557f59ecc40236e437abbbf5be251207bf --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: DQN_CNN + device: cuda + env_name: Breakout-v4 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_Mario_DQN_CNN_20221205-205203 + max_steps: 1000 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: envs.wrappers.MarioWrappers +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 diff --git a/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/logs/log.txt b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..2c8dd2aeb72cf29d38d62bb775899ee8c5e25b72 --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/logs/log.txt @@ -0,0 +1,242 @@ +2022-12-07 21:49:17 - r - INFO: - Hyperparameters: +2022-12-07 21:49:17 - r - INFO: - ================================================================================ +2022-12-07 21:49:17 - r - INFO: - Name Value Type +2022-12-07 21:49:17 - r - INFO: - env_name Breakout-v4 +2022-12-07 21:49:17 - r - INFO: - new_step_api 1 +2022-12-07 21:49:17 - r - INFO: - wrapper envs.wrappers.MarioWrappers +2022-12-07 21:49:17 - r - INFO: - render 0 +2022-12-07 21:49:17 - r - INFO: - algo_name DQN_CNN +2022-12-07 21:49:17 - r - INFO: - mode train +2022-12-07 21:49:17 - r - INFO: - seed 1 +2022-12-07 21:49:17 - r - INFO: - device cuda +2022-12-07 21:49:17 - r - INFO: - train_eps 200 +2022-12-07 21:49:17 - r - INFO: - test_eps 10 +2022-12-07 21:49:17 - r - INFO: - eval_eps 10 +2022-12-07 21:49:17 - r - INFO: - eval_per_episode 5 +2022-12-07 21:49:17 - r - INFO: - max_steps 1000 +2022-12-07 21:49:17 - r - INFO: - load_checkpoint 0 +2022-12-07 21:49:17 - r - INFO: - load_path Train_Mario_DQN_CNN_20221205-205203 +2022-12-07 21:49:17 - r - INFO: - show_fig 0 +2022-12-07 21:49:17 - r - INFO: - save_fig 1 +2022-12-07 21:49:17 - r - INFO: - epsilon_start 0.95 +2022-12-07 21:49:17 - r - INFO: - epsilon_end 0.01 +2022-12-07 21:49:17 - r - INFO: - epsilon_decay 500 +2022-12-07 21:49:17 - r - INFO: - gamma 0.99 +2022-12-07 21:49:17 - r - INFO: - lr 0.0001 +2022-12-07 21:49:17 - r - INFO: - buffer_size 100000 +2022-12-07 21:49:17 - r - INFO: - batch_size 64 +2022-12-07 21:49:17 - r - INFO: - target_update 4 +2022-12-07 21:49:17 - r - INFO: - ================================================================================ +2022-12-07 21:49:18 - r - INFO: - n_states: 4, n_actions: 4 +2022-12-07 21:49:21 - r - INFO: - Start training! +2022-12-07 21:49:21 - r - INFO: - Env: Breakout-v4, Algorithm: DQN_CNN, Device: cuda +2022-12-07 21:49:22 - r - INFO: - Episode: 1/200, Reward: 0.000, Step: 54 +2022-12-07 21:49:24 - r - INFO: - Episode: 2/200, Reward: 2.000, Step: 85 +2022-12-07 21:49:26 - r - INFO: - Episode: 3/200, Reward: 3.000, Step: 100 +2022-12-07 21:49:29 - r - INFO: - Episode: 4/200, Reward: 4.000, Step: 110 +2022-12-07 21:49:31 - r - INFO: - Episode: 5/200, Reward: 1.000, Step: 87 +2022-12-07 21:50:08 - r - INFO: - Current episode 5 has the best eval reward: 0.000 +2022-12-07 21:50:10 - r - INFO: - Episode: 6/200, Reward: 0.000, Step: 78 +2022-12-07 21:50:12 - r - INFO: - Episode: 7/200, Reward: 0.000, Step: 73 +2022-12-07 21:50:15 - r - INFO: - Episode: 8/200, Reward: 2.000, Step: 113 +2022-12-07 21:50:18 - r - INFO: - Episode: 9/200, Reward: 2.000, Step: 98 +2022-12-07 21:50:19 - r - INFO: - Episode: 10/200, Reward: 1.000, Step: 57 +2022-12-07 21:50:56 - r - INFO: - Current episode 10 has the best eval reward: 0.000 +2022-12-07 21:51:00 - r - INFO: - Episode: 11/200, Reward: 1.000, Step: 123 +2022-12-07 21:51:02 - r - INFO: - Episode: 12/200, Reward: 1.000, Step: 69 +2022-12-07 21:51:08 - r - INFO: - Episode: 13/200, Reward: 0.000, Step: 235 +2022-12-07 21:51:10 - r - INFO: - Episode: 14/200, Reward: 2.000, Step: 111 +2022-12-07 21:51:13 - r - INFO: - Episode: 15/200, Reward: 0.000, Step: 100 +2022-12-07 21:51:15 - r - INFO: - Current episode 15 has the best eval reward: 0.000 +2022-12-07 21:51:17 - r - INFO: - Episode: 16/200, Reward: 1.000, Step: 87 +2022-12-07 21:51:23 - r - INFO: - Episode: 17/200, Reward: 3.000, Step: 231 +2022-12-07 21:51:26 - r - INFO: - Episode: 18/200, Reward: 2.000, Step: 105 +2022-12-07 21:51:29 - r - INFO: - Episode: 19/200, Reward: 0.000, Step: 113 +2022-12-07 21:51:31 - r - INFO: - Episode: 20/200, Reward: 3.000, Step: 97 +2022-12-07 21:51:33 - r - INFO: - Current episode 20 has the best eval reward: 1.000 +2022-12-07 21:51:35 - r - INFO: - Episode: 21/200, Reward: 1.000, Step: 71 +2022-12-07 21:51:40 - r - INFO: - Episode: 22/200, Reward: 2.000, Step: 161 +2022-12-07 21:51:43 - r - INFO: - Episode: 23/200, Reward: 1.000, Step: 129 +2022-12-07 21:51:45 - r - INFO: - Episode: 24/200, Reward: 1.000, Step: 80 +2022-12-07 21:51:48 - r - INFO: - Episode: 25/200, Reward: 2.000, Step: 91 +2022-12-07 21:52:27 - r - INFO: - Episode: 26/200, Reward: 1.000, Step: 493 +2022-12-07 21:52:34 - r - INFO: - Episode: 27/200, Reward: 1.000, Step: 266 +2022-12-07 21:52:38 - r - INFO: - Episode: 28/200, Reward: 2.000, Step: 140 +2022-12-07 21:52:42 - r - INFO: - Episode: 29/200, Reward: 1.000, Step: 169 +2022-12-07 21:52:46 - r - INFO: - Episode: 30/200, Reward: 2.000, Step: 139 +2022-12-07 21:53:28 - r - INFO: - Episode: 31/200, Reward: 1.000, Step: 175 +2022-12-07 21:53:34 - r - INFO: - Episode: 32/200, Reward: 2.000, Step: 256 +2022-12-07 21:53:41 - r - INFO: - Episode: 33/200, Reward: 0.000, Step: 256 +2022-12-07 21:53:43 - r - INFO: - Episode: 34/200, Reward: 1.000, Step: 70 +2022-12-07 21:53:51 - r - INFO: - Episode: 35/200, Reward: 0.000, Step: 295 +2022-12-07 21:54:43 - r - INFO: - Episode: 36/200, Reward: 0.000, Step: 554 +2022-12-07 21:54:46 - r - INFO: - Episode: 37/200, Reward: 2.000, Step: 126 +2022-12-07 21:54:49 - r - INFO: - Episode: 38/200, Reward: 1.000, Step: 95 +2022-12-07 21:54:51 - r - INFO: - Episode: 39/200, Reward: 1.000, Step: 83 +2022-12-07 21:54:53 - r - INFO: - Episode: 40/200, Reward: 1.000, Step: 89 +2022-12-07 21:55:33 - r - INFO: - Episode: 41/200, Reward: 4.000, Step: 110 +2022-12-07 21:55:35 - r - INFO: - Episode: 42/200, Reward: 2.000, Step: 73 +2022-12-07 21:55:37 - r - INFO: - Episode: 43/200, Reward: 1.000, Step: 72 +2022-12-07 21:55:40 - r - INFO: - Episode: 44/200, Reward: 4.000, Step: 121 +2022-12-07 21:55:43 - r - INFO: - Episode: 45/200, Reward: 2.000, Step: 117 +2022-12-07 21:56:17 - r - INFO: - Current episode 45 has the best eval reward: 1.000 +2022-12-07 21:56:19 - r - INFO: - Episode: 46/200, Reward: 3.000, Step: 74 +2022-12-07 21:56:21 - r - INFO: - Episode: 47/200, Reward: 1.000, Step: 85 +2022-12-07 21:56:25 - r - INFO: - Episode: 48/200, Reward: 2.000, Step: 131 +2022-12-07 21:56:26 - r - INFO: - Episode: 49/200, Reward: 2.000, Step: 66 +2022-12-07 21:56:29 - r - INFO: - Episode: 50/200, Reward: 4.000, Step: 108 +2022-12-07 21:57:09 - r - INFO: - Episode: 51/200, Reward: 2.000, Step: 101 +2022-12-07 21:57:20 - r - INFO: - Episode: 52/200, Reward: 1.000, Step: 406 +2022-12-07 21:57:22 - r - INFO: - Episode: 53/200, Reward: 0.000, Step: 86 +2022-12-07 21:57:24 - r - INFO: - Episode: 54/200, Reward: 4.000, Step: 87 +2022-12-07 21:57:26 - r - INFO: - Episode: 55/200, Reward: 1.000, Step: 66 +2022-12-07 21:58:06 - r - INFO: - Episode: 56/200, Reward: 2.000, Step: 121 +2022-12-07 21:58:08 - r - INFO: - Episode: 57/200, Reward: 3.000, Step: 77 +2022-12-07 21:58:10 - r - INFO: - Episode: 58/200, Reward: 1.000, Step: 79 +2022-12-07 21:58:14 - r - INFO: - Episode: 59/200, Reward: 4.000, Step: 142 +2022-12-07 21:58:16 - r - INFO: - Episode: 60/200, Reward: 1.000, Step: 71 +2022-12-07 21:58:53 - r - INFO: - Episode: 61/200, Reward: 5.000, Step: 126 +2022-12-07 21:58:55 - r - INFO: - Episode: 62/200, Reward: 1.000, Step: 82 +2022-12-07 21:58:58 - r - INFO: - Episode: 63/200, Reward: 3.000, Step: 97 +2022-12-07 21:59:00 - r - INFO: - Episode: 64/200, Reward: 1.000, Step: 63 +2022-12-07 21:59:01 - r - INFO: - Episode: 65/200, Reward: 2.000, Step: 74 +2022-12-07 21:59:42 - r - INFO: - Episode: 66/200, Reward: 5.000, Step: 125 +2022-12-07 21:59:43 - r - INFO: - Episode: 67/200, Reward: 0.000, Step: 55 +2022-12-07 21:59:47 - r - INFO: - Episode: 68/200, Reward: 0.000, Step: 126 +2022-12-07 21:59:51 - r - INFO: - Episode: 69/200, Reward: 5.000, Step: 169 +2022-12-07 22:00:07 - r - INFO: - Episode: 70/200, Reward: 4.000, Step: 612 +2022-12-07 22:00:23 - r - INFO: - Current episode 70 has the best eval reward: 2.600 +2022-12-07 22:00:35 - r - INFO: - Episode: 71/200, Reward: 2.000, Step: 422 +2022-12-07 22:00:37 - r - INFO: - Episode: 72/200, Reward: 3.000, Step: 94 +2022-12-07 22:00:39 - r - INFO: - Episode: 73/200, Reward: 3.000, Step: 76 +2022-12-07 22:00:44 - r - INFO: - Episode: 74/200, Reward: 1.000, Step: 181 +2022-12-07 22:00:54 - r - INFO: - Episode: 75/200, Reward: 5.000, Step: 363 +2022-12-07 22:01:33 - r - INFO: - Episode: 76/200, Reward: 1.000, Step: 68 +2022-12-07 22:01:49 - r - INFO: - Episode: 77/200, Reward: 5.000, Step: 604 +2022-12-07 22:01:51 - r - INFO: - Episode: 78/200, Reward: 4.000, Step: 88 +2022-12-07 22:01:54 - r - INFO: - Episode: 79/200, Reward: 1.000, Step: 120 +2022-12-07 22:01:56 - r - INFO: - Episode: 80/200, Reward: 2.000, Step: 74 +2022-12-07 22:02:35 - r - INFO: - Episode: 81/200, Reward: 1.000, Step: 65 +2022-12-07 22:02:38 - r - INFO: - Episode: 82/200, Reward: 3.000, Step: 82 +2022-12-07 22:02:40 - r - INFO: - Episode: 83/200, Reward: 2.000, Step: 81 +2022-12-07 22:02:53 - r - INFO: - Episode: 84/200, Reward: 3.000, Step: 499 +2022-12-07 22:03:13 - r - INFO: - Episode: 85/200, Reward: 6.000, Step: 774 +2022-12-07 22:03:53 - r - INFO: - Episode: 86/200, Reward: 2.000, Step: 106 +2022-12-07 22:03:56 - r - INFO: - Episode: 87/200, Reward: 0.000, Step: 88 +2022-12-07 22:04:04 - r - INFO: - Episode: 88/200, Reward: 3.000, Step: 300 +2022-12-07 22:04:06 - r - INFO: - Episode: 89/200, Reward: 0.000, Step: 103 +2022-12-07 22:04:13 - r - INFO: - Episode: 90/200, Reward: 2.000, Step: 250 +2022-12-07 22:04:50 - r - INFO: - Episode: 91/200, Reward: 4.000, Step: 275 +2022-12-07 22:05:12 - r - INFO: - Episode: 92/200, Reward: 4.000, Step: 814 +2022-12-07 22:05:22 - r - INFO: - Episode: 93/200, Reward: 4.000, Step: 363 +2022-12-07 22:05:24 - r - INFO: - Episode: 94/200, Reward: 3.000, Step: 94 +2022-12-07 22:05:28 - r - INFO: - Episode: 95/200, Reward: 3.000, Step: 165 +2022-12-07 22:06:11 - r - INFO: - Episode: 96/200, Reward: 3.000, Step: 315 +2022-12-07 22:06:22 - r - INFO: - Episode: 97/200, Reward: 2.000, Step: 417 +2022-12-07 22:06:24 - r - INFO: - Episode: 98/200, Reward: 2.000, Step: 81 +2022-12-07 22:06:26 - r - INFO: - Episode: 99/200, Reward: 2.000, Step: 74 +2022-12-07 22:06:28 - r - INFO: - Episode: 100/200, Reward: 2.000, Step: 71 +2022-12-07 22:06:55 - r - INFO: - Current episode 100 has the best eval reward: 3.600 +2022-12-07 22:06:56 - r - INFO: - Episode: 101/200, Reward: 1.000, Step: 54 +2022-12-07 22:07:07 - r - INFO: - Episode: 102/200, Reward: 1.000, Step: 383 +2022-12-07 22:07:12 - r - INFO: - Episode: 103/200, Reward: 4.000, Step: 195 +2022-12-07 22:07:15 - r - INFO: - Episode: 104/200, Reward: 4.000, Step: 108 +2022-12-07 22:07:22 - r - INFO: - Episode: 105/200, Reward: 2.000, Step: 248 +2022-12-07 22:08:04 - r - INFO: - Episode: 106/200, Reward: 5.000, Step: 182 +2022-12-07 22:08:14 - r - INFO: - Episode: 107/200, Reward: 5.000, Step: 372 +2022-12-07 22:08:18 - r - INFO: - Episode: 108/200, Reward: 3.000, Step: 152 +2022-12-07 22:08:24 - r - INFO: - Episode: 109/200, Reward: 5.000, Step: 239 +2022-12-07 22:08:27 - r - INFO: - Episode: 110/200, Reward: 4.000, Step: 97 +2022-12-07 22:09:09 - r - INFO: - Episode: 111/200, Reward: 3.000, Step: 211 +2022-12-07 22:09:12 - r - INFO: - Episode: 112/200, Reward: 3.000, Step: 88 +2022-12-07 22:09:15 - r - INFO: - Episode: 113/200, Reward: 3.000, Step: 140 +2022-12-07 22:09:25 - r - INFO: - Episode: 114/200, Reward: 9.000, Step: 346 +2022-12-07 22:09:27 - r - INFO: - Episode: 115/200, Reward: 3.000, Step: 100 +2022-12-07 22:09:43 - r - INFO: - Episode: 116/200, Reward: 5.000, Step: 101 +2022-12-07 22:09:47 - r - INFO: - Episode: 117/200, Reward: 2.000, Step: 129 +2022-12-07 22:09:50 - r - INFO: - Episode: 118/200, Reward: 7.000, Step: 140 +2022-12-07 22:09:53 - r - INFO: - Episode: 119/200, Reward: 4.000, Step: 101 +2022-12-07 22:09:55 - r - INFO: - Episode: 120/200, Reward: 1.000, Step: 64 +2022-12-07 22:10:26 - r - INFO: - Episode: 121/200, Reward: 0.000, Step: 45 +2022-12-07 22:10:29 - r - INFO: - Episode: 122/200, Reward: 4.000, Step: 109 +2022-12-07 22:10:31 - r - INFO: - Episode: 123/200, Reward: 2.000, Step: 69 +2022-12-07 22:10:49 - r - INFO: - Episode: 124/200, Reward: 4.000, Step: 663 +2022-12-07 22:10:52 - r - INFO: - Episode: 125/200, Reward: 4.000, Step: 124 +2022-12-07 22:11:00 - r - INFO: - Episode: 126/200, Reward: 1.000, Step: 71 +2022-12-07 22:11:03 - r - INFO: - Episode: 127/200, Reward: 3.000, Step: 90 +2022-12-07 22:11:05 - r - INFO: - Episode: 128/200, Reward: 3.000, Step: 87 +2022-12-07 22:11:07 - r - INFO: - Episode: 129/200, Reward: 3.000, Step: 85 +2022-12-07 22:11:11 - r - INFO: - Episode: 130/200, Reward: 2.000, Step: 135 +2022-12-07 22:11:32 - r - INFO: - Episode: 131/200, Reward: 7.000, Step: 174 +2022-12-07 22:11:39 - r - INFO: - Episode: 132/200, Reward: 4.000, Step: 247 +2022-12-07 22:11:42 - r - INFO: - Episode: 133/200, Reward: 3.000, Step: 133 +2022-12-07 22:11:45 - r - INFO: - Episode: 134/200, Reward: 3.000, Step: 109 +2022-12-07 22:11:48 - r - INFO: - Episode: 135/200, Reward: 3.000, Step: 127 +2022-12-07 22:12:05 - r - INFO: - Current episode 135 has the best eval reward: 3.600 +2022-12-07 22:12:08 - r - INFO: - Episode: 136/200, Reward: 3.000, Step: 88 +2022-12-07 22:12:22 - r - INFO: - Episode: 137/200, Reward: 5.000, Step: 542 +2022-12-07 22:12:30 - r - INFO: - Episode: 138/200, Reward: 2.000, Step: 286 +2022-12-07 22:12:39 - r - INFO: - Episode: 139/200, Reward: 3.000, Step: 337 +2022-12-07 22:12:42 - r - INFO: - Episode: 140/200, Reward: 4.000, Step: 134 +2022-12-07 22:13:10 - r - INFO: - Episode: 141/200, Reward: 3.000, Step: 301 +2022-12-07 22:13:13 - r - INFO: - Episode: 142/200, Reward: 4.000, Step: 81 +2022-12-07 22:13:14 - r - INFO: - Episode: 143/200, Reward: 2.000, Step: 73 +2022-12-07 22:13:17 - r - INFO: - Episode: 144/200, Reward: 3.000, Step: 88 +2022-12-07 22:13:19 - r - INFO: - Episode: 145/200, Reward: 2.000, Step: 72 +2022-12-07 22:13:58 - r - INFO: - Episode: 146/200, Reward: 1.000, Step: 66 +2022-12-07 22:14:01 - r - INFO: - Episode: 147/200, Reward: 2.000, Step: 114 +2022-12-07 22:14:08 - r - INFO: - Episode: 148/200, Reward: 4.000, Step: 286 +2022-12-07 22:14:11 - r - INFO: - Episode: 149/200, Reward: 4.000, Step: 125 +2022-12-07 22:14:13 - r - INFO: - Episode: 150/200, Reward: 1.000, Step: 68 +2022-12-07 22:14:52 - r - INFO: - Episode: 151/200, Reward: 1.000, Step: 57 +2022-12-07 22:14:55 - r - INFO: - Episode: 152/200, Reward: 4.000, Step: 102 +2022-12-07 22:15:03 - r - INFO: - Episode: 153/200, Reward: 4.000, Step: 305 +2022-12-07 22:15:05 - r - INFO: - Episode: 154/200, Reward: 3.000, Step: 91 +2022-12-07 22:15:09 - r - INFO: - Episode: 155/200, Reward: 6.000, Step: 123 +2022-12-07 22:15:48 - r - INFO: - Episode: 156/200, Reward: 2.000, Step: 73 +2022-12-07 22:15:51 - r - INFO: - Episode: 157/200, Reward: 3.000, Step: 107 +2022-12-07 22:16:08 - r - INFO: - Episode: 158/200, Reward: 6.000, Step: 652 +2022-12-07 22:16:10 - r - INFO: - Episode: 159/200, Reward: 3.000, Step: 83 +2022-12-07 22:16:35 - r - INFO: - Episode: 160/200, Reward: 6.000, Step: 919 +2022-12-07 22:16:58 - r - INFO: - Current episode 160 has the best eval reward: 3.600 +2022-12-07 22:17:06 - r - INFO: - Episode: 161/200, Reward: 4.000, Step: 293 +2022-12-07 22:17:15 - r - INFO: - Episode: 162/200, Reward: 3.000, Step: 328 +2022-12-07 22:17:17 - r - INFO: - Episode: 163/200, Reward: 4.000, Step: 86 +2022-12-07 22:17:43 - r - INFO: - Episode: 164/200, Reward: 7.000, Step: 968 +2022-12-07 22:18:03 - r - INFO: - Episode: 165/200, Reward: 5.000, Step: 753 +2022-12-07 22:18:41 - r - INFO: - Episode: 166/200, Reward: 4.000, Step: 175 +2022-12-07 22:18:45 - r - INFO: - Episode: 167/200, Reward: 7.000, Step: 146 +2022-12-07 22:18:51 - r - INFO: - Episode: 168/200, Reward: 4.000, Step: 225 +2022-12-07 22:18:56 - r - INFO: - Episode: 169/200, Reward: 7.000, Step: 149 +2022-12-07 22:18:58 - r - INFO: - Episode: 170/200, Reward: 4.000, Step: 93 +2022-12-07 22:19:40 - r - INFO: - Episode: 171/200, Reward: 5.000, Step: 169 +2022-12-07 22:19:46 - r - INFO: - Episode: 172/200, Reward: 5.000, Step: 254 +2022-12-07 22:19:50 - r - INFO: - Episode: 173/200, Reward: 6.000, Step: 123 +2022-12-07 22:19:55 - r - INFO: - Episode: 174/200, Reward: 4.000, Step: 182 +2022-12-07 22:19:58 - r - INFO: - Episode: 175/200, Reward: 6.000, Step: 122 +2022-12-07 22:20:32 - r - INFO: - Episode: 176/200, Reward: 5.000, Step: 124 +2022-12-07 22:20:35 - r - INFO: - Episode: 177/200, Reward: 4.000, Step: 142 +2022-12-07 22:20:38 - r - INFO: - Episode: 178/200, Reward: 4.000, Step: 114 +2022-12-07 22:20:41 - r - INFO: - Episode: 179/200, Reward: 3.000, Step: 102 +2022-12-07 22:20:43 - r - INFO: - Episode: 180/200, Reward: 3.000, Step: 84 +2022-12-07 22:21:13 - r - INFO: - Episode: 181/200, Reward: 6.000, Step: 115 +2022-12-07 22:21:15 - r - INFO: - Episode: 182/200, Reward: 3.000, Step: 75 +2022-12-07 22:21:20 - r - INFO: - Episode: 183/200, Reward: 7.000, Step: 191 +2022-12-07 22:21:35 - r - INFO: - Episode: 184/200, Reward: 7.000, Step: 572 +2022-12-07 22:21:54 - r - INFO: - Episode: 185/200, Reward: 8.000, Step: 721 +2022-12-07 22:22:33 - r - INFO: - Episode: 186/200, Reward: 3.000, Step: 174 +2022-12-07 22:22:37 - r - INFO: - Episode: 187/200, Reward: 6.000, Step: 120 +2022-12-07 22:22:40 - r - INFO: - Episode: 188/200, Reward: 5.000, Step: 115 +2022-12-07 22:22:57 - r - INFO: - Episode: 189/200, Reward: 5.000, Step: 640 +2022-12-07 22:23:01 - r - INFO: - Episode: 190/200, Reward: 9.000, Step: 157 +2022-12-07 22:23:44 - r - INFO: - Episode: 191/200, Reward: 6.000, Step: 161 +2022-12-07 22:23:55 - r - INFO: - Episode: 192/200, Reward: 4.000, Step: 424 +2022-12-07 22:24:01 - r - INFO: - Episode: 193/200, Reward: 6.000, Step: 242 +2022-12-07 22:24:11 - r - INFO: - Episode: 194/200, Reward: 2.000, Step: 357 +2022-12-07 22:24:23 - r - INFO: - Episode: 195/200, Reward: 7.000, Step: 480 +2022-12-07 22:24:58 - r - INFO: - Episode: 196/200, Reward: 6.000, Step: 255 +2022-12-07 22:25:18 - r - INFO: - Episode: 197/200, Reward: 1.000, Step: 746 +2022-12-07 22:25:22 - r - INFO: - Episode: 198/200, Reward: 4.000, Step: 133 +2022-12-07 22:25:36 - r - INFO: - Episode: 199/200, Reward: 4.000, Step: 558 +2022-12-07 22:25:39 - r - INFO: - Episode: 200/200, Reward: 5.000, Step: 99 +2022-12-07 22:26:17 - r - INFO: - Finish training! diff --git a/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/models/checkpoint.pt b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e85d1f7235c6cab0ef445d6390d3d050331b0dfd --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28265c6f02e3c5690ab9faac6ebc6cb72c05a24d23d21259c73e197d3eea863a +size 6747719 diff --git a/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/results/learning_curve.png b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..502f01155f9d6b1df9714c255b70ed6daffe021e Binary files /dev/null and b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/results/learning_curve.png differ diff --git a/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/results/res.csv b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..a1ac8953811fb87c9b2caec02c479183dba9d143 --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_DQN_CNN_20221207-214917/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,0.0,54 +1,2.0,85 +2,3.0,100 +3,4.0,110 +4,1.0,87 +5,0.0,78 +6,0.0,73 +7,2.0,113 +8,2.0,98 +9,1.0,57 +10,1.0,123 +11,1.0,69 +12,0.0,235 +13,2.0,111 +14,0.0,100 +15,1.0,87 +16,3.0,231 +17,2.0,105 +18,0.0,113 +19,3.0,97 +20,1.0,71 +21,2.0,161 +22,1.0,129 +23,1.0,80 +24,2.0,91 +25,1.0,493 +26,1.0,266 +27,2.0,140 +28,1.0,169 +29,2.0,139 +30,1.0,175 +31,2.0,256 +32,0.0,256 +33,1.0,70 +34,0.0,295 +35,0.0,554 +36,2.0,126 +37,1.0,95 +38,1.0,83 +39,1.0,89 +40,4.0,110 +41,2.0,73 +42,1.0,72 +43,4.0,121 +44,2.0,117 +45,3.0,74 +46,1.0,85 +47,2.0,131 +48,2.0,66 +49,4.0,108 +50,2.0,101 +51,1.0,406 +52,0.0,86 +53,4.0,87 +54,1.0,66 +55,2.0,121 +56,3.0,77 +57,1.0,79 +58,4.0,142 +59,1.0,71 +60,5.0,126 +61,1.0,82 +62,3.0,97 +63,1.0,63 +64,2.0,74 +65,5.0,125 +66,0.0,55 +67,0.0,126 +68,5.0,169 +69,4.0,612 +70,2.0,422 +71,3.0,94 +72,3.0,76 +73,1.0,181 +74,5.0,363 +75,1.0,68 +76,5.0,604 +77,4.0,88 +78,1.0,120 +79,2.0,74 +80,1.0,65 +81,3.0,82 +82,2.0,81 +83,3.0,499 +84,6.0,774 +85,2.0,106 +86,0.0,88 +87,3.0,300 +88,0.0,103 +89,2.0,250 +90,4.0,275 +91,4.0,814 +92,4.0,363 +93,3.0,94 +94,3.0,165 +95,3.0,315 +96,2.0,417 +97,2.0,81 +98,2.0,74 +99,2.0,71 +100,1.0,54 +101,1.0,383 +102,4.0,195 +103,4.0,108 +104,2.0,248 +105,5.0,182 +106,5.0,372 +107,3.0,152 +108,5.0,239 +109,4.0,97 +110,3.0,211 +111,3.0,88 +112,3.0,140 +113,9.0,346 +114,3.0,100 +115,5.0,101 +116,2.0,129 +117,7.0,140 +118,4.0,101 +119,1.0,64 +120,0.0,45 +121,4.0,109 +122,2.0,69 +123,4.0,663 +124,4.0,124 +125,1.0,71 +126,3.0,90 +127,3.0,87 +128,3.0,85 +129,2.0,135 +130,7.0,174 +131,4.0,247 +132,3.0,133 +133,3.0,109 +134,3.0,127 +135,3.0,88 +136,5.0,542 +137,2.0,286 +138,3.0,337 +139,4.0,134 +140,3.0,301 +141,4.0,81 +142,2.0,73 +143,3.0,88 +144,2.0,72 +145,1.0,66 +146,2.0,114 +147,4.0,286 +148,4.0,125 +149,1.0,68 +150,1.0,57 +151,4.0,102 +152,4.0,305 +153,3.0,91 +154,6.0,123 +155,2.0,73 +156,3.0,107 +157,6.0,652 +158,3.0,83 +159,6.0,919 +160,4.0,293 +161,3.0,328 +162,4.0,86 +163,7.0,968 +164,5.0,753 +165,4.0,175 +166,7.0,146 +167,4.0,225 +168,7.0,149 +169,4.0,93 +170,5.0,169 +171,5.0,254 +172,6.0,123 +173,4.0,182 +174,6.0,122 +175,5.0,124 +176,4.0,142 +177,4.0,114 +178,3.0,102 +179,3.0,84 +180,6.0,115 +181,3.0,75 +182,7.0,191 +183,7.0,572 +184,8.0,721 +185,3.0,174 +186,6.0,120 +187,5.0,115 +188,5.0,640 +189,9.0,157 +190,6.0,161 +191,4.0,424 +192,6.0,242 +193,2.0,357 +194,7.0,480 +195,6.0,255 +196,1.0,746 +197,4.0,133 +198,4.0,558 +199,5.0,99 diff --git a/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/config.yaml b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10df6b1da3c81d803c8e203f12909c5b9843fed0 --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/config.yaml @@ -0,0 +1,28 @@ +general_cfg: + algo_name: DQN_CNN + device: cpu + env_name: Breakout-v4 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_Breakout-v4_DQN_CNN_20221207-214917 + max_steps: 500 + mode: train + n_workers: 4 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 500 + wrapper: envs.wrappers.MarioWrappers +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 diff --git a/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/logs/log.txt b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..75f02f7cce9f8d86afc0c0ac0d6acd7be74c47b4 --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/logs/log.txt @@ -0,0 +1,39 @@ +2023-03-20 08:48:01 - r - INFO: - Hyperparameters: +2023-03-20 08:48:01 - r - INFO: - ================================================================================ +2023-03-20 08:48:01 - r - INFO: - Name Value Type +2023-03-20 08:48:01 - r - INFO: - env_name Breakout-v4 +2023-03-20 08:48:01 - r - INFO: - new_step_api 1 +2023-03-20 08:48:01 - r - INFO: - wrapper envs.wrappers.MarioWrappers +2023-03-20 08:48:01 - r - INFO: - render 0 +2023-03-20 08:48:01 - r - INFO: - algo_name DQN_CNN +2023-03-20 08:48:01 - r - INFO: - mode train +2023-03-20 08:48:01 - r - INFO: - seed 1 +2023-03-20 08:48:01 - r - INFO: - device cpu +2023-03-20 08:48:01 - r - INFO: - train_eps 500 +2023-03-20 08:48:01 - r - INFO: - test_eps 10 +2023-03-20 08:48:01 - r - INFO: - eval_eps 10 +2023-03-20 08:48:01 - r - INFO: - eval_per_episode 5 +2023-03-20 08:48:01 - r - INFO: - max_steps 500 +2023-03-20 08:48:01 - r - INFO: - load_checkpoint 0 +2023-03-20 08:48:01 - r - INFO: - load_path Train_Breakout-v4_DQN_CNN_20221207-214917 +2023-03-20 08:48:01 - r - INFO: - show_fig 0 +2023-03-20 08:48:01 - r - INFO: - save_fig 1 +2023-03-20 08:48:01 - r - INFO: - n_workers 4 +2023-03-20 08:48:01 - r - INFO: - epsilon_start 0.95 +2023-03-20 08:48:01 - r - INFO: - epsilon_end 0.01 +2023-03-20 08:48:01 - r - INFO: - epsilon_decay 500 +2023-03-20 08:48:01 - r - INFO: - gamma 0.99 +2023-03-20 08:48:01 - r - INFO: - lr 0.0001 +2023-03-20 08:48:01 - r - INFO: - buffer_size 100000 +2023-03-20 08:48:01 - r - INFO: - batch_size 64 +2023-03-20 08:48:01 - r - INFO: - target_update 4 +2023-03-20 08:48:01 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_Breakout-v4_DQN_CNN_20230320-084801 +2023-03-20 08:48:01 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_Breakout-v4_DQN_CNN_20230320-084801/results +2023-03-20 08:48:01 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_Breakout-v4_DQN_CNN_20230320-084801/logs +2023-03-20 08:48:01 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_Breakout-v4_DQN_CNN_20230320-084801/traj +2023-03-20 08:48:01 - r - INFO: - tb_dir /media/disk/gsc/joyrl-offline/tasks/Train_Breakout-v4_DQN_CNN_20230320-084801/tb_logs +2023-03-20 08:48:01 - r - INFO: - ================================================================================ +2023-03-20 08:48:02 - r - INFO: - n_states: 4, n_actions: 4 +2023-03-20 08:48:02 - r - INFO: - Start training! +2023-03-20 08:48:02 - r - INFO: - Env: Breakout-v4, Algorithm: DQN_CNN, Device: cpu +2023-03-20 12:32:55 - r - INFO: - Finish training! diff --git a/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/models/checkpoint.pt b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..0162aea1bfaacfd68912e0e9037e59f37994c172 --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec74cc055c2f33ef16473f9aee7b4ec9e1229fcffc4a1be7ba4668bf97e56ad +size 6747591 diff --git a/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/results/learning_curve.png b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..339d484621e8229cc28de9d3cd2df3afefd0f383 Binary files /dev/null and b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/results/learning_curve.png differ diff --git a/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/results/res.csv b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..b0e3ee1e9fd2fe385a05fc233b96ebd27b28ac81 --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/results/res.csv @@ -0,0 +1,502 @@ +episodes,rewards +0,2.0 +1,1.0 +2,2.0 +3,0.0 +4,1.0 +5,0.0 +6,0.0 +7,1.0 +8,4.0 +9,2.0 +10,4.0 +11,1.0 +12,1.0 +13,3.0 +14,1.0 +15,1.0 +16,1.0 +17,5.0 +18,2.0 +19,2.0 +20,1.0 +21,2.0 +22,2.0 +23,0.0 +24,1.0 +25,0.0 +26,1.0 +27,2.0 +28,2.0 +29,2.0 +30,0.0 +31,0.0 +32,2.0 +33,1.0 +34,2.0 +35,1.0 +36,5.0 +37,0.0 +38,2.0 +39,1.0 +40,1.0 +41,1.0 +42,2.0 +43,2.0 +44,1.0 +45,1.0 +46,2.0 +47,2.0 +48,0.0 +49,2.0 +50,2.0 +51,1.0 +52,3.0 +53,3.0 +54,0.0 +55,1.0 +56,0.0 +57,1.0 +58,2.0 +59,0.0 +60,1.0 +61,0.0 +62,1.0 +63,1.0 +64,2.0 +65,0.0 +66,3.0 +67,0.0 +68,2.0 +69,1.0 +70,5.0 +71,2.0 +72,4.0 +73,3.0 +74,0.0 +75,1.0 +76,0.0 +77,0.0 +78,2.0 +79,2.0 +80,2.0 +81,4.0 +82,0.0 +83,1.0 +84,1.0 +85,3.0 +86,1.0 +87,2.0 +88,3.0 +89,1.0 +90,1.0 +91,1.0 +92,1.0 +93,2.0 +94,0.0 +95,1.0 +96,4.0 +97,0.0 +98,2.0 +99,0.0 +100,4.0 +101,1.0 +102,1.0 +103,0.0 +104,1.0 +105,1.0 +106,2.0 +107,0.0 +108,1.0 +109,2.0 +110,1.0 +111,1.0 +112,2.0 +113,1.0 +114,3.0 +115,2.0 +116,2.0 +117,0.0 +118,1.0 +119,0.0 +120,2.0 +121,0.0 +122,4.0 +123,0.0 +124,4.0 +125,3.0 +126,4.0 +127,0.0 +128,5.0 +129,1.0 +130,0.0 +131,2.0 +132,0.0 +133,0.0 +134,2.0 +135,0.0 +136,1.0 +137,2.0 +138,3.0 +139,4.0 +140,2.0 +141,3.0 +142,0.0 +143,2.0 +144,2.0 +145,2.0 +146,3.0 +147,1.0 +148,2.0 +149,1.0 +150,2.0 +151,0.0 +152,2.0 +153,4.0 +154,1.0 +155,4.0 +156,5.0 +157,1.0 +158,5.0 +159,0.0 +160,4.0 +161,3.0 +162,3.0 +163,3.0 +164,3.0 +165,4.0 +166,4.0 +167,3.0 +168,3.0 +169,2.0 +170,5.0 +171,2.0 +172,2.0 +173,4.0 +174,3.0 +175,1.0 +176,3.0 +177,4.0 +178,2.0 +179,3.0 +180,5.0 +181,2.0 +182,1.0 +183,3.0 +184,1.0 +185,2.0 +186,4.0 +187,3.0 +188,2.0 +189,4.0 +190,3.0 +191,3.0 +192,1.0 +193,3.0 +194,3.0 +195,4.0 +196,4.0 +197,3.0 +198,1.0 +199,4.0 +200,5.0 +201,2.0 +202,4.0 +203,3.0 +204,1.0 +205,6.0 +206,5.0 +207,3.0 +208,5.0 +209,3.0 +210,5.0 +211,1.0 +212,3.0 +213,1.0 +214,1.0 +215,5.0 +216,5.0 +217,3.0 +218,4.0 +219,5.0 +220,4.0 +221,4.0 +222,4.0 +223,3.0 +224,3.0 +225,4.0 +226,3.0 +227,5.0 +228,5.0 +229,4.0 +230,1.0 +231,4.0 +232,3.0 +233,3.0 +234,5.0 +235,4.0 +236,3.0 +237,2.0 +238,3.0 +239,5.0 +240,5.0 +241,3.0 +242,5.0 +243,2.0 +244,4.0 +245,4.0 +246,4.0 +247,6.0 +248,4.0 +249,7.0 +250,4.0 +251,4.0 +252,4.0 +253,3.0 +254,1.0 +255,2.0 +256,2.0 +257,4.0 +258,2.0 +259,6.0 +260,6.0 +261,2.0 +262,5.0 +263,6.0 +264,2.0 +265,4.0 +266,7.0 +267,2.0 +268,6.0 +269,5.0 +270,7.0 +271,5.0 +272,2.0 +273,3.0 +274,3.0 +275,3.0 +276,5.0 +277,2.0 +278,3.0 +279,3.0 +280,3.0 +281,4.0 +282,6.0 +283,5.0 +284,6.0 +285,3.0 +286,3.0 +287,8.0 +288,2.0 +289,3.0 +290,6.0 +291,7.0 +292,7.0 +293,7.0 +294,1.0 +295,3.0 +296,3.0 +297,8.0 +298,5.0 +299,3.0 +300,3.0 +301,4.0 +302,4.0 +303,3.0 +304,6.0 +305,7.0 +306,8.0 +307,4.0 +308,6.0 +309,5.0 +310,6.0 +311,3.0 +312,7.0 +313,5.0 +314,3.0 +315,5.0 +316,5.0 +317,3.0 +318,6.0 +319,5.0 +320,3.0 +321,6.0 +322,4.0 +323,6.0 +324,3.0 +325,3.0 +326,4.0 +327,6.0 +328,5.0 +329,3.0 +330,5.0 +331,5.0 +332,2.0 +333,5.0 +334,6.0 +335,4.0 +336,2.0 +337,3.0 +338,5.0 +339,6.0 +340,4.0 +341,6.0 +342,4.0 +343,7.0 +344,4.0 +345,5.0 +346,2.0 +347,5.0 +348,7.0 +349,3.0 +350,4.0 +351,4.0 +352,20.0 +353,6.0 +354,12.0 +355,9.0 +356,9.0 +357,6.0 +358,3.0 +359,5.0 +360,3.0 +361,11.0 +362,6.0 +363,6.0 +364,6.0 +365,6.0 +366,6.0 +367,4.0 +368,10.0 +369,8.0 +370,2.0 +371,6.0 +372,5.0 +373,11.0 +374,4.0 +375,8.0 +376,10.0 +377,7.0 +378,7.0 +379,8.0 +380,10.0 +381,8.0 +382,7.0 +383,9.0 +384,8.0 +385,11.0 +386,4.0 +387,5.0 +388,6.0 +389,7.0 +390,6.0 +391,8.0 +392,8.0 +393,7.0 +394,6.0 +395,10.0 +396,8.0 +397,6.0 +398,7.0 +399,8.0 +400,9.0 +401,7.0 +402,7.0 +403,7.0 +404,6.0 +405,5.0 +406,21.0 +407,7.0 +408,6.0 +409,9.0 +410,14.0 +411,8.0 +412,9.0 +413,10.0 +414,2.0 +415,9.0 +416,8.0 +417,8.0 +418,6.0 +419,10.0 +420,8.0 +421,8.0 +422,7.0 +423,7.0 +424,4.0 +425,8.0 +426,7.0 +427,7.0 +428,5.0 +429,9.0 +430,3.0 +431,9.0 +432,7.0 +433,7.0 +434,9.0 +435,8.0 +436,7.0 +437,7.0 +438,5.0 +439,12.0 +440,10.0 +441,7.0 +442,12.0 +443,6.0 +444,8.0 +445,10.0 +446,15.0 +447,11.0 +448,5.0 +449,3.0 +450,6.0 +451,12.0 +452,13.0 +453,10.0 +454,16.0 +455,8.0 +456,18.0 +457,7.0 +458,7.0 +459,6.0 +460,16.0 +461,6.0 +462,9.0 +463,5.0 +464,6.0 +465,6.0 +466,9.0 +467,10.0 +468,5.0 +469,8.0 +470,12.0 +471,9.0 +472,10.0 +473,9.0 +474,16.0 +475,5.0 +476,6.0 +477,4.0 +478,5.0 +479,8.0 +480,9.0 +481,10.0 +482,6.0 +483,10.0 +484,6.0 +485,8.0 +486,10.0 +487,3.0 +488,8.0 +489,7.0 +490,7.0 +491,7.0 +492,10.0 +493,6.0 +494,3.0 +495,17.0 +496,8.0 +497,4.0 +498,12.0 +499,12.0 +500,5.0 diff --git a/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/tb_logs/events.out.tfevents.1679273281.ML3090.295914.0 b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/tb_logs/events.out.tfevents.1679273281.ML3090.295914.0 new file mode 100644 index 0000000000000000000000000000000000000000..34b2c7a986dd849cbd2cf0625d9613e95f10b131 --- /dev/null +++ b/Breakout-v4/Train_Breakout-v4_mp_DQN_CNN_20230320-084801/tb_logs/events.out.tfevents.1679273281.ML3090.295914.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b8b2e0244b99670b7b034269f35a6d1900062180ff45e0d94a6becdcbf51fb8 +size 40 diff --git a/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/config.yaml b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4dee7d15fffdec8e9621408b007abb66d773477c --- /dev/null +++ b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/config.yaml @@ -0,0 +1,31 @@ +general_cfg: + algo_name: PPO + collect_eps: 100 + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_PPO_20221206-171046 + max_steps: 200 + min_reward: 195 + mode: collect + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.0003 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + update_freq: 100 diff --git a/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/logs/log.txt b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..de02f0bb214de9212e03344a071101c2c7dcfd4d --- /dev/null +++ b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/logs/log.txt @@ -0,0 +1,138 @@ +2022-12-06 17:32:22 - r - INFO: - Hyperparameters: +2022-12-06 17:32:22 - r - INFO: - ================================================================================ +2022-12-06 17:32:22 - r - INFO: - Name Value Type +2022-12-06 17:32:22 - r - INFO: - env_name CartPole-v1 +2022-12-06 17:32:22 - r - INFO: - new_step_api 1 +2022-12-06 17:32:22 - r - INFO: - wrapper None +2022-12-06 17:32:22 - r - INFO: - render 0 +2022-12-06 17:32:22 - r - INFO: - algo_name PPO +2022-12-06 17:32:22 - r - INFO: - mode collect +2022-12-06 17:32:22 - r - INFO: - seed 1 +2022-12-06 17:32:22 - r - INFO: - device cuda +2022-12-06 17:32:22 - r - INFO: - train_eps 200 +2022-12-06 17:32:22 - r - INFO: - test_eps 10 +2022-12-06 17:32:22 - r - INFO: - eval_eps 10 +2022-12-06 17:32:22 - r - INFO: - eval_per_episode 5 +2022-12-06 17:32:22 - r - INFO: - max_steps 200 +2022-12-06 17:32:22 - r - INFO: - load_checkpoint 1 +2022-12-06 17:32:22 - r - INFO: - load_path Train_CartPole-v1_PPO_20221206-171046 +2022-12-06 17:32:22 - r - INFO: - show_fig 0 +2022-12-06 17:32:22 - r - INFO: - save_fig 1 +2022-12-06 17:32:22 - r - INFO: - collect_eps 100 +2022-12-06 17:32:22 - r - INFO: - min_reward 195 +2022-12-06 17:32:22 - r - INFO: - continuous 0 +2022-12-06 17:32:22 - r - INFO: - gamma 0.99 +2022-12-06 17:32:22 - r - INFO: - k_epochs 4 +2022-12-06 17:32:22 - r - INFO: - actor_lr 0.0003 +2022-12-06 17:32:22 - r - INFO: - critic_lr 0.0003 +2022-12-06 17:32:22 - r - INFO: - eps_clip 0.2 +2022-12-06 17:32:22 - r - INFO: - entropy_coef 0.01 +2022-12-06 17:32:22 - r - INFO: - update_freq 100 +2022-12-06 17:32:22 - r - INFO: - actor_hidden_dim 256 +2022-12-06 17:32:22 - r - INFO: - critic_hidden_dim 256 +2022-12-06 17:32:22 - r - INFO: - ================================================================================ +2022-12-06 17:32:23 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-06 17:32:24 - r - INFO: - Start collecting! +2022-12-06 17:32:24 - r - INFO: - Env: CartPole-v1, Algorithm: PPO, Device: cuda +2022-12-06 17:32:35 - r - INFO: - trajectories 1 collected, reward 199.0 +2022-12-06 17:32:53 - r - INFO: - trajectories 2 collected, reward 200.0 +2022-12-06 17:33:06 - r - INFO: - trajectories 3 collected, reward 200.0 +2022-12-06 17:33:36 - r - INFO: - trajectories 4 collected, reward 197.0 +2022-12-06 17:34:15 - r - INFO: - trajectories 5 collected, reward 196.0 +2022-12-06 17:34:29 - r - INFO: - trajectories 6 collected, reward 200.0 +2022-12-06 17:34:36 - r - INFO: - trajectories 7 collected, reward 200.0 +2022-12-06 17:34:54 - r - INFO: - trajectories 8 collected, reward 198.0 +2022-12-06 17:35:42 - r - INFO: - trajectories 9 collected, reward 200.0 +2022-12-06 17:35:56 - r - INFO: - trajectories 10 collected, reward 197.0 +2022-12-06 17:36:04 - r - INFO: - trajectories 11 collected, reward 197.0 +2022-12-06 17:36:10 - r - INFO: - trajectories 12 collected, reward 200.0 +2022-12-06 17:36:17 - r - INFO: - trajectories 13 collected, reward 196.0 +2022-12-06 17:37:15 - r - INFO: - trajectories 14 collected, reward 196.0 +2022-12-06 17:38:00 - r - INFO: - trajectories 15 collected, reward 199.0 +2022-12-06 17:38:06 - r - INFO: - trajectories 16 collected, reward 197.0 +2022-12-06 17:38:22 - r - INFO: - trajectories 17 collected, reward 196.0 +2022-12-06 17:38:41 - r - INFO: - trajectories 18 collected, reward 198.0 +2022-12-06 17:39:07 - r - INFO: - trajectories 19 collected, reward 200.0 +2022-12-06 17:39:29 - r - INFO: - trajectories 20 collected, reward 197.0 +2022-12-06 17:39:38 - r - INFO: - trajectories 21 collected, reward 200.0 +2022-12-06 17:40:10 - r - INFO: - trajectories 22 collected, reward 198.0 +2022-12-06 17:40:20 - r - INFO: - trajectories 23 collected, reward 195.0 +2022-12-06 17:40:35 - r - INFO: - trajectories 24 collected, reward 199.0 +2022-12-06 17:40:53 - r - INFO: - trajectories 25 collected, reward 200.0 +2022-12-06 17:41:00 - r - INFO: - trajectories 26 collected, reward 198.0 +2022-12-06 17:41:53 - r - INFO: - trajectories 27 collected, reward 199.0 +2022-12-06 17:41:57 - r - INFO: - trajectories 28 collected, reward 200.0 +2022-12-06 17:42:46 - r - INFO: - trajectories 29 collected, reward 196.0 +2022-12-06 17:43:18 - r - INFO: - trajectories 30 collected, reward 198.0 +2022-12-06 17:43:32 - r - INFO: - trajectories 31 collected, reward 200.0 +2022-12-06 17:43:52 - r - INFO: - trajectories 32 collected, reward 198.0 +2022-12-06 17:44:21 - r - INFO: - trajectories 33 collected, reward 200.0 +2022-12-06 17:44:26 - r - INFO: - trajectories 34 collected, reward 199.0 +2022-12-06 17:44:47 - r - INFO: - trajectories 35 collected, reward 199.0 +2022-12-06 17:45:12 - r - INFO: - trajectories 36 collected, reward 196.0 +2022-12-06 17:45:23 - r - INFO: - trajectories 37 collected, reward 200.0 +2022-12-06 17:45:26 - r - INFO: - trajectories 38 collected, reward 196.0 +2022-12-06 17:45:39 - r - INFO: - trajectories 39 collected, reward 195.0 +2022-12-06 17:45:42 - r - INFO: - trajectories 40 collected, reward 196.0 +2022-12-06 17:45:48 - r - INFO: - trajectories 41 collected, reward 197.0 +2022-12-06 17:45:59 - r - INFO: - trajectories 42 collected, reward 200.0 +2022-12-06 17:46:20 - r - INFO: - trajectories 43 collected, reward 197.0 +2022-12-06 17:46:22 - r - INFO: - trajectories 44 collected, reward 198.0 +2022-12-06 17:46:25 - r - INFO: - trajectories 45 collected, reward 200.0 +2022-12-06 17:46:59 - r - INFO: - trajectories 46 collected, reward 197.0 +2022-12-06 17:47:17 - r - INFO: - trajectories 47 collected, reward 199.0 +2022-12-06 17:47:23 - r - INFO: - trajectories 48 collected, reward 198.0 +2022-12-06 17:47:30 - r - INFO: - trajectories 49 collected, reward 195.0 +2022-12-06 17:47:47 - r - INFO: - trajectories 50 collected, reward 198.0 +2022-12-06 17:47:59 - r - INFO: - trajectories 51 collected, reward 195.0 +2022-12-06 17:48:02 - r - INFO: - trajectories 52 collected, reward 196.0 +2022-12-06 17:48:16 - r - INFO: - trajectories 53 collected, reward 200.0 +2022-12-06 17:48:50 - r - INFO: - trajectories 54 collected, reward 198.0 +2022-12-06 17:48:55 - r - INFO: - trajectories 55 collected, reward 199.0 +2022-12-06 17:49:35 - r - INFO: - trajectories 56 collected, reward 195.0 +2022-12-06 17:49:46 - r - INFO: - trajectories 57 collected, reward 200.0 +2022-12-06 17:50:07 - r - INFO: - trajectories 58 collected, reward 198.0 +2022-12-06 17:50:50 - r - INFO: - trajectories 59 collected, reward 195.0 +2022-12-06 17:50:55 - r - INFO: - trajectories 60 collected, reward 198.0 +2022-12-06 17:51:43 - r - INFO: - trajectories 61 collected, reward 196.0 +2022-12-06 17:52:22 - r - INFO: - trajectories 62 collected, reward 200.0 +2022-12-06 17:52:24 - r - INFO: - trajectories 63 collected, reward 196.0 +2022-12-06 17:52:24 - r - INFO: - trajectories 64 collected, reward 195.0 +2022-12-06 17:52:39 - r - INFO: - trajectories 65 collected, reward 200.0 +2022-12-06 17:53:54 - r - INFO: - trajectories 66 collected, reward 200.0 +2022-12-06 17:53:58 - r - INFO: - trajectories 67 collected, reward 198.0 +2022-12-06 17:54:16 - r - INFO: - trajectories 68 collected, reward 195.0 +2022-12-06 17:55:00 - r - INFO: - trajectories 69 collected, reward 199.0 +2022-12-06 17:55:02 - r - INFO: - trajectories 70 collected, reward 199.0 +2022-12-06 17:55:42 - r - INFO: - trajectories 71 collected, reward 196.0 +2022-12-06 17:56:12 - r - INFO: - trajectories 72 collected, reward 199.0 +2022-12-06 17:57:33 - r - INFO: - trajectories 73 collected, reward 199.0 +2022-12-06 17:57:44 - r - INFO: - trajectories 74 collected, reward 196.0 +2022-12-06 17:58:08 - r - INFO: - trajectories 75 collected, reward 196.0 +2022-12-06 17:58:14 - r - INFO: - trajectories 76 collected, reward 198.0 +2022-12-06 17:58:24 - r - INFO: - trajectories 77 collected, reward 196.0 +2022-12-06 17:59:53 - r - INFO: - trajectories 78 collected, reward 200.0 +2022-12-06 18:00:34 - r - INFO: - trajectories 79 collected, reward 199.0 +2022-12-06 18:00:52 - r - INFO: - trajectories 80 collected, reward 199.0 +2022-12-06 18:01:03 - r - INFO: - trajectories 81 collected, reward 199.0 +2022-12-06 18:01:05 - r - INFO: - trajectories 82 collected, reward 195.0 +2022-12-06 18:01:13 - r - INFO: - trajectories 83 collected, reward 197.0 +2022-12-06 18:01:26 - r - INFO: - trajectories 84 collected, reward 196.0 +2022-12-06 18:02:51 - r - INFO: - trajectories 85 collected, reward 200.0 +2022-12-06 18:03:01 - r - INFO: - trajectories 86 collected, reward 195.0 +2022-12-06 18:03:02 - r - INFO: - trajectories 87 collected, reward 196.0 +2022-12-06 18:03:46 - r - INFO: - trajectories 88 collected, reward 195.0 +2022-12-06 18:05:09 - r - INFO: - trajectories 89 collected, reward 200.0 +2022-12-06 18:05:26 - r - INFO: - trajectories 90 collected, reward 198.0 +2022-12-06 18:06:06 - r - INFO: - trajectories 91 collected, reward 195.0 +2022-12-06 18:07:49 - r - INFO: - trajectories 92 collected, reward 199.0 +2022-12-06 18:07:55 - r - INFO: - trajectories 93 collected, reward 195.0 +2022-12-06 18:08:36 - r - INFO: - trajectories 94 collected, reward 195.0 +2022-12-06 18:08:37 - r - INFO: - trajectories 95 collected, reward 199.0 +2022-12-06 18:09:36 - r - INFO: - trajectories 96 collected, reward 200.0 +2022-12-06 18:10:16 - r - INFO: - trajectories 97 collected, reward 198.0 +2022-12-06 18:10:16 - r - INFO: - trajectories 98 collected, reward 198.0 +2022-12-06 18:10:17 - r - INFO: - trajectories 99 collected, reward 195.0 +2022-12-06 18:11:23 - r - INFO: - trajectories 100 collected, reward 197.0 +2022-12-06 18:11:23 - r - INFO: - trajectories saved to F:\rl-tutorials\joyrl/tasks/Collect_CartPole-v1_PPO_20221206-173222/traj/ +2022-12-06 18:11:23 - r - INFO: - Finish collecting! diff --git a/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/results/learning_curve.png b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..8fb3ed5d1c7fb489caf842ed99ef266236c5b6c6 Binary files /dev/null and b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/results/learning_curve.png differ diff --git a/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/results/res.csv b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..ed27e810ca337eda83352cea49c7577f36c39668 --- /dev/null +++ b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,199.0,200 +1,200.0,200 +2,200.0,200 +3,197.0,200 +4,196.0,200 +5,200.0,200 +6,200.0,200 +7,198.0,200 +8,200.0,200 +9,197.0,200 +10,197.0,200 +11,200.0,200 +12,196.0,200 +13,196.0,200 +14,199.0,200 +15,197.0,200 +16,196.0,200 +17,198.0,200 +18,200.0,200 +19,197.0,200 +20,200.0,200 +21,198.0,200 +22,195.0,200 +23,199.0,200 +24,200.0,200 +25,198.0,200 +26,199.0,200 +27,200.0,200 +28,196.0,200 +29,198.0,200 +30,200.0,200 +31,198.0,200 +32,200.0,200 +33,199.0,200 +34,199.0,200 +35,196.0,200 +36,200.0,200 +37,196.0,200 +38,195.0,200 +39,196.0,200 +40,197.0,200 +41,200.0,200 +42,197.0,200 +43,198.0,200 +44,200.0,200 +45,197.0,200 +46,199.0,200 +47,198.0,200 +48,195.0,200 +49,198.0,200 +50,195.0,200 +51,196.0,200 +52,200.0,200 +53,198.0,200 +54,199.0,200 +55,195.0,200 +56,200.0,200 +57,198.0,200 +58,195.0,200 +59,198.0,200 +60,196.0,200 +61,200.0,200 +62,196.0,200 +63,195.0,200 +64,200.0,200 +65,200.0,200 +66,198.0,200 +67,195.0,200 +68,199.0,200 +69,199.0,200 +70,196.0,200 +71,199.0,200 +72,199.0,200 +73,196.0,200 +74,196.0,200 +75,198.0,200 +76,196.0,200 +77,200.0,200 +78,199.0,200 +79,199.0,200 +80,199.0,200 +81,195.0,200 +82,197.0,200 +83,196.0,200 +84,200.0,200 +85,195.0,200 +86,196.0,200 +87,195.0,200 +88,200.0,200 +89,198.0,200 +90,195.0,200 +91,199.0,200 +92,195.0,200 +93,195.0,200 +94,199.0,200 +95,200.0,200 +96,198.0,200 +97,198.0,200 +98,195.0,200 +99,197.0,200 diff --git a/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/traj/traj.pkl b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/traj/traj.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7dc966f1909f210bd059ed3b560336be815f47dc --- /dev/null +++ b/CartPole-v1/Collect_CartPole-v1_PPO_20221206-173222/traj/traj.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a026d6558242e05988870823ae6c1a5c053fa2bff69866d985cfa4791ddce2b +size 1621269 diff --git a/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/config.yaml b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..315674c28834201face9f30b34793820bd550fdf --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: A2C + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_A2C_20221204-003439 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 1000 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.001 + gamma: 0.99 + update_freq: 20 diff --git a/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/logs/log.txt b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..934d32444f5b9dde426fb0051e84b8239cffba12 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/logs/log.txt @@ -0,0 +1,43 @@ +2022-12-04 00:36:59 - r - INFO: - Hyperparameters: +2022-12-04 00:36:59 - r - INFO: - ================================================================================ +2022-12-04 00:36:59 - r - INFO: - Name Value Type +2022-12-04 00:36:59 - r - INFO: - env_name CartPole-v1 +2022-12-04 00:36:59 - r - INFO: - new_step_api 1 +2022-12-04 00:36:59 - r - INFO: - wrapper None +2022-12-04 00:36:59 - r - INFO: - render 0 +2022-12-04 00:36:59 - r - INFO: - algo_name A2C +2022-12-04 00:36:59 - r - INFO: - mode test +2022-12-04 00:36:59 - r - INFO: - seed 1 +2022-12-04 00:36:59 - r - INFO: - device cpu +2022-12-04 00:36:59 - r - INFO: - train_eps 1000 +2022-12-04 00:36:59 - r - INFO: - test_eps 10 +2022-12-04 00:36:59 - r - INFO: - eval_eps 10 +2022-12-04 00:36:59 - r - INFO: - eval_per_episode 5 +2022-12-04 00:36:59 - r - INFO: - max_steps 200 +2022-12-04 00:36:59 - r - INFO: - load_checkpoint 1 +2022-12-04 00:36:59 - r - INFO: - load_path Train_CartPole-v1_A2C_20221204-003439 +2022-12-04 00:36:59 - r - INFO: - show_fig 0 +2022-12-04 00:36:59 - r - INFO: - save_fig 1 +2022-12-04 00:36:59 - r - INFO: - continuous 0 +2022-12-04 00:36:59 - r - INFO: - gamma 0.99 +2022-12-04 00:36:59 - r - INFO: - actor_lr 0.0003 +2022-12-04 00:36:59 - r - INFO: - critic_lr 0.001 +2022-12-04 00:36:59 - r - INFO: - actor_hidden_dim 256 +2022-12-04 00:36:59 - r - INFO: - critic_hidden_dim 256 +2022-12-04 00:36:59 - r - INFO: - entropy_coef 0.001 +2022-12-04 00:36:59 - r - INFO: - update_freq 20 +2022-12-04 00:36:59 - r - INFO: - ================================================================================ +2022-12-04 00:37:00 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-04 00:37:00 - r - INFO: - Start testing! +2022-12-04 00:37:00 - r - INFO: - Env: CartPole-v1, Algorithm: A2C, Device: cpu +2022-12-04 00:37:00 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2022-12-04 00:37:00 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/models/actor_checkpoint.pt b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4015ed746ab697417d75bf10b5945dd77cc8e6d2 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3e59a01cece98a09e8126a81cb4ba69f7d316bee4b6d6768a092489da7f5e2e +size 272151 diff --git a/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/models/critic_checkpoint.pt b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/models/critic_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d2d92943121e4cfb4e72259681f4a8587a8e20b --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/models/critic_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4dcb18c48518129aba10d1a34954e57ad6a73800b872800b93d920912aeaf05 +size 271127 diff --git a/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..fa56ad33858c18d841f06a2f72b4141fddbdd44f Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/results/res.csv b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A2C_20221204-003659/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/config.yaml b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be96119401e4d57adf15bec4389299684cde0a40 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/config.yaml @@ -0,0 +1,28 @@ +general_cfg: + algo_name: A3C + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_A3C_20230111-132542 + max_steps: 200 + mode: test + n_workers: 1 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.001 + gamma: 0.99 + update_freq: 20 diff --git a/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/logs/log.txt b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..0ce5d88fb5a14d07b8315f1c08e78e557b216a6b --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/logs/log.txt @@ -0,0 +1,49 @@ +2023-01-11 13:29:29 - r - INFO: - Hyperparameters: +2023-01-11 13:29:29 - r - INFO: - ================================================================================ +2023-01-11 13:29:29 - r - INFO: - Name Value Type +2023-01-11 13:29:29 - r - INFO: - env_name CartPole-v1 +2023-01-11 13:29:29 - r - INFO: - new_step_api 1 +2023-01-11 13:29:29 - r - INFO: - wrapper None +2023-01-11 13:29:29 - r - INFO: - render 0 +2023-01-11 13:29:29 - r - INFO: - algo_name A3C +2023-01-11 13:29:29 - r - INFO: - mode test +2023-01-11 13:29:29 - r - INFO: - seed 1 +2023-01-11 13:29:29 - r - INFO: - device cpu +2023-01-11 13:29:29 - r - INFO: - train_eps 400 +2023-01-11 13:29:29 - r - INFO: - test_eps 10 +2023-01-11 13:29:29 - r - INFO: - eval_eps 10 +2023-01-11 13:29:29 - r - INFO: - eval_per_episode 5 +2023-01-11 13:29:29 - r - INFO: - max_steps 200 +2023-01-11 13:29:29 - r - INFO: - load_checkpoint 1 +2023-01-11 13:29:29 - r - INFO: - load_path Train_CartPole-v1_A3C_20230111-132542 +2023-01-11 13:29:29 - r - INFO: - show_fig 0 +2023-01-11 13:29:29 - r - INFO: - save_fig 1 +2023-01-11 13:29:29 - r - INFO: - n_workers 1 +2023-01-11 13:29:29 - r - INFO: - gamma 0.99 +2023-01-11 13:29:29 - r - INFO: - actor_lr 0.0003 +2023-01-11 13:29:29 - r - INFO: - critic_lr 0.001 +2023-01-11 13:29:29 - r - INFO: - actor_hidden_dim 256 +2023-01-11 13:29:29 - r - INFO: - critic_hidden_dim 256 +2023-01-11 13:29:29 - r - INFO: - entropy_coef 0.001 +2023-01-11 13:29:29 - r - INFO: - update_freq 20 +2023-01-11 13:29:29 - r - INFO: - continuous 0 +2023-01-11 13:29:29 - r - INFO: - task_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Test_CartPole-v1_A3C_20230111-132929 +2023-01-11 13:29:29 - r - INFO: - model_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Test_CartPole-v1_A3C_20230111-132929/models +2023-01-11 13:29:29 - r - INFO: - res_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Test_CartPole-v1_A3C_20230111-132929/results +2023-01-11 13:29:29 - r - INFO: - log_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Test_CartPole-v1_A3C_20230111-132929/logs +2023-01-11 13:29:29 - r - INFO: - traj_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Test_CartPole-v1_A3C_20230111-132929/traj +2023-01-11 13:29:29 - r - INFO: - ================================================================================ +2023-01-11 13:29:29 - r - INFO: - n_states: 4, n_actions: 2 +2023-01-11 13:29:29 - r - INFO: - Start testing! +2023-01-11 13:29:29 - r - INFO: - Env: CartPole-v1, Algorithm: A3C, Device: cpu +2023-01-11 13:29:29 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-01-11 13:29:29 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/models/actor_checkpoint.pt b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2cb5fed16e3d7e33be76ba1d7a157eaa207a6a8 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39778e5ed902fd1548d304a25877444c0e5dade38d40ccbdc20f879a9f573986 +size 272151 diff --git a/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/models/critic_checkpoint.pt b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/models/critic_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..27c0ec962e1dfef43c90c71993f5187a3e100dec --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/models/critic_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:689822d7abbdfab3b4c0d061b5a2298637277eddb3b8e52469061857439df933 +size 271127 diff --git a/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..4bc5f7e995b4cbf754e439cb9c2c7d819e4682a5 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/results/res.csv b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_A3C_20230111-132929/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/config.yaml b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d4a2c9afbe32e1a9db1658ae23ee4694e2f6bfe --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/config.yaml @@ -0,0 +1,26 @@ +general_cfg: + algo_name: BC + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 1 + load_checkpoint: true + load_path: Train_CartPole-v1_BC_20230319-190431 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 1 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + batch_size: 256 + continuous: false + critic_hidden_dim: 256 + expert_path: tasks/Collect_CartPole-v1_PPO_20230319-170351/traj/traj.pkl + lr: 0.0003 + train_iterations: 500 diff --git a/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/logs/log.txt b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b6c38162ee0572dd130f580c5e768af0d0b6d6c --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/logs/log.txt @@ -0,0 +1,47 @@ +2023-03-19 19:09:43 - r - INFO: - Hyperparameters: +2023-03-19 19:09:43 - r - INFO: - ================================================================================ +2023-03-19 19:09:43 - r - INFO: - Name Value Type +2023-03-19 19:09:43 - r - INFO: - env_name CartPole-v1 +2023-03-19 19:09:43 - r - INFO: - new_step_api 1 +2023-03-19 19:09:43 - r - INFO: - wrapper None +2023-03-19 19:09:43 - r - INFO: - render 0 +2023-03-19 19:09:43 - r - INFO: - algo_name BC +2023-03-19 19:09:43 - r - INFO: - mode test +2023-03-19 19:09:43 - r - INFO: - seed 1 +2023-03-19 19:09:43 - r - INFO: - device cuda +2023-03-19 19:09:43 - r - INFO: - train_eps 1 +2023-03-19 19:09:43 - r - INFO: - test_eps 10 +2023-03-19 19:09:43 - r - INFO: - eval_eps 10 +2023-03-19 19:09:43 - r - INFO: - eval_per_episode 1 +2023-03-19 19:09:43 - r - INFO: - max_steps 200 +2023-03-19 19:09:43 - r - INFO: - load_checkpoint 1 +2023-03-19 19:09:43 - r - INFO: - load_path Train_CartPole-v1_BC_20230319-190431 +2023-03-19 19:09:43 - r - INFO: - show_fig 0 +2023-03-19 19:09:43 - r - INFO: - save_fig 1 +2023-03-19 19:09:43 - r - INFO: - continuous 0 +2023-03-19 19:09:43 - r - INFO: - lr 0.0003 +2023-03-19 19:09:43 - r - INFO: - batch_size 256 +2023-03-19 19:09:43 - r - INFO: - train_iterations 500 +2023-03-19 19:09:43 - r - INFO: - actor_hidden_dim 256 +2023-03-19 19:09:43 - r - INFO: - critic_hidden_dim 256 +2023-03-19 19:09:43 - r - INFO: - expert_path tasks/Collect_CartPole-v1_PPO_20230319-170351/traj/traj.pkl +2023-03-19 19:09:43 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_BC_20230319-190943 +2023-03-19 19:09:43 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_BC_20230319-190943/results +2023-03-19 19:09:43 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_BC_20230319-190943/logs +2023-03-19 19:09:43 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_BC_20230319-190943/traj +2023-03-19 19:09:43 - r - INFO: - tb_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_BC_20230319-190943/tb_logs +2023-03-19 19:09:43 - r - INFO: - ================================================================================ +2023-03-19 19:09:43 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-19 19:09:47 - r - INFO: - Start testing! +2023-03-19 19:09:47 - r - INFO: - Env: CartPole-v1, Algorithm: BC, Device: cuda +2023-03-19 19:09:48 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:48 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:49 - r - INFO: - Episode: 3/10, Reward: 149.000, Step: 149 +2023-03-19 19:09:50 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:50 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:51 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:51 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:52 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:53 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-03-19 19:09:53 - r - INFO: - Episode: 10/10, Reward: 177.000, Step: 177 +2023-03-19 19:09:53 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/models/model.pth b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/models/model.pth new file mode 100644 index 0000000000000000000000000000000000000000..0105809efa8822a61704bd401484457c74db821b --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/models/model.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fb33e7cea0214b2d6baa00a5e2e46ca4d423b1abd6929bb5ec4ad7fda648907 +size 272215 diff --git a/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..c5f2d94462cbb2df8d0f4aa1154d77ec4ca1498d Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/results/res.csv b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..a9ba318adedb0aef5173d12b837427d3d235b6ad --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,149.0,149 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,177.0,177 diff --git a/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/tb_logs/events.out.tfevents.1679224183.dell-Precision-5820-Tower.15407.0 b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/tb_logs/events.out.tfevents.1679224183.dell-Precision-5820-Tower.15407.0 new file mode 100644 index 0000000000000000000000000000000000000000..a3d2f9117a079a10d190d5d65c40458670c11e28 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_BC_20230319-190943/tb_logs/events.out.tfevents.1679224183.dell-Precision-5820-Tower.15407.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f43a14060ef1a1a8d8e88bf08ce8c87d2ca8667e0427be5fb7566a37290f1a95 +size 40 diff --git a/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/config.yaml b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97a4175b1c576224d6783b75ae27d949c9b461e1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/config.yaml @@ -0,0 +1,55 @@ +general_cfg: + Vmax: 200 + Vmin: 0 + algo_name: C51 + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_C51_20230114-222523 + max_steps: 200 + mode: test + n_atoms: 51 + new_step_api: true + render: false + save_fig: true + seed: 0 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + Vmax: 200.0 + Vmin: 0.0 + batch_size: 64 + buffer_size: 100000 + delta_z: 4.0 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + memory_capacity: 10000 + num_atoms: 51 + support: !!python/object/apply:torch._utils._rebuild_tensor_v2 + - !!python/object/apply:torch.storage._load_from_bytes + - !!binary | + gAKKCmz8nEb5IGqoUBkugAJN6QMugAJ9cQAoWBAAAABwcm90b2NvbF92ZXJzaW9ucQFN6QNYDQAA + AGxpdHRsZV9lbmRpYW5xAohYCgAAAHR5cGVfc2l6ZXNxA31xBChYBQAAAHNob3J0cQVLAlgDAAAA + aW50cQZLBFgEAAAAbG9uZ3EHSwR1dS6AAihYBwAAAHN0b3JhZ2VxAGN0b3JjaApGbG9hdFN0b3Jh + Z2UKcQFYCQAAADEyNzIxODM2OHECWAMAAABjcHVxA0szTnRxBFEugAJdcQBYCQAAADEyNzIxODM2 + OHEBYS4zAAAAAAAAAAAAAAAAAIBAAAAAQQAAQEEAAIBBAACgQQAAwEEAAOBBAAAAQgAAEEIAACBC + AAAwQgAAQEIAAFBCAABgQgAAcEIAAIBCAACIQgAAkEIAAJhCAACgQgAAqEIAALBCAAC4QgAAwEIA + AMhCAADQQgAA2EIAAOBCAADoQgAA8EIAAPhCAAAAQwAABEMAAAhDAAAMQwAAEEMAABRDAAAYQwAA + HEMAACBDAAAkQwAAKEMAACxDAAAwQwAANEMAADhDAAA8QwAAQEMAAERDAABIQw== + - 0 + - !!python/tuple + - 51 + - !!python/tuple + - 1 + - false + - !!python/object/apply:collections.OrderedDict + - [] + target_update: 4 + tau: 1.0 diff --git a/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/logs/log.txt b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..68bd583378f84cfd8e22dc7140719fe9eeea0e55 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/logs/log.txt @@ -0,0 +1,56 @@ +2023-01-14 22:27:28 - r - INFO: - Hyperparameters: +2023-01-14 22:27:28 - r - INFO: - ================================================================================ +2023-01-14 22:27:28 - r - INFO: - Name Value Type +2023-01-14 22:27:28 - r - INFO: - env_name CartPole-v1 +2023-01-14 22:27:28 - r - INFO: - new_step_api 1 +2023-01-14 22:27:28 - r - INFO: - wrapper None +2023-01-14 22:27:28 - r - INFO: - render 0 +2023-01-14 22:27:28 - r - INFO: - algo_name C51 +2023-01-14 22:27:28 - r - INFO: - mode test +2023-01-14 22:27:28 - r - INFO: - seed 0 +2023-01-14 22:27:28 - r - INFO: - device cuda +2023-01-14 22:27:28 - r - INFO: - train_eps 100 +2023-01-14 22:27:28 - r - INFO: - test_eps 10 +2023-01-14 22:27:28 - r - INFO: - eval_eps 10 +2023-01-14 22:27:28 - r - INFO: - eval_per_episode 5 +2023-01-14 22:27:28 - r - INFO: - max_steps 200 +2023-01-14 22:27:28 - r - INFO: - load_checkpoint 1 +2023-01-14 22:27:28 - r - INFO: - load_path Train_CartPole-v1_C51_20230114-222523 +2023-01-14 22:27:28 - r - INFO: - show_fig 0 +2023-01-14 22:27:28 - r - INFO: - save_fig 1 +2023-01-14 22:27:28 - r - INFO: - Vmin 0.0 +2023-01-14 22:27:28 - r - INFO: - Vmax 200.0 +2023-01-14 22:27:28 - r - INFO: - n_atoms 51 +2023-01-14 22:27:28 - r - INFO: - gamma 0.95 +2023-01-14 22:27:28 - r - INFO: - tau 1.0 +2023-01-14 22:27:28 - r - INFO: - num_atoms 51 +2023-01-14 22:27:28 - r - INFO: - support tensor(0.) +2023-01-14 22:27:28 - r - INFO: - delta_z 4.0 +2023-01-14 22:27:28 - r - INFO: - batch_size 64 +2023-01-14 22:27:28 - r - INFO: - lr 0.0001 +2023-01-14 22:27:28 - r - INFO: - target_update 4 +2023-01-14 22:27:28 - r - INFO: - memory_capacity 10000 +2023-01-14 22:27:28 - r - INFO: - epsilon_start 0.95 +2023-01-14 22:27:28 - r - INFO: - epsilon_end 0.01 +2023-01-14 22:27:28 - r - INFO: - epsilon_decay 500 +2023-01-14 22:27:28 - r - INFO: - buffer_size 100000 +2023-01-14 22:27:28 - r - INFO: - task_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_C51_20230114-222728 +2023-01-14 22:27:28 - r - INFO: - model_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_C51_20230114-222728/models +2023-01-14 22:27:28 - r - INFO: - res_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_C51_20230114-222728/results +2023-01-14 22:27:28 - r - INFO: - log_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_C51_20230114-222728/logs +2023-01-14 22:27:28 - r - INFO: - traj_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_C51_20230114-222728/traj +2023-01-14 22:27:28 - r - INFO: - ================================================================================ +2023-01-14 22:27:28 - r - INFO: - n_states: 4, n_actions: 2 +2023-01-14 22:27:30 - r - INFO: - Start testing! +2023-01-14 22:27:30 - r - INFO: - Env: CartPole-v1, Algorithm: C51, Device: cuda +2023-01-14 22:27:31 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:31 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:31 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:31 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:31 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:31 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:31 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:31 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:32 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:32 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-01-14 22:27:32 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5eab673c5154d7d54b0737705fe3889a5229e05e --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43646595614f98979d24634093626d35b9fab53c884a101bf17adf50691f5aed +size 123539 diff --git a/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..55785b12f27d95aa2e567da1dc5486d2bca25432 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/results/res.csv b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_C51_20230114-222728/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/config.yaml b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db154891f78b2875adebd43af5a29a8d4fc77e0e --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/config.yaml @@ -0,0 +1,46 @@ +general_cfg: + algo_name: CQL + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_CQL_20230408-183652 + max_steps: 200 + mode: test + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: true + test_eps: 10 + train_eps: 300 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.001 + target_update: 4 + tau: 0.001 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/logs/log.txt b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d5730dfc4eb3e94c9399828ffa418910b68b7891 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/logs/log.txt @@ -0,0 +1,52 @@ +2023-04-09 16:01:45 - r - INFO: - Hyperparameters: +2023-04-09 16:01:45 - r - INFO: - ================================================================================ +2023-04-09 16:01:45 - r - INFO: - Name Value Type +2023-04-09 16:01:45 - r - INFO: - env_name CartPole-v1 +2023-04-09 16:01:45 - r - INFO: - new_step_api 1 +2023-04-09 16:01:45 - r - INFO: - wrapper None +2023-04-09 16:01:45 - r - INFO: - render 0 +2023-04-09 16:01:45 - r - INFO: - render_mode human +2023-04-09 16:01:45 - r - INFO: - algo_name CQL +2023-04-09 16:01:45 - r - INFO: - mode test +2023-04-09 16:01:45 - r - INFO: - mp_backend mp +2023-04-09 16:01:45 - r - INFO: - seed 1 +2023-04-09 16:01:45 - r - INFO: - device cpu +2023-04-09 16:01:45 - r - INFO: - train_eps 300 +2023-04-09 16:01:45 - r - INFO: - test_eps 10 +2023-04-09 16:01:45 - r - INFO: - eval_eps 10 +2023-04-09 16:01:45 - r - INFO: - eval_per_episode 5 +2023-04-09 16:01:45 - r - INFO: - max_steps 200 +2023-04-09 16:01:45 - r - INFO: - load_checkpoint 1 +2023-04-09 16:01:45 - r - INFO: - load_path Train_CartPole-v1_CQL_20230408-183652 +2023-04-09 16:01:45 - r - INFO: - show_fig 1 +2023-04-09 16:01:45 - r - INFO: - save_fig 1 +2023-04-09 16:01:45 - r - INFO: - epsilon_start 0.95 +2023-04-09 16:01:45 - r - INFO: - epsilon_end 0.01 +2023-04-09 16:01:45 - r - INFO: - epsilon_decay 500 +2023-04-09 16:01:45 - r - INFO: - gamma 0.99 +2023-04-09 16:01:45 - r - INFO: - tau 0.001 +2023-04-09 16:01:45 - r - INFO: - lr 0.001 +2023-04-09 16:01:45 - r - INFO: - buffer_size 100000 +2023-04-09 16:01:45 - r - INFO: - batch_size 64 +2023-04-09 16:01:45 - r - INFO: - target_update 4 +2023-04-09 16:01:45 - r - INFO: - value_layers [{'activation': 'relu', 'layer_dim': ['n_states', 256], 'layer_type': 'linear'}, {'activation': 'relu', 'layer_dim': [256, 256], 'layer_type': 'linear'}, {'activation': 'none', 'layer_dim': [256, 'n_actions'], 'layer_type': 'linear'}] +2023-04-09 16:01:45 - r - INFO: - task_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_CartPole-v1_CQL_20230409-160145 +2023-04-09 16:01:45 - r - INFO: - res_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_CartPole-v1_CQL_20230409-160145/results +2023-04-09 16:01:45 - r - INFO: - log_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_CartPole-v1_CQL_20230409-160145/logs +2023-04-09 16:01:45 - r - INFO: - traj_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_CartPole-v1_CQL_20230409-160145/traj +2023-04-09 16:01:45 - r - INFO: - video_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_CartPole-v1_CQL_20230409-160145/videos +2023-04-09 16:01:45 - r - INFO: - ================================================================================ +2023-04-09 16:01:45 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-09 16:01:45 - r - INFO: - Start testing! +2023-04-09 16:01:45 - r - INFO: - Env: CartPole-v1, Algorithm: CQL, Device: cpu +2023-04-09 16:01:45 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:45 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:45 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:45 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:46 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:46 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:46 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:46 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:46 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:46 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-09 16:01:46 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbfa4e2127b4953d376c7070d13c81b5b9748a54 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6891f88ceb4ef1865723645afaae8ba271bcc8735d8809dd45322aeb2ca4a08b +size 272407 diff --git a/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..8f1d474a7c281ee8a172b31fe03b95ed8fa8cec7 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/results/res.csv b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_CQL_20230409-160145/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/config.yaml b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7214f9d653439ce70b5bd2ebb2da8bdbeeb70011 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/config.yaml @@ -0,0 +1,48 @@ +general_cfg: + algo_name: DRQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_DRQN_20230204-223146 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 0 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 0.995 + epsilon_end: 0.001 + epsilon_start: 0.1 + gamma: 0.99 + hidden_dim: 64 + lookup_step: 10 + lr: 0.001 + max_epi_len: 128 + max_epi_num: 100 + min_epi_num: 16 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 64 + layer_type: linear + - activation: relu + layer_dim: + - 64 + - 64 + layer_type: linear + - activation: none + layer_dim: + - 64 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/logs/log.txt b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3170efe60f0ad481ad71001bbdd17560724df255 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/logs/log.txt @@ -0,0 +1,54 @@ +2023-02-04 22:37:10 - r - INFO: - Hyperparameters: +2023-02-04 22:37:10 - r - INFO: - ================================================================================ +2023-02-04 22:37:10 - r - INFO: - Name Value Type +2023-02-04 22:37:10 - r - INFO: - env_name CartPole-v1 +2023-02-04 22:37:10 - r - INFO: - new_step_api 1 +2023-02-04 22:37:10 - r - INFO: - wrapper None +2023-02-04 22:37:10 - r - INFO: - render 0 +2023-02-04 22:37:10 - r - INFO: - algo_name DRQN +2023-02-04 22:37:10 - r - INFO: - mode test +2023-02-04 22:37:10 - r - INFO: - seed 0 +2023-02-04 22:37:10 - r - INFO: - device cuda +2023-02-04 22:37:10 - r - INFO: - train_eps 100 +2023-02-04 22:37:10 - r - INFO: - test_eps 10 +2023-02-04 22:37:10 - r - INFO: - eval_eps 10 +2023-02-04 22:37:10 - r - INFO: - eval_per_episode 5 +2023-02-04 22:37:10 - r - INFO: - max_steps 200 +2023-02-04 22:37:10 - r - INFO: - load_checkpoint 1 +2023-02-04 22:37:10 - r - INFO: - load_path Train_CartPole-v1_DRQN_20230204-223146 +2023-02-04 22:37:10 - r - INFO: - show_fig 0 +2023-02-04 22:37:10 - r - INFO: - save_fig 1 +2023-02-04 22:37:10 - r - INFO: - epsilon_start 0.1 +2023-02-04 22:37:10 - r - INFO: - epsilon_end 0.001 +2023-02-04 22:37:10 - r - INFO: - epsilon_decay 0.995 +2023-02-04 22:37:10 - r - INFO: - hidden_dim 64 +2023-02-04 22:37:10 - r - INFO: - gamma 0.99 +2023-02-04 22:37:10 - r - INFO: - lr 0.001 +2023-02-04 22:37:10 - r - INFO: - buffer_size 100000 +2023-02-04 22:37:10 - r - INFO: - batch_size 64 +2023-02-04 22:37:10 - r - INFO: - target_update 4 +2023-02-04 22:37:10 - r - INFO: - lookup_step 10 +2023-02-04 22:37:10 - r - INFO: - min_epi_num 16 +2023-02-04 22:37:10 - r - INFO: - max_epi_len 128 +2023-02-04 22:37:10 - r - INFO: - max_epi_num 100 +2023-02-04 22:37:10 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 64], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [64, 64], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [64, 'n_actions'], 'activation': 'none'}] +2023-02-04 22:37:10 - r - INFO: - task_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_DRQN_20230204-223710 +2023-02-04 22:37:10 - r - INFO: - model_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_DRQN_20230204-223710/models +2023-02-04 22:37:10 - r - INFO: - res_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_DRQN_20230204-223710/results +2023-02-04 22:37:10 - r - INFO: - log_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_DRQN_20230204-223710/logs +2023-02-04 22:37:10 - r - INFO: - traj_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_DRQN_20230204-223710/traj +2023-02-04 22:37:10 - r - INFO: - ================================================================================ +2023-02-04 22:37:10 - r - INFO: - n_states: 4, n_actions: 2 +2023-02-04 22:37:13 - r - INFO: - Start testing! +2023-02-04 22:37:13 - r - INFO: - Env: CartPole-v1, Algorithm: DRQN, Device: cuda +2023-02-04 22:37:13 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:13 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:14 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:14 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:14 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:14 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:14 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:15 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:15 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:15 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-02-04 22:37:15 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..6679e951c06ee4f3d5e9761e680a0a6a371338d5 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b13d91b609753478dd5b3e6faf4dcaa127e96b8e9d9ebd521f9c9589da41f8c +size 136859 diff --git a/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..47e1110cbb1d6ecc1b7514c960474d970e4bf35d Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/results/res.csv b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DRQN_20230204-223710/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/config.yaml b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6762719835c73d20bafa5a2e0c20cbffcbeb24f --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/config.yaml @@ -0,0 +1,40 @@ +general_cfg: + algo_name: DoubleDQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_DoubleDQN_20221122-125516 + max_steps: 200 + mode: test + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/logs/log.txt b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a40125e02a3b2b63bec1969abf59e09d8177382c --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/logs/log.txt @@ -0,0 +1,14 @@ +2022-11-22 12:56:12 - r - INFO: - n_states: 4, n_actions: 2 +2022-11-22 12:56:14 - r - INFO: - Start testing! +2022-11-22 12:56:14 - r - INFO: - Env: CartPole-v1, Algorithm: DoubleDQN, Device: cuda +2022-11-22 12:56:14 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 5/10, Reward: 138.000, Step: 138 +2022-11-22 12:56:15 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2022-11-22 12:56:15 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/models/checkpoint.pth b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/models/checkpoint.pth new file mode 100644 index 0000000000000000000000000000000000000000..27cfa80035673442926e5e8ae4a75214ae1e6aad --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/models/checkpoint.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c80e643e52ee0e109e55ba083247021287455374ec28f27c4f2705e51fee23 +size 272471 diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..e23e1854c6c6d9577ebf4b54c724d0238d167a37 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/results/res.csv b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..e167003140d6dce2789243c707e2d930452261d8 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_20221122-125611/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,138.0,138 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/config.yaml b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..072d7483adcca10b5dd189b74cf30ae7cc5aa7a2 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/config.yaml @@ -0,0 +1,46 @@ +general_cfg: + algo_name: DoubleDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_DoubleDQN_mp_20230406-160028 + max_steps: 200 + mode: test + mp_backend: mp + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/logs/log.txt b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d7d28ca8cfd26732536dd28d1f4e413731193035 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/logs/log.txt @@ -0,0 +1,52 @@ +2023-04-06 16:04:10 - r - INFO: - Hyperparameters: +2023-04-06 16:04:10 - r - INFO: - ================================================================================ +2023-04-06 16:04:10 - r - INFO: - Name Value Type +2023-04-06 16:04:10 - r - INFO: - env_name CartPole-v1 +2023-04-06 16:04:10 - r - INFO: - new_step_api 1 +2023-04-06 16:04:10 - r - INFO: - wrapper None +2023-04-06 16:04:10 - r - INFO: - render 0 +2023-04-06 16:04:10 - r - INFO: - render_mode human +2023-04-06 16:04:10 - r - INFO: - algo_name DoubleDQN +2023-04-06 16:04:10 - r - INFO: - mode test +2023-04-06 16:04:10 - r - INFO: - mp_backend mp +2023-04-06 16:04:10 - r - INFO: - seed 1 +2023-04-06 16:04:10 - r - INFO: - device cpu +2023-04-06 16:04:10 - r - INFO: - train_eps 400 +2023-04-06 16:04:10 - r - INFO: - test_eps 10 +2023-04-06 16:04:10 - r - INFO: - eval_eps 10 +2023-04-06 16:04:10 - r - INFO: - eval_per_episode 5 +2023-04-06 16:04:10 - r - INFO: - max_steps 200 +2023-04-06 16:04:10 - r - INFO: - load_checkpoint 1 +2023-04-06 16:04:10 - r - INFO: - load_path Train_CartPole-v1_DoubleDQN_mp_20230406-160028 +2023-04-06 16:04:10 - r - INFO: - show_fig 0 +2023-04-06 16:04:10 - r - INFO: - save_fig 1 +2023-04-06 16:04:10 - r - INFO: - n_workers 1 +2023-04-06 16:04:10 - r - INFO: - epsilon_start 0.95 +2023-04-06 16:04:10 - r - INFO: - epsilon_end 0.01 +2023-04-06 16:04:10 - r - INFO: - epsilon_decay 500 +2023-04-06 16:04:10 - r - INFO: - gamma 0.95 +2023-04-06 16:04:10 - r - INFO: - lr 0.0001 +2023-04-06 16:04:10 - r - INFO: - buffer_size 100000 +2023-04-06 16:04:10 - r - INFO: - batch_size 64 +2023-04-06 16:04:10 - r - INFO: - target_update 4 +2023-04-06 16:04:10 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-06 16:04:10 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-160410 +2023-04-06 16:04:10 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-160410/results +2023-04-06 16:04:10 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-160410/logs +2023-04-06 16:04:10 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-160410/traj +2023-04-06 16:04:10 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-160410/videos +2023-04-06 16:04:10 - r - INFO: - ================================================================================ +2023-04-06 16:04:10 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-06 16:04:10 - r - INFO: - Start testing! +2023-04-06 16:04:10 - r - INFO: - Env: CartPole-v1, Algorithm: DoubleDQN, Device: cpu +2023-04-06 16:04:10 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-04-06 16:04:10 - r - INFO: - Episode: 10/10, Reward: 198.000, Step: 198 +2023-04-06 16:04:10 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/models/checkpoint.pth b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/models/checkpoint.pth new file mode 100644 index 0000000000000000000000000000000000000000..a145e71eceffd117546543c32d5e5214414c6eb2 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/models/checkpoint.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cd537f6ab4ac0ff27caa323076685e3a3fff04b064dbdfc509baae76e9a9406 +size 272407 diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..25a6f1170c6b7006af7cbbeb89719d0473d5d424 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/results/res.csv b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..a5e22806d9977ec8a28d3da237cdd05d757ef4d9 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_mp_20230406-160410/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,198.0,198 diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/config.yaml b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b53ab4979916e32cdfbbb7f1af10a280b319963b --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/config.yaml @@ -0,0 +1,46 @@ +general_cfg: + algo_name: DoubleDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_DoubleDQN_ray_20230406-162938 + max_steps: 200 + mode: test + mp_backend: ray + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/logs/log.txt b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..2ffc7ffa9344bf08e6ef4582bc56185af96b4586 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/logs/log.txt @@ -0,0 +1,52 @@ +2023-04-06 17:03:48 - r - INFO: - Hyperparameters: +2023-04-06 17:03:48 - r - INFO: - ================================================================================ +2023-04-06 17:03:48 - r - INFO: - Name Value Type +2023-04-06 17:03:48 - r - INFO: - env_name CartPole-v1 +2023-04-06 17:03:48 - r - INFO: - new_step_api 1 +2023-04-06 17:03:48 - r - INFO: - wrapper None +2023-04-06 17:03:48 - r - INFO: - render 0 +2023-04-06 17:03:48 - r - INFO: - render_mode human +2023-04-06 17:03:48 - r - INFO: - algo_name DoubleDQN +2023-04-06 17:03:48 - r - INFO: - mode test +2023-04-06 17:03:48 - r - INFO: - mp_backend ray +2023-04-06 17:03:48 - r - INFO: - seed 1 +2023-04-06 17:03:48 - r - INFO: - device cpu +2023-04-06 17:03:48 - r - INFO: - train_eps 400 +2023-04-06 17:03:48 - r - INFO: - test_eps 10 +2023-04-06 17:03:48 - r - INFO: - eval_eps 10 +2023-04-06 17:03:48 - r - INFO: - eval_per_episode 5 +2023-04-06 17:03:48 - r - INFO: - max_steps 200 +2023-04-06 17:03:48 - r - INFO: - load_checkpoint 1 +2023-04-06 17:03:48 - r - INFO: - load_path Train_CartPole-v1_DoubleDQN_ray_20230406-162938 +2023-04-06 17:03:48 - r - INFO: - show_fig 0 +2023-04-06 17:03:48 - r - INFO: - save_fig 1 +2023-04-06 17:03:48 - r - INFO: - n_workers 1 +2023-04-06 17:03:48 - r - INFO: - epsilon_start 0.95 +2023-04-06 17:03:48 - r - INFO: - epsilon_end 0.01 +2023-04-06 17:03:48 - r - INFO: - epsilon_decay 500 +2023-04-06 17:03:48 - r - INFO: - gamma 0.95 +2023-04-06 17:03:48 - r - INFO: - lr 0.0001 +2023-04-06 17:03:48 - r - INFO: - buffer_size 100000 +2023-04-06 17:03:48 - r - INFO: - batch_size 64 +2023-04-06 17:03:48 - r - INFO: - target_update 4 +2023-04-06 17:03:48 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-06 17:03:48 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-170348 +2023-04-06 17:03:48 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-170348/results +2023-04-06 17:03:48 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-170348/logs +2023-04-06 17:03:48 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-170348/traj +2023-04-06 17:03:48 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DoubleDQN_20230406-170348/videos +2023-04-06 17:03:48 - r - INFO: - ================================================================================ +2023-04-06 17:03:48 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-06 17:03:48 - r - INFO: - Start testing! +2023-04-06 17:03:48 - r - INFO: - Env: CartPole-v1, Algorithm: DoubleDQN, Device: cpu +2023-04-06 17:03:48 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:48 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:49 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-06 17:03:49 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cfaf6fe119daae13b1e48521ac5f0a6dfd5731c --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3a724152cea263dec5b58d80bee101405e7b3268a34c265d414ebbd771c5ac +size 272407 diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..3b7b4f7de0438536dbc520bdf9b56573679c088e Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/results/res.csv b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DoubleDQN_ray_20230406-170348/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/config.yaml b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02c74e40f0a75da35a2e7df104d875ce793d3ec2 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/config.yaml @@ -0,0 +1,41 @@ +general_cfg: + algo_name: DuelingDQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_DuelingDQN_20221122-125403 + max_steps: 200 + mode: test + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/logs/log.txt b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..b75371ca5c4aafaa594dc83020cbf19def1cdbe0 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/logs/log.txt @@ -0,0 +1,14 @@ +2022-11-22 12:54:55 - r - INFO: - n_states: 4, n_actions: 2 +2022-11-22 12:54:58 - r - INFO: - Start testing! +2022-11-22 12:54:58 - r - INFO: - Env: CartPole-v1, Algorithm: DuelingDQN, Device: cuda +2022-11-22 12:54:58 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2022-11-22 12:54:58 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f8f1bb2765fa7bb055a8236cbc3aa26d8825a98 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39a64f85a8848a1f4c937e359c4370458cf318c2bffbae406915ce7ee449dc50 +size 539719 diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..5a8dd72d80717b4283b5c360717089e9874f50fc Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/results/res.csv b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_20221122-125455/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/config.yaml b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86012d297c175e34fdbca7d861d86c9f7fa97388 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/config.yaml @@ -0,0 +1,47 @@ +general_cfg: + algo_name: DuelingDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_DuelingDQN_mp_20230407-170853 + max_steps: 200 + mode: test + mp_backend: mp + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/logs/log.txt b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..012765caa73c0ed489bcb948cdce74da3cd45f69 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/logs/log.txt @@ -0,0 +1,53 @@ +2023-04-07 17:11:20 - r - INFO: - Hyperparameters: +2023-04-07 17:11:20 - r - INFO: - ================================================================================ +2023-04-07 17:11:20 - r - INFO: - Name Value Type +2023-04-07 17:11:20 - r - INFO: - env_name CartPole-v1 +2023-04-07 17:11:20 - r - INFO: - new_step_api 1 +2023-04-07 17:11:20 - r - INFO: - wrapper None +2023-04-07 17:11:20 - r - INFO: - render 0 +2023-04-07 17:11:20 - r - INFO: - render_mode human +2023-04-07 17:11:20 - r - INFO: - algo_name DuelingDQN +2023-04-07 17:11:20 - r - INFO: - mode test +2023-04-07 17:11:20 - r - INFO: - mp_backend mp +2023-04-07 17:11:20 - r - INFO: - seed 1 +2023-04-07 17:11:20 - r - INFO: - device cpu +2023-04-07 17:11:20 - r - INFO: - train_eps 200 +2023-04-07 17:11:20 - r - INFO: - test_eps 10 +2023-04-07 17:11:20 - r - INFO: - eval_eps 10 +2023-04-07 17:11:20 - r - INFO: - eval_per_episode 5 +2023-04-07 17:11:20 - r - INFO: - max_steps 200 +2023-04-07 17:11:20 - r - INFO: - load_checkpoint 1 +2023-04-07 17:11:20 - r - INFO: - load_path Train_CartPole-v1_DuelingDQN_mp_20230407-170853 +2023-04-07 17:11:20 - r - INFO: - show_fig 0 +2023-04-07 17:11:20 - r - INFO: - save_fig 1 +2023-04-07 17:11:20 - r - INFO: - n_workers 1 +2023-04-07 17:11:20 - r - INFO: - epsilon_start 0.95 +2023-04-07 17:11:20 - r - INFO: - epsilon_end 0.01 +2023-04-07 17:11:20 - r - INFO: - epsilon_decay 500 +2023-04-07 17:11:20 - r - INFO: - gamma 0.99 +2023-04-07 17:11:20 - r - INFO: - lr 0.0001 +2023-04-07 17:11:20 - r - INFO: - buffer_size 100000 +2023-04-07 17:11:20 - r - INFO: - batch_size 64 +2023-04-07 17:11:20 - r - INFO: - target_update 4 +2023-04-07 17:11:20 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-07 17:11:20 - r - INFO: - hidden_dim 256 +2023-04-07 17:11:20 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-171120 +2023-04-07 17:11:20 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-171120/results +2023-04-07 17:11:20 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-171120/logs +2023-04-07 17:11:20 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-171120/traj +2023-04-07 17:11:20 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-171120/videos +2023-04-07 17:11:20 - r - INFO: - ================================================================================ +2023-04-07 17:11:20 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-07 17:11:20 - r - INFO: - Start testing! +2023-04-07 17:11:20 - r - INFO: - Env: CartPole-v1, Algorithm: DuelingDQN, Device: cpu +2023-04-07 17:11:20 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-07 17:11:21 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-04-07 17:11:21 - r - INFO: - Episode: 3/10, Reward: 190.000, Step: 190 +2023-04-07 17:11:21 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-07 17:11:21 - r - INFO: - Episode: 5/10, Reward: 187.000, Step: 187 +2023-04-07 17:11:21 - r - INFO: - Episode: 6/10, Reward: 182.000, Step: 182 +2023-04-07 17:11:21 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-07 17:11:21 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-07 17:11:21 - r - INFO: - Episode: 9/10, Reward: 196.000, Step: 196 +2023-04-07 17:11:21 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-07 17:11:21 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f8c7f24c0c9fc6d4a576d479507b204876171ea --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87aab291b33d6423c3c54eced436183398700a290427c1913be9d65f5503b5ae +size 537607 diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..40f1167bff6ebfc47a6bf79a42d8a2e8f8f56014 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/results/res.csv b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac13e4c9167d41ac2fc6c3ed3640606a2eaae65b --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_mp_20230407-171120/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,190.0,190 +3,200.0,200 +4,187.0,187 +5,182.0,182 +6,200.0,200 +7,200.0,200 +8,196.0,196 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/config.yaml b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f96c847df597c3bec2d1dfd5303825cd1370157a --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/config.yaml @@ -0,0 +1,47 @@ +general_cfg: + algo_name: DuelingDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_DuelingDQN_20230407-153236 + max_steps: 200 + mode: test + mp_backend: ray + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/logs/log.txt b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d67567b83f689297f270ecc30ca910d4d40eeb46 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/logs/log.txt @@ -0,0 +1,53 @@ +2023-04-07 16:52:08 - r - INFO: - Hyperparameters: +2023-04-07 16:52:08 - r - INFO: - ================================================================================ +2023-04-07 16:52:08 - r - INFO: - Name Value Type +2023-04-07 16:52:08 - r - INFO: - env_name CartPole-v1 +2023-04-07 16:52:08 - r - INFO: - new_step_api 1 +2023-04-07 16:52:08 - r - INFO: - wrapper None +2023-04-07 16:52:08 - r - INFO: - render 0 +2023-04-07 16:52:08 - r - INFO: - render_mode human +2023-04-07 16:52:08 - r - INFO: - algo_name DuelingDQN +2023-04-07 16:52:08 - r - INFO: - mode test +2023-04-07 16:52:08 - r - INFO: - mp_backend ray +2023-04-07 16:52:08 - r - INFO: - seed 1 +2023-04-07 16:52:08 - r - INFO: - device cpu +2023-04-07 16:52:08 - r - INFO: - train_eps 400 +2023-04-07 16:52:08 - r - INFO: - test_eps 10 +2023-04-07 16:52:08 - r - INFO: - eval_eps 10 +2023-04-07 16:52:08 - r - INFO: - eval_per_episode 5 +2023-04-07 16:52:08 - r - INFO: - max_steps 200 +2023-04-07 16:52:08 - r - INFO: - load_checkpoint 1 +2023-04-07 16:52:08 - r - INFO: - load_path Train_CartPole-v1_DuelingDQN_20230407-153236 +2023-04-07 16:52:08 - r - INFO: - show_fig 0 +2023-04-07 16:52:08 - r - INFO: - save_fig 1 +2023-04-07 16:52:08 - r - INFO: - n_workers 1 +2023-04-07 16:52:08 - r - INFO: - epsilon_start 0.95 +2023-04-07 16:52:08 - r - INFO: - epsilon_end 0.01 +2023-04-07 16:52:08 - r - INFO: - epsilon_decay 500 +2023-04-07 16:52:08 - r - INFO: - gamma 0.99 +2023-04-07 16:52:08 - r - INFO: - lr 0.0001 +2023-04-07 16:52:08 - r - INFO: - buffer_size 100000 +2023-04-07 16:52:08 - r - INFO: - batch_size 64 +2023-04-07 16:52:08 - r - INFO: - target_update 4 +2023-04-07 16:52:08 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-07 16:52:08 - r - INFO: - hidden_dim 256 +2023-04-07 16:52:08 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-165208 +2023-04-07 16:52:08 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-165208/results +2023-04-07 16:52:08 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-165208/logs +2023-04-07 16:52:08 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-165208/traj +2023-04-07 16:52:08 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DuelingDQN_20230407-165208/videos +2023-04-07 16:52:08 - r - INFO: - ================================================================================ +2023-04-07 16:52:08 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-07 16:52:08 - r - INFO: - Start testing! +2023-04-07 16:52:08 - r - INFO: - Env: CartPole-v1, Algorithm: DuelingDQN, Device: cpu +2023-04-07 16:52:08 - r - INFO: - Episode: 1/10, Reward: 171.000, Step: 171 +2023-04-07 16:52:08 - r - INFO: - Episode: 2/10, Reward: 185.000, Step: 185 +2023-04-07 16:52:08 - r - INFO: - Episode: 3/10, Reward: 159.000, Step: 159 +2023-04-07 16:52:08 - r - INFO: - Episode: 4/10, Reward: 155.000, Step: 155 +2023-04-07 16:52:08 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-07 16:52:08 - r - INFO: - Episode: 6/10, Reward: 120.000, Step: 120 +2023-04-07 16:52:08 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-07 16:52:08 - r - INFO: - Episode: 8/10, Reward: 187.000, Step: 187 +2023-04-07 16:52:08 - r - INFO: - Episode: 9/10, Reward: 154.000, Step: 154 +2023-04-07 16:52:08 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-07 16:52:08 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..d64126c7ac0958c7567fbb3cc692d7d11ed40efc --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f1262598e3d636dd22e3b2fc0dfe52bf7a55348d54f51f02a8410682ec5a18 +size 537607 diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..9bb4b42a700435f1b8dfd21b5f3e6c03821f5281 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/results/res.csv b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..4b5323e1519c0179ab0a52f1d89956a4f4de8a96 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_DuelingDQN_ray_20230407-165208/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,171.0,171 +1,185.0,185 +2,159.0,159 +3,155.0,155 +4,200.0,200 +5,120.0,120 +6,200.0,200 +7,187.0,187 +8,154.0,154 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/config.yaml b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc87ded82c98f4be65f9bb7bf8f627f94982e972 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/config.yaml @@ -0,0 +1,38 @@ +general_cfg: + algo_name: GAIL + batch_size: 2048 + device: cuda + discount: 0.99 + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + hidden_dim: 32 + imitation_batch_size: 128 + imitation_epochs: 5 + imitation_replay_size: 4 + load_checkpoint: true + load_path: Train_CartPole-v1_GAIL_20221207-160945 + lr: 0.001 + max_steps: 200 + mode: test + new_step_api: true + ppo_epochs: 5 + r1_reg_coeff: 1 + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + batch_size: 2048 + critic_hidden_dim: 256 + critic_lr: 0.0003 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + update_freq: 2048 diff --git a/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/logs/log.txt b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..f6df111517d2a9220e0fd6ab853f75582a429c7a --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/logs/log.txt @@ -0,0 +1,53 @@ +2022-12-07 16:36:50 - r - INFO: - Hyperparameters: +2022-12-07 16:36:50 - r - INFO: - ================================================================================ +2022-12-07 16:36:50 - r - INFO: - Name Value Type +2022-12-07 16:36:50 - r - INFO: - env_name CartPole-v1 +2022-12-07 16:36:50 - r - INFO: - new_step_api 1 +2022-12-07 16:36:50 - r - INFO: - wrapper None +2022-12-07 16:36:50 - r - INFO: - render 0 +2022-12-07 16:36:50 - r - INFO: - algo_name GAIL +2022-12-07 16:36:50 - r - INFO: - mode test +2022-12-07 16:36:50 - r - INFO: - seed 1 +2022-12-07 16:36:50 - r - INFO: - device cuda +2022-12-07 16:36:50 - r - INFO: - train_eps 400 +2022-12-07 16:36:50 - r - INFO: - test_eps 10 +2022-12-07 16:36:50 - r - INFO: - eval_eps 10 +2022-12-07 16:36:50 - r - INFO: - eval_per_episode 5 +2022-12-07 16:36:50 - r - INFO: - max_steps 200 +2022-12-07 16:36:50 - r - INFO: - load_checkpoint 1 +2022-12-07 16:36:50 - r - INFO: - load_path Train_CartPole-v1_GAIL_20221207-160945 +2022-12-07 16:36:50 - r - INFO: - show_fig 0 +2022-12-07 16:36:50 - r - INFO: - save_fig 1 +2022-12-07 16:36:50 - r - INFO: - batch_size 2048 +2022-12-07 16:36:50 - r - INFO: - discount 0.99 +2022-12-07 16:36:50 - r - INFO: - hidden_dim 32 +2022-12-07 16:36:50 - r - INFO: - lr 0.001 +2022-12-07 16:36:50 - r - INFO: - imitation_replay_size 4 +2022-12-07 16:36:50 - r - INFO: - imitation_epochs 5 +2022-12-07 16:36:50 - r - INFO: - imitation_batch_size 128 +2022-12-07 16:36:50 - r - INFO: - ppo_epochs 5 +2022-12-07 16:36:50 - r - INFO: - r1_reg_coeff 1 +2022-12-07 16:36:50 - r - INFO: - gamma 0.99 +2022-12-07 16:36:50 - r - INFO: - k_epochs 4 +2022-12-07 16:36:50 - r - INFO: - actor_lr 0.0003 +2022-12-07 16:36:50 - r - INFO: - critic_lr 0.0003 +2022-12-07 16:36:50 - r - INFO: - eps_clip 0.2 +2022-12-07 16:36:50 - r - INFO: - entropy_coef 0.01 +2022-12-07 16:36:50 - r - INFO: - update_freq 2048 +2022-12-07 16:36:50 - r - INFO: - actor_hidden_dim 256 +2022-12-07 16:36:50 - r - INFO: - critic_hidden_dim 256 +2022-12-07 16:36:50 - r - INFO: - ================================================================================ +2022-12-07 16:36:51 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-07 16:36:52 - r - INFO: - Start testing! +2022-12-07 16:36:52 - r - INFO: - Env: CartPole-v1, Algorithm: GAIL, Device: cuda +2022-12-07 16:36:53 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:54 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:54 - r - INFO: - Episode: 3/10, Reward: 148.000, Step: 148 +2022-12-07 16:36:54 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:55 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:55 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:55 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:55 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:56 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:56 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2022-12-07 16:36:56 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/models/actor.pth b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..4cce6e4d2cd12fc5524c7db6123b2b8db3eca0d8 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a7cadc94484feb4feafb0b53f7fee416ba201906622cdce8e82bcfe2ab47ca8 +size 272215 diff --git a/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/models/critic.pth b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..8940c8a31d888cf8984afaa02c447d6eca453347 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2003a930fc6c0045c6933875066426140b002a1b6a74735854bae1c04bc5a99 +size 271191 diff --git a/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..04cc6d084e8005790fa996f7fc78de46a11b3b01 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/results/res.csv b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..96d443d04579c45cd1e5691cff917e07f0491f67 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_GAIL_20221207-163649/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,148.0,148 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/config.yaml b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b823e8bf27e5b4ebc61371d02b6ba01bb999c46e --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/config.yaml @@ -0,0 +1,29 @@ +general_cfg: + algo_name: NoisyDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_NoisyDQN_20230318-162139 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + tau: 1.0 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/logs/log.txt b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..2421d393873f81b8759b75b457b3599d6e9901f8 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/logs/log.txt @@ -0,0 +1,50 @@ +2023-03-18 16:26:30 - r - INFO: - Hyperparameters: +2023-03-18 16:26:30 - r - INFO: - ================================================================================ +2023-03-18 16:26:30 - r - INFO: - Name Value Type +2023-03-18 16:26:30 - r - INFO: - env_name CartPole-v1 +2023-03-18 16:26:30 - r - INFO: - new_step_api 1 +2023-03-18 16:26:30 - r - INFO: - wrapper None +2023-03-18 16:26:30 - r - INFO: - render 0 +2023-03-18 16:26:30 - r - INFO: - algo_name NoisyDQN +2023-03-18 16:26:30 - r - INFO: - mode test +2023-03-18 16:26:30 - r - INFO: - seed 1 +2023-03-18 16:26:30 - r - INFO: - device cpu +2023-03-18 16:26:30 - r - INFO: - train_eps 100 +2023-03-18 16:26:30 - r - INFO: - test_eps 10 +2023-03-18 16:26:30 - r - INFO: - eval_eps 10 +2023-03-18 16:26:30 - r - INFO: - eval_per_episode 5 +2023-03-18 16:26:30 - r - INFO: - max_steps 200 +2023-03-18 16:26:30 - r - INFO: - load_checkpoint 1 +2023-03-18 16:26:30 - r - INFO: - load_path Train_CartPole-v1_NoisyDQN_20230318-162139 +2023-03-18 16:26:30 - r - INFO: - show_fig 0 +2023-03-18 16:26:30 - r - INFO: - save_fig 1 +2023-03-18 16:26:30 - r - INFO: - epsilon_start 0.95 +2023-03-18 16:26:30 - r - INFO: - tau 1.0 +2023-03-18 16:26:30 - r - INFO: - epsilon_end 0.01 +2023-03-18 16:26:30 - r - INFO: - epsilon_decay 500 +2023-03-18 16:26:30 - r - INFO: - hidden_dim 256 +2023-03-18 16:26:30 - r - INFO: - gamma 0.95 +2023-03-18 16:26:30 - r - INFO: - lr 0.0001 +2023-03-18 16:26:30 - r - INFO: - buffer_size 100000 +2023-03-18 16:26:30 - r - INFO: - batch_size 64 +2023-03-18 16:26:30 - r - INFO: - target_update 4 +2023-03-18 16:26:30 - r - INFO: - task_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_NoisyDQN_20230318-162630 +2023-03-18 16:26:30 - r - INFO: - res_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_NoisyDQN_20230318-162630/results +2023-03-18 16:26:30 - r - INFO: - log_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_NoisyDQN_20230318-162630/logs +2023-03-18 16:26:30 - r - INFO: - traj_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_NoisyDQN_20230318-162630/traj +2023-03-18 16:26:30 - r - INFO: - tb_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_NoisyDQN_20230318-162630/tb_logs +2023-03-18 16:26:30 - r - INFO: - ================================================================================ +2023-03-18 16:26:30 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-18 16:26:30 - r - INFO: - Start testing! +2023-03-18 16:26:30 - r - INFO: - Env: CartPole-v1, Algorithm: NoisyDQN, Device: cpu +2023-03-18 16:26:30 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-03-18 16:26:30 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..3aa536480797ae052c46c05f6e9c38201887cdad --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c018ce117a1834aec6f585e6be780865631d49a8a81fb42ee4db9602b49793f5 +size 804667 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..210820b85c519eeeedf89c35f3ca7f5cd798f3ff Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/results/res.csv b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/tb_logs/events.out.tfevents.1679127990.DESKTOP-H34HQIQ.41784.0 b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/tb_logs/events.out.tfevents.1679127990.DESKTOP-H34HQIQ.41784.0 new file mode 100644 index 0000000000000000000000000000000000000000..47d238227c8026a005c1b4612304ebf1186af257 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_20230318-162630/tb_logs/events.out.tfevents.1679127990.DESKTOP-H34HQIQ.41784.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab9ab499eeb1c82fc01c7c5112993d777beb0303452a699ecf6b3327d1d5857 +size 40 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/config.yaml b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..804a5013b9280434a87a84bb56d702334e602c9a --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/config.yaml @@ -0,0 +1,36 @@ +general_cfg: + algo_name: NoisyDQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_NoisyDQN_mp_20230413-220639 + max_steps: 200 + mode: test + mp_backend: mp + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + tau: 1.0 +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/logs/log.txt b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..555e43072353e164e41ca8f8d12cf4a697c26cb6 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/logs/log.txt @@ -0,0 +1,54 @@ +2023-04-13 22:10:12 - r - INFO: - Hyperparameters: +2023-04-13 22:10:12 - r - INFO: - ================================================================================ +2023-04-13 22:10:12 - r - INFO: - Name Value Type +2023-04-13 22:10:12 - r - INFO: - env_name gym +2023-04-13 22:10:12 - r - INFO: - new_step_api 1 +2023-04-13 22:10:12 - r - INFO: - wrapper None +2023-04-13 22:10:12 - r - INFO: - render 0 +2023-04-13 22:10:12 - r - INFO: - render_mode None +2023-04-13 22:10:12 - r - INFO: - algo_name NoisyDQN +2023-04-13 22:10:12 - r - INFO: - mode test +2023-04-13 22:10:12 - r - INFO: - mp_backend mp +2023-04-13 22:10:12 - r - INFO: - seed 1 +2023-04-13 22:10:12 - r - INFO: - device cpu +2023-04-13 22:10:12 - r - INFO: - train_eps 200 +2023-04-13 22:10:12 - r - INFO: - test_eps 10 +2023-04-13 22:10:12 - r - INFO: - eval_eps 10 +2023-04-13 22:10:12 - r - INFO: - eval_per_episode 5 +2023-04-13 22:10:12 - r - INFO: - max_steps 200 +2023-04-13 22:10:12 - r - INFO: - load_checkpoint 1 +2023-04-13 22:10:12 - r - INFO: - load_path Train_CartPole-v1_NoisyDQN_mp_20230413-220639 +2023-04-13 22:10:12 - r - INFO: - show_fig 0 +2023-04-13 22:10:12 - r - INFO: - save_fig 1 +2023-04-13 22:10:12 - r - INFO: - n_workers 1 +2023-04-13 22:10:12 - r - INFO: - epsilon_start 0.95 +2023-04-13 22:10:12 - r - INFO: - tau 1.0 +2023-04-13 22:10:12 - r - INFO: - epsilon_end 0.01 +2023-04-13 22:10:12 - r - INFO: - epsilon_decay 500 +2023-04-13 22:10:12 - r - INFO: - hidden_dim 256 +2023-04-13 22:10:12 - r - INFO: - gamma 0.95 +2023-04-13 22:10:12 - r - INFO: - lr 0.0001 +2023-04-13 22:10:12 - r - INFO: - buffer_size 100000 +2023-04-13 22:10:12 - r - INFO: - batch_size 64 +2023-04-13 22:10:12 - r - INFO: - target_update 4 +2023-04-13 22:10:12 - r - INFO: - id CartPole-v1 +2023-04-13 22:10:12 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230413-221012 +2023-04-13 22:10:12 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230413-221012/results +2023-04-13 22:10:12 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230413-221012/logs +2023-04-13 22:10:12 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230413-221012/traj +2023-04-13 22:10:12 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230413-221012/videos +2023-04-13 22:10:12 - r - INFO: - ================================================================================ +2023-04-13 22:10:12 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-13 22:10:12 - r - INFO: - Start testing! +2023-04-13 22:10:12 - r - INFO: - Env: gym, Algorithm: NoisyDQN, Device: cpu +2023-04-13 22:10:12 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-13 22:10:12 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee646c7afec5e202c73770cd5fc016b932f986e6 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab846435c3ac88c2507622e5f1050d4be6bfc94e33845dc5d7de2fac6df2c92f +size 804667 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..0f2e6d031302ee37d97fa4072c63d9da2afba89c Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/results/res.csv b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_mp_20230413-221012/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/config.yaml b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..236f3f63962bceb5e2848d94130132aae8f58263 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/config.yaml @@ -0,0 +1,36 @@ +general_cfg: + algo_name: NoisyDQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_gym_NoisyDQN_20230414-101351 + max_steps: 200 + mode: test + mp_backend: ray + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + tau: 1.0 +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/logs/log.txt b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6d7b2d60ecb42fdac6fcc6cf6c99954ef6f26e6e --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/logs/log.txt @@ -0,0 +1,54 @@ +2023-04-14 10:19:26 - r - INFO: - Hyperparameters: +2023-04-14 10:19:26 - r - INFO: - ================================================================================ +2023-04-14 10:19:26 - r - INFO: - Name Value Type +2023-04-14 10:19:26 - r - INFO: - env_name gym +2023-04-14 10:19:26 - r - INFO: - new_step_api 1 +2023-04-14 10:19:26 - r - INFO: - wrapper None +2023-04-14 10:19:26 - r - INFO: - render 0 +2023-04-14 10:19:26 - r - INFO: - render_mode None +2023-04-14 10:19:26 - r - INFO: - algo_name NoisyDQN +2023-04-14 10:19:26 - r - INFO: - mode test +2023-04-14 10:19:26 - r - INFO: - mp_backend ray +2023-04-14 10:19:26 - r - INFO: - seed 1 +2023-04-14 10:19:26 - r - INFO: - device cpu +2023-04-14 10:19:26 - r - INFO: - train_eps 200 +2023-04-14 10:19:26 - r - INFO: - test_eps 10 +2023-04-14 10:19:26 - r - INFO: - eval_eps 10 +2023-04-14 10:19:26 - r - INFO: - eval_per_episode 5 +2023-04-14 10:19:26 - r - INFO: - max_steps 200 +2023-04-14 10:19:26 - r - INFO: - load_checkpoint 1 +2023-04-14 10:19:26 - r - INFO: - load_path Train_gym_NoisyDQN_20230414-101351 +2023-04-14 10:19:26 - r - INFO: - show_fig 0 +2023-04-14 10:19:26 - r - INFO: - save_fig 1 +2023-04-14 10:19:26 - r - INFO: - n_workers 1 +2023-04-14 10:19:26 - r - INFO: - epsilon_start 0.95 +2023-04-14 10:19:26 - r - INFO: - tau 1.0 +2023-04-14 10:19:26 - r - INFO: - epsilon_end 0.01 +2023-04-14 10:19:26 - r - INFO: - epsilon_decay 500 +2023-04-14 10:19:26 - r - INFO: - hidden_dim 256 +2023-04-14 10:19:26 - r - INFO: - gamma 0.95 +2023-04-14 10:19:26 - r - INFO: - lr 0.0001 +2023-04-14 10:19:26 - r - INFO: - buffer_size 100000 +2023-04-14 10:19:26 - r - INFO: - batch_size 64 +2023-04-14 10:19:26 - r - INFO: - target_update 4 +2023-04-14 10:19:26 - r - INFO: - id CartPole-v1 +2023-04-14 10:19:26 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230414-101926 +2023-04-14 10:19:26 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230414-101926/results +2023-04-14 10:19:26 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230414-101926/logs +2023-04-14 10:19:26 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230414-101926/traj +2023-04-14 10:19:26 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_NoisyDQN_20230414-101926/videos +2023-04-14 10:19:26 - r - INFO: - ================================================================================ +2023-04-14 10:19:26 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-14 10:19:26 - r - INFO: - Start testing! +2023-04-14 10:19:26 - r - INFO: - Env: gym, Algorithm: NoisyDQN, Device: cpu +2023-04-14 10:19:26 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:26 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:26 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:26 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:26 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:26 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:26 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:26 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:27 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:27 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-14 10:19:27 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c648ab8e0e4d608b462bcff135f583c4594ea2f --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96c4f99970f7ebbc667d28e3f1c8c05399296b0257a95025ab12b4eca17d3919 +size 804667 diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..0f2e6d031302ee37d97fa4072c63d9da2afba89c Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/results/res.csv b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_NoisyDQN_ray_20230414-101926/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/config.yaml b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91654883827265098bcc856e73636f35001bd9f4 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/config.yaml @@ -0,0 +1,48 @@ +general_cfg: + algo_name: PER_DQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_PER_DQN_20230331-225815 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + per_alpha: 0.6 + per_beta: 0.4 + per_beta_annealing: 0.001 + per_epsilon: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/logs/log.txt b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..2c2203b84e5cd69cc152d08d11073728d338cb4b --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/logs/log.txt @@ -0,0 +1,54 @@ +2023-03-31 23:14:42 - r - INFO: - Hyperparameters: +2023-03-31 23:14:42 - r - INFO: - ================================================================================ +2023-03-31 23:14:42 - r - INFO: - Name Value Type +2023-03-31 23:14:42 - r - INFO: - env_name CartPole-v1 +2023-03-31 23:14:42 - r - INFO: - new_step_api 1 +2023-03-31 23:14:42 - r - INFO: - wrapper None +2023-03-31 23:14:42 - r - INFO: - render 0 +2023-03-31 23:14:42 - r - INFO: - algo_name PER_DQN +2023-03-31 23:14:42 - r - INFO: - mode test +2023-03-31 23:14:42 - r - INFO: - seed 1 +2023-03-31 23:14:42 - r - INFO: - device cuda +2023-03-31 23:14:42 - r - INFO: - train_eps 100 +2023-03-31 23:14:42 - r - INFO: - test_eps 10 +2023-03-31 23:14:42 - r - INFO: - eval_eps 10 +2023-03-31 23:14:42 - r - INFO: - eval_per_episode 5 +2023-03-31 23:14:42 - r - INFO: - max_steps 200 +2023-03-31 23:14:42 - r - INFO: - load_checkpoint 1 +2023-03-31 23:14:42 - r - INFO: - load_path Train_CartPole-v1_PER_DQN_20230331-225815 +2023-03-31 23:14:42 - r - INFO: - show_fig 0 +2023-03-31 23:14:42 - r - INFO: - save_fig 1 +2023-03-31 23:14:42 - r - INFO: - epsilon_start 0.95 +2023-03-31 23:14:42 - r - INFO: - epsilon_end 0.01 +2023-03-31 23:14:42 - r - INFO: - epsilon_decay 500 +2023-03-31 23:14:42 - r - INFO: - hidden_dim 256 +2023-03-31 23:14:42 - r - INFO: - gamma 0.99 +2023-03-31 23:14:42 - r - INFO: - lr 0.0001 +2023-03-31 23:14:42 - r - INFO: - buffer_size 100000 +2023-03-31 23:14:42 - r - INFO: - per_alpha 0.6 +2023-03-31 23:14:42 - r - INFO: - per_beta 0.4 +2023-03-31 23:14:42 - r - INFO: - per_beta_annealing 0.001 +2023-03-31 23:14:42 - r - INFO: - per_epsilon 0.01 +2023-03-31 23:14:42 - r - INFO: - batch_size 64 +2023-03-31 23:14:42 - r - INFO: - target_update 4 +2023-03-31 23:14:42 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-03-31 23:14:42 - r - INFO: - task_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_PER_DQN_20230331-231442 +2023-03-31 23:14:42 - r - INFO: - res_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_PER_DQN_20230331-231442/results +2023-03-31 23:14:42 - r - INFO: - log_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_PER_DQN_20230331-231442/logs +2023-03-31 23:14:42 - r - INFO: - traj_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_PER_DQN_20230331-231442/traj +2023-03-31 23:14:42 - r - INFO: - tb_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CartPole-v1_PER_DQN_20230331-231442/tb_logs +2023-03-31 23:14:42 - r - INFO: - ================================================================================ +2023-03-31 23:14:42 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-31 23:14:43 - r - INFO: - Start testing! +2023-03-31 23:14:43 - r - INFO: - Env: CartPole-v1, Algorithm: PER_DQN, Device: cuda +2023-03-31 23:14:44 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-03-31 23:14:44 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..663b12f22263098d243896df93ba28f600d9a5fe --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c438616b97ca890557a9e9b1cd42decfc5decc64e5aee660d89158290e92683d +size 272471 diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..e6669f6e45a8470b88f8a09b7e3e7cee16f30d2c Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/results/res.csv b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/tb_logs/events.out.tfevents.1680275682.DESKTOP-H34HQIQ.290376.0 b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/tb_logs/events.out.tfevents.1680275682.DESKTOP-H34HQIQ.290376.0 new file mode 100644 index 0000000000000000000000000000000000000000..dca8765e122e7c8281d95e93e0c43e2faede67f4 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_20230331-231442/tb_logs/events.out.tfevents.1680275682.DESKTOP-H34HQIQ.290376.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5157a74a9c33c3bb078d1509c6bd0013251490c4413c68decd709494f31a8d60 +size 40 diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/config.yaml b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb754572a76f62b2f98f257e5b3d70cbfdd2bc31 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/config.yaml @@ -0,0 +1,55 @@ +general_cfg: + algo_name: PER_DQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_gym_PER_DQN_20230415-215002 + max_steps: 200 + mode: test + mp_backend: mp + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + per_alpha: 0.6 + per_beta: 0.4 + per_beta_annealing: 0.001 + per_epsilon: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/logs/log.txt b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d1217d2d110ec32b297fe04b77ca5858db432dbb --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/logs/log.txt @@ -0,0 +1,58 @@ +2023-04-15 21:51:47 - r - INFO: - Hyperparameters: +2023-04-15 21:51:47 - r - INFO: - ================================================================================ +2023-04-15 21:51:47 - r - INFO: - Name Value Type +2023-04-15 21:51:47 - r - INFO: - env_name gym +2023-04-15 21:51:47 - r - INFO: - new_step_api 1 +2023-04-15 21:51:47 - r - INFO: - wrapper None +2023-04-15 21:51:47 - r - INFO: - render 0 +2023-04-15 21:51:47 - r - INFO: - render_mode None +2023-04-15 21:51:47 - r - INFO: - algo_name PER_DQN +2023-04-15 21:51:47 - r - INFO: - mode test +2023-04-15 21:51:47 - r - INFO: - mp_backend mp +2023-04-15 21:51:47 - r - INFO: - seed 1 +2023-04-15 21:51:47 - r - INFO: - device cpu +2023-04-15 21:51:47 - r - INFO: - train_eps 200 +2023-04-15 21:51:47 - r - INFO: - test_eps 10 +2023-04-15 21:51:47 - r - INFO: - eval_eps 10 +2023-04-15 21:51:47 - r - INFO: - eval_per_episode 5 +2023-04-15 21:51:47 - r - INFO: - max_steps 200 +2023-04-15 21:51:47 - r - INFO: - load_checkpoint 1 +2023-04-15 21:51:47 - r - INFO: - load_path Train_gym_PER_DQN_20230415-215002 +2023-04-15 21:51:47 - r - INFO: - show_fig 0 +2023-04-15 21:51:47 - r - INFO: - save_fig 1 +2023-04-15 21:51:47 - r - INFO: - n_workers 1 +2023-04-15 21:51:47 - r - INFO: - epsilon_start 0.95 +2023-04-15 21:51:47 - r - INFO: - epsilon_end 0.01 +2023-04-15 21:51:47 - r - INFO: - epsilon_decay 500 +2023-04-15 21:51:47 - r - INFO: - hidden_dim 256 +2023-04-15 21:51:47 - r - INFO: - gamma 0.99 +2023-04-15 21:51:47 - r - INFO: - lr 0.0001 +2023-04-15 21:51:47 - r - INFO: - buffer_size 100000 +2023-04-15 21:51:47 - r - INFO: - per_alpha 0.6 +2023-04-15 21:51:47 - r - INFO: - per_beta 0.4 +2023-04-15 21:51:47 - r - INFO: - per_beta_annealing 0.001 +2023-04-15 21:51:47 - r - INFO: - per_epsilon 0.01 +2023-04-15 21:51:47 - r - INFO: - batch_size 64 +2023-04-15 21:51:47 - r - INFO: - target_update 4 +2023-04-15 21:51:47 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-15 21:51:47 - r - INFO: - id CartPole-v1 +2023-04-15 21:51:47 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-215147 +2023-04-15 21:51:47 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-215147/results +2023-04-15 21:51:47 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-215147/logs +2023-04-15 21:51:47 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-215147/traj +2023-04-15 21:51:47 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-215147/videos +2023-04-15 21:51:47 - r - INFO: - ================================================================================ +2023-04-15 21:51:47 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-15 21:51:47 - r - INFO: - Start testing! +2023-04-15 21:51:47 - r - INFO: - Env: gym, Algorithm: PER_DQN, Device: cpu +2023-04-15 21:51:47 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-15 21:51:47 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f68746694a6ae30f06b2351e23e70ef9deafc34 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4251c7f141686d5391c5c933b493b27a184102ccf1596bead1dccaa6cc0bd9a +size 272407 diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..3d1599fb5f661c59b34fc753553b016cbc1f9e75 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/results/res.csv b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_mp_20230415-215147/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/config.yaml b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51a2e800e946e5822c90c42de6eb847f8db744cc --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/config.yaml @@ -0,0 +1,55 @@ +general_cfg: + algo_name: PER_DQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_PER_DQN_ray_20230415-215738 + max_steps: 200 + mode: test + mp_backend: ray + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 250 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + per_alpha: 0.6 + per_beta: 0.4 + per_beta_annealing: 0.001 + per_epsilon: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/logs/log.txt b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d545c207f7126a8b9d0b68b035501e24fbc4c325 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/logs/log.txt @@ -0,0 +1,58 @@ +2023-04-15 22:05:40 - r - INFO: - Hyperparameters: +2023-04-15 22:05:40 - r - INFO: - ================================================================================ +2023-04-15 22:05:40 - r - INFO: - Name Value Type +2023-04-15 22:05:40 - r - INFO: - env_name gym +2023-04-15 22:05:40 - r - INFO: - new_step_api 1 +2023-04-15 22:05:40 - r - INFO: - wrapper None +2023-04-15 22:05:40 - r - INFO: - render 0 +2023-04-15 22:05:40 - r - INFO: - render_mode None +2023-04-15 22:05:40 - r - INFO: - algo_name PER_DQN +2023-04-15 22:05:40 - r - INFO: - mode test +2023-04-15 22:05:40 - r - INFO: - mp_backend ray +2023-04-15 22:05:40 - r - INFO: - seed 1 +2023-04-15 22:05:40 - r - INFO: - device cpu +2023-04-15 22:05:40 - r - INFO: - train_eps 250 +2023-04-15 22:05:40 - r - INFO: - test_eps 10 +2023-04-15 22:05:40 - r - INFO: - eval_eps 10 +2023-04-15 22:05:40 - r - INFO: - eval_per_episode 5 +2023-04-15 22:05:40 - r - INFO: - max_steps 200 +2023-04-15 22:05:40 - r - INFO: - load_checkpoint 1 +2023-04-15 22:05:40 - r - INFO: - load_path Train_CartPole-v1_PER_DQN_ray_20230415-215738 +2023-04-15 22:05:40 - r - INFO: - show_fig 0 +2023-04-15 22:05:40 - r - INFO: - save_fig 1 +2023-04-15 22:05:40 - r - INFO: - n_workers 1 +2023-04-15 22:05:40 - r - INFO: - epsilon_start 0.95 +2023-04-15 22:05:40 - r - INFO: - epsilon_end 0.01 +2023-04-15 22:05:40 - r - INFO: - epsilon_decay 500 +2023-04-15 22:05:40 - r - INFO: - hidden_dim 256 +2023-04-15 22:05:40 - r - INFO: - gamma 0.99 +2023-04-15 22:05:40 - r - INFO: - lr 0.0001 +2023-04-15 22:05:40 - r - INFO: - buffer_size 100000 +2023-04-15 22:05:40 - r - INFO: - per_alpha 0.6 +2023-04-15 22:05:40 - r - INFO: - per_beta 0.4 +2023-04-15 22:05:40 - r - INFO: - per_beta_annealing 0.001 +2023-04-15 22:05:40 - r - INFO: - per_epsilon 0.01 +2023-04-15 22:05:40 - r - INFO: - batch_size 64 +2023-04-15 22:05:40 - r - INFO: - target_update 4 +2023-04-15 22:05:40 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-15 22:05:40 - r - INFO: - id CartPole-v1 +2023-04-15 22:05:40 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-220540 +2023-04-15 22:05:40 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-220540/results +2023-04-15 22:05:40 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-220540/logs +2023-04-15 22:05:40 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-220540/traj +2023-04-15 22:05:40 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Test_gym_PER_DQN_20230415-220540/videos +2023-04-15 22:05:40 - r - INFO: - ================================================================================ +2023-04-15 22:05:40 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-15 22:05:40 - r - INFO: - Start testing! +2023-04-15 22:05:40 - r - INFO: - Env: gym, Algorithm: PER_DQN, Device: cpu +2023-04-15 22:05:40 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:40 - r - INFO: - Episode: 2/10, Reward: 199.000, Step: 199 +2023-04-15 22:05:40 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:40 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:40 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:40 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:40 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:40 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:41 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:41 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-04-15 22:05:41 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..75946dbb92769075d2500d272c9ae471ca3675eb --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0efe3ec576afef2311748067e61af0fe6c939f7a2c2a1500001987a5d0092ce3 +size 272407 diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..04f725dd5c226a7baa0918e354e668324718445f Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/results/res.csv b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..08489d2627812baa1fb01515831f31b697056794 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PER_DQN_ray_20230415-220540/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,199.0,199 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/config.yaml b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7883d3417c8d79ca1626b299eb661c42e71373ff --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/config.yaml @@ -0,0 +1,35 @@ +general_cfg: + algo_name: PPO + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_PPO_20221217-204003 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + kl_alpha: 2 + kl_beta: 1.5 + kl_lambda: 0.5 + kl_target: 0.01 + ppo_type: kl + sgd_batch_size: 64 + train_batch_size: 100 diff --git a/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/logs/log.txt b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..bc860c3d9260de19df00c8fced3665870e2ba0cf --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/logs/log.txt @@ -0,0 +1,51 @@ +2022-12-17 20:42:14 - r - INFO: - Hyperparameters: +2022-12-17 20:42:14 - r - INFO: - ================================================================================ +2022-12-17 20:42:14 - r - INFO: - Name Value Type +2022-12-17 20:42:14 - r - INFO: - env_name CartPole-v1 +2022-12-17 20:42:14 - r - INFO: - new_step_api 1 +2022-12-17 20:42:14 - r - INFO: - wrapper None +2022-12-17 20:42:14 - r - INFO: - render 0 +2022-12-17 20:42:14 - r - INFO: - algo_name PPO +2022-12-17 20:42:14 - r - INFO: - mode test +2022-12-17 20:42:14 - r - INFO: - seed 1 +2022-12-17 20:42:14 - r - INFO: - device cuda +2022-12-17 20:42:14 - r - INFO: - train_eps 200 +2022-12-17 20:42:14 - r - INFO: - test_eps 10 +2022-12-17 20:42:14 - r - INFO: - eval_eps 10 +2022-12-17 20:42:14 - r - INFO: - eval_per_episode 5 +2022-12-17 20:42:14 - r - INFO: - max_steps 200 +2022-12-17 20:42:14 - r - INFO: - load_checkpoint 1 +2022-12-17 20:42:14 - r - INFO: - load_path Train_CartPole-v1_PPO_20221217-204003 +2022-12-17 20:42:14 - r - INFO: - show_fig 0 +2022-12-17 20:42:14 - r - INFO: - save_fig 1 +2022-12-17 20:42:14 - r - INFO: - ppo_type kl +2022-12-17 20:42:14 - r - INFO: - continuous 0 +2022-12-17 20:42:14 - r - INFO: - gamma 0.99 +2022-12-17 20:42:14 - r - INFO: - k_epochs 4 +2022-12-17 20:42:14 - r - INFO: - actor_lr 0.0003 +2022-12-17 20:42:14 - r - INFO: - critic_lr 0.001 +2022-12-17 20:42:14 - r - INFO: - eps_clip 0.2 +2022-12-17 20:42:14 - r - INFO: - entropy_coef 0.01 +2022-12-17 20:42:14 - r - INFO: - train_batch_size 100 +2022-12-17 20:42:14 - r - INFO: - sgd_batch_size 64 +2022-12-17 20:42:14 - r - INFO: - actor_hidden_dim 256 +2022-12-17 20:42:14 - r - INFO: - critic_hidden_dim 256 +2022-12-17 20:42:14 - r - INFO: - kl_alpha 2 +2022-12-17 20:42:14 - r - INFO: - kl_beta 1.5 +2022-12-17 20:42:14 - r - INFO: - kl_lambda 0.5 +2022-12-17 20:42:14 - r - INFO: - kl_target 0.01 +2022-12-17 20:42:14 - r - INFO: - ================================================================================ +2022-12-17 20:42:15 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-17 20:42:16 - r - INFO: - Start testing! +2022-12-17 20:42:16 - r - INFO: - Env: CartPole-v1, Algorithm: PPO, Device: cuda +2022-12-17 20:42:17 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:17 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:18 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:18 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:18 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:19 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:19 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:19 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:19 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:20 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2022-12-17 20:42:20 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/models/actor.pth b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..1fd56fde4dab4d2521f264a2a50ec4fb8f0cbf39 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffb5b77fd6cfb12b05d0565beee7740f4fa416356f845f856f5af472262d6726 +size 272215 diff --git a/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/models/critic.pth b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..02a05142e10e68007354dd5be7e7429fb112339e --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e913c00d51a429d08ba5956d4ee79c8a4c4ad029072c539bb53b684c874cdd +size 271191 diff --git a/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..0d8920b9f1d449c6a8cd94bc8d8b7f42b36c93ed Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/results/res.csv b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO-KL_20221217-204214/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/config.yaml b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57391cdbb48d80fdae488309494743fa69da6337 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/config.yaml @@ -0,0 +1,31 @@ +general_cfg: + algo_name: PPO + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_PPO_20230220-212959 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + ppo_type: clip + sgd_batch_size: 128 + train_batch_size: 256 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/logs/log.txt b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..80604ec63f8080029ce8aabc3a6a04ffa95e0286 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/logs/log.txt @@ -0,0 +1,52 @@ +2023-02-20 21:31:53 - r - INFO: - Hyperparameters: +2023-02-20 21:31:53 - r - INFO: - ================================================================================ +2023-02-20 21:31:53 - r - INFO: - Name Value Type +2023-02-20 21:31:53 - r - INFO: - env_name CartPole-v1 +2023-02-20 21:31:53 - r - INFO: - new_step_api 1 +2023-02-20 21:31:53 - r - INFO: - wrapper None +2023-02-20 21:31:53 - r - INFO: - render 0 +2023-02-20 21:31:53 - r - INFO: - algo_name PPO +2023-02-20 21:31:53 - r - INFO: - mode test +2023-02-20 21:31:53 - r - INFO: - seed 1 +2023-02-20 21:31:53 - r - INFO: - device cuda +2023-02-20 21:31:53 - r - INFO: - train_eps 200 +2023-02-20 21:31:53 - r - INFO: - test_eps 10 +2023-02-20 21:31:53 - r - INFO: - eval_eps 10 +2023-02-20 21:31:53 - r - INFO: - eval_per_episode 5 +2023-02-20 21:31:53 - r - INFO: - max_steps 200 +2023-02-20 21:31:53 - r - INFO: - load_checkpoint 1 +2023-02-20 21:31:53 - r - INFO: - load_path Train_CartPole-v1_PPO_20230220-212959 +2023-02-20 21:31:53 - r - INFO: - show_fig 0 +2023-02-20 21:31:53 - r - INFO: - save_fig 1 +2023-02-20 21:31:53 - r - INFO: - ppo_type clip +2023-02-20 21:31:53 - r - INFO: - continuous 0 +2023-02-20 21:31:53 - r - INFO: - gamma 0.99 +2023-02-20 21:31:53 - r - INFO: - k_epochs 4 +2023-02-20 21:31:53 - r - INFO: - actor_lr 0.0003 +2023-02-20 21:31:53 - r - INFO: - critic_lr 0.001 +2023-02-20 21:31:53 - r - INFO: - eps_clip 0.2 +2023-02-20 21:31:53 - r - INFO: - entropy_coef 0.01 +2023-02-20 21:31:53 - r - INFO: - train_batch_size 256 +2023-02-20 21:31:53 - r - INFO: - sgd_batch_size 128 +2023-02-20 21:31:53 - r - INFO: - actor_hidden_dim 256 +2023-02-20 21:31:53 - r - INFO: - critic_hidden_dim 256 +2023-02-20 21:31:53 - r - INFO: - task_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Test_CartPole-v1_PPO_20230220-213153 +2023-02-20 21:31:53 - r - INFO: - model_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Test_CartPole-v1_PPO_20230220-213153/models +2023-02-20 21:31:53 - r - INFO: - res_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Test_CartPole-v1_PPO_20230220-213153/results +2023-02-20 21:31:53 - r - INFO: - log_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Test_CartPole-v1_PPO_20230220-213153/logs +2023-02-20 21:31:53 - r - INFO: - traj_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Test_CartPole-v1_PPO_20230220-213153/traj +2023-02-20 21:31:53 - r - INFO: - ================================================================================ +2023-02-20 21:31:53 - r - INFO: - n_states: 4, n_actions: 2 +2023-02-20 21:31:54 - r - INFO: - Start testing! +2023-02-20 21:31:54 - r - INFO: - Env: CartPole-v1, Algorithm: PPO, Device: cuda +2023-02-20 21:31:55 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:55 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:56 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:56 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:56 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:56 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:56 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:57 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:57 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-02-20 21:31:57 - r - INFO: - Episode: 10/10, Reward: 189.000, Step: 189 +2023-02-20 21:31:57 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/models/actor.pth b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..22cd1f4300c6f64826192a7b43471008d9044122 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d07e4388597f766e04099380da27cb55fd877e8d26cdd14eab48bc097525216 +size 272215 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/models/critic.pth b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..373c9387a423d3545d43deac3722143e84a8b455 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6743877f9534c272b5e6d8bae3cbc87b1fa32bb21af2935c28e503122e042c2d +size 271191 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..bb57a32335ac171f22b4e66e9c7f669e1d61696c Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/results/res.csv b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..a5a032c537a17f2951a4be5dc6855af42a5964d4 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230220-213153/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,189.0,189 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/config.yaml b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e97fabd1b45800cd7198c72bb0a58004debfe1c --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: PPO + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_mp_PPO_20230401-223204 + max_steps: 200 + mode: test + n_workers: 1 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 300 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + ppo_type: clip + sgd_batch_size: 128 + train_batch_size: 256 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/logs/log.txt b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..63b8038106b36760bcaeca84724bc40f2b8a5626 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/logs/log.txt @@ -0,0 +1,53 @@ +2023-04-01 22:34:12 - r - INFO: - Hyperparameters: +2023-04-01 22:34:12 - r - INFO: - ================================================================================ +2023-04-01 22:34:12 - r - INFO: - Name Value Type +2023-04-01 22:34:12 - r - INFO: - env_name CartPole-v1 +2023-04-01 22:34:12 - r - INFO: - new_step_api 1 +2023-04-01 22:34:12 - r - INFO: - wrapper None +2023-04-01 22:34:12 - r - INFO: - render 0 +2023-04-01 22:34:12 - r - INFO: - algo_name PPO +2023-04-01 22:34:12 - r - INFO: - mode test +2023-04-01 22:34:12 - r - INFO: - seed 1 +2023-04-01 22:34:12 - r - INFO: - device cpu +2023-04-01 22:34:12 - r - INFO: - train_eps 300 +2023-04-01 22:34:12 - r - INFO: - test_eps 10 +2023-04-01 22:34:12 - r - INFO: - eval_eps 10 +2023-04-01 22:34:12 - r - INFO: - eval_per_episode 5 +2023-04-01 22:34:12 - r - INFO: - max_steps 200 +2023-04-01 22:34:12 - r - INFO: - load_checkpoint 1 +2023-04-01 22:34:12 - r - INFO: - load_path Train_CartPole-v1_mp_PPO_20230401-223204 +2023-04-01 22:34:12 - r - INFO: - show_fig 0 +2023-04-01 22:34:12 - r - INFO: - save_fig 1 +2023-04-01 22:34:12 - r - INFO: - n_workers 1 +2023-04-01 22:34:12 - r - INFO: - ppo_type clip +2023-04-01 22:34:12 - r - INFO: - continuous 0 +2023-04-01 22:34:12 - r - INFO: - gamma 0.99 +2023-04-01 22:34:12 - r - INFO: - k_epochs 4 +2023-04-01 22:34:12 - r - INFO: - actor_lr 0.0003 +2023-04-01 22:34:12 - r - INFO: - critic_lr 0.001 +2023-04-01 22:34:12 - r - INFO: - eps_clip 0.2 +2023-04-01 22:34:12 - r - INFO: - entropy_coef 0.01 +2023-04-01 22:34:12 - r - INFO: - train_batch_size 256 +2023-04-01 22:34:12 - r - INFO: - sgd_batch_size 128 +2023-04-01 22:34:12 - r - INFO: - actor_hidden_dim 256 +2023-04-01 22:34:12 - r - INFO: - critic_hidden_dim 256 +2023-04-01 22:34:12 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_PPO_20230401-223412 +2023-04-01 22:34:12 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_PPO_20230401-223412/results +2023-04-01 22:34:12 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_PPO_20230401-223412/logs +2023-04-01 22:34:12 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_PPO_20230401-223412/traj +2023-04-01 22:34:12 - r - INFO: - tb_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_PPO_20230401-223412/tb_logs +2023-04-01 22:34:12 - r - INFO: - ================================================================================ +2023-04-01 22:34:12 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-01 22:34:12 - r - INFO: - Start testing! +2023-04-01 22:34:12 - r - INFO: - Env: CartPole-v1, Algorithm: PPO, Device: cpu +2023-04-01 22:34:12 - r - INFO: - Episode: 1/10, Reward: 136.000, Step: 136 +2023-04-01 22:34:12 - r - INFO: - Episode: 2/10, Reward: 136.000, Step: 136 +2023-04-01 22:34:12 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-04-01 22:34:12 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-04-01 22:34:12 - r - INFO: - Episode: 5/10, Reward: 187.000, Step: 187 +2023-04-01 22:34:13 - r - INFO: - Episode: 6/10, Reward: 192.000, Step: 192 +2023-04-01 22:34:13 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-04-01 22:34:13 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-04-01 22:34:13 - r - INFO: - Episode: 9/10, Reward: 159.000, Step: 159 +2023-04-01 22:34:13 - r - INFO: - Episode: 10/10, Reward: 124.000, Step: 124 +2023-04-01 22:34:13 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/models/actor.pth b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..a22cff925f29e43f1ef84a9742bcc9a77b29ce68 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a3e48d551bcba327ff4c5d3cc464a6a94b83eda543a54d231016e021e8cbd3 +size 272151 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/models/critic.pth b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..10365bebec659f3c79f3521957f2814d8dfe1646 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4db7aeb3805e1deb11428a34a600a40068a0f711986f38fdf9e0f9895f8a45c +size 271127 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..0abc4cb2c132e0a346005674ee7e988276896c37 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/results/res.csv b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..93934dfcb62a6b2e7d25c554d02ab0bc6eb06682 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,136.0,136 +1,136.0,136 +2,200.0,200 +3,200.0,200 +4,187.0,187 +5,192.0,192 +6,200.0,200 +7,200.0,200 +8,159.0,159 +9,124.0,124 diff --git a/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/tb_logs/events.out.tfevents.1680359652.dell-Precision-5820-Tower.4337.0 b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/tb_logs/events.out.tfevents.1680359652.dell-Precision-5820-Tower.4337.0 new file mode 100644 index 0000000000000000000000000000000000000000..7c4c7884bbc5985b34631fe41664ffdc34062a19 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_PPO_20230401-223412/tb_logs/events.out.tfevents.1680359652.dell-Precision-5820-Tower.4337.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ddf29ee69cf62909c79397c263b36e21520613cb43c5bfe70a96b2ac2ff0871 +size 40 diff --git a/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/config.yaml b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dfe83b668b0d71b0b8c2404dcdcfb388794d145 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/config.yaml @@ -0,0 +1,23 @@ +general_cfg: + algo_name: REINFORCE + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_REINFORCE_20221203-143307 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + gamma: 0.99 + hidden_dim: 36 + lr: 0.01 + update_freq: 200 diff --git a/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/logs/log.txt b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..ca89ffcc60379190dfa00d5fd579fe90253bdb62 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/logs/log.txt @@ -0,0 +1,39 @@ +2022-12-03 14:34:30 - r - INFO: - Hyperparameters: +2022-12-03 14:34:30 - r - INFO: - ================================================================================ +2022-12-03 14:34:30 - r - INFO: - Name Value Type +2022-12-03 14:34:30 - r - INFO: - env_name CartPole-v1 +2022-12-03 14:34:30 - r - INFO: - new_step_api 1 +2022-12-03 14:34:30 - r - INFO: - wrapper None +2022-12-03 14:34:30 - r - INFO: - render 0 +2022-12-03 14:34:30 - r - INFO: - algo_name REINFORCE +2022-12-03 14:34:30 - r - INFO: - mode test +2022-12-03 14:34:30 - r - INFO: - seed 1 +2022-12-03 14:34:30 - r - INFO: - device cpu +2022-12-03 14:34:30 - r - INFO: - train_eps 200 +2022-12-03 14:34:30 - r - INFO: - test_eps 10 +2022-12-03 14:34:30 - r - INFO: - eval_eps 10 +2022-12-03 14:34:30 - r - INFO: - eval_per_episode 5 +2022-12-03 14:34:30 - r - INFO: - max_steps 200 +2022-12-03 14:34:30 - r - INFO: - load_checkpoint 1 +2022-12-03 14:34:30 - r - INFO: - load_path Train_CartPole-v1_REINFORCE_20221203-143307 +2022-12-03 14:34:30 - r - INFO: - show_fig 0 +2022-12-03 14:34:30 - r - INFO: - save_fig 1 +2022-12-03 14:34:30 - r - INFO: - lr 0.01 +2022-12-03 14:34:30 - r - INFO: - gamma 0.99 +2022-12-03 14:34:30 - r - INFO: - hidden_dim 36 +2022-12-03 14:34:30 - r - INFO: - update_freq 200 +2022-12-03 14:34:30 - r - INFO: - ================================================================================ +2022-12-03 14:34:31 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-03 14:34:31 - r - INFO: - Start testing! +2022-12-03 14:34:31 - r - INFO: - Env: CartPole-v1, Algorithm: REINFORCE, Device: cpu +2022-12-03 14:34:31 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:31 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:31 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:31 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:31 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:31 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:31 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:32 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:32 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:32 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2022-12-03 14:34:32 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2c9ea3eed609bf1b815f0f560ea071eec183e17 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9965ae728a6d4aa17b0a11c659cefd12037acb45bcf73265de391a87c69698c8 +size 8279 diff --git a/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..70abd7401f34cbc4a7d1f27d8674c5fa49d01a43 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/results/res.csv b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_REINFORCE_20221203-143430/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/config.yaml b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c4a5cb770a667edfe41e5450c812cd1e6d1a237 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/config.yaml @@ -0,0 +1,57 @@ +general_cfg: + Vmax: 200 + Vmin: 0 + algo_name: RainbowDQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_RainbowDQN_20230114-222012 + max_steps: 200 + mode: test + n_atoms: 51 + new_step_api: true + render: false + save_fig: true + seed: 0 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + Vmax: 200.0 + Vmin: 0.0 + batch_size: 64 + buffer_size: 100000 + delta_z: 4.0 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + memory_capacity: 10000 + n_step: 1 + num_atoms: 51 + support: !!python/object/apply:torch._utils._rebuild_tensor_v2 + - !!python/object/apply:torch.storage._load_from_bytes + - !!binary | + gAKKCmz8nEb5IGqoUBkugAJN6QMugAJ9cQAoWBAAAABwcm90b2NvbF92ZXJzaW9ucQFN6QNYDQAA + AGxpdHRsZV9lbmRpYW5xAohYCgAAAHR5cGVfc2l6ZXNxA31xBChYBQAAAHNob3J0cQVLAlgDAAAA + aW50cQZLBFgEAAAAbG9uZ3EHSwR1dS6AAihYBwAAAHN0b3JhZ2VxAGN0b3JjaApGbG9hdFN0b3Jh + Z2UKcQFYCQAAADExNzgyODk2MHECWAMAAABjcHVxA0szTnRxBFEugAJdcQBYCQAAADExNzgyODk2 + MHEBYS4zAAAAAAAAAAAAAAAAAIBAAAAAQQAAQEEAAIBBAACgQQAAwEEAAOBBAAAAQgAAEEIAACBC + AAAwQgAAQEIAAFBCAABgQgAAcEIAAIBCAACIQgAAkEIAAJhCAACgQgAAqEIAALBCAAC4QgAAwEIA + AMhCAADQQgAA2EIAAOBCAADoQgAA8EIAAPhCAAAAQwAABEMAAAhDAAAMQwAAEEMAABRDAAAYQwAA + HEMAACBDAAAkQwAAKEMAACxDAAAwQwAANEMAADhDAAA8QwAAQEMAAERDAABIQw== + - 0 + - !!python/tuple + - 51 + - !!python/tuple + - 1 + - false + - !!python/object/apply:collections.OrderedDict + - [] + target_update: 4 + tau: 1.0 diff --git a/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/logs/log.txt b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4558b623a5e1be441d8a4a74cbdba042c80b7a43 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/logs/log.txt @@ -0,0 +1,58 @@ +2023-01-14 22:24:46 - r - INFO: - Hyperparameters: +2023-01-14 22:24:46 - r - INFO: - ================================================================================ +2023-01-14 22:24:46 - r - INFO: - Name Value Type +2023-01-14 22:24:46 - r - INFO: - env_name CartPole-v1 +2023-01-14 22:24:46 - r - INFO: - new_step_api 1 +2023-01-14 22:24:46 - r - INFO: - wrapper None +2023-01-14 22:24:46 - r - INFO: - render 0 +2023-01-14 22:24:46 - r - INFO: - algo_name RainbowDQN +2023-01-14 22:24:46 - r - INFO: - mode test +2023-01-14 22:24:46 - r - INFO: - seed 0 +2023-01-14 22:24:46 - r - INFO: - device cuda +2023-01-14 22:24:46 - r - INFO: - train_eps 100 +2023-01-14 22:24:46 - r - INFO: - test_eps 10 +2023-01-14 22:24:46 - r - INFO: - eval_eps 10 +2023-01-14 22:24:46 - r - INFO: - eval_per_episode 5 +2023-01-14 22:24:46 - r - INFO: - max_steps 200 +2023-01-14 22:24:46 - r - INFO: - load_checkpoint 1 +2023-01-14 22:24:46 - r - INFO: - load_path Train_CartPole-v1_RainbowDQN_20230114-222012 +2023-01-14 22:24:46 - r - INFO: - show_fig 0 +2023-01-14 22:24:46 - r - INFO: - save_fig 1 +2023-01-14 22:24:46 - r - INFO: - Vmin 0.0 +2023-01-14 22:24:46 - r - INFO: - Vmax 200.0 +2023-01-14 22:24:46 - r - INFO: - n_atoms 51 +2023-01-14 22:24:46 - r - INFO: - gamma 0.95 +2023-01-14 22:24:46 - r - INFO: - tau 1.0 +2023-01-14 22:24:46 - r - INFO: - hidden_dim 256 +2023-01-14 22:24:46 - r - INFO: - num_atoms 51 +2023-01-14 22:24:46 - r - INFO: - support tensor(0.) +2023-01-14 22:24:46 - r - INFO: - delta_z 4.0 +2023-01-14 22:24:46 - r - INFO: - n_step 1 +2023-01-14 22:24:46 - r - INFO: - batch_size 64 +2023-01-14 22:24:46 - r - INFO: - lr 0.0001 +2023-01-14 22:24:46 - r - INFO: - target_update 4 +2023-01-14 22:24:46 - r - INFO: - memory_capacity 10000 +2023-01-14 22:24:46 - r - INFO: - epsilon_start 0.95 +2023-01-14 22:24:46 - r - INFO: - epsilon_end 0.01 +2023-01-14 22:24:46 - r - INFO: - epsilon_decay 500 +2023-01-14 22:24:46 - r - INFO: - buffer_size 100000 +2023-01-14 22:24:46 - r - INFO: - task_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_RainbowDQN_20230114-222446 +2023-01-14 22:24:46 - r - INFO: - model_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_RainbowDQN_20230114-222446/models +2023-01-14 22:24:46 - r - INFO: - res_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_RainbowDQN_20230114-222446/results +2023-01-14 22:24:46 - r - INFO: - log_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_RainbowDQN_20230114-222446/logs +2023-01-14 22:24:46 - r - INFO: - traj_dir /home/dingli/rl-tutorials/joyrl/tasks/Test_CartPole-v1_RainbowDQN_20230114-222446/traj +2023-01-14 22:24:46 - r - INFO: - ================================================================================ +2023-01-14 22:24:46 - r - INFO: - n_states: 4, n_actions: 2 +2023-01-14 22:24:48 - r - INFO: - Start testing! +2023-01-14 22:24:48 - r - INFO: - Env: CartPole-v1, Algorithm: RainbowDQN, Device: cuda +2023-01-14 22:24:48 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:49 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:49 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:49 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:49 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:49 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:49 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:49 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:50 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:50 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-01-14 22:24:50 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c388dd5bae6b0ccde7d78dd46998d02b7a96fac --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:510ec8e389f33f05d770df84af4fdfca682c7e83dd02da6c1374a578b9ce9ffd +size 2064020 diff --git a/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..6375dd095af0b705197fb95b026d9273afb61512 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/results/res.csv b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_RainbowDQN_20230114-222446/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/config.yaml b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12ea00e6c855dc1b22f80d3ee6c165695044793a --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: SAC_D + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_SAC_D_20230305-112849 + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 0 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + alpha: 0.2 + automatic_entropy_tuning: false + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + n_epochs: 1 + target_update: 1 + tau: 0.005 diff --git a/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/logs/log.txt b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..45f1cddd55da6fb92dd94dba272b4d28006a6e89 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/logs/log.txt @@ -0,0 +1,53 @@ +2023-03-05 11:32:38 - r - INFO: - Hyperparameters: +2023-03-05 11:32:38 - r - INFO: - ================================================================================ +2023-03-05 11:32:38 - r - INFO: - Name Value Type +2023-03-05 11:32:38 - r - INFO: - env_name CartPole-v1 +2023-03-05 11:32:38 - r - INFO: - new_step_api 1 +2023-03-05 11:32:38 - r - INFO: - wrapper None +2023-03-05 11:32:38 - r - INFO: - render 0 +2023-03-05 11:32:38 - r - INFO: - algo_name SAC_D +2023-03-05 11:32:38 - r - INFO: - mode test +2023-03-05 11:32:38 - r - INFO: - seed 0 +2023-03-05 11:32:38 - r - INFO: - device cuda +2023-03-05 11:32:38 - r - INFO: - train_eps 200 +2023-03-05 11:32:38 - r - INFO: - test_eps 10 +2023-03-05 11:32:38 - r - INFO: - eval_eps 10 +2023-03-05 11:32:38 - r - INFO: - eval_per_episode 5 +2023-03-05 11:32:38 - r - INFO: - max_steps 200 +2023-03-05 11:32:38 - r - INFO: - load_checkpoint 1 +2023-03-05 11:32:38 - r - INFO: - load_path Train_CartPole-v1_SAC_D_20230305-112849 +2023-03-05 11:32:38 - r - INFO: - show_fig 0 +2023-03-05 11:32:38 - r - INFO: - save_fig 1 +2023-03-05 11:32:38 - r - INFO: - epsilon_start 0.95 +2023-03-05 11:32:38 - r - INFO: - epsilon_end 0.01 +2023-03-05 11:32:38 - r - INFO: - epsilon_decay 500 +2023-03-05 11:32:38 - r - INFO: - lr 0.0001 +2023-03-05 11:32:38 - r - INFO: - gamma 0.95 +2023-03-05 11:32:38 - r - INFO: - tau 0.005 +2023-03-05 11:32:38 - r - INFO: - alpha 0.2 +2023-03-05 11:32:38 - r - INFO: - automatic_entropy_tuning 0 +2023-03-05 11:32:38 - r - INFO: - batch_size 64 +2023-03-05 11:32:38 - r - INFO: - hidden_dim 256 +2023-03-05 11:32:38 - r - INFO: - n_epochs 1 +2023-03-05 11:32:38 - r - INFO: - target_update 1 +2023-03-05 11:32:38 - r - INFO: - buffer_size 100000 +2023-03-05 11:32:38 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_SAC_D_20230305-113238 +2023-03-05 11:32:38 - r - INFO: - model_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_SAC_D_20230305-113238/models +2023-03-05 11:32:38 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_SAC_D_20230305-113238/results +2023-03-05 11:32:38 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_SAC_D_20230305-113238/logs +2023-03-05 11:32:38 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Test_CartPole-v1_SAC_D_20230305-113238/traj +2023-03-05 11:32:38 - r - INFO: - ================================================================================ +2023-03-05 11:32:38 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-05 11:32:40 - r - INFO: - Start testing! +2023-03-05 11:32:40 - r - INFO: - Env: CartPole-v1, Algorithm: SAC_D, Device: cuda +2023-03-05 11:32:41 - r - INFO: - Episode: 1/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 2/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 3/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 4/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 5/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 6/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 7/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 8/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 9/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Episode: 10/10, Reward: 200.000, Step: 200 +2023-03-05 11:32:41 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..c65db22c12e78156882175497b41be1c6ebff0aa --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360f671a85e43af869fcdcf35133cbea2f63d6c1c609967d3f3b457962e16f73 +size 2993695 diff --git a/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..90d54a240fe3a0436d14465102f5e9511fd2384e Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/results/res.csv b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbbcf2eb2cccfce2f3060e96b3484890fe578ac1 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_SAC_D_20230305-113238/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/config.yaml b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e7a4cc61a537d3d4acb61a0b165811120d673fc --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/config.yaml @@ -0,0 +1,44 @@ +general_cfg: + algo_name: DQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CartPole-v1_mp_DQN_20230319-215340 + max_steps: 200 + mode: test + n_workers: 1 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 20 + train_eps: 300 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/logs/log.txt b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..01d405051cfaa47b789f7f82ef4390cc5327b883 --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/logs/log.txt @@ -0,0 +1,60 @@ +2023-03-19 21:58:06 - r - INFO: - Hyperparameters: +2023-03-19 21:58:06 - r - INFO: - ================================================================================ +2023-03-19 21:58:06 - r - INFO: - Name Value Type +2023-03-19 21:58:06 - r - INFO: - env_name CartPole-v1 +2023-03-19 21:58:06 - r - INFO: - new_step_api 1 +2023-03-19 21:58:06 - r - INFO: - wrapper None +2023-03-19 21:58:06 - r - INFO: - render 0 +2023-03-19 21:58:06 - r - INFO: - algo_name DQN +2023-03-19 21:58:06 - r - INFO: - mode test +2023-03-19 21:58:06 - r - INFO: - seed 1 +2023-03-19 21:58:06 - r - INFO: - device cpu +2023-03-19 21:58:06 - r - INFO: - train_eps 300 +2023-03-19 21:58:06 - r - INFO: - test_eps 20 +2023-03-19 21:58:06 - r - INFO: - eval_eps 10 +2023-03-19 21:58:06 - r - INFO: - eval_per_episode 5 +2023-03-19 21:58:06 - r - INFO: - max_steps 200 +2023-03-19 21:58:06 - r - INFO: - load_checkpoint 1 +2023-03-19 21:58:06 - r - INFO: - load_path Train_CartPole-v1_mp_DQN_20230319-215340 +2023-03-19 21:58:06 - r - INFO: - show_fig 0 +2023-03-19 21:58:06 - r - INFO: - save_fig 1 +2023-03-19 21:58:06 - r - INFO: - n_workers 1 +2023-03-19 21:58:06 - r - INFO: - epsilon_start 0.95 +2023-03-19 21:58:06 - r - INFO: - epsilon_end 0.01 +2023-03-19 21:58:06 - r - INFO: - epsilon_decay 500 +2023-03-19 21:58:06 - r - INFO: - gamma 0.95 +2023-03-19 21:58:06 - r - INFO: - lr 0.0001 +2023-03-19 21:58:06 - r - INFO: - buffer_size 100000 +2023-03-19 21:58:06 - r - INFO: - batch_size 64 +2023-03-19 21:58:06 - r - INFO: - target_update 4 +2023-03-19 21:58:06 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-03-19 21:58:06 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DQN_20230319-215806 +2023-03-19 21:58:06 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DQN_20230319-215806/results +2023-03-19 21:58:06 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DQN_20230319-215806/logs +2023-03-19 21:58:06 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DQN_20230319-215806/traj +2023-03-19 21:58:06 - r - INFO: - tb_dir /media/disk/gsc/joyrl-offline/tasks/Test_CartPole-v1_DQN_20230319-215806/tb_logs +2023-03-19 21:58:06 - r - INFO: - ================================================================================ +2023-03-19 21:58:06 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-19 21:58:06 - r - INFO: - Start testing! +2023-03-19 21:58:06 - r - INFO: - Env: CartPole-v1, Algorithm: DQN, Device: cpu +2023-03-19 21:58:07 - r - INFO: - Episode: 1/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 2/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 3/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 4/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 5/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 6/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 7/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 8/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 9/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 10/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 11/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 12/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 13/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 14/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 15/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 16/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 17/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 18/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 19/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Episode: 20/20, Reward: 200.000, Step: 200 +2023-03-19 21:58:07 - r - INFO: - Finish testing! diff --git a/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/models/checkpoint.pt b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e20bc5cbeafb154ae2c3816a36507e9e8c5b34ae --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:308eccb972f5142d25c00579d63ecd9ae50df94fcdd9bb172d00217e9db3dd12 +size 272407 diff --git a/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/results/learning_curve.png b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..5084beef5e93d281c8ae9ad28e4f0f7934e7b925 Binary files /dev/null and b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/results/learning_curve.png differ diff --git a/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/results/res.csv b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..bbcf538967643440c57b300a16b93f8465e2ab9f --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,200.0,200 +1,200.0,200 +2,200.0,200 +3,200.0,200 +4,200.0,200 +5,200.0,200 +6,200.0,200 +7,200.0,200 +8,200.0,200 +9,200.0,200 +10,200.0,200 +11,200.0,200 +12,200.0,200 +13,200.0,200 +14,200.0,200 +15,200.0,200 +16,200.0,200 +17,200.0,200 +18,200.0,200 +19,200.0,200 diff --git a/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/tb_logs/events.out.tfevents.1679234286.ML3090.3788443.0 b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/tb_logs/events.out.tfevents.1679234286.ML3090.3788443.0 new file mode 100644 index 0000000000000000000000000000000000000000..aeb11a4cee5596929ed39f058b33a74f0791349c --- /dev/null +++ b/CartPole-v1/Test_CartPole-v1_mp_DQN_20230319-215806/tb_logs/events.out.tfevents.1679234286.ML3090.3788443.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86913021f0ed612b76dea0115a72ea9ca414e36e4a9b8ba768ae086473f616d8 +size 40 diff --git a/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/config.yaml b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f21a1ee07dea61df45e6e3f90ed256afb9bca707 --- /dev/null +++ b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/config.yaml @@ -0,0 +1,45 @@ +general_cfg: + algo_name: DQN + collect_traj: true + device: cpu + env_name: gym + load_checkpoint: true + load_model_step: best + load_path: Train_single_CartPole-v1_DQN_20230515-211721 + max_episode: 10 + max_step: 200 + mode: test + model_save_fre: 500 + mp_backend: single + n_workers: 1 + online_eval: true + online_eval_episode: 10 + save_fig: true + seed: 1 + show_fig: false +algo_cfg: + batch_size: 64 + buffer_size: 100000 + buffer_type: REPLAY_QUE + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + layer_type: linear +env_cfg: + id: CartPole-v1 + ignore_params: + - wrapper + - ignore_params + render_mode: rgb_array + wrapper: null diff --git a/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/logs/log.txt b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..57a99180430ba1e6fe64fd3bd09dfb7e904e299f --- /dev/null +++ b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/logs/log.txt @@ -0,0 +1,57 @@ +2023-05-15 21:19:26 - SimpleLog - INFO: - General Configs: +2023-05-15 21:19:26 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:19:26 - SimpleLog - INFO: - Name Value Type +2023-05-15 21:19:26 - SimpleLog - INFO: - env_name gym +2023-05-15 21:19:26 - SimpleLog - INFO: - algo_name DQN +2023-05-15 21:19:26 - SimpleLog - INFO: - mode test +2023-05-15 21:19:26 - SimpleLog - INFO: - collect_traj 1 +2023-05-15 21:19:26 - SimpleLog - INFO: - mp_backend single +2023-05-15 21:19:26 - SimpleLog - INFO: - n_workers 1 +2023-05-15 21:19:26 - SimpleLog - INFO: - seed 1 +2023-05-15 21:19:26 - SimpleLog - INFO: - device cpu +2023-05-15 21:19:26 - SimpleLog - INFO: - max_episode 10 +2023-05-15 21:19:26 - SimpleLog - INFO: - max_step 200 +2023-05-15 21:19:26 - SimpleLog - INFO: - online_eval 1 +2023-05-15 21:19:26 - SimpleLog - INFO: - online_eval_episode 10 +2023-05-15 21:19:26 - SimpleLog - INFO: - load_checkpoint 1 +2023-05-15 21:19:26 - SimpleLog - INFO: - load_path Train_single_CartPole-v1_DQN_20230515-211721 +2023-05-15 21:19:26 - SimpleLog - INFO: - show_fig 0 +2023-05-15 21:19:26 - SimpleLog - INFO: - save_fig 1 +2023-05-15 21:19:26 - SimpleLog - INFO: - load_model_step best +2023-05-15 21:19:26 - SimpleLog - INFO: - model_save_fre 500 +2023-05-15 21:19:26 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:19:26 - SimpleLog - INFO: - Algo Configs: +2023-05-15 21:19:26 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:19:26 - SimpleLog - INFO: - Name Value Type +2023-05-15 21:19:26 - SimpleLog - INFO: - epsilon_start 0.95 +2023-05-15 21:19:26 - SimpleLog - INFO: - epsilon_end 0.01 +2023-05-15 21:19:26 - SimpleLog - INFO: - epsilon_decay 500 +2023-05-15 21:19:26 - SimpleLog - INFO: - gamma 0.95 +2023-05-15 21:19:26 - SimpleLog - INFO: - lr 0.0001 +2023-05-15 21:19:26 - SimpleLog - INFO: - buffer_size 100000 +2023-05-15 21:19:26 - SimpleLog - INFO: - batch_size 64 +2023-05-15 21:19:26 - SimpleLog - INFO: - target_update 4 +2023-05-15 21:19:26 - SimpleLog - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': [256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256], 'activation': 'relu'}] +2023-05-15 21:19:26 - SimpleLog - INFO: - buffer_type REPLAY_QUE +2023-05-15 21:19:26 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:19:26 - SimpleLog - INFO: - Env Configs: +2023-05-15 21:19:26 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:19:26 - SimpleLog - INFO: - Name Value Type +2023-05-15 21:19:26 - SimpleLog - INFO: - id CartPole-v1 +2023-05-15 21:19:26 - SimpleLog - INFO: - render_mode rgb_array +2023-05-15 21:19:26 - SimpleLog - INFO: - wrapper None +2023-05-15 21:19:26 - SimpleLog - INFO: - ignore_params ['wrapper', 'ignore_params'] +2023-05-15 21:19:26 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:19:26 - SimpleLog - INFO: - obs_space: Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32), n_actions: Discrete(2) +2023-05-15 21:19:26 - SimpleLog - INFO: - Start testing! +2023-05-15 21:19:26 - SimpleLog - INFO: - episode: 0, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:30 - SimpleLog - INFO: - episode: 1, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:30 - SimpleLog - INFO: - episode: 2, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:31 - SimpleLog - INFO: - episode: 3, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:31 - SimpleLog - INFO: - episode: 4, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:31 - SimpleLog - INFO: - episode: 5, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:32 - SimpleLog - INFO: - episode: 6, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:32 - SimpleLog - INFO: - episode: 7, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:33 - SimpleLog - INFO: - episode: 8, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:33 - SimpleLog - INFO: - episode: 9, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:19:33 - SimpleLog - INFO: - Finish testing! total time consumed: 7.28s diff --git a/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/results/trajs_0.pkl b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/results/trajs_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8d7e2a9ab7a8025ba357a61f7c7ba89c0cacb387 --- /dev/null +++ b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/results/trajs_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fae5aa5ceb51833f761621229159f743bbc8e8a6766007136b3f2af48a1a001 +size 130746 diff --git a/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/tb_logs/interact/events.out.tfevents.1684156766.DESKTOP-H34HQIQ.16348.0 b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/tb_logs/interact/events.out.tfevents.1684156766.DESKTOP-H34HQIQ.16348.0 new file mode 100644 index 0000000000000000000000000000000000000000..a5eb29cf540657759cc2b373b27e7a5ce08946a0 --- /dev/null +++ b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/tb_logs/interact/events.out.tfevents.1684156766.DESKTOP-H34HQIQ.16348.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f5348239a851416ae5ead991b86721f16f71d5541ce3e49671133bb408edec2 +size 1056 diff --git a/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/tb_logs/model/events.out.tfevents.1684156766.DESKTOP-H34HQIQ.16348.1 b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/tb_logs/model/events.out.tfevents.1684156766.DESKTOP-H34HQIQ.16348.1 new file mode 100644 index 0000000000000000000000000000000000000000..06fc4e645766b00692487094cb074c5d3dead96d --- /dev/null +++ b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/tb_logs/model/events.out.tfevents.1684156766.DESKTOP-H34HQIQ.16348.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:778bf4752bfe3bce34855fa51be3e7fdeb15c8d13d02779f6ba433435fa2fdf4 +size 40 diff --git a/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/videos/video.gif b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/videos/video.gif new file mode 100644 index 0000000000000000000000000000000000000000..2c5a427744aa816b8fe87013db7f1df59bf0a040 --- /dev/null +++ b/CartPole-v1/Test_single_CartPole-v1_DQN_20230515-211926/videos/video.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a417ad8def4a6907872c3de9cd2883536a4b41e1c0d36bb98af3830d6eb76739 +size 131807 diff --git a/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/config.yaml b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce9d3a72a0d4766d2b95375fbfa66a27861a79c9 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: A2C + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 1000 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.001 + gamma: 0.99 + update_freq: 20 diff --git a/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/logs/log.txt b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..713bd6209da366b2c1ac2f97ad98c5e66623f2c6 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/logs/log.txt @@ -0,0 +1,1080 @@ +2022-12-04 00:34:39 - r - INFO: - Hyperparameters: +2022-12-04 00:34:39 - r - INFO: - ================================================================================ +2022-12-04 00:34:39 - r - INFO: - Name Value Type +2022-12-04 00:34:39 - r - INFO: - env_name CartPole-v1 +2022-12-04 00:34:39 - r - INFO: - new_step_api 1 +2022-12-04 00:34:39 - r - INFO: - wrapper None +2022-12-04 00:34:39 - r - INFO: - render 0 +2022-12-04 00:34:39 - r - INFO: - algo_name A2C +2022-12-04 00:34:39 - r - INFO: - mode train +2022-12-04 00:34:39 - r - INFO: - seed 1 +2022-12-04 00:34:39 - r - INFO: - device cpu +2022-12-04 00:34:39 - r - INFO: - train_eps 1000 +2022-12-04 00:34:39 - r - INFO: - test_eps 10 +2022-12-04 00:34:39 - r - INFO: - eval_eps 10 +2022-12-04 00:34:39 - r - INFO: - eval_per_episode 5 +2022-12-04 00:34:39 - r - INFO: - max_steps 200 +2022-12-04 00:34:39 - r - INFO: - load_checkpoint 0 +2022-12-04 00:34:39 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2022-12-04 00:34:39 - r - INFO: - show_fig 0 +2022-12-04 00:34:39 - r - INFO: - save_fig 1 +2022-12-04 00:34:39 - r - INFO: - continuous 0 +2022-12-04 00:34:39 - r - INFO: - gamma 0.99 +2022-12-04 00:34:39 - r - INFO: - actor_lr 0.0003 +2022-12-04 00:34:39 - r - INFO: - critic_lr 0.001 +2022-12-04 00:34:39 - r - INFO: - actor_hidden_dim 256 +2022-12-04 00:34:39 - r - INFO: - critic_hidden_dim 256 +2022-12-04 00:34:39 - r - INFO: - entropy_coef 0.001 +2022-12-04 00:34:39 - r - INFO: - update_freq 20 +2022-12-04 00:34:39 - r - INFO: - ================================================================================ +2022-12-04 00:34:40 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-04 00:34:40 - r - INFO: - Start training! +2022-12-04 00:34:40 - r - INFO: - Env: CartPole-v1, Algorithm: A2C, Device: cpu +2022-12-04 00:34:40 - r - INFO: - Episode: 1/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:40 - r - INFO: - Episode: 2/1000, Reward: 30.000, Step: 30 +2022-12-04 00:34:40 - r - INFO: - Episode: 3/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:40 - r - INFO: - Episode: 4/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:40 - r - INFO: - Episode: 5/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:40 - r - INFO: - Current episode 5 has the best eval reward: 18.800 +2022-12-04 00:34:40 - r - INFO: - Episode: 6/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:40 - r - INFO: - Episode: 7/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:40 - r - INFO: - Episode: 8/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:40 - r - INFO: - Episode: 9/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:40 - r - INFO: - Episode: 10/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:40 - r - INFO: - Current episode 10 has the best eval reward: 20.700 +2022-12-04 00:34:40 - r - INFO: - Episode: 11/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:40 - r - INFO: - Episode: 12/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:41 - r - INFO: - Episode: 13/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:41 - r - INFO: - Episode: 14/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:41 - r - INFO: - Episode: 15/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:41 - r - INFO: - Current episode 15 has the best eval reward: 24.200 +2022-12-04 00:34:41 - r - INFO: - Episode: 16/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:41 - r - INFO: - Episode: 17/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:41 - r - INFO: - Episode: 18/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:41 - r - INFO: - Episode: 19/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:41 - r - INFO: - Episode: 20/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:41 - r - INFO: - Episode: 21/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:41 - r - INFO: - Episode: 22/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:41 - r - INFO: - Episode: 23/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:41 - r - INFO: - Episode: 24/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:41 - r - INFO: - Episode: 25/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:41 - r - INFO: - Current episode 25 has the best eval reward: 24.600 +2022-12-04 00:34:41 - r - INFO: - Episode: 26/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:41 - r - INFO: - Episode: 27/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:41 - r - INFO: - Episode: 28/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:41 - r - INFO: - Episode: 29/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:41 - r - INFO: - Episode: 30/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:41 - r - INFO: - Episode: 31/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:41 - r - INFO: - Episode: 32/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:41 - r - INFO: - Episode: 33/1000, Reward: 30.000, Step: 30 +2022-12-04 00:34:41 - r - INFO: - Episode: 34/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:41 - r - INFO: - Episode: 35/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:41 - r - INFO: - Episode: 36/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:41 - r - INFO: - Episode: 37/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:41 - r - INFO: - Episode: 38/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:41 - r - INFO: - Episode: 39/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:41 - r - INFO: - Episode: 40/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:41 - r - INFO: - Episode: 41/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:41 - r - INFO: - Episode: 42/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:41 - r - INFO: - Episode: 43/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:41 - r - INFO: - Episode: 44/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:41 - r - INFO: - Episode: 45/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:41 - r - INFO: - Episode: 46/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:41 - r - INFO: - Episode: 47/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:42 - r - INFO: - Episode: 48/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:42 - r - INFO: - Episode: 49/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:42 - r - INFO: - Episode: 50/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:42 - r - INFO: - Episode: 51/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:42 - r - INFO: - Episode: 52/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:42 - r - INFO: - Episode: 53/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:42 - r - INFO: - Episode: 54/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:42 - r - INFO: - Episode: 55/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:42 - r - INFO: - Episode: 56/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:42 - r - INFO: - Episode: 57/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:42 - r - INFO: - Episode: 58/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:42 - r - INFO: - Episode: 59/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:42 - r - INFO: - Episode: 60/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:42 - r - INFO: - Current episode 60 has the best eval reward: 26.700 +2022-12-04 00:34:42 - r - INFO: - Episode: 61/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:42 - r - INFO: - Episode: 62/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:42 - r - INFO: - Episode: 63/1000, Reward: 34.000, Step: 34 +2022-12-04 00:34:42 - r - INFO: - Episode: 64/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:42 - r - INFO: - Episode: 65/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:42 - r - INFO: - Episode: 66/1000, Reward: 40.000, Step: 40 +2022-12-04 00:34:42 - r - INFO: - Episode: 67/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:42 - r - INFO: - Episode: 68/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:42 - r - INFO: - Episode: 69/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:42 - r - INFO: - Episode: 70/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:42 - r - INFO: - Current episode 70 has the best eval reward: 26.800 +2022-12-04 00:34:42 - r - INFO: - Episode: 71/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:42 - r - INFO: - Episode: 72/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:42 - r - INFO: - Episode: 73/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:42 - r - INFO: - Episode: 74/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:42 - r - INFO: - Episode: 75/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:42 - r - INFO: - Episode: 76/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:42 - r - INFO: - Episode: 77/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:42 - r - INFO: - Episode: 78/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:42 - r - INFO: - Episode: 79/1000, Reward: 43.000, Step: 43 +2022-12-04 00:34:42 - r - INFO: - Episode: 80/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:42 - r - INFO: - Episode: 81/1000, Reward: 43.000, Step: 43 +2022-12-04 00:34:42 - r - INFO: - Episode: 82/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:42 - r - INFO: - Episode: 83/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:42 - r - INFO: - Episode: 84/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:42 - r - INFO: - Episode: 85/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:43 - r - INFO: - Current episode 85 has the best eval reward: 26.800 +2022-12-04 00:34:43 - r - INFO: - Episode: 86/1000, Reward: 40.000, Step: 40 +2022-12-04 00:34:43 - r - INFO: - Episode: 87/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:43 - r - INFO: - Episode: 88/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:43 - r - INFO: - Episode: 89/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:43 - r - INFO: - Episode: 90/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:43 - r - INFO: - Episode: 91/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:43 - r - INFO: - Episode: 92/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:43 - r - INFO: - Episode: 93/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:43 - r - INFO: - Episode: 94/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:43 - r - INFO: - Episode: 95/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:43 - r - INFO: - Episode: 96/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:43 - r - INFO: - Episode: 97/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:43 - r - INFO: - Episode: 98/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:43 - r - INFO: - Episode: 99/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:43 - r - INFO: - Episode: 100/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:43 - r - INFO: - Episode: 101/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:43 - r - INFO: - Episode: 102/1000, Reward: 34.000, Step: 34 +2022-12-04 00:34:43 - r - INFO: - Episode: 103/1000, Reward: 47.000, Step: 47 +2022-12-04 00:34:43 - r - INFO: - Episode: 104/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:43 - r - INFO: - Episode: 105/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:43 - r - INFO: - Episode: 106/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:43 - r - INFO: - Episode: 107/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:43 - r - INFO: - Episode: 108/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:43 - r - INFO: - Episode: 109/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:43 - r - INFO: - Episode: 110/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:43 - r - INFO: - Episode: 111/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:43 - r - INFO: - Episode: 112/1000, Reward: 27.000, Step: 27 +2022-12-04 00:34:43 - r - INFO: - Episode: 113/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:43 - r - INFO: - Episode: 114/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:43 - r - INFO: - Episode: 115/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:43 - r - INFO: - Episode: 116/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:43 - r - INFO: - Episode: 117/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:43 - r - INFO: - Episode: 118/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:43 - r - INFO: - Episode: 119/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:43 - r - INFO: - Episode: 120/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:43 - r - INFO: - Episode: 121/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:43 - r - INFO: - Episode: 122/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:43 - r - INFO: - Episode: 123/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:43 - r - INFO: - Episode: 124/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:43 - r - INFO: - Episode: 125/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:44 - r - INFO: - Episode: 126/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:44 - r - INFO: - Episode: 127/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:44 - r - INFO: - Episode: 128/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:44 - r - INFO: - Episode: 129/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:44 - r - INFO: - Episode: 130/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 131/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 132/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:44 - r - INFO: - Episode: 133/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 134/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:44 - r - INFO: - Episode: 135/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:44 - r - INFO: - Episode: 136/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:44 - r - INFO: - Episode: 137/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 138/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:44 - r - INFO: - Episode: 139/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 140/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:44 - r - INFO: - Episode: 141/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:44 - r - INFO: - Episode: 142/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:44 - r - INFO: - Episode: 143/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:44 - r - INFO: - Episode: 144/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 145/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 146/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:44 - r - INFO: - Episode: 147/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:44 - r - INFO: - Episode: 148/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:44 - r - INFO: - Episode: 149/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:44 - r - INFO: - Episode: 150/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 151/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:44 - r - INFO: - Episode: 152/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:44 - r - INFO: - Episode: 153/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:44 - r - INFO: - Episode: 154/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 155/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 156/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:44 - r - INFO: - Episode: 157/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 158/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 159/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:44 - r - INFO: - Episode: 160/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 161/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:44 - r - INFO: - Episode: 162/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 163/1000, Reward: 47.000, Step: 47 +2022-12-04 00:34:44 - r - INFO: - Episode: 164/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:44 - r - INFO: - Episode: 165/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:44 - r - INFO: - Episode: 166/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:44 - r - INFO: - Episode: 167/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:44 - r - INFO: - Episode: 168/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:45 - r - INFO: - Episode: 169/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:45 - r - INFO: - Episode: 170/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:45 - r - INFO: - Episode: 171/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:45 - r - INFO: - Episode: 172/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:45 - r - INFO: - Episode: 173/1000, Reward: 18.000, Step: 18 +2022-12-04 00:34:45 - r - INFO: - Episode: 174/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:45 - r - INFO: - Episode: 175/1000, Reward: 18.000, Step: 18 +2022-12-04 00:34:45 - r - INFO: - Episode: 176/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:45 - r - INFO: - Episode: 177/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:45 - r - INFO: - Episode: 178/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:45 - r - INFO: - Episode: 179/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:45 - r - INFO: - Episode: 180/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:45 - r - INFO: - Episode: 181/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:45 - r - INFO: - Episode: 182/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:45 - r - INFO: - Episode: 183/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:45 - r - INFO: - Episode: 184/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:45 - r - INFO: - Episode: 185/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:45 - r - INFO: - Episode: 186/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:45 - r - INFO: - Episode: 187/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:45 - r - INFO: - Episode: 188/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:45 - r - INFO: - Episode: 189/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:45 - r - INFO: - Episode: 190/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:45 - r - INFO: - Episode: 191/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:45 - r - INFO: - Episode: 192/1000, Reward: 29.000, Step: 29 +2022-12-04 00:34:45 - r - INFO: - Episode: 193/1000, Reward: 43.000, Step: 43 +2022-12-04 00:34:45 - r - INFO: - Episode: 194/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:45 - r - INFO: - Episode: 195/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:45 - r - INFO: - Episode: 196/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:45 - r - INFO: - Episode: 197/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:45 - r - INFO: - Episode: 198/1000, Reward: 49.000, Step: 49 +2022-12-04 00:34:45 - r - INFO: - Episode: 199/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:45 - r - INFO: - Episode: 200/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:45 - r - INFO: - Episode: 201/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:45 - r - INFO: - Episode: 202/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:45 - r - INFO: - Episode: 203/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:45 - r - INFO: - Episode: 204/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:45 - r - INFO: - Episode: 205/1000, Reward: 29.000, Step: 29 +2022-12-04 00:34:45 - r - INFO: - Episode: 206/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:45 - r - INFO: - Episode: 207/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:45 - r - INFO: - Episode: 208/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:45 - r - INFO: - Episode: 209/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:45 - r - INFO: - Episode: 210/1000, Reward: 9.000, Step: 9 +2022-12-04 00:34:45 - r - INFO: - Episode: 211/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:45 - r - INFO: - Episode: 212/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:45 - r - INFO: - Episode: 213/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:45 - r - INFO: - Episode: 214/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:45 - r - INFO: - Episode: 215/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:46 - r - INFO: - Episode: 216/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:46 - r - INFO: - Episode: 217/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:46 - r - INFO: - Episode: 218/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:46 - r - INFO: - Episode: 219/1000, Reward: 18.000, Step: 18 +2022-12-04 00:34:46 - r - INFO: - Episode: 220/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:46 - r - INFO: - Episode: 221/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:46 - r - INFO: - Episode: 222/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:46 - r - INFO: - Episode: 223/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:46 - r - INFO: - Episode: 224/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:46 - r - INFO: - Episode: 225/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:46 - r - INFO: - Current episode 225 has the best eval reward: 27.300 +2022-12-04 00:34:46 - r - INFO: - Episode: 226/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:46 - r - INFO: - Episode: 227/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:46 - r - INFO: - Episode: 228/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:46 - r - INFO: - Episode: 229/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:46 - r - INFO: - Episode: 230/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:46 - r - INFO: - Episode: 231/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:46 - r - INFO: - Episode: 232/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:46 - r - INFO: - Episode: 233/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:46 - r - INFO: - Episode: 234/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:46 - r - INFO: - Episode: 235/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:46 - r - INFO: - Episode: 236/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:46 - r - INFO: - Episode: 237/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:46 - r - INFO: - Episode: 238/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:46 - r - INFO: - Episode: 239/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:46 - r - INFO: - Episode: 240/1000, Reward: 33.000, Step: 33 +2022-12-04 00:34:46 - r - INFO: - Current episode 240 has the best eval reward: 28.300 +2022-12-04 00:34:46 - r - INFO: - Episode: 241/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:46 - r - INFO: - Episode: 242/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:46 - r - INFO: - Episode: 243/1000, Reward: 32.000, Step: 32 +2022-12-04 00:34:46 - r - INFO: - Episode: 244/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:46 - r - INFO: - Episode: 245/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:46 - r - INFO: - Episode: 246/1000, Reward: 42.000, Step: 42 +2022-12-04 00:34:46 - r - INFO: - Episode: 247/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:46 - r - INFO: - Episode: 248/1000, Reward: 30.000, Step: 30 +2022-12-04 00:34:46 - r - INFO: - Episode: 249/1000, Reward: 27.000, Step: 27 +2022-12-04 00:34:46 - r - INFO: - Episode: 250/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:46 - r - INFO: - Episode: 251/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:46 - r - INFO: - Episode: 252/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:46 - r - INFO: - Episode: 253/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:46 - r - INFO: - Episode: 254/1000, Reward: 18.000, Step: 18 +2022-12-04 00:34:47 - r - INFO: - Episode: 255/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:47 - r - INFO: - Current episode 255 has the best eval reward: 28.800 +2022-12-04 00:34:47 - r - INFO: - Episode: 256/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:47 - r - INFO: - Episode: 257/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:47 - r - INFO: - Episode: 258/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:47 - r - INFO: - Episode: 259/1000, Reward: 51.000, Step: 51 +2022-12-04 00:34:47 - r - INFO: - Episode: 260/1000, Reward: 53.000, Step: 53 +2022-12-04 00:34:47 - r - INFO: - Current episode 260 has the best eval reward: 29.100 +2022-12-04 00:34:47 - r - INFO: - Episode: 261/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:47 - r - INFO: - Episode: 262/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:47 - r - INFO: - Episode: 263/1000, Reward: 47.000, Step: 47 +2022-12-04 00:34:47 - r - INFO: - Episode: 264/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:47 - r - INFO: - Episode: 265/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:47 - r - INFO: - Episode: 266/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:47 - r - INFO: - Episode: 267/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:47 - r - INFO: - Episode: 268/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:47 - r - INFO: - Episode: 269/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:47 - r - INFO: - Episode: 270/1000, Reward: 56.000, Step: 56 +2022-12-04 00:34:47 - r - INFO: - Current episode 270 has the best eval reward: 31.600 +2022-12-04 00:34:47 - r - INFO: - Episode: 271/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:47 - r - INFO: - Episode: 272/1000, Reward: 44.000, Step: 44 +2022-12-04 00:34:47 - r - INFO: - Episode: 273/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:47 - r - INFO: - Episode: 274/1000, Reward: 46.000, Step: 46 +2022-12-04 00:34:47 - r - INFO: - Episode: 275/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:47 - r - INFO: - Episode: 276/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:47 - r - INFO: - Episode: 277/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:47 - r - INFO: - Episode: 278/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:47 - r - INFO: - Episode: 279/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:47 - r - INFO: - Episode: 280/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:47 - r - INFO: - Episode: 281/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:47 - r - INFO: - Episode: 282/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:47 - r - INFO: - Episode: 283/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:47 - r - INFO: - Episode: 284/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:47 - r - INFO: - Episode: 285/1000, Reward: 36.000, Step: 36 +2022-12-04 00:34:48 - r - INFO: - Episode: 286/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:48 - r - INFO: - Episode: 287/1000, Reward: 51.000, Step: 51 +2022-12-04 00:34:48 - r - INFO: - Episode: 288/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:48 - r - INFO: - Episode: 289/1000, Reward: 36.000, Step: 36 +2022-12-04 00:34:48 - r - INFO: - Episode: 290/1000, Reward: 49.000, Step: 49 +2022-12-04 00:34:48 - r - INFO: - Current episode 290 has the best eval reward: 39.400 +2022-12-04 00:34:48 - r - INFO: - Episode: 291/1000, Reward: 54.000, Step: 54 +2022-12-04 00:34:48 - r - INFO: - Episode: 292/1000, Reward: 47.000, Step: 47 +2022-12-04 00:34:48 - r - INFO: - Episode: 293/1000, Reward: 62.000, Step: 62 +2022-12-04 00:34:48 - r - INFO: - Episode: 294/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:48 - r - INFO: - Episode: 295/1000, Reward: 54.000, Step: 54 +2022-12-04 00:34:48 - r - INFO: - Episode: 296/1000, Reward: 30.000, Step: 30 +2022-12-04 00:34:48 - r - INFO: - Episode: 297/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:48 - r - INFO: - Episode: 298/1000, Reward: 12.000, Step: 12 +2022-12-04 00:34:48 - r - INFO: - Episode: 299/1000, Reward: 18.000, Step: 18 +2022-12-04 00:34:48 - r - INFO: - Episode: 300/1000, Reward: 61.000, Step: 61 +2022-12-04 00:34:48 - r - INFO: - Current episode 300 has the best eval reward: 40.500 +2022-12-04 00:34:48 - r - INFO: - Episode: 301/1000, Reward: 30.000, Step: 30 +2022-12-04 00:34:48 - r - INFO: - Episode: 302/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:48 - r - INFO: - Episode: 303/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:48 - r - INFO: - Episode: 304/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:48 - r - INFO: - Episode: 305/1000, Reward: 54.000, Step: 54 +2022-12-04 00:34:48 - r - INFO: - Episode: 306/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:48 - r - INFO: - Episode: 307/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:48 - r - INFO: - Episode: 308/1000, Reward: 50.000, Step: 50 +2022-12-04 00:34:48 - r - INFO: - Episode: 309/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:48 - r - INFO: - Episode: 310/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:48 - r - INFO: - Episode: 311/1000, Reward: 46.000, Step: 46 +2022-12-04 00:34:48 - r - INFO: - Episode: 312/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:48 - r - INFO: - Episode: 313/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:48 - r - INFO: - Episode: 314/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:48 - r - INFO: - Episode: 315/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:49 - r - INFO: - Episode: 316/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:49 - r - INFO: - Episode: 317/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:49 - r - INFO: - Episode: 318/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:49 - r - INFO: - Episode: 319/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:49 - r - INFO: - Episode: 320/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:49 - r - INFO: - Episode: 321/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:49 - r - INFO: - Episode: 322/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:49 - r - INFO: - Episode: 323/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:49 - r - INFO: - Episode: 324/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:49 - r - INFO: - Episode: 325/1000, Reward: 32.000, Step: 32 +2022-12-04 00:34:49 - r - INFO: - Episode: 326/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:49 - r - INFO: - Episode: 327/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:49 - r - INFO: - Episode: 328/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:49 - r - INFO: - Episode: 329/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:49 - r - INFO: - Episode: 330/1000, Reward: 40.000, Step: 40 +2022-12-04 00:34:49 - r - INFO: - Episode: 331/1000, Reward: 36.000, Step: 36 +2022-12-04 00:34:49 - r - INFO: - Episode: 332/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:49 - r - INFO: - Episode: 333/1000, Reward: 30.000, Step: 30 +2022-12-04 00:34:49 - r - INFO: - Episode: 334/1000, Reward: 30.000, Step: 30 +2022-12-04 00:34:49 - r - INFO: - Episode: 335/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:49 - r - INFO: - Episode: 336/1000, Reward: 33.000, Step: 33 +2022-12-04 00:34:49 - r - INFO: - Episode: 337/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:49 - r - INFO: - Episode: 338/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:49 - r - INFO: - Episode: 339/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:49 - r - INFO: - Episode: 340/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:49 - r - INFO: - Episode: 341/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:49 - r - INFO: - Episode: 342/1000, Reward: 27.000, Step: 27 +2022-12-04 00:34:49 - r - INFO: - Episode: 343/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:49 - r - INFO: - Episode: 344/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:49 - r - INFO: - Episode: 345/1000, Reward: 14.000, Step: 14 +2022-12-04 00:34:49 - r - INFO: - Episode: 346/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:49 - r - INFO: - Episode: 347/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:49 - r - INFO: - Episode: 348/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:49 - r - INFO: - Episode: 349/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:49 - r - INFO: - Episode: 350/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:49 - r - INFO: - Episode: 351/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:49 - r - INFO: - Episode: 352/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:49 - r - INFO: - Episode: 353/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:49 - r - INFO: - Episode: 354/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:49 - r - INFO: - Episode: 355/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:50 - r - INFO: - Episode: 356/1000, Reward: 29.000, Step: 29 +2022-12-04 00:34:50 - r - INFO: - Episode: 357/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:50 - r - INFO: - Episode: 358/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:50 - r - INFO: - Episode: 359/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:50 - r - INFO: - Episode: 360/1000, Reward: 13.000, Step: 13 +2022-12-04 00:34:50 - r - INFO: - Episode: 361/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:50 - r - INFO: - Episode: 362/1000, Reward: 11.000, Step: 11 +2022-12-04 00:34:50 - r - INFO: - Episode: 363/1000, Reward: 27.000, Step: 27 +2022-12-04 00:34:50 - r - INFO: - Episode: 364/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:50 - r - INFO: - Episode: 365/1000, Reward: 78.000, Step: 78 +2022-12-04 00:34:50 - r - INFO: - Episode: 366/1000, Reward: 18.000, Step: 18 +2022-12-04 00:34:50 - r - INFO: - Episode: 367/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:50 - r - INFO: - Episode: 368/1000, Reward: 27.000, Step: 27 +2022-12-04 00:34:50 - r - INFO: - Episode: 369/1000, Reward: 10.000, Step: 10 +2022-12-04 00:34:50 - r - INFO: - Episode: 370/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:50 - r - INFO: - Episode: 371/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:50 - r - INFO: - Episode: 372/1000, Reward: 44.000, Step: 44 +2022-12-04 00:34:50 - r - INFO: - Episode: 373/1000, Reward: 77.000, Step: 77 +2022-12-04 00:34:50 - r - INFO: - Episode: 374/1000, Reward: 57.000, Step: 57 +2022-12-04 00:34:50 - r - INFO: - Episode: 375/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:50 - r - INFO: - Episode: 376/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:50 - r - INFO: - Episode: 377/1000, Reward: 44.000, Step: 44 +2022-12-04 00:34:50 - r - INFO: - Episode: 378/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:50 - r - INFO: - Episode: 379/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:50 - r - INFO: - Episode: 380/1000, Reward: 27.000, Step: 27 +2022-12-04 00:34:50 - r - INFO: - Episode: 381/1000, Reward: 43.000, Step: 43 +2022-12-04 00:34:50 - r - INFO: - Episode: 382/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:50 - r - INFO: - Episode: 383/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:50 - r - INFO: - Episode: 384/1000, Reward: 34.000, Step: 34 +2022-12-04 00:34:50 - r - INFO: - Episode: 385/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:51 - r - INFO: - Episode: 386/1000, Reward: 48.000, Step: 48 +2022-12-04 00:34:51 - r - INFO: - Episode: 387/1000, Reward: 18.000, Step: 18 +2022-12-04 00:34:51 - r - INFO: - Episode: 388/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:51 - r - INFO: - Episode: 389/1000, Reward: 58.000, Step: 58 +2022-12-04 00:34:51 - r - INFO: - Episode: 390/1000, Reward: 84.000, Step: 84 +2022-12-04 00:34:51 - r - INFO: - Episode: 391/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:51 - r - INFO: - Episode: 392/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:51 - r - INFO: - Episode: 393/1000, Reward: 48.000, Step: 48 +2022-12-04 00:34:51 - r - INFO: - Episode: 394/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:51 - r - INFO: - Episode: 395/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:51 - r - INFO: - Episode: 396/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:51 - r - INFO: - Episode: 397/1000, Reward: 54.000, Step: 54 +2022-12-04 00:34:51 - r - INFO: - Episode: 398/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:51 - r - INFO: - Episode: 399/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:51 - r - INFO: - Episode: 400/1000, Reward: 44.000, Step: 44 +2022-12-04 00:34:51 - r - INFO: - Current episode 400 has the best eval reward: 42.400 +2022-12-04 00:34:51 - r - INFO: - Episode: 401/1000, Reward: 60.000, Step: 60 +2022-12-04 00:34:51 - r - INFO: - Episode: 402/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:51 - r - INFO: - Episode: 403/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:51 - r - INFO: - Episode: 404/1000, Reward: 29.000, Step: 29 +2022-12-04 00:34:51 - r - INFO: - Episode: 405/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:51 - r - INFO: - Episode: 406/1000, Reward: 36.000, Step: 36 +2022-12-04 00:34:51 - r - INFO: - Episode: 407/1000, Reward: 32.000, Step: 32 +2022-12-04 00:34:51 - r - INFO: - Episode: 408/1000, Reward: 54.000, Step: 54 +2022-12-04 00:34:51 - r - INFO: - Episode: 409/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:51 - r - INFO: - Episode: 410/1000, Reward: 75.000, Step: 75 +2022-12-04 00:34:52 - r - INFO: - Episode: 411/1000, Reward: 74.000, Step: 74 +2022-12-04 00:34:52 - r - INFO: - Episode: 412/1000, Reward: 68.000, Step: 68 +2022-12-04 00:34:52 - r - INFO: - Episode: 413/1000, Reward: 114.000, Step: 114 +2022-12-04 00:34:52 - r - INFO: - Episode: 414/1000, Reward: 38.000, Step: 38 +2022-12-04 00:34:52 - r - INFO: - Episode: 415/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:52 - r - INFO: - Current episode 415 has the best eval reward: 48.400 +2022-12-04 00:34:52 - r - INFO: - Episode: 416/1000, Reward: 41.000, Step: 41 +2022-12-04 00:34:52 - r - INFO: - Episode: 417/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:52 - r - INFO: - Episode: 418/1000, Reward: 67.000, Step: 67 +2022-12-04 00:34:52 - r - INFO: - Episode: 419/1000, Reward: 16.000, Step: 16 +2022-12-04 00:34:52 - r - INFO: - Episode: 420/1000, Reward: 33.000, Step: 33 +2022-12-04 00:34:52 - r - INFO: - Current episode 420 has the best eval reward: 61.900 +2022-12-04 00:34:52 - r - INFO: - Episode: 421/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:52 - r - INFO: - Episode: 422/1000, Reward: 78.000, Step: 78 +2022-12-04 00:34:52 - r - INFO: - Episode: 423/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:52 - r - INFO: - Episode: 424/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:52 - r - INFO: - Episode: 425/1000, Reward: 37.000, Step: 37 +2022-12-04 00:34:52 - r - INFO: - Episode: 426/1000, Reward: 20.000, Step: 20 +2022-12-04 00:34:52 - r - INFO: - Episode: 427/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:52 - r - INFO: - Episode: 428/1000, Reward: 94.000, Step: 94 +2022-12-04 00:34:52 - r - INFO: - Episode: 429/1000, Reward: 184.000, Step: 184 +2022-12-04 00:34:52 - r - INFO: - Episode: 430/1000, Reward: 61.000, Step: 61 +2022-12-04 00:34:53 - r - INFO: - Current episode 430 has the best eval reward: 63.200 +2022-12-04 00:34:53 - r - INFO: - Episode: 431/1000, Reward: 36.000, Step: 36 +2022-12-04 00:34:53 - r - INFO: - Episode: 432/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:53 - r - INFO: - Episode: 433/1000, Reward: 132.000, Step: 132 +2022-12-04 00:34:53 - r - INFO: - Episode: 434/1000, Reward: 70.000, Step: 70 +2022-12-04 00:34:53 - r - INFO: - Episode: 435/1000, Reward: 52.000, Step: 52 +2022-12-04 00:34:53 - r - INFO: - Episode: 436/1000, Reward: 23.000, Step: 23 +2022-12-04 00:34:53 - r - INFO: - Episode: 437/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:53 - r - INFO: - Episode: 438/1000, Reward: 41.000, Step: 41 +2022-12-04 00:34:53 - r - INFO: - Episode: 439/1000, Reward: 61.000, Step: 61 +2022-12-04 00:34:53 - r - INFO: - Episode: 440/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:53 - r - INFO: - Episode: 441/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:53 - r - INFO: - Episode: 442/1000, Reward: 99.000, Step: 99 +2022-12-04 00:34:53 - r - INFO: - Episode: 443/1000, Reward: 82.000, Step: 82 +2022-12-04 00:34:53 - r - INFO: - Episode: 444/1000, Reward: 63.000, Step: 63 +2022-12-04 00:34:53 - r - INFO: - Episode: 445/1000, Reward: 68.000, Step: 68 +2022-12-04 00:34:53 - r - INFO: - Current episode 445 has the best eval reward: 74.500 +2022-12-04 00:34:53 - r - INFO: - Episode: 446/1000, Reward: 51.000, Step: 51 +2022-12-04 00:34:53 - r - INFO: - Episode: 447/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:54 - r - INFO: - Episode: 448/1000, Reward: 79.000, Step: 79 +2022-12-04 00:34:54 - r - INFO: - Episode: 449/1000, Reward: 61.000, Step: 61 +2022-12-04 00:34:54 - r - INFO: - Episode: 450/1000, Reward: 46.000, Step: 46 +2022-12-04 00:34:54 - r - INFO: - Episode: 451/1000, Reward: 49.000, Step: 49 +2022-12-04 00:34:54 - r - INFO: - Episode: 452/1000, Reward: 110.000, Step: 110 +2022-12-04 00:34:54 - r - INFO: - Episode: 453/1000, Reward: 69.000, Step: 69 +2022-12-04 00:34:54 - r - INFO: - Episode: 454/1000, Reward: 27.000, Step: 27 +2022-12-04 00:34:54 - r - INFO: - Episode: 455/1000, Reward: 52.000, Step: 52 +2022-12-04 00:34:54 - r - INFO: - Episode: 456/1000, Reward: 44.000, Step: 44 +2022-12-04 00:34:54 - r - INFO: - Episode: 457/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:54 - r - INFO: - Episode: 458/1000, Reward: 25.000, Step: 25 +2022-12-04 00:34:54 - r - INFO: - Episode: 459/1000, Reward: 34.000, Step: 34 +2022-12-04 00:34:54 - r - INFO: - Episode: 460/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:54 - r - INFO: - Episode: 461/1000, Reward: 34.000, Step: 34 +2022-12-04 00:34:54 - r - INFO: - Episode: 462/1000, Reward: 29.000, Step: 29 +2022-12-04 00:34:54 - r - INFO: - Episode: 463/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:54 - r - INFO: - Episode: 464/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:54 - r - INFO: - Episode: 465/1000, Reward: 52.000, Step: 52 +2022-12-04 00:34:54 - r - INFO: - Episode: 466/1000, Reward: 49.000, Step: 49 +2022-12-04 00:34:54 - r - INFO: - Episode: 467/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:54 - r - INFO: - Episode: 468/1000, Reward: 35.000, Step: 35 +2022-12-04 00:34:54 - r - INFO: - Episode: 469/1000, Reward: 45.000, Step: 45 +2022-12-04 00:34:54 - r - INFO: - Episode: 470/1000, Reward: 55.000, Step: 55 +2022-12-04 00:34:55 - r - INFO: - Episode: 471/1000, Reward: 17.000, Step: 17 +2022-12-04 00:34:55 - r - INFO: - Episode: 472/1000, Reward: 61.000, Step: 61 +2022-12-04 00:34:55 - r - INFO: - Episode: 473/1000, Reward: 55.000, Step: 55 +2022-12-04 00:34:55 - r - INFO: - Episode: 474/1000, Reward: 31.000, Step: 31 +2022-12-04 00:34:55 - r - INFO: - Episode: 475/1000, Reward: 43.000, Step: 43 +2022-12-04 00:34:55 - r - INFO: - Episode: 476/1000, Reward: 77.000, Step: 77 +2022-12-04 00:34:55 - r - INFO: - Episode: 477/1000, Reward: 75.000, Step: 75 +2022-12-04 00:34:55 - r - INFO: - Episode: 478/1000, Reward: 146.000, Step: 146 +2022-12-04 00:34:55 - r - INFO: - Episode: 479/1000, Reward: 19.000, Step: 19 +2022-12-04 00:34:55 - r - INFO: - Episode: 480/1000, Reward: 61.000, Step: 61 +2022-12-04 00:34:55 - r - INFO: - Episode: 481/1000, Reward: 44.000, Step: 44 +2022-12-04 00:34:55 - r - INFO: - Episode: 482/1000, Reward: 53.000, Step: 53 +2022-12-04 00:34:55 - r - INFO: - Episode: 483/1000, Reward: 124.000, Step: 124 +2022-12-04 00:34:55 - r - INFO: - Episode: 484/1000, Reward: 45.000, Step: 45 +2022-12-04 00:34:55 - r - INFO: - Episode: 485/1000, Reward: 97.000, Step: 97 +2022-12-04 00:34:55 - r - INFO: - Current episode 485 has the best eval reward: 76.400 +2022-12-04 00:34:56 - r - INFO: - Episode: 486/1000, Reward: 98.000, Step: 98 +2022-12-04 00:34:56 - r - INFO: - Episode: 487/1000, Reward: 101.000, Step: 101 +2022-12-04 00:34:56 - r - INFO: - Episode: 488/1000, Reward: 42.000, Step: 42 +2022-12-04 00:34:56 - r - INFO: - Episode: 489/1000, Reward: 86.000, Step: 86 +2022-12-04 00:34:56 - r - INFO: - Episode: 490/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:56 - r - INFO: - Current episode 490 has the best eval reward: 96.000 +2022-12-04 00:34:56 - r - INFO: - Episode: 491/1000, Reward: 96.000, Step: 96 +2022-12-04 00:34:56 - r - INFO: - Episode: 492/1000, Reward: 177.000, Step: 177 +2022-12-04 00:34:56 - r - INFO: - Episode: 493/1000, Reward: 46.000, Step: 46 +2022-12-04 00:34:56 - r - INFO: - Episode: 494/1000, Reward: 196.000, Step: 196 +2022-12-04 00:34:56 - r - INFO: - Episode: 495/1000, Reward: 157.000, Step: 157 +2022-12-04 00:34:56 - r - INFO: - Episode: 496/1000, Reward: 34.000, Step: 34 +2022-12-04 00:34:56 - r - INFO: - Episode: 497/1000, Reward: 60.000, Step: 60 +2022-12-04 00:34:57 - r - INFO: - Episode: 498/1000, Reward: 102.000, Step: 102 +2022-12-04 00:34:57 - r - INFO: - Episode: 499/1000, Reward: 53.000, Step: 53 +2022-12-04 00:34:57 - r - INFO: - Episode: 500/1000, Reward: 52.000, Step: 52 +2022-12-04 00:34:57 - r - INFO: - Episode: 501/1000, Reward: 32.000, Step: 32 +2022-12-04 00:34:57 - r - INFO: - Episode: 502/1000, Reward: 114.000, Step: 114 +2022-12-04 00:34:57 - r - INFO: - Episode: 503/1000, Reward: 33.000, Step: 33 +2022-12-04 00:34:57 - r - INFO: - Episode: 504/1000, Reward: 112.000, Step: 112 +2022-12-04 00:34:57 - r - INFO: - Episode: 505/1000, Reward: 59.000, Step: 59 +2022-12-04 00:34:57 - r - INFO: - Episode: 506/1000, Reward: 122.000, Step: 122 +2022-12-04 00:34:57 - r - INFO: - Episode: 507/1000, Reward: 128.000, Step: 128 +2022-12-04 00:34:57 - r - INFO: - Episode: 508/1000, Reward: 99.000, Step: 99 +2022-12-04 00:34:57 - r - INFO: - Episode: 509/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:57 - r - INFO: - Episode: 510/1000, Reward: 200.000, Step: 200 +2022-12-04 00:34:58 - r - INFO: - Current episode 510 has the best eval reward: 101.700 +2022-12-04 00:34:58 - r - INFO: - Episode: 511/1000, Reward: 137.000, Step: 137 +2022-12-04 00:34:58 - r - INFO: - Episode: 512/1000, Reward: 24.000, Step: 24 +2022-12-04 00:34:58 - r - INFO: - Episode: 513/1000, Reward: 101.000, Step: 101 +2022-12-04 00:34:58 - r - INFO: - Episode: 514/1000, Reward: 150.000, Step: 150 +2022-12-04 00:34:58 - r - INFO: - Episode: 515/1000, Reward: 15.000, Step: 15 +2022-12-04 00:34:58 - r - INFO: - Episode: 516/1000, Reward: 43.000, Step: 43 +2022-12-04 00:34:58 - r - INFO: - Episode: 517/1000, Reward: 39.000, Step: 39 +2022-12-04 00:34:58 - r - INFO: - Episode: 518/1000, Reward: 43.000, Step: 43 +2022-12-04 00:34:58 - r - INFO: - Episode: 519/1000, Reward: 26.000, Step: 26 +2022-12-04 00:34:58 - r - INFO: - Episode: 520/1000, Reward: 22.000, Step: 22 +2022-12-04 00:34:58 - r - INFO: - Episode: 521/1000, Reward: 51.000, Step: 51 +2022-12-04 00:34:58 - r - INFO: - Episode: 522/1000, Reward: 21.000, Step: 21 +2022-12-04 00:34:58 - r - INFO: - Episode: 523/1000, Reward: 71.000, Step: 71 +2022-12-04 00:34:58 - r - INFO: - Episode: 524/1000, Reward: 53.000, Step: 53 +2022-12-04 00:34:58 - r - INFO: - Episode: 525/1000, Reward: 164.000, Step: 164 +2022-12-04 00:34:59 - r - INFO: - Episode: 526/1000, Reward: 36.000, Step: 36 +2022-12-04 00:34:59 - r - INFO: - Episode: 527/1000, Reward: 97.000, Step: 97 +2022-12-04 00:34:59 - r - INFO: - Episode: 528/1000, Reward: 58.000, Step: 58 +2022-12-04 00:34:59 - r - INFO: - Episode: 529/1000, Reward: 28.000, Step: 28 +2022-12-04 00:34:59 - r - INFO: - Episode: 530/1000, Reward: 116.000, Step: 116 +2022-12-04 00:34:59 - r - INFO: - Current episode 530 has the best eval reward: 125.400 +2022-12-04 00:34:59 - r - INFO: - Episode: 531/1000, Reward: 123.000, Step: 123 +2022-12-04 00:34:59 - r - INFO: - Episode: 532/1000, Reward: 138.000, Step: 138 +2022-12-04 00:34:59 - r - INFO: - Episode: 533/1000, Reward: 60.000, Step: 60 +2022-12-04 00:34:59 - r - INFO: - Episode: 534/1000, Reward: 126.000, Step: 126 +2022-12-04 00:34:59 - r - INFO: - Episode: 535/1000, Reward: 133.000, Step: 133 +2022-12-04 00:35:00 - r - INFO: - Current episode 535 has the best eval reward: 128.800 +2022-12-04 00:35:00 - r - INFO: - Episode: 536/1000, Reward: 145.000, Step: 145 +2022-12-04 00:35:00 - r - INFO: - Episode: 537/1000, Reward: 39.000, Step: 39 +2022-12-04 00:35:00 - r - INFO: - Episode: 538/1000, Reward: 64.000, Step: 64 +2022-12-04 00:35:00 - r - INFO: - Episode: 539/1000, Reward: 106.000, Step: 106 +2022-12-04 00:35:00 - r - INFO: - Episode: 540/1000, Reward: 54.000, Step: 54 +2022-12-04 00:35:00 - r - INFO: - Current episode 540 has the best eval reward: 137.700 +2022-12-04 00:35:00 - r - INFO: - Episode: 541/1000, Reward: 51.000, Step: 51 +2022-12-04 00:35:00 - r - INFO: - Episode: 542/1000, Reward: 131.000, Step: 131 +2022-12-04 00:35:00 - r - INFO: - Episode: 543/1000, Reward: 140.000, Step: 140 +2022-12-04 00:35:00 - r - INFO: - Episode: 544/1000, Reward: 138.000, Step: 138 +2022-12-04 00:35:00 - r - INFO: - Episode: 545/1000, Reward: 123.000, Step: 123 +2022-12-04 00:35:01 - r - INFO: - Episode: 546/1000, Reward: 145.000, Step: 145 +2022-12-04 00:35:01 - r - INFO: - Episode: 547/1000, Reward: 173.000, Step: 173 +2022-12-04 00:35:01 - r - INFO: - Episode: 548/1000, Reward: 77.000, Step: 77 +2022-12-04 00:35:01 - r - INFO: - Episode: 549/1000, Reward: 159.000, Step: 159 +2022-12-04 00:35:01 - r - INFO: - Episode: 550/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:01 - r - INFO: - Current episode 550 has the best eval reward: 166.700 +2022-12-04 00:35:02 - r - INFO: - Episode: 551/1000, Reward: 89.000, Step: 89 +2022-12-04 00:35:02 - r - INFO: - Episode: 552/1000, Reward: 180.000, Step: 180 +2022-12-04 00:35:02 - r - INFO: - Episode: 553/1000, Reward: 138.000, Step: 138 +2022-12-04 00:35:02 - r - INFO: - Episode: 554/1000, Reward: 65.000, Step: 65 +2022-12-04 00:35:02 - r - INFO: - Episode: 555/1000, Reward: 64.000, Step: 64 +2022-12-04 00:35:02 - r - INFO: - Episode: 556/1000, Reward: 120.000, Step: 120 +2022-12-04 00:35:02 - r - INFO: - Episode: 557/1000, Reward: 50.000, Step: 50 +2022-12-04 00:35:02 - r - INFO: - Episode: 558/1000, Reward: 105.000, Step: 105 +2022-12-04 00:35:02 - r - INFO: - Episode: 559/1000, Reward: 46.000, Step: 46 +2022-12-04 00:35:02 - r - INFO: - Episode: 560/1000, Reward: 128.000, Step: 128 +2022-12-04 00:35:02 - r - INFO: - Episode: 561/1000, Reward: 62.000, Step: 62 +2022-12-04 00:35:03 - r - INFO: - Episode: 562/1000, Reward: 92.000, Step: 92 +2022-12-04 00:35:03 - r - INFO: - Episode: 563/1000, Reward: 125.000, Step: 125 +2022-12-04 00:35:03 - r - INFO: - Episode: 564/1000, Reward: 62.000, Step: 62 +2022-12-04 00:35:03 - r - INFO: - Episode: 565/1000, Reward: 60.000, Step: 60 +2022-12-04 00:35:03 - r - INFO: - Episode: 566/1000, Reward: 78.000, Step: 78 +2022-12-04 00:35:03 - r - INFO: - Episode: 567/1000, Reward: 48.000, Step: 48 +2022-12-04 00:35:03 - r - INFO: - Episode: 568/1000, Reward: 68.000, Step: 68 +2022-12-04 00:35:03 - r - INFO: - Episode: 569/1000, Reward: 126.000, Step: 126 +2022-12-04 00:35:03 - r - INFO: - Episode: 570/1000, Reward: 75.000, Step: 75 +2022-12-04 00:35:03 - r - INFO: - Episode: 571/1000, Reward: 84.000, Step: 84 +2022-12-04 00:35:03 - r - INFO: - Episode: 572/1000, Reward: 87.000, Step: 87 +2022-12-04 00:35:03 - r - INFO: - Episode: 573/1000, Reward: 68.000, Step: 68 +2022-12-04 00:35:03 - r - INFO: - Episode: 574/1000, Reward: 27.000, Step: 27 +2022-12-04 00:35:03 - r - INFO: - Episode: 575/1000, Reward: 55.000, Step: 55 +2022-12-04 00:35:04 - r - INFO: - Episode: 576/1000, Reward: 123.000, Step: 123 +2022-12-04 00:35:04 - r - INFO: - Episode: 577/1000, Reward: 137.000, Step: 137 +2022-12-04 00:35:04 - r - INFO: - Episode: 578/1000, Reward: 48.000, Step: 48 +2022-12-04 00:35:04 - r - INFO: - Episode: 579/1000, Reward: 17.000, Step: 17 +2022-12-04 00:35:04 - r - INFO: - Episode: 580/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:04 - r - INFO: - Episode: 581/1000, Reward: 22.000, Step: 22 +2022-12-04 00:35:04 - r - INFO: - Episode: 582/1000, Reward: 26.000, Step: 26 +2022-12-04 00:35:04 - r - INFO: - Episode: 583/1000, Reward: 40.000, Step: 40 +2022-12-04 00:35:04 - r - INFO: - Episode: 584/1000, Reward: 14.000, Step: 14 +2022-12-04 00:35:04 - r - INFO: - Episode: 585/1000, Reward: 130.000, Step: 130 +2022-12-04 00:35:04 - r - INFO: - Episode: 586/1000, Reward: 118.000, Step: 118 +2022-12-04 00:35:04 - r - INFO: - Episode: 587/1000, Reward: 46.000, Step: 46 +2022-12-04 00:35:04 - r - INFO: - Episode: 588/1000, Reward: 125.000, Step: 125 +2022-12-04 00:35:05 - r - INFO: - Episode: 589/1000, Reward: 33.000, Step: 33 +2022-12-04 00:35:05 - r - INFO: - Episode: 590/1000, Reward: 158.000, Step: 158 +2022-12-04 00:35:05 - r - INFO: - Episode: 591/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:05 - r - INFO: - Episode: 592/1000, Reward: 141.000, Step: 141 +2022-12-04 00:35:05 - r - INFO: - Episode: 593/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:05 - r - INFO: - Episode: 594/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:05 - r - INFO: - Episode: 595/1000, Reward: 122.000, Step: 122 +2022-12-04 00:35:06 - r - INFO: - Episode: 596/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:06 - r - INFO: - Episode: 597/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:06 - r - INFO: - Episode: 598/1000, Reward: 162.000, Step: 162 +2022-12-04 00:35:06 - r - INFO: - Episode: 599/1000, Reward: 27.000, Step: 27 +2022-12-04 00:35:06 - r - INFO: - Episode: 600/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:06 - r - INFO: - Episode: 601/1000, Reward: 28.000, Step: 28 +2022-12-04 00:35:06 - r - INFO: - Episode: 602/1000, Reward: 164.000, Step: 164 +2022-12-04 00:35:06 - r - INFO: - Episode: 603/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:06 - r - INFO: - Episode: 604/1000, Reward: 11.000, Step: 11 +2022-12-04 00:35:06 - r - INFO: - Episode: 605/1000, Reward: 146.000, Step: 146 +2022-12-04 00:35:07 - r - INFO: - Episode: 606/1000, Reward: 128.000, Step: 128 +2022-12-04 00:35:07 - r - INFO: - Episode: 607/1000, Reward: 13.000, Step: 13 +2022-12-04 00:35:07 - r - INFO: - Episode: 608/1000, Reward: 15.000, Step: 15 +2022-12-04 00:35:07 - r - INFO: - Episode: 609/1000, Reward: 131.000, Step: 131 +2022-12-04 00:35:07 - r - INFO: - Episode: 610/1000, Reward: 191.000, Step: 191 +2022-12-04 00:35:07 - r - INFO: - Episode: 611/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:07 - r - INFO: - Episode: 612/1000, Reward: 101.000, Step: 101 +2022-12-04 00:35:07 - r - INFO: - Episode: 613/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:07 - r - INFO: - Episode: 614/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:07 - r - INFO: - Episode: 615/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:08 - r - INFO: - Episode: 616/1000, Reward: 65.000, Step: 65 +2022-12-04 00:35:08 - r - INFO: - Episode: 617/1000, Reward: 172.000, Step: 172 +2022-12-04 00:35:08 - r - INFO: - Episode: 618/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:08 - r - INFO: - Episode: 619/1000, Reward: 190.000, Step: 190 +2022-12-04 00:35:08 - r - INFO: - Episode: 620/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:08 - r - INFO: - Episode: 621/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:09 - r - INFO: - Episode: 622/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:09 - r - INFO: - Episode: 623/1000, Reward: 58.000, Step: 58 +2022-12-04 00:35:09 - r - INFO: - Episode: 624/1000, Reward: 57.000, Step: 57 +2022-12-04 00:35:09 - r - INFO: - Episode: 625/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:09 - r - INFO: - Episode: 626/1000, Reward: 74.000, Step: 74 +2022-12-04 00:35:09 - r - INFO: - Episode: 627/1000, Reward: 61.000, Step: 61 +2022-12-04 00:35:09 - r - INFO: - Episode: 628/1000, Reward: 129.000, Step: 129 +2022-12-04 00:35:09 - r - INFO: - Episode: 629/1000, Reward: 114.000, Step: 114 +2022-12-04 00:35:09 - r - INFO: - Episode: 630/1000, Reward: 157.000, Step: 157 +2022-12-04 00:35:10 - r - INFO: - Episode: 631/1000, Reward: 62.000, Step: 62 +2022-12-04 00:35:10 - r - INFO: - Episode: 632/1000, Reward: 36.000, Step: 36 +2022-12-04 00:35:10 - r - INFO: - Episode: 633/1000, Reward: 37.000, Step: 37 +2022-12-04 00:35:10 - r - INFO: - Episode: 634/1000, Reward: 103.000, Step: 103 +2022-12-04 00:35:10 - r - INFO: - Episode: 635/1000, Reward: 190.000, Step: 190 +2022-12-04 00:35:10 - r - INFO: - Current episode 635 has the best eval reward: 167.400 +2022-12-04 00:35:10 - r - INFO: - Episode: 636/1000, Reward: 74.000, Step: 74 +2022-12-04 00:35:10 - r - INFO: - Episode: 637/1000, Reward: 141.000, Step: 141 +2022-12-04 00:35:10 - r - INFO: - Episode: 638/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:11 - r - INFO: - Episode: 639/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:11 - r - INFO: - Episode: 640/1000, Reward: 143.000, Step: 143 +2022-12-04 00:35:11 - r - INFO: - Episode: 641/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:11 - r - INFO: - Episode: 642/1000, Reward: 141.000, Step: 141 +2022-12-04 00:35:11 - r - INFO: - Episode: 643/1000, Reward: 135.000, Step: 135 +2022-12-04 00:35:11 - r - INFO: - Episode: 644/1000, Reward: 144.000, Step: 144 +2022-12-04 00:35:11 - r - INFO: - Episode: 645/1000, Reward: 46.000, Step: 46 +2022-12-04 00:35:12 - r - INFO: - Episode: 646/1000, Reward: 59.000, Step: 59 +2022-12-04 00:35:12 - r - INFO: - Episode: 647/1000, Reward: 63.000, Step: 63 +2022-12-04 00:35:12 - r - INFO: - Episode: 648/1000, Reward: 105.000, Step: 105 +2022-12-04 00:35:12 - r - INFO: - Episode: 649/1000, Reward: 101.000, Step: 101 +2022-12-04 00:35:12 - r - INFO: - Episode: 650/1000, Reward: 92.000, Step: 92 +2022-12-04 00:35:12 - r - INFO: - Episode: 651/1000, Reward: 40.000, Step: 40 +2022-12-04 00:35:12 - r - INFO: - Episode: 652/1000, Reward: 40.000, Step: 40 +2022-12-04 00:35:12 - r - INFO: - Episode: 653/1000, Reward: 58.000, Step: 58 +2022-12-04 00:35:12 - r - INFO: - Episode: 654/1000, Reward: 101.000, Step: 101 +2022-12-04 00:35:12 - r - INFO: - Episode: 655/1000, Reward: 38.000, Step: 38 +2022-12-04 00:35:12 - r - INFO: - Episode: 656/1000, Reward: 40.000, Step: 40 +2022-12-04 00:35:12 - r - INFO: - Episode: 657/1000, Reward: 32.000, Step: 32 +2022-12-04 00:35:12 - r - INFO: - Episode: 658/1000, Reward: 50.000, Step: 50 +2022-12-04 00:35:12 - r - INFO: - Episode: 659/1000, Reward: 49.000, Step: 49 +2022-12-04 00:35:12 - r - INFO: - Episode: 660/1000, Reward: 38.000, Step: 38 +2022-12-04 00:35:12 - r - INFO: - Episode: 661/1000, Reward: 61.000, Step: 61 +2022-12-04 00:35:12 - r - INFO: - Episode: 662/1000, Reward: 40.000, Step: 40 +2022-12-04 00:35:12 - r - INFO: - Episode: 663/1000, Reward: 13.000, Step: 13 +2022-12-04 00:35:12 - r - INFO: - Episode: 664/1000, Reward: 44.000, Step: 44 +2022-12-04 00:35:12 - r - INFO: - Episode: 665/1000, Reward: 30.000, Step: 30 +2022-12-04 00:35:13 - r - INFO: - Episode: 666/1000, Reward: 43.000, Step: 43 +2022-12-04 00:35:13 - r - INFO: - Episode: 667/1000, Reward: 37.000, Step: 37 +2022-12-04 00:35:13 - r - INFO: - Episode: 668/1000, Reward: 34.000, Step: 34 +2022-12-04 00:35:13 - r - INFO: - Episode: 669/1000, Reward: 42.000, Step: 42 +2022-12-04 00:35:13 - r - INFO: - Episode: 670/1000, Reward: 35.000, Step: 35 +2022-12-04 00:35:13 - r - INFO: - Episode: 671/1000, Reward: 67.000, Step: 67 +2022-12-04 00:35:13 - r - INFO: - Episode: 672/1000, Reward: 16.000, Step: 16 +2022-12-04 00:35:13 - r - INFO: - Episode: 673/1000, Reward: 41.000, Step: 41 +2022-12-04 00:35:13 - r - INFO: - Episode: 674/1000, Reward: 23.000, Step: 23 +2022-12-04 00:35:13 - r - INFO: - Episode: 675/1000, Reward: 40.000, Step: 40 +2022-12-04 00:35:13 - r - INFO: - Episode: 676/1000, Reward: 37.000, Step: 37 +2022-12-04 00:35:13 - r - INFO: - Episode: 677/1000, Reward: 41.000, Step: 41 +2022-12-04 00:35:13 - r - INFO: - Episode: 678/1000, Reward: 47.000, Step: 47 +2022-12-04 00:35:13 - r - INFO: - Episode: 679/1000, Reward: 87.000, Step: 87 +2022-12-04 00:35:13 - r - INFO: - Episode: 680/1000, Reward: 40.000, Step: 40 +2022-12-04 00:35:13 - r - INFO: - Episode: 681/1000, Reward: 82.000, Step: 82 +2022-12-04 00:35:13 - r - INFO: - Episode: 682/1000, Reward: 50.000, Step: 50 +2022-12-04 00:35:13 - r - INFO: - Episode: 683/1000, Reward: 67.000, Step: 67 +2022-12-04 00:35:13 - r - INFO: - Episode: 684/1000, Reward: 28.000, Step: 28 +2022-12-04 00:35:13 - r - INFO: - Episode: 685/1000, Reward: 144.000, Step: 144 +2022-12-04 00:35:14 - r - INFO: - Episode: 686/1000, Reward: 36.000, Step: 36 +2022-12-04 00:35:14 - r - INFO: - Episode: 687/1000, Reward: 51.000, Step: 51 +2022-12-04 00:35:14 - r - INFO: - Episode: 688/1000, Reward: 69.000, Step: 69 +2022-12-04 00:35:14 - r - INFO: - Episode: 689/1000, Reward: 48.000, Step: 48 +2022-12-04 00:35:14 - r - INFO: - Episode: 690/1000, Reward: 44.000, Step: 44 +2022-12-04 00:35:14 - r - INFO: - Episode: 691/1000, Reward: 46.000, Step: 46 +2022-12-04 00:35:14 - r - INFO: - Episode: 692/1000, Reward: 42.000, Step: 42 +2022-12-04 00:35:14 - r - INFO: - Episode: 693/1000, Reward: 42.000, Step: 42 +2022-12-04 00:35:14 - r - INFO: - Episode: 694/1000, Reward: 41.000, Step: 41 +2022-12-04 00:35:14 - r - INFO: - Episode: 695/1000, Reward: 161.000, Step: 161 +2022-12-04 00:35:14 - r - INFO: - Episode: 696/1000, Reward: 56.000, Step: 56 +2022-12-04 00:35:14 - r - INFO: - Episode: 697/1000, Reward: 66.000, Step: 66 +2022-12-04 00:35:14 - r - INFO: - Episode: 698/1000, Reward: 63.000, Step: 63 +2022-12-04 00:35:14 - r - INFO: - Episode: 699/1000, Reward: 98.000, Step: 98 +2022-12-04 00:35:14 - r - INFO: - Episode: 700/1000, Reward: 130.000, Step: 130 +2022-12-04 00:35:15 - r - INFO: - Episode: 701/1000, Reward: 54.000, Step: 54 +2022-12-04 00:35:15 - r - INFO: - Episode: 702/1000, Reward: 125.000, Step: 125 +2022-12-04 00:35:15 - r - INFO: - Episode: 703/1000, Reward: 75.000, Step: 75 +2022-12-04 00:35:15 - r - INFO: - Episode: 704/1000, Reward: 115.000, Step: 115 +2022-12-04 00:35:15 - r - INFO: - Episode: 705/1000, Reward: 110.000, Step: 110 +2022-12-04 00:35:15 - r - INFO: - Episode: 706/1000, Reward: 73.000, Step: 73 +2022-12-04 00:35:15 - r - INFO: - Episode: 707/1000, Reward: 120.000, Step: 120 +2022-12-04 00:35:15 - r - INFO: - Episode: 708/1000, Reward: 62.000, Step: 62 +2022-12-04 00:35:15 - r - INFO: - Episode: 709/1000, Reward: 134.000, Step: 134 +2022-12-04 00:35:15 - r - INFO: - Episode: 710/1000, Reward: 53.000, Step: 53 +2022-12-04 00:35:15 - r - INFO: - Episode: 711/1000, Reward: 45.000, Step: 45 +2022-12-04 00:35:15 - r - INFO: - Episode: 712/1000, Reward: 82.000, Step: 82 +2022-12-04 00:35:15 - r - INFO: - Episode: 713/1000, Reward: 47.000, Step: 47 +2022-12-04 00:35:15 - r - INFO: - Episode: 714/1000, Reward: 60.000, Step: 60 +2022-12-04 00:35:16 - r - INFO: - Episode: 715/1000, Reward: 121.000, Step: 121 +2022-12-04 00:35:16 - r - INFO: - Episode: 716/1000, Reward: 100.000, Step: 100 +2022-12-04 00:35:16 - r - INFO: - Episode: 717/1000, Reward: 29.000, Step: 29 +2022-12-04 00:35:16 - r - INFO: - Episode: 718/1000, Reward: 33.000, Step: 33 +2022-12-04 00:35:16 - r - INFO: - Episode: 719/1000, Reward: 60.000, Step: 60 +2022-12-04 00:35:16 - r - INFO: - Episode: 720/1000, Reward: 117.000, Step: 117 +2022-12-04 00:35:16 - r - INFO: - Episode: 721/1000, Reward: 71.000, Step: 71 +2022-12-04 00:35:16 - r - INFO: - Episode: 722/1000, Reward: 15.000, Step: 15 +2022-12-04 00:35:16 - r - INFO: - Episode: 723/1000, Reward: 130.000, Step: 130 +2022-12-04 00:35:16 - r - INFO: - Episode: 724/1000, Reward: 115.000, Step: 115 +2022-12-04 00:35:16 - r - INFO: - Episode: 725/1000, Reward: 27.000, Step: 27 +2022-12-04 00:35:16 - r - INFO: - Episode: 726/1000, Reward: 28.000, Step: 28 +2022-12-04 00:35:16 - r - INFO: - Episode: 727/1000, Reward: 19.000, Step: 19 +2022-12-04 00:35:16 - r - INFO: - Episode: 728/1000, Reward: 22.000, Step: 22 +2022-12-04 00:35:16 - r - INFO: - Episode: 729/1000, Reward: 22.000, Step: 22 +2022-12-04 00:35:16 - r - INFO: - Episode: 730/1000, Reward: 21.000, Step: 21 +2022-12-04 00:35:16 - r - INFO: - Episode: 731/1000, Reward: 14.000, Step: 14 +2022-12-04 00:35:16 - r - INFO: - Episode: 732/1000, Reward: 22.000, Step: 22 +2022-12-04 00:35:16 - r - INFO: - Episode: 733/1000, Reward: 16.000, Step: 16 +2022-12-04 00:35:16 - r - INFO: - Episode: 734/1000, Reward: 21.000, Step: 21 +2022-12-04 00:35:16 - r - INFO: - Episode: 735/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:17 - r - INFO: - Episode: 736/1000, Reward: 21.000, Step: 21 +2022-12-04 00:35:17 - r - INFO: - Episode: 737/1000, Reward: 17.000, Step: 17 +2022-12-04 00:35:17 - r - INFO: - Episode: 738/1000, Reward: 16.000, Step: 16 +2022-12-04 00:35:17 - r - INFO: - Episode: 739/1000, Reward: 97.000, Step: 97 +2022-12-04 00:35:17 - r - INFO: - Episode: 740/1000, Reward: 20.000, Step: 20 +2022-12-04 00:35:17 - r - INFO: - Episode: 741/1000, Reward: 15.000, Step: 15 +2022-12-04 00:35:17 - r - INFO: - Episode: 742/1000, Reward: 17.000, Step: 17 +2022-12-04 00:35:17 - r - INFO: - Episode: 743/1000, Reward: 14.000, Step: 14 +2022-12-04 00:35:17 - r - INFO: - Episode: 744/1000, Reward: 20.000, Step: 20 +2022-12-04 00:35:17 - r - INFO: - Episode: 745/1000, Reward: 20.000, Step: 20 +2022-12-04 00:35:17 - r - INFO: - Episode: 746/1000, Reward: 13.000, Step: 13 +2022-12-04 00:35:17 - r - INFO: - Episode: 747/1000, Reward: 17.000, Step: 17 +2022-12-04 00:35:17 - r - INFO: - Episode: 748/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:17 - r - INFO: - Episode: 749/1000, Reward: 30.000, Step: 30 +2022-12-04 00:35:17 - r - INFO: - Episode: 750/1000, Reward: 125.000, Step: 125 +2022-12-04 00:35:17 - r - INFO: - Episode: 751/1000, Reward: 25.000, Step: 25 +2022-12-04 00:35:17 - r - INFO: - Episode: 752/1000, Reward: 117.000, Step: 117 +2022-12-04 00:35:17 - r - INFO: - Episode: 753/1000, Reward: 94.000, Step: 94 +2022-12-04 00:35:17 - r - INFO: - Episode: 754/1000, Reward: 134.000, Step: 134 +2022-12-04 00:35:17 - r - INFO: - Episode: 755/1000, Reward: 22.000, Step: 22 +2022-12-04 00:35:17 - r - INFO: - Episode: 756/1000, Reward: 101.000, Step: 101 +2022-12-04 00:35:17 - r - INFO: - Episode: 757/1000, Reward: 134.000, Step: 134 +2022-12-04 00:35:17 - r - INFO: - Episode: 758/1000, Reward: 98.000, Step: 98 +2022-12-04 00:35:18 - r - INFO: - Episode: 759/1000, Reward: 167.000, Step: 167 +2022-12-04 00:35:18 - r - INFO: - Episode: 760/1000, Reward: 137.000, Step: 137 +2022-12-04 00:35:18 - r - INFO: - Episode: 761/1000, Reward: 153.000, Step: 153 +2022-12-04 00:35:18 - r - INFO: - Episode: 762/1000, Reward: 125.000, Step: 125 +2022-12-04 00:35:18 - r - INFO: - Episode: 763/1000, Reward: 146.000, Step: 146 +2022-12-04 00:35:18 - r - INFO: - Episode: 764/1000, Reward: 123.000, Step: 123 +2022-12-04 00:35:18 - r - INFO: - Episode: 765/1000, Reward: 131.000, Step: 131 +2022-12-04 00:35:18 - r - INFO: - Episode: 766/1000, Reward: 123.000, Step: 123 +2022-12-04 00:35:19 - r - INFO: - Episode: 767/1000, Reward: 38.000, Step: 38 +2022-12-04 00:35:19 - r - INFO: - Episode: 768/1000, Reward: 30.000, Step: 30 +2022-12-04 00:35:19 - r - INFO: - Episode: 769/1000, Reward: 126.000, Step: 126 +2022-12-04 00:35:19 - r - INFO: - Episode: 770/1000, Reward: 128.000, Step: 128 +2022-12-04 00:35:19 - r - INFO: - Episode: 771/1000, Reward: 178.000, Step: 178 +2022-12-04 00:35:19 - r - INFO: - Episode: 772/1000, Reward: 164.000, Step: 164 +2022-12-04 00:35:19 - r - INFO: - Episode: 773/1000, Reward: 145.000, Step: 145 +2022-12-04 00:35:19 - r - INFO: - Episode: 774/1000, Reward: 175.000, Step: 175 +2022-12-04 00:35:19 - r - INFO: - Episode: 775/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:20 - r - INFO: - Current episode 775 has the best eval reward: 200.000 +2022-12-04 00:35:20 - r - INFO: - Episode: 776/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:20 - r - INFO: - Episode: 777/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:20 - r - INFO: - Episode: 778/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:20 - r - INFO: - Episode: 779/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:20 - r - INFO: - Episode: 780/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:21 - r - INFO: - Episode: 781/1000, Reward: 161.000, Step: 161 +2022-12-04 00:35:21 - r - INFO: - Episode: 782/1000, Reward: 124.000, Step: 124 +2022-12-04 00:35:21 - r - INFO: - Episode: 783/1000, Reward: 110.000, Step: 110 +2022-12-04 00:35:21 - r - INFO: - Episode: 784/1000, Reward: 79.000, Step: 79 +2022-12-04 00:35:21 - r - INFO: - Episode: 785/1000, Reward: 107.000, Step: 107 +2022-12-04 00:35:21 - r - INFO: - Episode: 786/1000, Reward: 88.000, Step: 88 +2022-12-04 00:35:22 - r - INFO: - Episode: 787/1000, Reward: 79.000, Step: 79 +2022-12-04 00:35:22 - r - INFO: - Episode: 788/1000, Reward: 113.000, Step: 113 +2022-12-04 00:35:22 - r - INFO: - Episode: 789/1000, Reward: 104.000, Step: 104 +2022-12-04 00:35:22 - r - INFO: - Episode: 790/1000, Reward: 122.000, Step: 122 +2022-12-04 00:35:22 - r - INFO: - Episode: 791/1000, Reward: 135.000, Step: 135 +2022-12-04 00:35:22 - r - INFO: - Episode: 792/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:22 - r - INFO: - Episode: 793/1000, Reward: 117.000, Step: 117 +2022-12-04 00:35:22 - r - INFO: - Episode: 794/1000, Reward: 99.000, Step: 99 +2022-12-04 00:35:23 - r - INFO: - Episode: 795/1000, Reward: 103.000, Step: 103 +2022-12-04 00:35:23 - r - INFO: - Episode: 796/1000, Reward: 35.000, Step: 35 +2022-12-04 00:35:23 - r - INFO: - Episode: 797/1000, Reward: 85.000, Step: 85 +2022-12-04 00:35:23 - r - INFO: - Episode: 798/1000, Reward: 68.000, Step: 68 +2022-12-04 00:35:23 - r - INFO: - Episode: 799/1000, Reward: 79.000, Step: 79 +2022-12-04 00:35:23 - r - INFO: - Episode: 800/1000, Reward: 104.000, Step: 104 +2022-12-04 00:35:23 - r - INFO: - Episode: 801/1000, Reward: 100.000, Step: 100 +2022-12-04 00:35:23 - r - INFO: - Episode: 802/1000, Reward: 143.000, Step: 143 +2022-12-04 00:35:23 - r - INFO: - Episode: 803/1000, Reward: 184.000, Step: 184 +2022-12-04 00:35:23 - r - INFO: - Episode: 804/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:24 - r - INFO: - Episode: 805/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:24 - r - INFO: - Current episode 805 has the best eval reward: 200.000 +2022-12-04 00:35:24 - r - INFO: - Episode: 806/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:24 - r - INFO: - Episode: 807/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:24 - r - INFO: - Episode: 808/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:24 - r - INFO: - Episode: 809/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:25 - r - INFO: - Episode: 810/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:25 - r - INFO: - Current episode 810 has the best eval reward: 200.000 +2022-12-04 00:35:25 - r - INFO: - Episode: 811/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:25 - r - INFO: - Episode: 812/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:25 - r - INFO: - Episode: 813/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:25 - r - INFO: - Episode: 814/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:26 - r - INFO: - Episode: 815/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:26 - r - INFO: - Current episode 815 has the best eval reward: 200.000 +2022-12-04 00:35:26 - r - INFO: - Episode: 816/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:26 - r - INFO: - Episode: 817/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:26 - r - INFO: - Episode: 818/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:26 - r - INFO: - Episode: 819/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:27 - r - INFO: - Episode: 820/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:27 - r - INFO: - Current episode 820 has the best eval reward: 200.000 +2022-12-04 00:35:27 - r - INFO: - Episode: 821/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:27 - r - INFO: - Episode: 822/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:27 - r - INFO: - Episode: 823/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:27 - r - INFO: - Episode: 824/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:28 - r - INFO: - Episode: 825/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:28 - r - INFO: - Current episode 825 has the best eval reward: 200.000 +2022-12-04 00:35:28 - r - INFO: - Episode: 826/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:28 - r - INFO: - Episode: 827/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:28 - r - INFO: - Episode: 828/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:28 - r - INFO: - Episode: 829/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:29 - r - INFO: - Episode: 830/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:29 - r - INFO: - Current episode 830 has the best eval reward: 200.000 +2022-12-04 00:35:29 - r - INFO: - Episode: 831/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:29 - r - INFO: - Episode: 832/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:29 - r - INFO: - Episode: 833/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:29 - r - INFO: - Episode: 834/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:30 - r - INFO: - Episode: 835/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:30 - r - INFO: - Current episode 835 has the best eval reward: 200.000 +2022-12-04 00:35:30 - r - INFO: - Episode: 836/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:30 - r - INFO: - Episode: 837/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:31 - r - INFO: - Episode: 838/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:31 - r - INFO: - Episode: 839/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:31 - r - INFO: - Episode: 840/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:31 - r - INFO: - Current episode 840 has the best eval reward: 200.000 +2022-12-04 00:35:31 - r - INFO: - Episode: 841/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:32 - r - INFO: - Episode: 842/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:32 - r - INFO: - Episode: 843/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:32 - r - INFO: - Episode: 844/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:32 - r - INFO: - Episode: 845/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:32 - r - INFO: - Current episode 845 has the best eval reward: 200.000 +2022-12-04 00:35:32 - r - INFO: - Episode: 846/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:33 - r - INFO: - Episode: 847/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:33 - r - INFO: - Episode: 848/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:33 - r - INFO: - Episode: 849/1000, Reward: 192.000, Step: 192 +2022-12-04 00:35:33 - r - INFO: - Episode: 850/1000, Reward: 184.000, Step: 184 +2022-12-04 00:35:33 - r - INFO: - Episode: 851/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:34 - r - INFO: - Episode: 852/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:34 - r - INFO: - Episode: 853/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:34 - r - INFO: - Episode: 854/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:34 - r - INFO: - Episode: 855/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:35 - r - INFO: - Episode: 856/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:35 - r - INFO: - Episode: 857/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:35 - r - INFO: - Episode: 858/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:35 - r - INFO: - Episode: 859/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:36 - r - INFO: - Episode: 860/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:36 - r - INFO: - Current episode 860 has the best eval reward: 200.000 +2022-12-04 00:35:36 - r - INFO: - Episode: 861/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:37 - r - INFO: - Episode: 862/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:37 - r - INFO: - Episode: 863/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:37 - r - INFO: - Episode: 864/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:37 - r - INFO: - Episode: 865/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:38 - r - INFO: - Current episode 865 has the best eval reward: 200.000 +2022-12-04 00:35:38 - r - INFO: - Episode: 866/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:38 - r - INFO: - Episode: 867/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:38 - r - INFO: - Episode: 868/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:38 - r - INFO: - Episode: 869/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:38 - r - INFO: - Episode: 870/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:39 - r - INFO: - Episode: 871/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:39 - r - INFO: - Episode: 872/1000, Reward: 190.000, Step: 190 +2022-12-04 00:35:39 - r - INFO: - Episode: 873/1000, Reward: 124.000, Step: 124 +2022-12-04 00:35:39 - r - INFO: - Episode: 874/1000, Reward: 133.000, Step: 133 +2022-12-04 00:35:40 - r - INFO: - Episode: 875/1000, Reward: 161.000, Step: 161 +2022-12-04 00:35:40 - r - INFO: - Episode: 876/1000, Reward: 21.000, Step: 21 +2022-12-04 00:35:40 - r - INFO: - Episode: 877/1000, Reward: 139.000, Step: 139 +2022-12-04 00:35:40 - r - INFO: - Episode: 878/1000, Reward: 128.000, Step: 128 +2022-12-04 00:35:40 - r - INFO: - Episode: 879/1000, Reward: 122.000, Step: 122 +2022-12-04 00:35:40 - r - INFO: - Episode: 880/1000, Reward: 170.000, Step: 170 +2022-12-04 00:35:41 - r - INFO: - Episode: 881/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:41 - r - INFO: - Episode: 882/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:41 - r - INFO: - Episode: 883/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:41 - r - INFO: - Episode: 884/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:41 - r - INFO: - Episode: 885/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:42 - r - INFO: - Episode: 886/1000, Reward: 106.000, Step: 106 +2022-12-04 00:35:42 - r - INFO: - Episode: 887/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:42 - r - INFO: - Episode: 888/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:42 - r - INFO: - Episode: 889/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:42 - r - INFO: - Episode: 890/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:42 - r - INFO: - Current episode 890 has the best eval reward: 200.000 +2022-12-04 00:35:43 - r - INFO: - Episode: 891/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:43 - r - INFO: - Episode: 892/1000, Reward: 169.000, Step: 169 +2022-12-04 00:35:43 - r - INFO: - Episode: 893/1000, Reward: 132.000, Step: 132 +2022-12-04 00:35:43 - r - INFO: - Episode: 894/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:43 - r - INFO: - Episode: 895/1000, Reward: 132.000, Step: 132 +2022-12-04 00:35:43 - r - INFO: - Episode: 896/1000, Reward: 150.000, Step: 150 +2022-12-04 00:35:43 - r - INFO: - Episode: 897/1000, Reward: 25.000, Step: 25 +2022-12-04 00:35:43 - r - INFO: - Episode: 898/1000, Reward: 161.000, Step: 161 +2022-12-04 00:35:43 - r - INFO: - Episode: 899/1000, Reward: 187.000, Step: 187 +2022-12-04 00:35:44 - r - INFO: - Episode: 900/1000, Reward: 161.000, Step: 161 +2022-12-04 00:35:44 - r - INFO: - Episode: 901/1000, Reward: 146.000, Step: 146 +2022-12-04 00:35:44 - r - INFO: - Episode: 902/1000, Reward: 19.000, Step: 19 +2022-12-04 00:35:44 - r - INFO: - Episode: 903/1000, Reward: 119.000, Step: 119 +2022-12-04 00:35:44 - r - INFO: - Episode: 904/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:44 - r - INFO: - Episode: 905/1000, Reward: 18.000, Step: 18 +2022-12-04 00:35:44 - r - INFO: - Episode: 906/1000, Reward: 39.000, Step: 39 +2022-12-04 00:35:44 - r - INFO: - Episode: 907/1000, Reward: 139.000, Step: 139 +2022-12-04 00:35:44 - r - INFO: - Episode: 908/1000, Reward: 14.000, Step: 14 +2022-12-04 00:35:44 - r - INFO: - Episode: 909/1000, Reward: 129.000, Step: 129 +2022-12-04 00:35:44 - r - INFO: - Episode: 910/1000, Reward: 12.000, Step: 12 +2022-12-04 00:35:45 - r - INFO: - Episode: 911/1000, Reward: 129.000, Step: 129 +2022-12-04 00:35:45 - r - INFO: - Episode: 912/1000, Reward: 112.000, Step: 112 +2022-12-04 00:35:45 - r - INFO: - Episode: 913/1000, Reward: 122.000, Step: 122 +2022-12-04 00:35:45 - r - INFO: - Episode: 914/1000, Reward: 161.000, Step: 161 +2022-12-04 00:35:45 - r - INFO: - Episode: 915/1000, Reward: 174.000, Step: 174 +2022-12-04 00:35:45 - r - INFO: - Episode: 916/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:46 - r - INFO: - Episode: 917/1000, Reward: 198.000, Step: 198 +2022-12-04 00:35:46 - r - INFO: - Episode: 918/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:46 - r - INFO: - Episode: 919/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:46 - r - INFO: - Episode: 920/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:46 - r - INFO: - Episode: 921/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:47 - r - INFO: - Episode: 922/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:47 - r - INFO: - Episode: 923/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:47 - r - INFO: - Episode: 924/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:47 - r - INFO: - Episode: 925/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:47 - r - INFO: - Episode: 926/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:47 - r - INFO: - Episode: 927/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:48 - r - INFO: - Episode: 928/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:48 - r - INFO: - Episode: 929/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:48 - r - INFO: - Episode: 930/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:48 - r - INFO: - Current episode 930 has the best eval reward: 200.000 +2022-12-04 00:35:48 - r - INFO: - Episode: 931/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:48 - r - INFO: - Episode: 932/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:49 - r - INFO: - Episode: 933/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:49 - r - INFO: - Episode: 934/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:49 - r - INFO: - Episode: 935/1000, Reward: 22.000, Step: 22 +2022-12-04 00:35:49 - r - INFO: - Episode: 936/1000, Reward: 187.000, Step: 187 +2022-12-04 00:35:49 - r - INFO: - Episode: 937/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:49 - r - INFO: - Episode: 938/1000, Reward: 186.000, Step: 186 +2022-12-04 00:35:49 - r - INFO: - Episode: 939/1000, Reward: 183.000, Step: 183 +2022-12-04 00:35:50 - r - INFO: - Episode: 940/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:50 - r - INFO: - Current episode 940 has the best eval reward: 200.000 +2022-12-04 00:35:50 - r - INFO: - Episode: 941/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:50 - r - INFO: - Episode: 942/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:50 - r - INFO: - Episode: 943/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:50 - r - INFO: - Episode: 944/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:50 - r - INFO: - Episode: 945/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:51 - r - INFO: - Current episode 945 has the best eval reward: 200.000 +2022-12-04 00:35:51 - r - INFO: - Episode: 946/1000, Reward: 88.000, Step: 88 +2022-12-04 00:35:51 - r - INFO: - Episode: 947/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:51 - r - INFO: - Episode: 948/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:51 - r - INFO: - Episode: 949/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:51 - r - INFO: - Episode: 950/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:52 - r - INFO: - Current episode 950 has the best eval reward: 200.000 +2022-12-04 00:35:52 - r - INFO: - Episode: 951/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:52 - r - INFO: - Episode: 952/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:52 - r - INFO: - Episode: 953/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:52 - r - INFO: - Episode: 954/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:52 - r - INFO: - Episode: 955/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:53 - r - INFO: - Current episode 955 has the best eval reward: 200.000 +2022-12-04 00:35:53 - r - INFO: - Episode: 956/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:53 - r - INFO: - Episode: 957/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:53 - r - INFO: - Episode: 958/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:53 - r - INFO: - Episode: 959/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:53 - r - INFO: - Episode: 960/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:54 - r - INFO: - Current episode 960 has the best eval reward: 200.000 +2022-12-04 00:35:54 - r - INFO: - Episode: 961/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:54 - r - INFO: - Episode: 962/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:54 - r - INFO: - Episode: 963/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:54 - r - INFO: - Episode: 964/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:54 - r - INFO: - Episode: 965/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:55 - r - INFO: - Episode: 966/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:55 - r - INFO: - Episode: 967/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:55 - r - INFO: - Episode: 968/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:55 - r - INFO: - Episode: 969/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:55 - r - INFO: - Episode: 970/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:56 - r - INFO: - Episode: 971/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:56 - r - INFO: - Episode: 972/1000, Reward: 22.000, Step: 22 +2022-12-04 00:35:56 - r - INFO: - Episode: 973/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:56 - r - INFO: - Episode: 974/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:56 - r - INFO: - Episode: 975/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:56 - r - INFO: - Episode: 976/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:57 - r - INFO: - Episode: 977/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:57 - r - INFO: - Episode: 978/1000, Reward: 142.000, Step: 142 +2022-12-04 00:35:57 - r - INFO: - Episode: 979/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:57 - r - INFO: - Episode: 980/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:57 - r - INFO: - Episode: 981/1000, Reward: 179.000, Step: 179 +2022-12-04 00:35:58 - r - INFO: - Episode: 982/1000, Reward: 140.000, Step: 140 +2022-12-04 00:35:58 - r - INFO: - Episode: 983/1000, Reward: 109.000, Step: 109 +2022-12-04 00:35:58 - r - INFO: - Episode: 984/1000, Reward: 114.000, Step: 114 +2022-12-04 00:35:58 - r - INFO: - Episode: 985/1000, Reward: 14.000, Step: 14 +2022-12-04 00:35:58 - r - INFO: - Episode: 986/1000, Reward: 25.000, Step: 25 +2022-12-04 00:35:58 - r - INFO: - Episode: 987/1000, Reward: 12.000, Step: 12 +2022-12-04 00:35:58 - r - INFO: - Episode: 988/1000, Reward: 129.000, Step: 129 +2022-12-04 00:35:58 - r - INFO: - Episode: 989/1000, Reward: 150.000, Step: 150 +2022-12-04 00:35:58 - r - INFO: - Episode: 990/1000, Reward: 140.000, Step: 140 +2022-12-04 00:35:58 - r - INFO: - Episode: 991/1000, Reward: 178.000, Step: 178 +2022-12-04 00:35:59 - r - INFO: - Episode: 992/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:59 - r - INFO: - Episode: 993/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:59 - r - INFO: - Episode: 994/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:59 - r - INFO: - Episode: 995/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:59 - r - INFO: - Current episode 995 has the best eval reward: 200.000 +2022-12-04 00:35:59 - r - INFO: - Episode: 996/1000, Reward: 200.000, Step: 200 +2022-12-04 00:35:59 - r - INFO: - Episode: 997/1000, Reward: 197.000, Step: 197 +2022-12-04 00:36:00 - r - INFO: - Episode: 998/1000, Reward: 163.000, Step: 163 +2022-12-04 00:36:00 - r - INFO: - Episode: 999/1000, Reward: 169.000, Step: 169 +2022-12-04 00:36:00 - r - INFO: - Episode: 1000/1000, Reward: 12.000, Step: 12 +2022-12-04 00:36:00 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/models/actor_checkpoint.pt b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4015ed746ab697417d75bf10b5945dd77cc8e6d2 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3e59a01cece98a09e8126a81cb4ba69f7d316bee4b6d6768a092489da7f5e2e +size 272151 diff --git a/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/models/critic_checkpoint.pt b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/models/critic_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d2d92943121e4cfb4e72259681f4a8587a8e20b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/models/critic_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4dcb18c48518129aba10d1a34954e57ad6a73800b872800b93d920912aeaf05 +size 271127 diff --git a/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..0e10b7e83ec2be189f4f656c7c24b4499ce6f5c5 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/results/res.csv b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..97d8dec57bd4f8f981ec77255d232a7dfb1e2924 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A2C_20221204-003439/results/res.csv @@ -0,0 +1,1001 @@ +episodes,rewards,steps +0,13.0,13 +1,30.0,30 +2,15.0,15 +3,10.0,10 +4,38.0,38 +5,10.0,10 +6,20.0,20 +7,16.0,16 +8,21.0,21 +9,15.0,15 +10,21.0,21 +11,14.0,14 +12,14.0,14 +13,21.0,21 +14,26.0,26 +15,25.0,25 +16,15.0,15 +17,16.0,16 +18,23.0,23 +19,35.0,35 +20,16.0,16 +21,16.0,16 +22,13.0,13 +23,20.0,20 +24,15.0,15 +25,15.0,15 +26,9.0,9 +27,37.0,37 +28,25.0,25 +29,10.0,10 +30,10.0,10 +31,25.0,25 +32,30.0,30 +33,31.0,31 +34,9.0,9 +35,31.0,31 +36,24.0,24 +37,21.0,21 +38,14.0,14 +39,14.0,14 +40,10.0,10 +41,17.0,17 +42,21.0,21 +43,16.0,16 +44,20.0,20 +45,15.0,15 +46,11.0,11 +47,17.0,17 +48,12.0,12 +49,11.0,11 +50,13.0,13 +51,13.0,13 +52,16.0,16 +53,16.0,16 +54,13.0,13 +55,12.0,12 +56,24.0,24 +57,38.0,38 +58,39.0,39 +59,17.0,17 +60,23.0,23 +61,24.0,24 +62,34.0,34 +63,12.0,12 +64,11.0,11 +65,40.0,40 +66,15.0,15 +67,22.0,22 +68,23.0,23 +69,10.0,10 +70,10.0,10 +71,24.0,24 +72,16.0,16 +73,24.0,24 +74,24.0,24 +75,20.0,20 +76,11.0,11 +77,21.0,21 +78,43.0,43 +79,9.0,9 +80,43.0,43 +81,15.0,15 +82,14.0,14 +83,9.0,9 +84,15.0,15 +85,40.0,40 +86,35.0,35 +87,10.0,10 +88,22.0,22 +89,11.0,11 +90,26.0,26 +91,15.0,15 +92,11.0,11 +93,11.0,11 +94,17.0,17 +95,19.0,19 +96,17.0,17 +97,12.0,12 +98,15.0,15 +99,22.0,22 +100,10.0,10 +101,34.0,34 +102,47.0,47 +103,24.0,24 +104,28.0,28 +105,13.0,13 +106,14.0,14 +107,13.0,13 +108,17.0,17 +109,11.0,11 +110,14.0,14 +111,27.0,27 +112,12.0,12 +113,20.0,20 +114,12.0,12 +115,37.0,37 +116,9.0,9 +117,23.0,23 +118,17.0,17 +119,14.0,14 +120,39.0,39 +121,19.0,19 +122,20.0,20 +123,17.0,17 +124,13.0,13 +125,12.0,12 +126,12.0,12 +127,12.0,12 +128,15.0,15 +129,11.0,11 +130,11.0,11 +131,13.0,13 +132,11.0,11 +133,13.0,13 +134,26.0,26 +135,10.0,10 +136,9.0,9 +137,16.0,16 +138,9.0,9 +139,31.0,31 +140,20.0,20 +141,20.0,20 +142,13.0,13 +143,9.0,9 +144,9.0,9 +145,26.0,26 +146,14.0,14 +147,21.0,21 +148,25.0,25 +149,11.0,11 +150,20.0,20 +151,16.0,16 +152,14.0,14 +153,9.0,9 +154,11.0,11 +155,14.0,14 +156,11.0,11 +157,11.0,11 +158,11.0,11 +159,9.0,9 +160,31.0,31 +161,9.0,9 +162,47.0,47 +163,10.0,10 +164,9.0,9 +165,12.0,12 +166,12.0,12 +167,15.0,15 +168,12.0,12 +169,11.0,11 +170,19.0,19 +171,16.0,16 +172,18.0,18 +173,13.0,13 +174,18.0,18 +175,15.0,15 +176,14.0,14 +177,11.0,11 +178,13.0,13 +179,13.0,13 +180,13.0,13 +181,26.0,26 +182,10.0,10 +183,21.0,21 +184,9.0,9 +185,16.0,16 +186,16.0,16 +187,10.0,10 +188,17.0,17 +189,12.0,12 +190,9.0,9 +191,29.0,29 +192,43.0,43 +193,15.0,15 +194,35.0,35 +195,31.0,31 +196,13.0,13 +197,49.0,49 +198,9.0,9 +199,26.0,26 +200,12.0,12 +201,13.0,13 +202,11.0,11 +203,12.0,12 +204,29.0,29 +205,11.0,11 +206,25.0,25 +207,23.0,23 +208,17.0,17 +209,9.0,9 +210,13.0,13 +211,31.0,31 +212,14.0,14 +213,12.0,12 +214,14.0,14 +215,26.0,26 +216,19.0,19 +217,26.0,26 +218,18.0,18 +219,11.0,11 +220,19.0,19 +221,24.0,24 +222,13.0,13 +223,13.0,13 +224,20.0,20 +225,13.0,13 +226,24.0,24 +227,15.0,15 +228,15.0,15 +229,28.0,28 +230,26.0,26 +231,17.0,17 +232,26.0,26 +233,19.0,19 +234,14.0,14 +235,14.0,14 +236,25.0,25 +237,14.0,14 +238,24.0,24 +239,33.0,33 +240,31.0,31 +241,37.0,37 +242,32.0,32 +243,21.0,21 +244,16.0,16 +245,42.0,42 +246,21.0,21 +247,30.0,30 +248,27.0,27 +249,35.0,35 +250,11.0,11 +251,22.0,22 +252,10.0,10 +253,18.0,18 +254,25.0,25 +255,12.0,12 +256,19.0,19 +257,17.0,17 +258,51.0,51 +259,53.0,53 +260,35.0,35 +261,26.0,26 +262,47.0,47 +263,24.0,24 +264,25.0,25 +265,22.0,22 +266,17.0,17 +267,14.0,14 +268,15.0,15 +269,56.0,56 +270,38.0,38 +271,44.0,44 +272,38.0,38 +273,46.0,46 +274,35.0,35 +275,35.0,35 +276,13.0,13 +277,37.0,37 +278,12.0,12 +279,20.0,20 +280,19.0,19 +281,21.0,21 +282,16.0,16 +283,16.0,16 +284,36.0,36 +285,24.0,24 +286,51.0,51 +287,19.0,19 +288,36.0,36 +289,49.0,49 +290,54.0,54 +291,47.0,47 +292,62.0,62 +293,13.0,13 +294,54.0,54 +295,30.0,30 +296,21.0,21 +297,12.0,12 +298,18.0,18 +299,61.0,61 +300,30.0,30 +301,10.0,10 +302,24.0,24 +303,10.0,10 +304,54.0,54 +305,17.0,17 +306,13.0,13 +307,50.0,50 +308,21.0,21 +309,10.0,10 +310,46.0,46 +311,10.0,10 +312,28.0,28 +313,28.0,28 +314,21.0,21 +315,22.0,22 +316,35.0,35 +317,16.0,16 +318,20.0,20 +319,37.0,37 +320,17.0,17 +321,20.0,20 +322,20.0,20 +323,39.0,39 +324,32.0,32 +325,15.0,15 +326,15.0,15 +327,35.0,35 +328,22.0,22 +329,40.0,40 +330,36.0,36 +331,23.0,23 +332,30.0,30 +333,30.0,30 +334,15.0,15 +335,33.0,33 +336,38.0,38 +337,15.0,15 +338,25.0,25 +339,13.0,13 +340,15.0,15 +341,27.0,27 +342,15.0,15 +343,13.0,13 +344,14.0,14 +345,35.0,35 +346,22.0,22 +347,11.0,11 +348,13.0,13 +349,38.0,38 +350,15.0,15 +351,21.0,21 +352,17.0,17 +353,17.0,17 +354,23.0,23 +355,29.0,29 +356,19.0,19 +357,35.0,35 +358,39.0,39 +359,13.0,13 +360,24.0,24 +361,11.0,11 +362,27.0,27 +363,38.0,38 +364,78.0,78 +365,18.0,18 +366,28.0,28 +367,27.0,27 +368,10.0,10 +369,31.0,31 +370,22.0,22 +371,44.0,44 +372,77.0,77 +373,57.0,57 +374,23.0,23 +375,25.0,25 +376,44.0,44 +377,19.0,19 +378,22.0,22 +379,27.0,27 +380,43.0,43 +381,28.0,28 +382,28.0,28 +383,34.0,34 +384,16.0,16 +385,48.0,48 +386,18.0,18 +387,39.0,39 +388,58.0,58 +389,84.0,84 +390,28.0,28 +391,26.0,26 +392,48.0,48 +393,23.0,23 +394,24.0,24 +395,31.0,31 +396,54.0,54 +397,37.0,37 +398,31.0,31 +399,44.0,44 +400,60.0,60 +401,20.0,20 +402,25.0,25 +403,29.0,29 +404,16.0,16 +405,36.0,36 +406,32.0,32 +407,54.0,54 +408,21.0,21 +409,75.0,75 +410,74.0,74 +411,68.0,68 +412,114.0,114 +413,38.0,38 +414,22.0,22 +415,41.0,41 +416,37.0,37 +417,67.0,67 +418,16.0,16 +419,33.0,33 +420,21.0,21 +421,78.0,78 +422,24.0,24 +423,22.0,22 +424,37.0,37 +425,20.0,20 +426,39.0,39 +427,94.0,94 +428,184.0,184 +429,61.0,61 +430,36.0,36 +431,28.0,28 +432,132.0,132 +433,70.0,70 +434,52.0,52 +435,23.0,23 +436,28.0,28 +437,41.0,41 +438,61.0,61 +439,21.0,21 +440,22.0,22 +441,99.0,99 +442,82.0,82 +443,63.0,63 +444,68.0,68 +445,51.0,51 +446,28.0,28 +447,79.0,79 +448,61.0,61 +449,46.0,46 +450,49.0,49 +451,110.0,110 +452,69.0,69 +453,27.0,27 +454,52.0,52 +455,44.0,44 +456,35.0,35 +457,25.0,25 +458,34.0,34 +459,26.0,26 +460,34.0,34 +461,29.0,29 +462,21.0,21 +463,28.0,28 +464,52.0,52 +465,49.0,49 +466,31.0,31 +467,35.0,35 +468,45.0,45 +469,55.0,55 +470,17.0,17 +471,61.0,61 +472,55.0,55 +473,31.0,31 +474,43.0,43 +475,77.0,77 +476,75.0,75 +477,146.0,146 +478,19.0,19 +479,61.0,61 +480,44.0,44 +481,53.0,53 +482,124.0,124 +483,45.0,45 +484,97.0,97 +485,98.0,98 +486,101.0,101 +487,42.0,42 +488,86.0,86 +489,39.0,39 +490,96.0,96 +491,177.0,177 +492,46.0,46 +493,196.0,196 +494,157.0,157 +495,34.0,34 +496,60.0,60 +497,102.0,102 +498,53.0,53 +499,52.0,52 +500,32.0,32 +501,114.0,114 +502,33.0,33 +503,112.0,112 +504,59.0,59 +505,122.0,122 +506,128.0,128 +507,99.0,99 +508,28.0,28 +509,200.0,200 +510,137.0,137 +511,24.0,24 +512,101.0,101 +513,150.0,150 +514,15.0,15 +515,43.0,43 +516,39.0,39 +517,43.0,43 +518,26.0,26 +519,22.0,22 +520,51.0,51 +521,21.0,21 +522,71.0,71 +523,53.0,53 +524,164.0,164 +525,36.0,36 +526,97.0,97 +527,58.0,58 +528,28.0,28 +529,116.0,116 +530,123.0,123 +531,138.0,138 +532,60.0,60 +533,126.0,126 +534,133.0,133 +535,145.0,145 +536,39.0,39 +537,64.0,64 +538,106.0,106 +539,54.0,54 +540,51.0,51 +541,131.0,131 +542,140.0,140 +543,138.0,138 +544,123.0,123 +545,145.0,145 +546,173.0,173 +547,77.0,77 +548,159.0,159 +549,200.0,200 +550,89.0,89 +551,180.0,180 +552,138.0,138 +553,65.0,65 +554,64.0,64 +555,120.0,120 +556,50.0,50 +557,105.0,105 +558,46.0,46 +559,128.0,128 +560,62.0,62 +561,92.0,92 +562,125.0,125 +563,62.0,62 +564,60.0,60 +565,78.0,78 +566,48.0,48 +567,68.0,68 +568,126.0,126 +569,75.0,75 +570,84.0,84 +571,87.0,87 +572,68.0,68 +573,27.0,27 +574,55.0,55 +575,123.0,123 +576,137.0,137 +577,48.0,48 +578,17.0,17 +579,18.0,18 +580,22.0,22 +581,26.0,26 +582,40.0,40 +583,14.0,14 +584,130.0,130 +585,118.0,118 +586,46.0,46 +587,125.0,125 +588,33.0,33 +589,158.0,158 +590,200.0,200 +591,141.0,141 +592,200.0,200 +593,200.0,200 +594,122.0,122 +595,200.0,200 +596,200.0,200 +597,162.0,162 +598,27.0,27 +599,18.0,18 +600,28.0,28 +601,164.0,164 +602,18.0,18 +603,11.0,11 +604,146.0,146 +605,128.0,128 +606,13.0,13 +607,15.0,15 +608,131.0,131 +609,191.0,191 +610,18.0,18 +611,101.0,101 +612,200.0,200 +613,200.0,200 +614,200.0,200 +615,65.0,65 +616,172.0,172 +617,200.0,200 +618,190.0,190 +619,200.0,200 +620,200.0,200 +621,200.0,200 +622,58.0,58 +623,57.0,57 +624,200.0,200 +625,74.0,74 +626,61.0,61 +627,129.0,129 +628,114.0,114 +629,157.0,157 +630,62.0,62 +631,36.0,36 +632,37.0,37 +633,103.0,103 +634,190.0,190 +635,74.0,74 +636,141.0,141 +637,200.0,200 +638,200.0,200 +639,143.0,143 +640,200.0,200 +641,141.0,141 +642,135.0,135 +643,144.0,144 +644,46.0,46 +645,59.0,59 +646,63.0,63 +647,105.0,105 +648,101.0,101 +649,92.0,92 +650,40.0,40 +651,40.0,40 +652,58.0,58 +653,101.0,101 +654,38.0,38 +655,40.0,40 +656,32.0,32 +657,50.0,50 +658,49.0,49 +659,38.0,38 +660,61.0,61 +661,40.0,40 +662,13.0,13 +663,44.0,44 +664,30.0,30 +665,43.0,43 +666,37.0,37 +667,34.0,34 +668,42.0,42 +669,35.0,35 +670,67.0,67 +671,16.0,16 +672,41.0,41 +673,23.0,23 +674,40.0,40 +675,37.0,37 +676,41.0,41 +677,47.0,47 +678,87.0,87 +679,40.0,40 +680,82.0,82 +681,50.0,50 +682,67.0,67 +683,28.0,28 +684,144.0,144 +685,36.0,36 +686,51.0,51 +687,69.0,69 +688,48.0,48 +689,44.0,44 +690,46.0,46 +691,42.0,42 +692,42.0,42 +693,41.0,41 +694,161.0,161 +695,56.0,56 +696,66.0,66 +697,63.0,63 +698,98.0,98 +699,130.0,130 +700,54.0,54 +701,125.0,125 +702,75.0,75 +703,115.0,115 +704,110.0,110 +705,73.0,73 +706,120.0,120 +707,62.0,62 +708,134.0,134 +709,53.0,53 +710,45.0,45 +711,82.0,82 +712,47.0,47 +713,60.0,60 +714,121.0,121 +715,100.0,100 +716,29.0,29 +717,33.0,33 +718,60.0,60 +719,117.0,117 +720,71.0,71 +721,15.0,15 +722,130.0,130 +723,115.0,115 +724,27.0,27 +725,28.0,28 +726,19.0,19 +727,22.0,22 +728,22.0,22 +729,21.0,21 +730,14.0,14 +731,22.0,22 +732,16.0,16 +733,21.0,21 +734,18.0,18 +735,21.0,21 +736,17.0,17 +737,16.0,16 +738,97.0,97 +739,20.0,20 +740,15.0,15 +741,17.0,17 +742,14.0,14 +743,20.0,20 +744,20.0,20 +745,13.0,13 +746,17.0,17 +747,18.0,18 +748,30.0,30 +749,125.0,125 +750,25.0,25 +751,117.0,117 +752,94.0,94 +753,134.0,134 +754,22.0,22 +755,101.0,101 +756,134.0,134 +757,98.0,98 +758,167.0,167 +759,137.0,137 +760,153.0,153 +761,125.0,125 +762,146.0,146 +763,123.0,123 +764,131.0,131 +765,123.0,123 +766,38.0,38 +767,30.0,30 +768,126.0,126 +769,128.0,128 +770,178.0,178 +771,164.0,164 +772,145.0,145 +773,175.0,175 +774,200.0,200 +775,200.0,200 +776,200.0,200 +777,200.0,200 +778,200.0,200 +779,200.0,200 +780,161.0,161 +781,124.0,124 +782,110.0,110 +783,79.0,79 +784,107.0,107 +785,88.0,88 +786,79.0,79 +787,113.0,113 +788,104.0,104 +789,122.0,122 +790,135.0,135 +791,18.0,18 +792,117.0,117 +793,99.0,99 +794,103.0,103 +795,35.0,35 +796,85.0,85 +797,68.0,68 +798,79.0,79 +799,104.0,104 +800,100.0,100 +801,143.0,143 +802,184.0,184 +803,200.0,200 +804,200.0,200 +805,200.0,200 +806,200.0,200 +807,200.0,200 +808,200.0,200 +809,200.0,200 +810,200.0,200 +811,200.0,200 +812,200.0,200 +813,200.0,200 +814,200.0,200 +815,200.0,200 +816,200.0,200 +817,200.0,200 +818,200.0,200 +819,200.0,200 +820,200.0,200 +821,200.0,200 +822,200.0,200 +823,200.0,200 +824,200.0,200 +825,200.0,200 +826,200.0,200 +827,200.0,200 +828,200.0,200 +829,200.0,200 +830,200.0,200 +831,200.0,200 +832,200.0,200 +833,200.0,200 +834,200.0,200 +835,200.0,200 +836,200.0,200 +837,200.0,200 +838,200.0,200 +839,200.0,200 +840,200.0,200 +841,200.0,200 +842,200.0,200 +843,200.0,200 +844,200.0,200 +845,200.0,200 +846,200.0,200 +847,200.0,200 +848,192.0,192 +849,184.0,184 +850,200.0,200 +851,200.0,200 +852,200.0,200 +853,200.0,200 +854,200.0,200 +855,200.0,200 +856,200.0,200 +857,200.0,200 +858,200.0,200 +859,200.0,200 +860,200.0,200 +861,200.0,200 +862,200.0,200 +863,200.0,200 +864,200.0,200 +865,200.0,200 +866,200.0,200 +867,200.0,200 +868,200.0,200 +869,200.0,200 +870,200.0,200 +871,190.0,190 +872,124.0,124 +873,133.0,133 +874,161.0,161 +875,21.0,21 +876,139.0,139 +877,128.0,128 +878,122.0,122 +879,170.0,170 +880,200.0,200 +881,200.0,200 +882,200.0,200 +883,200.0,200 +884,200.0,200 +885,106.0,106 +886,200.0,200 +887,200.0,200 +888,200.0,200 +889,200.0,200 +890,200.0,200 +891,169.0,169 +892,132.0,132 +893,18.0,18 +894,132.0,132 +895,150.0,150 +896,25.0,25 +897,161.0,161 +898,187.0,187 +899,161.0,161 +900,146.0,146 +901,19.0,19 +902,119.0,119 +903,18.0,18 +904,18.0,18 +905,39.0,39 +906,139.0,139 +907,14.0,14 +908,129.0,129 +909,12.0,12 +910,129.0,129 +911,112.0,112 +912,122.0,122 +913,161.0,161 +914,174.0,174 +915,200.0,200 +916,198.0,198 +917,200.0,200 +918,200.0,200 +919,200.0,200 +920,200.0,200 +921,200.0,200 +922,200.0,200 +923,200.0,200 +924,200.0,200 +925,200.0,200 +926,200.0,200 +927,200.0,200 +928,200.0,200 +929,200.0,200 +930,200.0,200 +931,200.0,200 +932,200.0,200 +933,200.0,200 +934,22.0,22 +935,187.0,187 +936,200.0,200 +937,186.0,186 +938,183.0,183 +939,200.0,200 +940,200.0,200 +941,200.0,200 +942,200.0,200 +943,200.0,200 +944,200.0,200 +945,88.0,88 +946,200.0,200 +947,200.0,200 +948,200.0,200 +949,200.0,200 +950,200.0,200 +951,200.0,200 +952,200.0,200 +953,200.0,200 +954,200.0,200 +955,200.0,200 +956,200.0,200 +957,200.0,200 +958,200.0,200 +959,200.0,200 +960,200.0,200 +961,200.0,200 +962,200.0,200 +963,200.0,200 +964,200.0,200 +965,200.0,200 +966,200.0,200 +967,200.0,200 +968,200.0,200 +969,200.0,200 +970,200.0,200 +971,22.0,22 +972,200.0,200 +973,200.0,200 +974,200.0,200 +975,200.0,200 +976,200.0,200 +977,142.0,142 +978,200.0,200 +979,200.0,200 +980,179.0,179 +981,140.0,140 +982,109.0,109 +983,114.0,114 +984,14.0,14 +985,25.0,25 +986,12.0,12 +987,129.0,129 +988,150.0,150 +989,140.0,140 +990,178.0,178 +991,200.0,200 +992,200.0,200 +993,200.0,200 +994,200.0,200 +995,200.0,200 +996,197.0,197 +997,163.0,163 +998,169.0,169 +999,12.0,12 diff --git a/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/config.yaml b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2cdf43a1df066c090ed33a70c280c8bb08f71a2 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/config.yaml @@ -0,0 +1,34 @@ +general_cfg: + algo_name: A3C + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + n_workers: 10 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + device: cpu + entropy_coef: 0.001 + env_name: CartPole-v1 + gamma: 0.99 + max_steps: 200 + n_workers: 10 + seed: 1 + train_eps: 1000 + update_freq: 20 diff --git a/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/logs/log.txt b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..5d15cf1b6e129ff9852bac99bee50eba0b8cf43a --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/logs/log.txt @@ -0,0 +1,39 @@ +2023-01-11 13:25:42 - r - INFO: - Hyperparameters: +2023-01-11 13:25:42 - r - INFO: - ================================================================================ +2023-01-11 13:25:42 - r - INFO: - Name Value Type +2023-01-11 13:25:42 - r - INFO: - env_name CartPole-v1 +2023-01-11 13:25:42 - r - INFO: - new_step_api 1 +2023-01-11 13:25:42 - r - INFO: - wrapper None +2023-01-11 13:25:42 - r - INFO: - render 0 +2023-01-11 13:25:42 - r - INFO: - algo_name A3C +2023-01-11 13:25:42 - r - INFO: - mode train +2023-01-11 13:25:42 - r - INFO: - seed 1 +2023-01-11 13:25:42 - r - INFO: - device cpu +2023-01-11 13:25:42 - r - INFO: - train_eps 1000 +2023-01-11 13:25:42 - r - INFO: - test_eps 10 +2023-01-11 13:25:42 - r - INFO: - eval_eps 10 +2023-01-11 13:25:42 - r - INFO: - eval_per_episode 5 +2023-01-11 13:25:42 - r - INFO: - max_steps 200 +2023-01-11 13:25:42 - r - INFO: - load_checkpoint 0 +2023-01-11 13:25:42 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-01-11 13:25:42 - r - INFO: - show_fig 0 +2023-01-11 13:25:42 - r - INFO: - save_fig 1 +2023-01-11 13:25:42 - r - INFO: - n_workers 10 +2023-01-11 13:25:42 - r - INFO: - gamma 0.99 +2023-01-11 13:25:42 - r - INFO: - actor_lr 0.0003 +2023-01-11 13:25:42 - r - INFO: - critic_lr 0.001 +2023-01-11 13:25:42 - r - INFO: - actor_hidden_dim 256 +2023-01-11 13:25:42 - r - INFO: - critic_hidden_dim 256 +2023-01-11 13:25:42 - r - INFO: - entropy_coef 0.001 +2023-01-11 13:25:42 - r - INFO: - update_freq 20 +2023-01-11 13:25:42 - r - INFO: - continuous 0 +2023-01-11 13:25:42 - r - INFO: - task_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Train_CartPole-v1_A3C_20230111-132542 +2023-01-11 13:25:42 - r - INFO: - model_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Train_CartPole-v1_A3C_20230111-132542/models +2023-01-11 13:25:42 - r - INFO: - res_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Train_CartPole-v1_A3C_20230111-132542/results +2023-01-11 13:25:42 - r - INFO: - log_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Train_CartPole-v1_A3C_20230111-132542/logs +2023-01-11 13:25:42 - r - INFO: - traj_dir /Users/jj/Desktop/rl-tutorials/joyrl/tasks/Train_CartPole-v1_A3C_20230111-132542/traj +2023-01-11 13:25:42 - r - INFO: - ================================================================================ +2023-01-11 13:25:42 - r - INFO: - n_states: 4, n_actions: 2 +2023-01-11 13:25:43 - r - INFO: - Start training! +2023-01-11 13:25:43 - r - INFO: - Env: CartPole-v1, Algorithm: A3C, Device: cpu +2023-01-11 13:26:03 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/models/actor_checkpoint.pt b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2cb5fed16e3d7e33be76ba1d7a157eaa207a6a8 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39778e5ed902fd1548d304a25877444c0e5dade38d40ccbdc20f879a9f573986 +size 272151 diff --git a/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/models/critic_checkpoint.pt b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/models/critic_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..27c0ec962e1dfef43c90c71993f5187a3e100dec --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/models/critic_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:689822d7abbdfab3b4c0d061b5a2298637277eddb3b8e52469061857439df933 +size 271127 diff --git a/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..802be16574f397de6ad0db1f96a7675a471846c6 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/results/res.csv b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..e3113f98eec48be376ca483d9ee7f4e589bc9d18 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_A3C_20230111-132542/results/res.csv @@ -0,0 +1,1002 @@ +episodes,rewards +0,10.0 +1,39.0 +2,28.0 +3,10.0 +4,8.0 +5,10.0 +6,22.0 +7,13.0 +8,26.0 +9,58.0 +10,9.0 +11,13.0 +12,18.0 +13,11.0 +14,28.0 +15,28.0 +16,9.0 +17,11.0 +18,12.0 +19,38.0 +20,65.0 +21,23.0 +22,20.0 +23,9.0 +24,20.0 +25,14.0 +26,11.0 +27,20.0 +28,15.0 +29,17.0 +30,29.0 +31,17.0 +32,18.0 +33,38.0 +34,10.0 +35,13.0 +36,14.0 +37,16.0 +38,18.0 +39,18.0 +40,17.0 +41,36.0 +42,13.0 +43,17.0 +44,14.0 +45,21.0 +46,11.0 +47,15.0 +48,28.0 +49,11.0 +50,14.0 +51,19.0 +52,18.0 +53,15.0 +54,11.0 +55,20.0 +56,13.0 +57,14.0 +58,26.0 +59,16.0 +60,17.0 +61,20.0 +62,14.0 +63,11.0 +64,10.0 +65,19.0 +66,12.0 +67,11.0 +68,14.0 +69,25.0 +70,27.0 +71,9.0 +72,10.0 +73,22.0 +74,31.0 +75,13.0 +76,26.0 +77,11.0 +78,16.0 +79,12.0 +80,16.0 +81,29.0 +82,11.0 +83,15.0 +84,20.0 +85,15.0 +86,16.0 +87,11.0 +88,14.0 +89,13.0 +90,19.0 +91,9.0 +92,17.0 +93,32.0 +94,10.0 +95,39.0 +96,20.0 +97,18.0 +98,20.0 +99,40.0 +100,9.0 +101,14.0 +102,15.0 +103,18.0 +104,11.0 +105,15.0 +106,16.0 +107,27.0 +108,39.0 +109,12.0 +110,24.0 +111,19.0 +112,23.0 +113,11.0 +114,14.0 +115,16.0 +116,15.0 +117,11.0 +118,33.0 +119,11.0 +120,19.0 +121,21.0 +122,22.0 +123,42.0 +124,12.0 +125,19.0 +126,34.0 +127,21.0 +128,12.0 +129,12.0 +130,23.0 +131,20.0 +132,20.0 +133,17.0 +134,15.0 +135,13.0 +136,12.0 +137,24.0 +138,21.0 +139,21.0 +140,46.0 +141,9.0 +142,34.0 +143,23.0 +144,46.0 +145,13.0 +146,27.0 +147,87.0 +148,33.0 +149,27.0 +150,36.0 +151,26.0 +152,24.0 +153,15.0 +154,13.0 +155,13.0 +156,20.0 +157,11.0 +158,72.0 +159,26.0 +160,16.0 +161,23.0 +162,43.0 +163,23.0 +164,26.0 +165,65.0 +166,14.0 +167,14.0 +168,14.0 +169,33.0 +170,28.0 +171,10.0 +172,42.0 +173,12.0 +174,38.0 +175,19.0 +176,51.0 +177,11.0 +178,82.0 +179,56.0 +180,19.0 +181,18.0 +182,13.0 +183,28.0 +184,23.0 +185,19.0 +186,14.0 +187,16.0 +188,35.0 +189,19.0 +190,22.0 +191,21.0 +192,13.0 +193,54.0 +194,79.0 +195,44.0 +196,61.0 +197,51.0 +198,24.0 +199,31.0 +200,28.0 +201,83.0 +202,14.0 +203,86.0 +204,25.0 +205,28.0 +206,23.0 +207,54.0 +208,26.0 +209,51.0 +210,43.0 +211,32.0 +212,21.0 +213,54.0 +214,28.0 +215,76.0 +216,38.0 +217,70.0 +218,70.0 +219,23.0 +220,19.0 +221,114.0 +222,47.0 +223,25.0 +224,25.0 +225,73.0 +226,39.0 +227,62.0 +228,47.0 +229,34.0 +230,30.0 +231,96.0 +232,86.0 +233,138.0 +234,23.0 +235,33.0 +236,25.0 +237,47.0 +238,48.0 +239,46.0 +240,24.0 +241,16.0 +242,100.0 +243,61.0 +244,98.0 +245,35.0 +246,23.0 +247,38.0 +248,33.0 +249,91.0 +250,74.0 +251,51.0 +252,18.0 +253,81.0 +254,19.0 +255,61.0 +256,26.0 +257,124.0 +258,36.0 +259,32.0 +260,113.0 +261,34.0 +262,151.0 +263,39.0 +264,41.0 +265,55.0 +266,12.0 +267,16.0 +268,14.0 +269,58.0 +270,77.0 +271,28.0 +272,20.0 +273,10.0 +274,11.0 +275,49.0 +276,62.0 +277,24.0 +278,38.0 +279,19.0 +280,13.0 +281,58.0 +282,24.0 +283,72.0 +284,15.0 +285,22.0 +286,10.0 +287,14.0 +288,19.0 +289,21.0 +290,52.0 +291,44.0 +292,22.0 +293,28.0 +294,17.0 +295,75.0 +296,26.0 +297,35.0 +298,107.0 +299,69.0 +300,11.0 +301,29.0 +302,37.0 +303,107.0 +304,60.0 +305,61.0 +306,168.0 +307,37.0 +308,41.0 +309,58.0 +310,31.0 +311,102.0 +312,27.0 +313,18.0 +314,34.0 +315,28.0 +316,27.0 +317,32.0 +318,92.0 +319,26.0 +320,18.0 +321,26.0 +322,44.0 +323,15.0 +324,27.0 +325,37.0 +326,29.0 +327,22.0 +328,23.0 +329,31.0 +330,33.0 +331,23.0 +332,18.0 +333,31.0 +334,32.0 +335,29.0 +336,18.0 +337,31.0 +338,19.0 +339,30.0 +340,31.0 +341,40.0 +342,17.0 +343,26.0 +344,23.0 +345,31.0 +346,24.0 +347,30.0 +348,33.0 +349,31.0 +350,30.0 +351,30.0 +352,21.0 +353,28.0 +354,25.0 +355,67.0 +356,33.0 +357,34.0 +358,23.0 +359,28.0 +360,39.0 +361,26.0 +362,20.0 +363,59.0 +364,59.0 +365,40.0 +366,63.0 +367,36.0 +368,62.0 +369,38.0 +370,63.0 +371,109.0 +372,52.0 +373,72.0 +374,81.0 +375,32.0 +376,57.0 +377,28.0 +378,20.0 +379,16.0 +380,28.0 +381,31.0 +382,25.0 +383,100.0 +384,133.0 +385,92.0 +386,43.0 +387,17.0 +388,79.0 +389,42.0 +390,123.0 +391,64.0 +392,112.0 +393,66.0 +394,86.0 +395,30.0 +396,21.0 +397,64.0 +398,45.0 +399,15.0 +400,89.0 +401,15.0 +402,18.0 +403,37.0 +404,31.0 +405,20.0 +406,17.0 +407,16.0 +408,106.0 +409,70.0 +410,104.0 +411,34.0 +412,130.0 +413,38.0 +414,28.0 +415,42.0 +416,156.0 +417,42.0 +418,102.0 +419,26.0 +420,85.0 +421,36.0 +422,161.0 +423,43.0 +424,122.0 +425,45.0 +426,22.0 +427,25.0 +428,29.0 +429,13.0 +430,33.0 +431,21.0 +432,20.0 +433,17.0 +434,16.0 +435,35.0 +436,28.0 +437,22.0 +438,26.0 +439,122.0 +440,163.0 +441,23.0 +442,28.0 +443,38.0 +444,29.0 +445,35.0 +446,45.0 +447,16.0 +448,35.0 +449,26.0 +450,36.0 +451,66.0 +452,53.0 +453,37.0 +454,23.0 +455,50.0 +456,68.0 +457,94.0 +458,32.0 +459,110.0 +460,89.0 +461,29.0 +462,71.0 +463,43.0 +464,44.0 +465,43.0 +466,45.0 +467,32.0 +468,26.0 +469,51.0 +470,50.0 +471,48.0 +472,42.0 +473,28.0 +474,25.0 +475,25.0 +476,64.0 +477,43.0 +478,44.0 +479,34.0 +480,46.0 +481,49.0 +482,63.0 +483,68.0 +484,76.0 +485,100.0 +486,126.0 +487,66.0 +488,31.0 +489,200.0 +490,132.0 +491,200.0 +492,200.0 +493,175.0 +494,200.0 +495,200.0 +496,193.0 +497,22.0 +498,60.0 +499,68.0 +500,179.0 +501,127.0 +502,200.0 +503,123.0 +504,189.0 +505,183.0 +506,200.0 +507,48.0 +508,105.0 +509,200.0 +510,109.0 +511,105.0 +512,121.0 +513,200.0 +514,167.0 +515,108.0 +516,119.0 +517,138.0 +518,97.0 +519,21.0 +520,76.0 +521,103.0 +522,148.0 +523,80.0 +524,58.0 +525,155.0 +526,119.0 +527,109.0 +528,200.0 +529,85.0 +530,132.0 +531,34.0 +532,200.0 +533,200.0 +534,141.0 +535,60.0 +536,133.0 +537,81.0 +538,121.0 +539,114.0 +540,29.0 +541,162.0 +542,120.0 +543,92.0 +544,108.0 +545,101.0 +546,56.0 +547,25.0 +548,70.0 +549,90.0 +550,110.0 +551,116.0 +552,150.0 +553,141.0 +554,138.0 +555,191.0 +556,200.0 +557,195.0 +558,42.0 +559,200.0 +560,139.0 +561,200.0 +562,186.0 +563,200.0 +564,139.0 +565,145.0 +566,200.0 +567,200.0 +568,124.0 +569,120.0 +570,151.0 +571,112.0 +572,171.0 +573,200.0 +574,34.0 +575,120.0 +576,27.0 +577,73.0 +578,28.0 +579,45.0 +580,51.0 +581,68.0 +582,108.0 +583,95.0 +584,117.0 +585,111.0 +586,134.0 +587,100.0 +588,95.0 +589,125.0 +590,154.0 +591,200.0 +592,138.0 +593,183.0 +594,159.0 +595,200.0 +596,45.0 +597,200.0 +598,200.0 +599,200.0 +600,194.0 +601,171.0 +602,200.0 +603,200.0 +604,199.0 +605,200.0 +606,200.0 +607,200.0 +608,200.0 +609,184.0 +610,200.0 +611,200.0 +612,72.0 +613,200.0 +614,200.0 +615,200.0 +616,200.0 +617,200.0 +618,170.0 +619,138.0 +620,186.0 +621,169.0 +622,147.0 +623,90.0 +624,30.0 +625,116.0 +626,95.0 +627,136.0 +628,36.0 +629,70.0 +630,46.0 +631,105.0 +632,26.0 +633,118.0 +634,131.0 +635,139.0 +636,95.0 +637,147.0 +638,200.0 +639,200.0 +640,197.0 +641,162.0 +642,200.0 +643,158.0 +644,200.0 +645,200.0 +646,165.0 +647,200.0 +648,183.0 +649,177.0 +650,200.0 +651,194.0 +652,200.0 +653,200.0 +654,200.0 +655,200.0 +656,174.0 +657,200.0 +658,200.0 +659,200.0 +660,200.0 +661,200.0 +662,200.0 +663,200.0 +664,200.0 +665,200.0 +666,200.0 +667,200.0 +668,200.0 +669,200.0 +670,200.0 +671,200.0 +672,200.0 +673,200.0 +674,200.0 +675,200.0 +676,200.0 +677,196.0 +678,200.0 +679,121.0 +680,200.0 +681,123.0 +682,200.0 +683,111.0 +684,200.0 +685,53.0 +686,200.0 +687,200.0 +688,12.0 +689,165.0 +690,86.0 +691,102.0 +692,177.0 +693,200.0 +694,200.0 +695,200.0 +696,200.0 +697,200.0 +698,200.0 +699,200.0 +700,200.0 +701,200.0 +702,200.0 +703,200.0 +704,200.0 +705,200.0 +706,120.0 +707,200.0 +708,47.0 +709,200.0 +710,119.0 +711,200.0 +712,200.0 +713,89.0 +714,70.0 +715,131.0 +716,51.0 +717,59.0 +718,75.0 +719,159.0 +720,126.0 +721,82.0 +722,200.0 +723,58.0 +724,187.0 +725,133.0 +726,104.0 +727,98.0 +728,39.0 +729,61.0 +730,61.0 +731,134.0 +732,55.0 +733,136.0 +734,35.0 +735,48.0 +736,29.0 +737,57.0 +738,34.0 +739,52.0 +740,69.0 +741,17.0 +742,54.0 +743,157.0 +744,30.0 +745,39.0 +746,36.0 +747,75.0 +748,58.0 +749,14.0 +750,95.0 +751,66.0 +752,32.0 +753,47.0 +754,136.0 +755,53.0 +756,113.0 +757,81.0 +758,63.0 +759,51.0 +760,30.0 +761,114.0 +762,57.0 +763,95.0 +764,40.0 +765,38.0 +766,95.0 +767,67.0 +768,48.0 +769,50.0 +770,41.0 +771,65.0 +772,104.0 +773,115.0 +774,18.0 +775,27.0 +776,200.0 +777,154.0 +778,200.0 +779,144.0 +780,200.0 +781,141.0 +782,83.0 +783,161.0 +784,200.0 +785,189.0 +786,200.0 +787,155.0 +788,200.0 +789,200.0 +790,200.0 +791,197.0 +792,52.0 +793,75.0 +794,176.0 +795,183.0 +796,19.0 +797,124.0 +798,77.0 +799,200.0 +800,131.0 +801,200.0 +802,90.0 +803,175.0 +804,200.0 +805,200.0 +806,123.0 +807,64.0 +808,84.0 +809,126.0 +810,47.0 +811,200.0 +812,200.0 +813,200.0 +814,97.0 +815,143.0 +816,200.0 +817,200.0 +818,188.0 +819,145.0 +820,18.0 +821,120.0 +822,200.0 +823,95.0 +824,95.0 +825,138.0 +826,111.0 +827,200.0 +828,81.0 +829,200.0 +830,200.0 +831,200.0 +832,200.0 +833,94.0 +834,200.0 +835,200.0 +836,200.0 +837,200.0 +838,200.0 +839,200.0 +840,200.0 +841,200.0 +842,200.0 +843,200.0 +844,200.0 +845,200.0 +846,200.0 +847,200.0 +848,200.0 +849,200.0 +850,200.0 +851,200.0 +852,200.0 +853,200.0 +854,200.0 +855,200.0 +856,200.0 +857,200.0 +858,200.0 +859,200.0 +860,200.0 +861,200.0 +862,200.0 +863,200.0 +864,200.0 +865,200.0 +866,200.0 +867,200.0 +868,200.0 +869,200.0 +870,200.0 +871,200.0 +872,200.0 +873,200.0 +874,200.0 +875,200.0 +876,17.0 +877,200.0 +878,200.0 +879,200.0 +880,200.0 +881,200.0 +882,57.0 +883,129.0 +884,200.0 +885,200.0 +886,200.0 +887,200.0 +888,200.0 +889,200.0 +890,200.0 +891,200.0 +892,200.0 +893,200.0 +894,195.0 +895,200.0 +896,200.0 +897,200.0 +898,200.0 +899,200.0 +900,200.0 +901,200.0 +902,200.0 +903,200.0 +904,151.0 +905,200.0 +906,200.0 +907,183.0 +908,200.0 +909,200.0 +910,200.0 +911,200.0 +912,200.0 +913,200.0 +914,200.0 +915,200.0 +916,200.0 +917,200.0 +918,200.0 +919,200.0 +920,200.0 +921,200.0 +922,200.0 +923,200.0 +924,200.0 +925,200.0 +926,200.0 +927,48.0 +928,37.0 +929,167.0 +930,200.0 +931,200.0 +932,200.0 +933,200.0 +934,200.0 +935,200.0 +936,200.0 +937,200.0 +938,200.0 +939,200.0 +940,200.0 +941,200.0 +942,200.0 +943,200.0 +944,200.0 +945,200.0 +946,200.0 +947,200.0 +948,200.0 +949,200.0 +950,192.0 +951,117.0 +952,200.0 +953,200.0 +954,200.0 +955,200.0 +956,200.0 +957,176.0 +958,145.0 +959,200.0 +960,200.0 +961,199.0 +962,200.0 +963,200.0 +964,200.0 +965,200.0 +966,200.0 +967,165.0 +968,200.0 +969,128.0 +970,155.0 +971,160.0 +972,45.0 +973,161.0 +974,181.0 +975,53.0 +976,33.0 +977,158.0 +978,181.0 +979,105.0 +980,33.0 +981,38.0 +982,72.0 +983,102.0 +984,95.0 +985,28.0 +986,56.0 +987,65.0 +988,18.0 +989,12.0 +990,11.0 +991,15.0 +992,9.0 +993,51.0 +994,62.0 +995,77.0 +996,14.0 +997,14.0 +998,128.0 +999,200.0 +1000,63.0 diff --git a/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/config.yaml b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81ddfb3120ae72ef9cb40048f824dd64abd9cffd --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/config.yaml @@ -0,0 +1,26 @@ +general_cfg: + algo_name: BC + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 1 + load_checkpoint: false + load_path: Train_CartPole-v1_BC_20230319-114100 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 1 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + batch_size: 128 + continuous: false + critic_hidden_dim: 256 + expert_path: tasks/Collect_CartPole-v1_PPO_20230319-170351/traj/traj.pkl + lr: 0.0003 + train_iterations: 500 diff --git a/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/logs/log.txt b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..554148bc35c1504dafc82b7848c6336efbdafb03 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/logs/log.txt @@ -0,0 +1,39 @@ +2023-03-19 19:04:31 - r - INFO: - Hyperparameters: +2023-03-19 19:04:31 - r - INFO: - ================================================================================ +2023-03-19 19:04:31 - r - INFO: - Name Value Type +2023-03-19 19:04:31 - r - INFO: - env_name CartPole-v1 +2023-03-19 19:04:31 - r - INFO: - new_step_api 1 +2023-03-19 19:04:31 - r - INFO: - wrapper None +2023-03-19 19:04:31 - r - INFO: - render 0 +2023-03-19 19:04:31 - r - INFO: - algo_name BC +2023-03-19 19:04:31 - r - INFO: - mode train +2023-03-19 19:04:31 - r - INFO: - seed 1 +2023-03-19 19:04:31 - r - INFO: - device cuda +2023-03-19 19:04:31 - r - INFO: - train_eps 1 +2023-03-19 19:04:31 - r - INFO: - test_eps 10 +2023-03-19 19:04:31 - r - INFO: - eval_eps 10 +2023-03-19 19:04:31 - r - INFO: - eval_per_episode 1 +2023-03-19 19:04:31 - r - INFO: - max_steps 200 +2023-03-19 19:04:31 - r - INFO: - load_checkpoint 0 +2023-03-19 19:04:31 - r - INFO: - load_path Train_CartPole-v1_BC_20230319-114100 +2023-03-19 19:04:31 - r - INFO: - show_fig 0 +2023-03-19 19:04:31 - r - INFO: - save_fig 1 +2023-03-19 19:04:31 - r - INFO: - continuous 0 +2023-03-19 19:04:31 - r - INFO: - lr 0.0003 +2023-03-19 19:04:31 - r - INFO: - batch_size 128 +2023-03-19 19:04:31 - r - INFO: - train_iterations 500 +2023-03-19 19:04:31 - r - INFO: - actor_hidden_dim 256 +2023-03-19 19:04:31 - r - INFO: - critic_hidden_dim 256 +2023-03-19 19:04:31 - r - INFO: - expert_path tasks/Collect_CartPole-v1_PPO_20230319-170351/traj/traj.pkl +2023-03-19 19:04:31 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_BC_20230319-190431 +2023-03-19 19:04:31 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_BC_20230319-190431/results +2023-03-19 19:04:31 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_BC_20230319-190431/logs +2023-03-19 19:04:31 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_BC_20230319-190431/traj +2023-03-19 19:04:31 - r - INFO: - tb_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_BC_20230319-190431/tb_logs +2023-03-19 19:04:31 - r - INFO: - ================================================================================ +2023-03-19 19:04:31 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-19 19:04:35 - r - INFO: - Start training! +2023-03-19 19:04:35 - r - INFO: - Env: CartPole-v1, Algorithm: BC, Device: cuda +2023-03-19 19:09:04 - r - INFO: - Episode: 1/1, Reward: 200.000, Step: 200 +2023-03-19 19:09:10 - r - INFO: - Current episode 1 has the best eval reward: 189.100 +2023-03-19 19:09:10 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/models/model.pth b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/models/model.pth new file mode 100644 index 0000000000000000000000000000000000000000..0105809efa8822a61704bd401484457c74db821b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/models/model.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fb33e7cea0214b2d6baa00a5e2e46ca4d423b1abd6929bb5ec4ad7fda648907 +size 272215 diff --git a/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..eb4c9bc199119b8df97b96b0958e14d181f04d78 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/results/res.csv b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..c96d10309a0fe10b5dec8363c5827326bf0da5c0 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/results/res.csv @@ -0,0 +1,2 @@ +episodes,rewards,steps +0,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/tb_logs/events.out.tfevents.1679223871.dell-Precision-5820-Tower.11356.0 b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/tb_logs/events.out.tfevents.1679223871.dell-Precision-5820-Tower.11356.0 new file mode 100644 index 0000000000000000000000000000000000000000..b6293cd0afe9cf2767bf03830c88a67be6901aac --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_BC_20230319-190431/tb_logs/events.out.tfevents.1679223871.dell-Precision-5820-Tower.11356.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba2902977562043f2cb16556d9bb00665683ac41b6cca3109d290eef5dfff0ee +size 40 diff --git a/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/config.yaml b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04b89c35d4eb2dc68f64ae6c34b2a64320746bd8 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/config.yaml @@ -0,0 +1,71 @@ +general_cfg: + Vmax: 200 + Vmin: 0 + algo_name: C51 + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_C51_20221026-054757 + max_steps: 200 + mode: train + n_atoms: 51 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + Vmax: 200.0 + Vmin: 0.0 + batch_size: 64 + buffer_size: 100000 + delta_z: 4.0 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + memory_capacity: 10000 + num_atoms: 51 + support: !!python/object/apply:torch._utils._rebuild_tensor_v2 + - !!python/object/apply:torch.storage._load_from_bytes + - !!binary | + gAKKCmz8nEb5IGqoUBkugAJN6QMugAJ9cQAoWBAAAABwcm90b2NvbF92ZXJzaW9ucQFN6QNYDQAA + AGxpdHRsZV9lbmRpYW5xAohYCgAAAHR5cGVfc2l6ZXNxA31xBChYBQAAAHNob3J0cQVLAlgDAAAA + aW50cQZLBFgEAAAAbG9uZ3EHSwR1dS6AAihYBwAAAHN0b3JhZ2VxAGN0b3JjaApGbG9hdFN0b3Jh + Z2UKcQFYCQAAADExMzU3NDYyNHECWAMAAABjcHVxA0szTnRxBFEugAJdcQBYCQAAADExMzU3NDYy + NHEBYS4zAAAAAAAAAAAAAAAAAIBAAAAAQQAAQEEAAIBBAACgQQAAwEEAAOBBAAAAQgAAEEIAACBC + AAAwQgAAQEIAAFBCAABgQgAAcEIAAIBCAACIQgAAkEIAAJhCAACgQgAAqEIAALBCAAC4QgAAwEIA + AMhCAADQQgAA2EIAAOBCAADoQgAA8EIAAPhCAAAAQwAABEMAAAhDAAAMQwAAEEMAABRDAAAYQwAA + HEMAACBDAAAkQwAAKEMAACxDAAAwQwAANEMAADhDAAA8QwAAQEMAAERDAABIQw== + - 0 + - !!python/tuple + - 51 + - !!python/tuple + - 1 + - false + - !!python/object/apply:collections.OrderedDict + - [] + target_update: 4 + tau: 1.0 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/logs/log.txt b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4e59aeed4075894e669a88fe4f16432eb21aa44b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/logs/log.txt @@ -0,0 +1,161 @@ +2023-01-14 22:25:23 - r - INFO: - Hyperparameters: +2023-01-14 22:25:23 - r - INFO: - ================================================================================ +2023-01-14 22:25:23 - r - INFO: - Name Value Type +2023-01-14 22:25:23 - r - INFO: - env_name CartPole-v1 +2023-01-14 22:25:23 - r - INFO: - new_step_api 1 +2023-01-14 22:25:23 - r - INFO: - wrapper None +2023-01-14 22:25:23 - r - INFO: - render 0 +2023-01-14 22:25:23 - r - INFO: - algo_name C51 +2023-01-14 22:25:23 - r - INFO: - mode train +2023-01-14 22:25:23 - r - INFO: - seed 1 +2023-01-14 22:25:23 - r - INFO: - device cuda +2023-01-14 22:25:23 - r - INFO: - train_eps 100 +2023-01-14 22:25:23 - r - INFO: - test_eps 10 +2023-01-14 22:25:23 - r - INFO: - eval_eps 10 +2023-01-14 22:25:23 - r - INFO: - eval_per_episode 5 +2023-01-14 22:25:23 - r - INFO: - max_steps 200 +2023-01-14 22:25:23 - r - INFO: - load_checkpoint 0 +2023-01-14 22:25:23 - r - INFO: - load_path Train_CartPole-v1_C51_20221026-054757 +2023-01-14 22:25:23 - r - INFO: - show_fig 0 +2023-01-14 22:25:23 - r - INFO: - save_fig 1 +2023-01-14 22:25:23 - r - INFO: - Vmin 0.0 +2023-01-14 22:25:23 - r - INFO: - Vmax 200.0 +2023-01-14 22:25:23 - r - INFO: - n_atoms 51 +2023-01-14 22:25:23 - r - INFO: - gamma 0.95 +2023-01-14 22:25:23 - r - INFO: - tau 1.0 +2023-01-14 22:25:23 - r - INFO: - num_atoms 51 +2023-01-14 22:25:23 - r - INFO: - support tensor(0.) +2023-01-14 22:25:23 - r - INFO: - delta_z 4.0 +2023-01-14 22:25:23 - r - INFO: - batch_size 64 +2023-01-14 22:25:23 - r - INFO: - lr 0.0001 +2023-01-14 22:25:23 - r - INFO: - target_update 4 +2023-01-14 22:25:23 - r - INFO: - memory_capacity 10000 +2023-01-14 22:25:23 - r - INFO: - epsilon_start 0.95 +2023-01-14 22:25:23 - r - INFO: - epsilon_end 0.01 +2023-01-14 22:25:23 - r - INFO: - epsilon_decay 500 +2023-01-14 22:25:23 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-01-14 22:25:23 - r - INFO: - buffer_size 100000 +2023-01-14 22:25:23 - r - INFO: - task_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_C51_20230114-222523 +2023-01-14 22:25:23 - r - INFO: - model_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_C51_20230114-222523/models +2023-01-14 22:25:23 - r - INFO: - res_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_C51_20230114-222523/results +2023-01-14 22:25:23 - r - INFO: - log_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_C51_20230114-222523/logs +2023-01-14 22:25:23 - r - INFO: - traj_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_C51_20230114-222523/traj +2023-01-14 22:25:23 - r - INFO: - ================================================================================ +2023-01-14 22:25:23 - r - INFO: - n_states: 4, n_actions: 2 +2023-01-14 22:25:25 - r - INFO: - Start training! +2023-01-14 22:25:25 - r - INFO: - Env: CartPole-v1, Algorithm: C51, Device: cuda +2023-01-14 22:25:25 - r - INFO: - Episode: 1/100, Reward: 18.000, Step: 18 +2023-01-14 22:25:25 - r - INFO: - Episode: 2/100, Reward: 35.000, Step: 35 +2023-01-14 22:25:25 - r - INFO: - Episode: 3/100, Reward: 12.000, Step: 12 +2023-01-14 22:25:25 - r - INFO: - Episode: 4/100, Reward: 13.000, Step: 13 +2023-01-14 22:25:25 - r - INFO: - Episode: 5/100, Reward: 26.000, Step: 26 +2023-01-14 22:25:25 - r - INFO: - Current episode 5 has the best eval reward: 8.800 +2023-01-14 22:25:25 - r - INFO: - Episode: 6/100, Reward: 10.000, Step: 10 +2023-01-14 22:25:25 - r - INFO: - Episode: 7/100, Reward: 9.000, Step: 9 +2023-01-14 22:25:25 - r - INFO: - Episode: 8/100, Reward: 10.000, Step: 10 +2023-01-14 22:25:25 - r - INFO: - Episode: 9/100, Reward: 23.000, Step: 23 +2023-01-14 22:25:26 - r - INFO: - Episode: 10/100, Reward: 36.000, Step: 36 +2023-01-14 22:25:26 - r - INFO: - Current episode 10 has the best eval reward: 10.400 +2023-01-14 22:25:26 - r - INFO: - Episode: 11/100, Reward: 16.000, Step: 16 +2023-01-14 22:25:26 - r - INFO: - Episode: 12/100, Reward: 14.000, Step: 14 +2023-01-14 22:25:26 - r - INFO: - Episode: 13/100, Reward: 14.000, Step: 14 +2023-01-14 22:25:26 - r - INFO: - Episode: 14/100, Reward: 13.000, Step: 13 +2023-01-14 22:25:26 - r - INFO: - Episode: 15/100, Reward: 21.000, Step: 21 +2023-01-14 22:25:26 - r - INFO: - Episode: 16/100, Reward: 14.000, Step: 14 +2023-01-14 22:25:26 - r - INFO: - Episode: 17/100, Reward: 8.000, Step: 8 +2023-01-14 22:25:26 - r - INFO: - Episode: 18/100, Reward: 29.000, Step: 29 +2023-01-14 22:25:26 - r - INFO: - Episode: 19/100, Reward: 22.000, Step: 22 +2023-01-14 22:25:26 - r - INFO: - Episode: 20/100, Reward: 22.000, Step: 22 +2023-01-14 22:25:27 - r - INFO: - Current episode 20 has the best eval reward: 30.700 +2023-01-14 22:25:27 - r - INFO: - Episode: 21/100, Reward: 22.000, Step: 22 +2023-01-14 22:25:27 - r - INFO: - Episode: 22/100, Reward: 42.000, Step: 42 +2023-01-14 22:25:27 - r - INFO: - Episode: 23/100, Reward: 64.000, Step: 64 +2023-01-14 22:25:27 - r - INFO: - Episode: 24/100, Reward: 53.000, Step: 53 +2023-01-14 22:25:28 - r - INFO: - Episode: 25/100, Reward: 55.000, Step: 55 +2023-01-14 22:25:28 - r - INFO: - Current episode 25 has the best eval reward: 110.200 +2023-01-14 22:25:28 - r - INFO: - Episode: 26/100, Reward: 61.000, Step: 61 +2023-01-14 22:25:29 - r - INFO: - Episode: 27/100, Reward: 96.000, Step: 96 +2023-01-14 22:25:29 - r - INFO: - Episode: 28/100, Reward: 55.000, Step: 55 +2023-01-14 22:25:29 - r - INFO: - Episode: 29/100, Reward: 49.000, Step: 49 +2023-01-14 22:25:29 - r - INFO: - Episode: 30/100, Reward: 83.000, Step: 83 +2023-01-14 22:25:30 - r - INFO: - Episode: 31/100, Reward: 67.000, Step: 67 +2023-01-14 22:25:31 - r - INFO: - Episode: 32/100, Reward: 140.000, Step: 140 +2023-01-14 22:25:31 - r - INFO: - Episode: 33/100, Reward: 65.000, Step: 65 +2023-01-14 22:25:31 - r - INFO: - Episode: 34/100, Reward: 55.000, Step: 55 +2023-01-14 22:25:31 - r - INFO: - Episode: 35/100, Reward: 63.000, Step: 63 +2023-01-14 22:25:32 - r - INFO: - Episode: 36/100, Reward: 70.000, Step: 70 +2023-01-14 22:25:32 - r - INFO: - Episode: 37/100, Reward: 65.000, Step: 65 +2023-01-14 22:25:32 - r - INFO: - Episode: 38/100, Reward: 44.000, Step: 44 +2023-01-14 22:25:33 - r - INFO: - Episode: 39/100, Reward: 99.000, Step: 99 +2023-01-14 22:25:33 - r - INFO: - Episode: 40/100, Reward: 106.000, Step: 106 +2023-01-14 22:25:34 - r - INFO: - Current episode 40 has the best eval reward: 130.100 +2023-01-14 22:25:34 - r - INFO: - Episode: 41/100, Reward: 158.000, Step: 158 +2023-01-14 22:25:35 - r - INFO: - Episode: 42/100, Reward: 44.000, Step: 44 +2023-01-14 22:25:35 - r - INFO: - Episode: 43/100, Reward: 66.000, Step: 66 +2023-01-14 22:25:35 - r - INFO: - Episode: 44/100, Reward: 43.000, Step: 43 +2023-01-14 22:25:36 - r - INFO: - Episode: 45/100, Reward: 86.000, Step: 86 +2023-01-14 22:25:36 - r - INFO: - Episode: 46/100, Reward: 71.000, Step: 71 +2023-01-14 22:25:37 - r - INFO: - Episode: 47/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:38 - r - INFO: - Episode: 48/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:39 - r - INFO: - Episode: 49/100, Reward: 190.000, Step: 190 +2023-01-14 22:25:39 - r - INFO: - Episode: 50/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:40 - r - INFO: - Current episode 50 has the best eval reward: 200.000 +2023-01-14 22:25:41 - r - INFO: - Episode: 51/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:42 - r - INFO: - Episode: 52/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:43 - r - INFO: - Episode: 53/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:44 - r - INFO: - Episode: 54/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:45 - r - INFO: - Episode: 55/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:45 - r - INFO: - Current episode 55 has the best eval reward: 200.000 +2023-01-14 22:25:46 - r - INFO: - Episode: 56/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:47 - r - INFO: - Episode: 57/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:48 - r - INFO: - Episode: 58/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:49 - r - INFO: - Episode: 59/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:50 - r - INFO: - Episode: 60/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:50 - r - INFO: - Current episode 60 has the best eval reward: 200.000 +2023-01-14 22:25:51 - r - INFO: - Episode: 61/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:52 - r - INFO: - Episode: 62/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:53 - r - INFO: - Episode: 63/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:54 - r - INFO: - Episode: 64/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:55 - r - INFO: - Episode: 65/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:56 - r - INFO: - Current episode 65 has the best eval reward: 200.000 +2023-01-14 22:25:56 - r - INFO: - Episode: 66/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:57 - r - INFO: - Episode: 67/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:58 - r - INFO: - Episode: 68/100, Reward: 200.000, Step: 200 +2023-01-14 22:25:59 - r - INFO: - Episode: 69/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:00 - r - INFO: - Episode: 70/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:02 - r - INFO: - Episode: 71/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:02 - r - INFO: - Episode: 72/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:03 - r - INFO: - Episode: 73/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:04 - r - INFO: - Episode: 74/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:05 - r - INFO: - Episode: 75/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:07 - r - INFO: - Episode: 76/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:08 - r - INFO: - Episode: 77/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:08 - r - INFO: - Episode: 78/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:09 - r - INFO: - Episode: 79/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:10 - r - INFO: - Episode: 80/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:11 - r - INFO: - Current episode 80 has the best eval reward: 200.000 +2023-01-14 22:26:12 - r - INFO: - Episode: 81/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:13 - r - INFO: - Episode: 82/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:14 - r - INFO: - Episode: 83/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:15 - r - INFO: - Episode: 84/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:15 - r - INFO: - Episode: 85/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:16 - r - INFO: - Current episode 85 has the best eval reward: 200.000 +2023-01-14 22:26:17 - r - INFO: - Episode: 86/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:18 - r - INFO: - Episode: 87/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:19 - r - INFO: - Episode: 88/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:20 - r - INFO: - Episode: 89/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:21 - r - INFO: - Episode: 90/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:22 - r - INFO: - Current episode 90 has the best eval reward: 200.000 +2023-01-14 22:26:22 - r - INFO: - Episode: 91/100, Reward: 198.000, Step: 198 +2023-01-14 22:26:23 - r - INFO: - Episode: 92/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:24 - r - INFO: - Episode: 93/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:25 - r - INFO: - Episode: 94/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:26 - r - INFO: - Episode: 95/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:27 - r - INFO: - Current episode 95 has the best eval reward: 200.000 +2023-01-14 22:26:28 - r - INFO: - Episode: 96/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:28 - r - INFO: - Episode: 97/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:29 - r - INFO: - Episode: 98/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:30 - r - INFO: - Episode: 99/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:31 - r - INFO: - Episode: 100/100, Reward: 200.000, Step: 200 +2023-01-14 22:26:32 - r - INFO: - Current episode 100 has the best eval reward: 200.000 +2023-01-14 22:26:32 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5eab673c5154d7d54b0737705fe3889a5229e05e --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43646595614f98979d24634093626d35b9fab53c884a101bf17adf50691f5aed +size 123539 diff --git a/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..ed3c13045b2de04809c616f268463a9ccef909f1 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/results/res.csv b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cd65fc3cd17cff0aa7e5d6d717e1aa38404de88e --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_C51_20230114-222523/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,18.0,18 +1,35.0,35 +2,12.0,12 +3,13.0,13 +4,26.0,26 +5,10.0,10 +6,9.0,9 +7,10.0,10 +8,23.0,23 +9,36.0,36 +10,16.0,16 +11,14.0,14 +12,14.0,14 +13,13.0,13 +14,21.0,21 +15,14.0,14 +16,8.0,8 +17,29.0,29 +18,22.0,22 +19,22.0,22 +20,22.0,22 +21,42.0,42 +22,64.0,64 +23,53.0,53 +24,55.0,55 +25,61.0,61 +26,96.0,96 +27,55.0,55 +28,49.0,49 +29,83.0,83 +30,67.0,67 +31,140.0,140 +32,65.0,65 +33,55.0,55 +34,63.0,63 +35,70.0,70 +36,65.0,65 +37,44.0,44 +38,99.0,99 +39,106.0,106 +40,158.0,158 +41,44.0,44 +42,66.0,66 +43,43.0,43 +44,86.0,86 +45,71.0,71 +46,200.0,200 +47,200.0,200 +48,190.0,190 +49,200.0,200 +50,200.0,200 +51,200.0,200 +52,200.0,200 +53,200.0,200 +54,200.0,200 +55,200.0,200 +56,200.0,200 +57,200.0,200 +58,200.0,200 +59,200.0,200 +60,200.0,200 +61,200.0,200 +62,200.0,200 +63,200.0,200 +64,200.0,200 +65,200.0,200 +66,200.0,200 +67,200.0,200 +68,200.0,200 +69,200.0,200 +70,200.0,200 +71,200.0,200 +72,200.0,200 +73,200.0,200 +74,200.0,200 +75,200.0,200 +76,200.0,200 +77,200.0,200 +78,200.0,200 +79,200.0,200 +80,200.0,200 +81,200.0,200 +82,200.0,200 +83,200.0,200 +84,200.0,200 +85,200.0,200 +86,200.0,200 +87,200.0,200 +88,200.0,200 +89,200.0,200 +90,198.0,198 +91,200.0,200 +92,200.0,200 +93,200.0,200 +94,200.0,200 +95,200.0,200 +96,200.0,200 +97,200.0,200 +98,200.0,200 +99,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/config.yaml b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d66b4a090bef3342b1218593b4816872411d7dcf --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/config.yaml @@ -0,0 +1,44 @@ +general_cfg: + algo_name: CQL + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: tasks + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: true + test_eps: 20 + train_eps: 300 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.001 + target_update: 4 + tau: 0.001 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/logs/log.txt b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..7197aa050e2db4cbb635af09f6adb36cc5a315cd --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/logs/log.txt @@ -0,0 +1,363 @@ +2023-04-08 18:36:52 - r - INFO: - Hyperparameters: +2023-04-08 18:36:52 - r - INFO: - ================================================================================ +2023-04-08 18:36:52 - r - INFO: - Name Value Type +2023-04-08 18:36:52 - r - INFO: - env_name CartPole-v1 +2023-04-08 18:36:52 - r - INFO: - new_step_api 1 +2023-04-08 18:36:52 - r - INFO: - wrapper None +2023-04-08 18:36:52 - r - INFO: - render 0 +2023-04-08 18:36:52 - r - INFO: - algo_name CQL +2023-04-08 18:36:52 - r - INFO: - mode train +2023-04-08 18:36:52 - r - INFO: - seed 1 +2023-04-08 18:36:52 - r - INFO: - device cpu +2023-04-08 18:36:52 - r - INFO: - train_eps 300 +2023-04-08 18:36:52 - r - INFO: - test_eps 20 +2023-04-08 18:36:52 - r - INFO: - eval_eps 10 +2023-04-08 18:36:52 - r - INFO: - eval_per_episode 5 +2023-04-08 18:36:52 - r - INFO: - max_steps 200 +2023-04-08 18:36:52 - r - INFO: - load_checkpoint 0 +2023-04-08 18:36:52 - r - INFO: - load_path tasks +2023-04-08 18:36:52 - r - INFO: - show_fig 1 +2023-04-08 18:36:52 - r - INFO: - save_fig 1 +2023-04-08 18:36:52 - r - INFO: - epsilon_start 0.95 +2023-04-08 18:36:52 - r - INFO: - epsilon_end 0.01 +2023-04-08 18:36:52 - r - INFO: - epsilon_decay 500 +2023-04-08 18:36:52 - r - INFO: - gamma 0.99 +2023-04-08 18:36:52 - r - INFO: - tau 0.001 +2023-04-08 18:36:52 - r - INFO: - lr 0.001 +2023-04-08 18:36:52 - r - INFO: - buffer_size 100000 +2023-04-08 18:36:52 - r - INFO: - batch_size 64 +2023-04-08 18:36:52 - r - INFO: - target_update 4 +2023-04-08 18:36:52 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-08 18:36:52 - r - INFO: - task_dir D:\MyDoc\joyrl\rl-tutorials-master-newest\rl-tutorials-master\joyrl/tasks/Train_CartPole-v1_CQL_20230408-183652 +2023-04-08 18:36:52 - r - INFO: - model_dir D:\MyDoc\joyrl\rl-tutorials-master-newest\rl-tutorials-master\joyrl/tasks/Train_CartPole-v1_CQL_20230408-183652/models +2023-04-08 18:36:52 - r - INFO: - res_dir D:\MyDoc\joyrl\rl-tutorials-master-newest\rl-tutorials-master\joyrl/tasks/Train_CartPole-v1_CQL_20230408-183652/results +2023-04-08 18:36:52 - r - INFO: - log_dir D:\MyDoc\joyrl\rl-tutorials-master-newest\rl-tutorials-master\joyrl/tasks/Train_CartPole-v1_CQL_20230408-183652/logs +2023-04-08 18:36:52 - r - INFO: - traj_dir D:\MyDoc\joyrl\rl-tutorials-master-newest\rl-tutorials-master\joyrl/tasks/Train_CartPole-v1_CQL_20230408-183652/traj +2023-04-08 18:36:52 - r - INFO: - ================================================================================ +2023-04-08 18:36:52 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-08 18:36:52 - r - INFO: - Start training! +2023-04-08 18:36:52 - r - INFO: - Env: CartPole-v1, Algorithm: CQL, Device: cpu +2023-04-08 18:36:52 - r - INFO: - Episode: 1/300, Reward: 18.000, Step: 18 +2023-04-08 18:36:52 - r - INFO: - Episode: 2/300, Reward: 28.000, Step: 28 +2023-04-08 18:36:52 - r - INFO: - Episode: 3/300, Reward: 15.000, Step: 15 +2023-04-08 18:36:52 - r - INFO: - Episode: 4/300, Reward: 15.000, Step: 15 +2023-04-08 18:36:52 - r - INFO: - Episode: 5/300, Reward: 12.000, Step: 12 +2023-04-08 18:36:52 - r - INFO: - Current episode 5 has the best eval reward: 10.000 +2023-04-08 18:36:52 - r - INFO: - Episode: 6/300, Reward: 35.000, Step: 35 +2023-04-08 18:36:52 - r - INFO: - Episode: 7/300, Reward: 12.000, Step: 12 +2023-04-08 18:36:53 - r - INFO: - Episode: 8/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:53 - r - INFO: - Episode: 9/300, Reward: 17.000, Step: 17 +2023-04-08 18:36:53 - r - INFO: - Episode: 10/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:53 - r - INFO: - Episode: 11/300, Reward: 13.000, Step: 13 +2023-04-08 18:36:53 - r - INFO: - Episode: 12/300, Reward: 22.000, Step: 22 +2023-04-08 18:36:53 - r - INFO: - Episode: 13/300, Reward: 29.000, Step: 29 +2023-04-08 18:36:53 - r - INFO: - Episode: 14/300, Reward: 25.000, Step: 25 +2023-04-08 18:36:53 - r - INFO: - Episode: 15/300, Reward: 16.000, Step: 16 +2023-04-08 18:36:54 - r - INFO: - Episode: 16/300, Reward: 25.000, Step: 25 +2023-04-08 18:36:54 - r - INFO: - Episode: 17/300, Reward: 19.000, Step: 19 +2023-04-08 18:36:54 - r - INFO: - Episode: 18/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:54 - r - INFO: - Episode: 19/300, Reward: 28.000, Step: 28 +2023-04-08 18:36:54 - r - INFO: - Episode: 20/300, Reward: 22.000, Step: 22 +2023-04-08 18:36:54 - r - INFO: - Current episode 20 has the best eval reward: 10.000 +2023-04-08 18:36:54 - r - INFO: - Episode: 21/300, Reward: 18.000, Step: 18 +2023-04-08 18:36:54 - r - INFO: - Episode: 22/300, Reward: 9.000, Step: 9 +2023-04-08 18:36:54 - r - INFO: - Episode: 23/300, Reward: 21.000, Step: 21 +2023-04-08 18:36:54 - r - INFO: - Episode: 24/300, Reward: 9.000, Step: 9 +2023-04-08 18:36:54 - r - INFO: - Episode: 25/300, Reward: 12.000, Step: 12 +2023-04-08 18:36:54 - r - INFO: - Current episode 25 has the best eval reward: 10.000 +2023-04-08 18:36:55 - r - INFO: - Episode: 26/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:55 - r - INFO: - Episode: 27/300, Reward: 17.000, Step: 17 +2023-04-08 18:36:55 - r - INFO: - Episode: 28/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:55 - r - INFO: - Episode: 29/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:55 - r - INFO: - Episode: 30/300, Reward: 14.000, Step: 14 +2023-04-08 18:36:55 - r - INFO: - Episode: 31/300, Reward: 28.000, Step: 28 +2023-04-08 18:36:55 - r - INFO: - Episode: 32/300, Reward: 15.000, Step: 15 +2023-04-08 18:36:55 - r - INFO: - Episode: 33/300, Reward: 9.000, Step: 9 +2023-04-08 18:36:55 - r - INFO: - Episode: 34/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:56 - r - INFO: - Episode: 35/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:56 - r - INFO: - Episode: 36/300, Reward: 9.000, Step: 9 +2023-04-08 18:36:56 - r - INFO: - Episode: 37/300, Reward: 16.000, Step: 16 +2023-04-08 18:36:56 - r - INFO: - Episode: 38/300, Reward: 12.000, Step: 12 +2023-04-08 18:36:56 - r - INFO: - Episode: 39/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:56 - r - INFO: - Episode: 40/300, Reward: 30.000, Step: 30 +2023-04-08 18:36:56 - r - INFO: - Current episode 40 has the best eval reward: 10.000 +2023-04-08 18:36:56 - r - INFO: - Episode: 41/300, Reward: 15.000, Step: 15 +2023-04-08 18:36:56 - r - INFO: - Episode: 42/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:56 - r - INFO: - Episode: 43/300, Reward: 33.000, Step: 33 +2023-04-08 18:36:56 - r - INFO: - Episode: 44/300, Reward: 18.000, Step: 18 +2023-04-08 18:36:57 - r - INFO: - Episode: 45/300, Reward: 13.000, Step: 13 +2023-04-08 18:36:57 - r - INFO: - Current episode 45 has the best eval reward: 10.000 +2023-04-08 18:36:57 - r - INFO: - Episode: 46/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:57 - r - INFO: - Episode: 47/300, Reward: 26.000, Step: 26 +2023-04-08 18:36:57 - r - INFO: - Episode: 48/300, Reward: 15.000, Step: 15 +2023-04-08 18:36:57 - r - INFO: - Episode: 49/300, Reward: 14.000, Step: 14 +2023-04-08 18:36:57 - r - INFO: - Episode: 50/300, Reward: 18.000, Step: 18 +2023-04-08 18:36:57 - r - INFO: - Current episode 50 has the best eval reward: 10.000 +2023-04-08 18:36:57 - r - INFO: - Episode: 51/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:57 - r - INFO: - Episode: 52/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:58 - r - INFO: - Episode: 53/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:58 - r - INFO: - Episode: 54/300, Reward: 11.000, Step: 11 +2023-04-08 18:36:58 - r - INFO: - Episode: 55/300, Reward: 23.000, Step: 23 +2023-04-08 18:36:58 - r - INFO: - Episode: 56/300, Reward: 9.000, Step: 9 +2023-04-08 18:36:58 - r - INFO: - Episode: 57/300, Reward: 12.000, Step: 12 +2023-04-08 18:36:58 - r - INFO: - Episode: 58/300, Reward: 13.000, Step: 13 +2023-04-08 18:36:58 - r - INFO: - Episode: 59/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:58 - r - INFO: - Episode: 60/300, Reward: 34.000, Step: 34 +2023-04-08 18:36:58 - r - INFO: - Episode: 61/300, Reward: 12.000, Step: 12 +2023-04-08 18:36:59 - r - INFO: - Episode: 62/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:59 - r - INFO: - Episode: 63/300, Reward: 14.000, Step: 14 +2023-04-08 18:36:59 - r - INFO: - Episode: 64/300, Reward: 14.000, Step: 14 +2023-04-08 18:36:59 - r - INFO: - Episode: 65/300, Reward: 9.000, Step: 9 +2023-04-08 18:36:59 - r - INFO: - Episode: 66/300, Reward: 13.000, Step: 13 +2023-04-08 18:36:59 - r - INFO: - Episode: 67/300, Reward: 17.000, Step: 17 +2023-04-08 18:36:59 - r - INFO: - Episode: 68/300, Reward: 10.000, Step: 10 +2023-04-08 18:36:59 - r - INFO: - Episode: 69/300, Reward: 9.000, Step: 9 +2023-04-08 18:36:59 - r - INFO: - Episode: 70/300, Reward: 28.000, Step: 28 +2023-04-08 18:36:59 - r - INFO: - Current episode 70 has the best eval reward: 10.000 +2023-04-08 18:36:59 - r - INFO: - Episode: 71/300, Reward: 13.000, Step: 13 +2023-04-08 18:36:59 - r - INFO: - Episode: 72/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:00 - r - INFO: - Episode: 73/300, Reward: 26.000, Step: 26 +2023-04-08 18:37:00 - r - INFO: - Episode: 74/300, Reward: 20.000, Step: 20 +2023-04-08 18:37:00 - r - INFO: - Episode: 75/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:00 - r - INFO: - Current episode 75 has the best eval reward: 10.000 +2023-04-08 18:37:00 - r - INFO: - Episode: 76/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:00 - r - INFO: - Episode: 77/300, Reward: 39.000, Step: 39 +2023-04-08 18:37:00 - r - INFO: - Episode: 78/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:00 - r - INFO: - Episode: 79/300, Reward: 23.000, Step: 23 +2023-04-08 18:37:00 - r - INFO: - Episode: 80/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:00 - r - INFO: - Episode: 81/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:00 - r - INFO: - Episode: 82/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:01 - r - INFO: - Episode: 83/300, Reward: 14.000, Step: 14 +2023-04-08 18:37:01 - r - INFO: - Episode: 84/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:01 - r - INFO: - Episode: 85/300, Reward: 18.000, Step: 18 +2023-04-08 18:37:01 - r - INFO: - Current episode 85 has the best eval reward: 10.000 +2023-04-08 18:37:01 - r - INFO: - Episode: 86/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:01 - r - INFO: - Episode: 87/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:01 - r - INFO: - Episode: 88/300, Reward: 14.000, Step: 14 +2023-04-08 18:37:01 - r - INFO: - Episode: 89/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:01 - r - INFO: - Episode: 90/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:01 - r - INFO: - Current episode 90 has the best eval reward: 10.000 +2023-04-08 18:37:01 - r - INFO: - Episode: 91/300, Reward: 22.000, Step: 22 +2023-04-08 18:37:01 - r - INFO: - Episode: 92/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:01 - r - INFO: - Episode: 93/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:02 - r - INFO: - Episode: 94/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:02 - r - INFO: - Episode: 95/300, Reward: 19.000, Step: 19 +2023-04-08 18:37:02 - r - INFO: - Current episode 95 has the best eval reward: 10.000 +2023-04-08 18:37:02 - r - INFO: - Episode: 96/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:02 - r - INFO: - Episode: 97/300, Reward: 25.000, Step: 25 +2023-04-08 18:37:02 - r - INFO: - Episode: 98/300, Reward: 31.000, Step: 31 +2023-04-08 18:37:02 - r - INFO: - Episode: 99/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:02 - r - INFO: - Episode: 100/300, Reward: 18.000, Step: 18 +2023-04-08 18:37:02 - r - INFO: - Episode: 101/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:03 - r - INFO: - Episode: 102/300, Reward: 15.000, Step: 15 +2023-04-08 18:37:03 - r - INFO: - Episode: 103/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:03 - r - INFO: - Episode: 104/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:03 - r - INFO: - Episode: 105/300, Reward: 21.000, Step: 21 +2023-04-08 18:37:03 - r - INFO: - Current episode 105 has the best eval reward: 10.000 +2023-04-08 18:37:03 - r - INFO: - Episode: 106/300, Reward: 26.000, Step: 26 +2023-04-08 18:37:03 - r - INFO: - Episode: 107/300, Reward: 22.000, Step: 22 +2023-04-08 18:37:03 - r - INFO: - Episode: 108/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:03 - r - INFO: - Episode: 109/300, Reward: 13.000, Step: 13 +2023-04-08 18:37:03 - r - INFO: - Episode: 110/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:03 - r - INFO: - Current episode 110 has the best eval reward: 10.000 +2023-04-08 18:37:04 - r - INFO: - Episode: 111/300, Reward: 23.000, Step: 23 +2023-04-08 18:37:04 - r - INFO: - Episode: 112/300, Reward: 20.000, Step: 20 +2023-04-08 18:37:04 - r - INFO: - Episode: 113/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:04 - r - INFO: - Episode: 114/300, Reward: 14.000, Step: 14 +2023-04-08 18:37:04 - r - INFO: - Episode: 115/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:04 - r - INFO: - Current episode 115 has the best eval reward: 10.000 +2023-04-08 18:37:04 - r - INFO: - Episode: 116/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:04 - r - INFO: - Episode: 117/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:04 - r - INFO: - Episode: 118/300, Reward: 65.000, Step: 65 +2023-04-08 18:37:04 - r - INFO: - Episode: 119/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:05 - r - INFO: - Episode: 120/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:05 - r - INFO: - Episode: 121/300, Reward: 30.000, Step: 30 +2023-04-08 18:37:05 - r - INFO: - Episode: 122/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:05 - r - INFO: - Episode: 123/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:05 - r - INFO: - Episode: 124/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:05 - r - INFO: - Episode: 125/300, Reward: 31.000, Step: 31 +2023-04-08 18:37:05 - r - INFO: - Current episode 125 has the best eval reward: 12.000 +2023-04-08 18:37:05 - r - INFO: - Episode: 126/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:05 - r - INFO: - Episode: 127/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:05 - r - INFO: - Episode: 128/300, Reward: 17.000, Step: 17 +2023-04-08 18:37:05 - r - INFO: - Episode: 129/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:05 - r - INFO: - Episode: 130/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:06 - r - INFO: - Current episode 130 has the best eval reward: 15.000 +2023-04-08 18:37:06 - r - INFO: - Episode: 131/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:06 - r - INFO: - Episode: 132/300, Reward: 17.000, Step: 17 +2023-04-08 18:37:06 - r - INFO: - Episode: 133/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:06 - r - INFO: - Episode: 134/300, Reward: 51.000, Step: 51 +2023-04-08 18:37:06 - r - INFO: - Episode: 135/300, Reward: 25.000, Step: 25 +2023-04-08 18:37:06 - r - INFO: - Episode: 136/300, Reward: 15.000, Step: 15 +2023-04-08 18:37:06 - r - INFO: - Episode: 137/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:07 - r - INFO: - Episode: 138/300, Reward: 27.000, Step: 27 +2023-04-08 18:37:07 - r - INFO: - Episode: 139/300, Reward: 21.000, Step: 21 +2023-04-08 18:37:07 - r - INFO: - Episode: 140/300, Reward: 17.000, Step: 17 +2023-04-08 18:37:07 - r - INFO: - Episode: 141/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:07 - r - INFO: - Episode: 142/300, Reward: 17.000, Step: 17 +2023-04-08 18:37:07 - r - INFO: - Episode: 143/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:07 - r - INFO: - Episode: 144/300, Reward: 23.000, Step: 23 +2023-04-08 18:37:07 - r - INFO: - Episode: 145/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:07 - r - INFO: - Episode: 146/300, Reward: 20.000, Step: 20 +2023-04-08 18:37:07 - r - INFO: - Episode: 147/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:07 - r - INFO: - Episode: 148/300, Reward: 17.000, Step: 17 +2023-04-08 18:37:08 - r - INFO: - Episode: 149/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:08 - r - INFO: - Episode: 150/300, Reward: 31.000, Step: 31 +2023-04-08 18:37:08 - r - INFO: - Episode: 151/300, Reward: 27.000, Step: 27 +2023-04-08 18:37:08 - r - INFO: - Episode: 152/300, Reward: 34.000, Step: 34 +2023-04-08 18:37:08 - r - INFO: - Episode: 153/300, Reward: 16.000, Step: 16 +2023-04-08 18:37:08 - r - INFO: - Episode: 154/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:09 - r - INFO: - Episode: 155/300, Reward: 41.000, Step: 41 +2023-04-08 18:37:09 - r - INFO: - Episode: 156/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:09 - r - INFO: - Episode: 157/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:09 - r - INFO: - Episode: 158/300, Reward: 60.000, Step: 60 +2023-04-08 18:37:09 - r - INFO: - Episode: 159/300, Reward: 17.000, Step: 17 +2023-04-08 18:37:09 - r - INFO: - Episode: 160/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:09 - r - INFO: - Episode: 161/300, Reward: 17.000, Step: 17 +2023-04-08 18:37:09 - r - INFO: - Episode: 162/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:10 - r - INFO: - Episode: 163/300, Reward: 31.000, Step: 31 +2023-04-08 18:37:10 - r - INFO: - Episode: 164/300, Reward: 65.000, Step: 65 +2023-04-08 18:37:10 - r - INFO: - Episode: 165/300, Reward: 9.000, Step: 9 +2023-04-08 18:37:10 - r - INFO: - Episode: 166/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:10 - r - INFO: - Episode: 167/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:10 - r - INFO: - Episode: 168/300, Reward: 10.000, Step: 10 +2023-04-08 18:37:11 - r - INFO: - Episode: 169/300, Reward: 71.000, Step: 71 +2023-04-08 18:37:11 - r - INFO: - Episode: 170/300, Reward: 41.000, Step: 41 +2023-04-08 18:37:11 - r - INFO: - Episode: 171/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:11 - r - INFO: - Episode: 172/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:11 - r - INFO: - Episode: 173/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:11 - r - INFO: - Episode: 174/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:11 - r - INFO: - Episode: 175/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:11 - r - INFO: - Episode: 176/300, Reward: 14.000, Step: 14 +2023-04-08 18:37:11 - r - INFO: - Episode: 177/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:12 - r - INFO: - Episode: 178/300, Reward: 57.000, Step: 57 +2023-04-08 18:37:12 - r - INFO: - Episode: 179/300, Reward: 13.000, Step: 13 +2023-04-08 18:37:12 - r - INFO: - Episode: 180/300, Reward: 14.000, Step: 14 +2023-04-08 18:37:12 - r - INFO: - Episode: 181/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:12 - r - INFO: - Episode: 182/300, Reward: 49.000, Step: 49 +2023-04-08 18:37:13 - r - INFO: - Episode: 183/300, Reward: 33.000, Step: 33 +2023-04-08 18:37:13 - r - INFO: - Episode: 184/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:13 - r - INFO: - Episode: 185/300, Reward: 64.000, Step: 64 +2023-04-08 18:37:13 - r - INFO: - Episode: 186/300, Reward: 40.000, Step: 40 +2023-04-08 18:37:13 - r - INFO: - Episode: 187/300, Reward: 12.000, Step: 12 +2023-04-08 18:37:13 - r - INFO: - Episode: 188/300, Reward: 14.000, Step: 14 +2023-04-08 18:37:14 - r - INFO: - Episode: 189/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:14 - r - INFO: - Episode: 190/300, Reward: 64.000, Step: 64 +2023-04-08 18:37:14 - r - INFO: - Episode: 191/300, Reward: 53.000, Step: 53 +2023-04-08 18:37:15 - r - INFO: - Episode: 192/300, Reward: 64.000, Step: 64 +2023-04-08 18:37:15 - r - INFO: - Episode: 193/300, Reward: 54.000, Step: 54 +2023-04-08 18:37:16 - r - INFO: - Episode: 194/300, Reward: 83.000, Step: 83 +2023-04-08 18:37:16 - r - INFO: - Episode: 195/300, Reward: 53.000, Step: 53 +2023-04-08 18:37:16 - r - INFO: - Episode: 196/300, Reward: 27.000, Step: 27 +2023-04-08 18:37:16 - r - INFO: - Episode: 197/300, Reward: 39.000, Step: 39 +2023-04-08 18:37:17 - r - INFO: - Episode: 198/300, Reward: 93.000, Step: 93 +2023-04-08 18:37:17 - r - INFO: - Episode: 199/300, Reward: 11.000, Step: 11 +2023-04-08 18:37:17 - r - INFO: - Episode: 200/300, Reward: 13.000, Step: 13 +2023-04-08 18:37:17 - r - INFO: - Current episode 200 has the best eval reward: 18.000 +2023-04-08 18:37:18 - r - INFO: - Episode: 201/300, Reward: 127.000, Step: 127 +2023-04-08 18:37:18 - r - INFO: - Episode: 202/300, Reward: 120.000, Step: 120 +2023-04-08 18:37:19 - r - INFO: - Episode: 203/300, Reward: 60.000, Step: 60 +2023-04-08 18:37:20 - r - INFO: - Episode: 204/300, Reward: 157.000, Step: 157 +2023-04-08 18:37:20 - r - INFO: - Episode: 205/300, Reward: 63.000, Step: 63 +2023-04-08 18:37:20 - r - INFO: - Current episode 205 has the best eval reward: 89.000 +2023-04-08 18:37:20 - r - INFO: - Episode: 206/300, Reward: 26.000, Step: 26 +2023-04-08 18:37:21 - r - INFO: - Episode: 207/300, Reward: 45.000, Step: 45 +2023-04-08 18:37:21 - r - INFO: - Episode: 208/300, Reward: 80.000, Step: 80 +2023-04-08 18:37:22 - r - INFO: - Episode: 209/300, Reward: 101.000, Step: 101 +2023-04-08 18:37:22 - r - INFO: - Episode: 210/300, Reward: 96.000, Step: 96 +2023-04-08 18:37:23 - r - INFO: - Episode: 211/300, Reward: 81.000, Step: 81 +2023-04-08 18:37:23 - r - INFO: - Episode: 212/300, Reward: 80.000, Step: 80 +2023-04-08 18:37:24 - r - INFO: - Episode: 213/300, Reward: 193.000, Step: 193 +2023-04-08 18:37:25 - r - INFO: - Episode: 214/300, Reward: 148.000, Step: 148 +2023-04-08 18:37:26 - r - INFO: - Episode: 215/300, Reward: 138.000, Step: 138 +2023-04-08 18:37:27 - r - INFO: - Episode: 216/300, Reward: 152.000, Step: 152 +2023-04-08 18:37:28 - r - INFO: - Episode: 217/300, Reward: 173.000, Step: 173 +2023-04-08 18:37:29 - r - INFO: - Episode: 218/300, Reward: 86.000, Step: 86 +2023-04-08 18:37:29 - r - INFO: - Episode: 219/300, Reward: 107.000, Step: 107 +2023-04-08 18:37:30 - r - INFO: - Episode: 220/300, Reward: 152.000, Step: 152 +2023-04-08 18:37:31 - r - INFO: - Episode: 221/300, Reward: 100.000, Step: 100 +2023-04-08 18:37:31 - r - INFO: - Episode: 222/300, Reward: 97.000, Step: 97 +2023-04-08 18:37:32 - r - INFO: - Episode: 223/300, Reward: 104.000, Step: 104 +2023-04-08 18:37:33 - r - INFO: - Episode: 224/300, Reward: 93.000, Step: 93 +2023-04-08 18:37:33 - r - INFO: - Episode: 225/300, Reward: 138.000, Step: 138 +2023-04-08 18:37:35 - r - INFO: - Episode: 226/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:36 - r - INFO: - Episode: 227/300, Reward: 120.000, Step: 120 +2023-04-08 18:37:36 - r - INFO: - Episode: 228/300, Reward: 133.000, Step: 133 +2023-04-08 18:37:37 - r - INFO: - Episode: 229/300, Reward: 105.000, Step: 105 +2023-04-08 18:37:38 - r - INFO: - Episode: 230/300, Reward: 96.000, Step: 96 +2023-04-08 18:37:38 - r - INFO: - Current episode 230 has the best eval reward: 91.000 +2023-04-08 18:37:39 - r - INFO: - Episode: 231/300, Reward: 166.000, Step: 166 +2023-04-08 18:37:40 - r - INFO: - Episode: 232/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:41 - r - INFO: - Episode: 233/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:42 - r - INFO: - Episode: 234/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:43 - r - INFO: - Episode: 235/300, Reward: 129.000, Step: 129 +2023-04-08 18:37:43 - r - INFO: - Current episode 235 has the best eval reward: 193.000 +2023-04-08 18:37:44 - r - INFO: - Episode: 236/300, Reward: 135.000, Step: 135 +2023-04-08 18:37:45 - r - INFO: - Episode: 237/300, Reward: 128.000, Step: 128 +2023-04-08 18:37:46 - r - INFO: - Episode: 238/300, Reward: 141.000, Step: 141 +2023-04-08 18:37:47 - r - INFO: - Episode: 239/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:48 - r - INFO: - Episode: 240/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:49 - r - INFO: - Episode: 241/300, Reward: 137.000, Step: 137 +2023-04-08 18:37:50 - r - INFO: - Episode: 242/300, Reward: 172.000, Step: 172 +2023-04-08 18:37:51 - r - INFO: - Episode: 243/300, Reward: 149.000, Step: 149 +2023-04-08 18:37:53 - r - INFO: - Episode: 244/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:53 - r - INFO: - Episode: 245/300, Reward: 154.000, Step: 154 +2023-04-08 18:37:55 - r - INFO: - Episode: 246/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:56 - r - INFO: - Episode: 247/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:57 - r - INFO: - Episode: 248/300, Reward: 200.000, Step: 200 +2023-04-08 18:37:59 - r - INFO: - Episode: 249/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:00 - r - INFO: - Episode: 250/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:01 - r - INFO: - Episode: 251/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:03 - r - INFO: - Episode: 252/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:04 - r - INFO: - Episode: 253/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:05 - r - INFO: - Episode: 254/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:06 - r - INFO: - Episode: 255/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:08 - r - INFO: - Episode: 256/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:09 - r - INFO: - Episode: 257/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:10 - r - INFO: - Episode: 258/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:11 - r - INFO: - Episode: 259/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:13 - r - INFO: - Episode: 260/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:14 - r - INFO: - Episode: 261/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:15 - r - INFO: - Episode: 262/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:17 - r - INFO: - Episode: 263/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:18 - r - INFO: - Episode: 264/300, Reward: 195.000, Step: 195 +2023-04-08 18:38:19 - r - INFO: - Episode: 265/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:21 - r - INFO: - Episode: 266/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:22 - r - INFO: - Episode: 267/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:23 - r - INFO: - Episode: 268/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:24 - r - INFO: - Episode: 269/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:25 - r - INFO: - Episode: 270/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:27 - r - INFO: - Episode: 271/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:28 - r - INFO: - Episode: 272/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:30 - r - INFO: - Episode: 273/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:31 - r - INFO: - Episode: 274/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:33 - r - INFO: - Episode: 275/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:34 - r - INFO: - Episode: 276/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:36 - r - INFO: - Episode: 277/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:37 - r - INFO: - Episode: 278/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:38 - r - INFO: - Episode: 279/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:39 - r - INFO: - Episode: 280/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:40 - r - INFO: - Current episode 280 has the best eval reward: 200.000 +2023-04-08 18:38:41 - r - INFO: - Episode: 281/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:42 - r - INFO: - Episode: 282/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:43 - r - INFO: - Episode: 283/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:45 - r - INFO: - Episode: 284/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:46 - r - INFO: - Episode: 285/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:46 - r - INFO: - Current episode 285 has the best eval reward: 200.000 +2023-04-08 18:38:47 - r - INFO: - Episode: 286/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:49 - r - INFO: - Episode: 287/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:50 - r - INFO: - Episode: 288/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:51 - r - INFO: - Episode: 289/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:52 - r - INFO: - Episode: 290/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:53 - r - INFO: - Current episode 290 has the best eval reward: 200.000 +2023-04-08 18:38:54 - r - INFO: - Episode: 291/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:55 - r - INFO: - Episode: 292/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:57 - r - INFO: - Episode: 293/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:58 - r - INFO: - Episode: 294/300, Reward: 200.000, Step: 200 +2023-04-08 18:38:59 - r - INFO: - Episode: 295/300, Reward: 200.000, Step: 200 +2023-04-08 18:39:00 - r - INFO: - Episode: 296/300, Reward: 200.000, Step: 200 +2023-04-08 18:39:02 - r - INFO: - Episode: 297/300, Reward: 200.000, Step: 200 +2023-04-08 18:39:03 - r - INFO: - Episode: 298/300, Reward: 200.000, Step: 200 +2023-04-08 18:39:04 - r - INFO: - Episode: 299/300, Reward: 200.000, Step: 200 +2023-04-08 18:39:05 - r - INFO: - Episode: 300/300, Reward: 200.000, Step: 200 +2023-04-08 18:39:06 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbfa4e2127b4953d376c7070d13c81b5b9748a54 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6891f88ceb4ef1865723645afaae8ba271bcc8735d8809dd45322aeb2ca4a08b +size 272407 diff --git a/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..865aba99883e2a878e0bdc5fa233fa9fbf841497 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/results/res.csv b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..696a52678a7aa94bbc0c5c2a789be840b822ef19 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_CQL_20230408-183652/results/res.csv @@ -0,0 +1,301 @@ +episodes,rewards,steps +0,18.0,18 +1,28.0,28 +2,15.0,15 +3,15.0,15 +4,12.0,12 +5,35.0,35 +6,12.0,12 +7,10.0,10 +8,17.0,17 +9,10.0,10 +10,13.0,13 +11,22.0,22 +12,29.0,29 +13,25.0,25 +14,16.0,16 +15,25.0,25 +16,19.0,19 +17,10.0,10 +18,28.0,28 +19,22.0,22 +20,18.0,18 +21,9.0,9 +22,21.0,21 +23,9.0,9 +24,12.0,12 +25,11.0,11 +26,17.0,17 +27,11.0,11 +28,10.0,10 +29,14.0,14 +30,28.0,28 +31,15.0,15 +32,9.0,9 +33,10.0,10 +34,11.0,11 +35,9.0,9 +36,16.0,16 +37,12.0,12 +38,10.0,10 +39,30.0,30 +40,15.0,15 +41,11.0,11 +42,33.0,33 +43,18.0,18 +44,13.0,13 +45,11.0,11 +46,26.0,26 +47,15.0,15 +48,14.0,14 +49,18.0,18 +50,10.0,10 +51,11.0,11 +52,11.0,11 +53,11.0,11 +54,23.0,23 +55,9.0,9 +56,12.0,12 +57,13.0,13 +58,10.0,10 +59,34.0,34 +60,12.0,12 +61,10.0,10 +62,14.0,14 +63,14.0,14 +64,9.0,9 +65,13.0,13 +66,17.0,17 +67,10.0,10 +68,9.0,9 +69,28.0,28 +70,13.0,13 +71,11.0,11 +72,26.0,26 +73,20.0,20 +74,9.0,9 +75,10.0,10 +76,39.0,39 +77,10.0,10 +78,23.0,23 +79,11.0,11 +80,9.0,9 +81,10.0,10 +82,14.0,14 +83,10.0,10 +84,18.0,18 +85,10.0,10 +86,10.0,10 +87,14.0,14 +88,11.0,11 +89,10.0,10 +90,22.0,22 +91,10.0,10 +92,9.0,9 +93,12.0,12 +94,19.0,19 +95,10.0,10 +96,25.0,25 +97,31.0,31 +98,9.0,9 +99,18.0,18 +100,9.0,9 +101,15.0,15 +102,11.0,11 +103,10.0,10 +104,21.0,21 +105,26.0,26 +106,22.0,22 +107,10.0,10 +108,13.0,13 +109,10.0,10 +110,23.0,23 +111,20.0,20 +112,10.0,10 +113,14.0,14 +114,10.0,10 +115,11.0,11 +116,10.0,10 +117,65.0,65 +118,10.0,10 +119,9.0,9 +120,30.0,30 +121,11.0,11 +122,9.0,9 +123,10.0,10 +124,31.0,31 +125,9.0,9 +126,12.0,12 +127,17.0,17 +128,10.0,10 +129,11.0,11 +130,10.0,10 +131,17.0,17 +132,10.0,10 +133,51.0,51 +134,25.0,25 +135,15.0,15 +136,10.0,10 +137,27.0,27 +138,21.0,21 +139,17.0,17 +140,10.0,10 +141,17.0,17 +142,10.0,10 +143,23.0,23 +144,10.0,10 +145,20.0,20 +146,10.0,10 +147,17.0,17 +148,10.0,10 +149,31.0,31 +150,27.0,27 +151,34.0,34 +152,16.0,16 +153,12.0,12 +154,41.0,41 +155,9.0,9 +156,12.0,12 +157,60.0,60 +158,17.0,17 +159,10.0,10 +160,17.0,17 +161,10.0,10 +162,31.0,31 +163,65.0,65 +164,9.0,9 +165,12.0,12 +166,12.0,12 +167,10.0,10 +168,71.0,71 +169,41.0,41 +170,11.0,11 +171,12.0,12 +172,11.0,11 +173,12.0,12 +174,11.0,11 +175,14.0,14 +176,11.0,11 +177,57.0,57 +178,13.0,13 +179,14.0,14 +180,11.0,11 +181,49.0,49 +182,33.0,33 +183,11.0,11 +184,64.0,64 +185,40.0,40 +186,12.0,12 +187,14.0,14 +188,11.0,11 +189,64.0,64 +190,53.0,53 +191,64.0,64 +192,54.0,54 +193,83.0,83 +194,53.0,53 +195,27.0,27 +196,39.0,39 +197,93.0,93 +198,11.0,11 +199,13.0,13 +200,127.0,127 +201,120.0,120 +202,60.0,60 +203,157.0,157 +204,63.0,63 +205,26.0,26 +206,45.0,45 +207,80.0,80 +208,101.0,101 +209,96.0,96 +210,81.0,81 +211,80.0,80 +212,193.0,193 +213,148.0,148 +214,138.0,138 +215,152.0,152 +216,173.0,173 +217,86.0,86 +218,107.0,107 +219,152.0,152 +220,100.0,100 +221,97.0,97 +222,104.0,104 +223,93.0,93 +224,138.0,138 +225,200.0,200 +226,120.0,120 +227,133.0,133 +228,105.0,105 +229,96.0,96 +230,166.0,166 +231,200.0,200 +232,200.0,200 +233,200.0,200 +234,129.0,129 +235,135.0,135 +236,128.0,128 +237,141.0,141 +238,200.0,200 +239,200.0,200 +240,137.0,137 +241,172.0,172 +242,149.0,149 +243,200.0,200 +244,154.0,154 +245,200.0,200 +246,200.0,200 +247,200.0,200 +248,200.0,200 +249,200.0,200 +250,200.0,200 +251,200.0,200 +252,200.0,200 +253,200.0,200 +254,200.0,200 +255,200.0,200 +256,200.0,200 +257,200.0,200 +258,200.0,200 +259,200.0,200 +260,200.0,200 +261,200.0,200 +262,200.0,200 +263,195.0,195 +264,200.0,200 +265,200.0,200 +266,200.0,200 +267,200.0,200 +268,200.0,200 +269,200.0,200 +270,200.0,200 +271,200.0,200 +272,200.0,200 +273,200.0,200 +274,200.0,200 +275,200.0,200 +276,200.0,200 +277,200.0,200 +278,200.0,200 +279,200.0,200 +280,200.0,200 +281,200.0,200 +282,200.0,200 +283,200.0,200 +284,200.0,200 +285,200.0,200 +286,200.0,200 +287,200.0,200 +288,200.0,200 +289,200.0,200 +290,200.0,200 +291,200.0,200 +292,200.0,200 +293,200.0,200 +294,200.0,200 +295,200.0,200 +296,200.0,200 +297,200.0,200 +298,200.0,200 +299,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/config.yaml b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..202434a21a6b717d08e24907b0847c09377506f6 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/config.yaml @@ -0,0 +1,48 @@ +general_cfg: + algo_name: DRQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_ + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 8 + buffer_size: 100000 + epsilon_decay: 0.995 + epsilon_end: 0.001 + epsilon_start: 0.1 + gamma: 0.99 + hidden_dim: 64 + lookup_step: 10 + lr: 0.001 + max_epi_len: 100 + max_epi_num: 100 + min_epi_num: 16 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 64 + layer_type: linear + - activation: relu + layer_dim: + - 64 + - 64 + layer_type: linear + - activation: none + layer_dim: + - 64 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/logs/log.txt b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..62c670bdd61d9f2fe1f66721263b03d9b897e4c3 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/logs/log.txt @@ -0,0 +1,259 @@ +2023-02-04 22:31:46 - r - INFO: - Hyperparameters: +2023-02-04 22:31:46 - r - INFO: - ================================================================================ +2023-02-04 22:31:46 - r - INFO: - Name Value Type +2023-02-04 22:31:46 - r - INFO: - env_name CartPole-v1 +2023-02-04 22:31:46 - r - INFO: - new_step_api 1 +2023-02-04 22:31:46 - r - INFO: - wrapper None +2023-02-04 22:31:46 - r - INFO: - render 0 +2023-02-04 22:31:46 - r - INFO: - algo_name DRQN +2023-02-04 22:31:46 - r - INFO: - mode train +2023-02-04 22:31:46 - r - INFO: - seed 1 +2023-02-04 22:31:46 - r - INFO: - device cuda +2023-02-04 22:31:46 - r - INFO: - train_eps 200 +2023-02-04 22:31:46 - r - INFO: - test_eps 10 +2023-02-04 22:31:46 - r - INFO: - eval_eps 10 +2023-02-04 22:31:46 - r - INFO: - eval_per_episode 5 +2023-02-04 22:31:46 - r - INFO: - max_steps 200 +2023-02-04 22:31:46 - r - INFO: - load_checkpoint 0 +2023-02-04 22:31:46 - r - INFO: - load_path Train_CartPole-v1_ +2023-02-04 22:31:46 - r - INFO: - show_fig 0 +2023-02-04 22:31:46 - r - INFO: - save_fig 1 +2023-02-04 22:31:46 - r - INFO: - epsilon_start 0.1 +2023-02-04 22:31:46 - r - INFO: - epsilon_end 0.001 +2023-02-04 22:31:46 - r - INFO: - epsilon_decay 0.995 +2023-02-04 22:31:46 - r - INFO: - hidden_dim 64 +2023-02-04 22:31:46 - r - INFO: - gamma 0.99 +2023-02-04 22:31:46 - r - INFO: - lr 0.001 +2023-02-04 22:31:46 - r - INFO: - buffer_size 100000 +2023-02-04 22:31:46 - r - INFO: - batch_size 8 +2023-02-04 22:31:46 - r - INFO: - target_update 4 +2023-02-04 22:31:46 - r - INFO: - lookup_step 10 +2023-02-04 22:31:46 - r - INFO: - min_epi_num 16 +2023-02-04 22:31:46 - r - INFO: - max_epi_len 100 +2023-02-04 22:31:46 - r - INFO: - max_epi_num 100 +2023-02-04 22:31:46 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 64], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [64, 64], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [64, 'n_actions'], 'activation': 'none'}] +2023-02-04 22:31:46 - r - INFO: - task_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_DRQN_20230204-223146 +2023-02-04 22:31:46 - r - INFO: - model_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_DRQN_20230204-223146/models +2023-02-04 22:31:46 - r - INFO: - res_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_DRQN_20230204-223146/results +2023-02-04 22:31:46 - r - INFO: - log_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_DRQN_20230204-223146/logs +2023-02-04 22:31:46 - r - INFO: - traj_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_DRQN_20230204-223146/traj +2023-02-04 22:31:46 - r - INFO: - ================================================================================ +2023-02-04 22:31:46 - r - INFO: - n_states: 4, n_actions: 2 +2023-02-04 22:31:48 - r - INFO: - Start training! +2023-02-04 22:31:48 - r - INFO: - Env: CartPole-v1, Algorithm: DRQN, Device: cuda +2023-02-04 22:31:49 - r - INFO: - Episode: 1/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 2/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 3/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 4/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 5/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Current episode 5 has the best eval reward: 10.000 +2023-02-04 22:31:49 - r - INFO: - Episode: 6/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 7/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 8/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 9/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 10/200, Reward: 12.000, Step: 12 +2023-02-04 22:31:49 - r - INFO: - Current episode 10 has the best eval reward: 10.000 +2023-02-04 22:31:49 - r - INFO: - Episode: 11/200, Reward: 11.000, Step: 11 +2023-02-04 22:31:49 - r - INFO: - Episode: 12/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 13/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 14/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:49 - r - INFO: - Episode: 15/200, Reward: 11.000, Step: 11 +2023-02-04 22:31:50 - r - INFO: - Current episode 15 has the best eval reward: 10.000 +2023-02-04 22:31:50 - r - INFO: - Episode: 16/200, Reward: 10.000, Step: 10 +2023-02-04 22:31:50 - r - INFO: - Episode: 17/200, Reward: 13.000, Step: 13 +2023-02-04 22:31:50 - r - INFO: - Episode: 18/200, Reward: 14.000, Step: 14 +2023-02-04 22:31:50 - r - INFO: - Episode: 19/200, Reward: 18.000, Step: 18 +2023-02-04 22:31:51 - r - INFO: - Episode: 20/200, Reward: 13.000, Step: 13 +2023-02-04 22:31:51 - r - INFO: - Current episode 20 has the best eval reward: 11.000 +2023-02-04 22:31:51 - r - INFO: - Episode: 21/200, Reward: 12.000, Step: 12 +2023-02-04 22:31:51 - r - INFO: - Episode: 22/200, Reward: 12.000, Step: 12 +2023-02-04 22:31:51 - r - INFO: - Episode: 23/200, Reward: 14.000, Step: 14 +2023-02-04 22:31:51 - r - INFO: - Episode: 24/200, Reward: 11.000, Step: 11 +2023-02-04 22:31:51 - r - INFO: - Episode: 25/200, Reward: 9.000, Step: 9 +2023-02-04 22:31:52 - r - INFO: - Episode: 26/200, Reward: 20.000, Step: 20 +2023-02-04 22:31:52 - r - INFO: - Episode: 27/200, Reward: 19.000, Step: 19 +2023-02-04 22:31:52 - r - INFO: - Episode: 28/200, Reward: 13.000, Step: 13 +2023-02-04 22:31:52 - r - INFO: - Episode: 29/200, Reward: 16.000, Step: 16 +2023-02-04 22:31:53 - r - INFO: - Episode: 30/200, Reward: 17.000, Step: 17 +2023-02-04 22:31:54 - r - INFO: - Current episode 30 has the best eval reward: 102.000 +2023-02-04 22:31:55 - r - INFO: - Episode: 31/200, Reward: 74.000, Step: 74 +2023-02-04 22:31:56 - r - INFO: - Episode: 32/200, Reward: 87.000, Step: 87 +2023-02-04 22:31:58 - r - INFO: - Episode: 33/200, Reward: 142.000, Step: 142 +2023-02-04 22:31:58 - r - INFO: - Episode: 34/200, Reward: 53.000, Step: 53 +2023-02-04 22:31:59 - r - INFO: - Episode: 35/200, Reward: 56.000, Step: 56 +2023-02-04 22:32:00 - r - INFO: - Episode: 36/200, Reward: 57.000, Step: 57 +2023-02-04 22:32:01 - r - INFO: - Episode: 37/200, Reward: 24.000, Step: 24 +2023-02-04 22:32:01 - r - INFO: - Episode: 38/200, Reward: 56.000, Step: 56 +2023-02-04 22:32:03 - r - INFO: - Episode: 39/200, Reward: 110.000, Step: 110 +2023-02-04 22:32:03 - r - INFO: - Episode: 40/200, Reward: 35.000, Step: 35 +2023-02-04 22:32:04 - r - INFO: - Episode: 41/200, Reward: 27.000, Step: 27 +2023-02-04 22:32:04 - r - INFO: - Episode: 42/200, Reward: 31.000, Step: 31 +2023-02-04 22:32:05 - r - INFO: - Episode: 43/200, Reward: 28.000, Step: 28 +2023-02-04 22:32:05 - r - INFO: - Episode: 44/200, Reward: 27.000, Step: 27 +2023-02-04 22:32:05 - r - INFO: - Episode: 45/200, Reward: 38.000, Step: 38 +2023-02-04 22:32:06 - r - INFO: - Episode: 46/200, Reward: 27.000, Step: 27 +2023-02-04 22:32:07 - r - INFO: - Episode: 47/200, Reward: 30.000, Step: 30 +2023-02-04 22:32:07 - r - INFO: - Episode: 48/200, Reward: 25.000, Step: 25 +2023-02-04 22:32:07 - r - INFO: - Episode: 49/200, Reward: 22.000, Step: 22 +2023-02-04 22:32:08 - r - INFO: - Episode: 50/200, Reward: 30.000, Step: 30 +2023-02-04 22:32:08 - r - INFO: - Episode: 51/200, Reward: 26.000, Step: 26 +2023-02-04 22:32:09 - r - INFO: - Episode: 52/200, Reward: 29.000, Step: 29 +2023-02-04 22:32:09 - r - INFO: - Episode: 53/200, Reward: 34.000, Step: 34 +2023-02-04 22:32:10 - r - INFO: - Episode: 54/200, Reward: 30.000, Step: 30 +2023-02-04 22:32:10 - r - INFO: - Episode: 55/200, Reward: 30.000, Step: 30 +2023-02-04 22:32:11 - r - INFO: - Episode: 56/200, Reward: 25.000, Step: 25 +2023-02-04 22:32:12 - r - INFO: - Episode: 57/200, Reward: 78.000, Step: 78 +2023-02-04 22:32:12 - r - INFO: - Episode: 58/200, Reward: 23.000, Step: 23 +2023-02-04 22:32:13 - r - INFO: - Episode: 59/200, Reward: 53.000, Step: 53 +2023-02-04 22:32:13 - r - INFO: - Episode: 60/200, Reward: 46.000, Step: 46 +2023-02-04 22:32:14 - r - INFO: - Episode: 61/200, Reward: 39.000, Step: 39 +2023-02-04 22:32:15 - r - INFO: - Episode: 62/200, Reward: 33.000, Step: 33 +2023-02-04 22:32:15 - r - INFO: - Episode: 63/200, Reward: 31.000, Step: 31 +2023-02-04 22:32:16 - r - INFO: - Episode: 64/200, Reward: 34.000, Step: 34 +2023-02-04 22:32:16 - r - INFO: - Episode: 65/200, Reward: 57.000, Step: 57 +2023-02-04 22:32:17 - r - INFO: - Episode: 66/200, Reward: 52.000, Step: 52 +2023-02-04 22:32:18 - r - INFO: - Episode: 67/200, Reward: 90.000, Step: 90 +2023-02-04 22:32:19 - r - INFO: - Episode: 68/200, Reward: 22.000, Step: 22 +2023-02-04 22:32:19 - r - INFO: - Episode: 69/200, Reward: 21.000, Step: 21 +2023-02-04 22:32:19 - r - INFO: - Episode: 70/200, Reward: 39.000, Step: 39 +2023-02-04 22:32:20 - r - INFO: - Episode: 71/200, Reward: 21.000, Step: 21 +2023-02-04 22:32:20 - r - INFO: - Episode: 72/200, Reward: 25.000, Step: 25 +2023-02-04 22:32:21 - r - INFO: - Episode: 73/200, Reward: 18.000, Step: 18 +2023-02-04 22:32:21 - r - INFO: - Episode: 74/200, Reward: 18.000, Step: 18 +2023-02-04 22:32:22 - r - INFO: - Episode: 75/200, Reward: 69.000, Step: 69 +2023-02-04 22:32:22 - r - INFO: - Episode: 76/200, Reward: 28.000, Step: 28 +2023-02-04 22:32:23 - r - INFO: - Episode: 77/200, Reward: 28.000, Step: 28 +2023-02-04 22:32:23 - r - INFO: - Episode: 78/200, Reward: 18.000, Step: 18 +2023-02-04 22:32:23 - r - INFO: - Episode: 79/200, Reward: 18.000, Step: 18 +2023-02-04 22:32:23 - r - INFO: - Episode: 80/200, Reward: 20.000, Step: 20 +2023-02-04 22:32:24 - r - INFO: - Episode: 81/200, Reward: 41.000, Step: 41 +2023-02-04 22:32:25 - r - INFO: - Episode: 82/200, Reward: 69.000, Step: 69 +2023-02-04 22:32:25 - r - INFO: - Episode: 83/200, Reward: 18.000, Step: 18 +2023-02-04 22:32:26 - r - INFO: - Episode: 84/200, Reward: 27.000, Step: 27 +2023-02-04 22:32:26 - r - INFO: - Episode: 85/200, Reward: 30.000, Step: 30 +2023-02-04 22:32:27 - r - INFO: - Episode: 86/200, Reward: 28.000, Step: 28 +2023-02-04 22:32:29 - r - INFO: - Episode: 87/200, Reward: 131.000, Step: 131 +2023-02-04 22:32:30 - r - INFO: - Episode: 88/200, Reward: 83.000, Step: 83 +2023-02-04 22:32:31 - r - INFO: - Episode: 89/200, Reward: 103.000, Step: 103 +2023-02-04 22:32:32 - r - INFO: - Episode: 90/200, Reward: 30.000, Step: 30 +2023-02-04 22:32:32 - r - INFO: - Episode: 91/200, Reward: 30.000, Step: 30 +2023-02-04 22:32:33 - r - INFO: - Episode: 92/200, Reward: 31.000, Step: 31 +2023-02-04 22:32:34 - r - INFO: - Episode: 93/200, Reward: 115.000, Step: 115 +2023-02-04 22:32:36 - r - INFO: - Episode: 94/200, Reward: 160.000, Step: 160 +2023-02-04 22:32:37 - r - INFO: - Episode: 95/200, Reward: 64.000, Step: 64 +2023-02-04 22:32:39 - r - INFO: - Episode: 96/200, Reward: 143.000, Step: 143 +2023-02-04 22:32:41 - r - INFO: - Episode: 97/200, Reward: 88.000, Step: 88 +2023-02-04 22:32:42 - r - INFO: - Episode: 98/200, Reward: 111.000, Step: 111 +2023-02-04 22:32:43 - r - INFO: - Episode: 99/200, Reward: 103.000, Step: 103 +2023-02-04 22:32:45 - r - INFO: - Episode: 100/200, Reward: 120.000, Step: 120 +2023-02-04 22:32:46 - r - INFO: - Current episode 100 has the best eval reward: 118.000 +2023-02-04 22:32:48 - r - INFO: - Episode: 101/200, Reward: 112.000, Step: 112 +2023-02-04 22:32:49 - r - INFO: - Episode: 102/200, Reward: 98.000, Step: 98 +2023-02-04 22:32:51 - r - INFO: - Episode: 103/200, Reward: 100.000, Step: 100 +2023-02-04 22:32:52 - r - INFO: - Episode: 104/200, Reward: 91.000, Step: 91 +2023-02-04 22:32:53 - r - INFO: - Episode: 105/200, Reward: 112.000, Step: 112 +2023-02-04 22:32:56 - r - INFO: - Episode: 106/200, Reward: 107.000, Step: 107 +2023-02-04 22:32:57 - r - INFO: - Episode: 107/200, Reward: 123.000, Step: 123 +2023-02-04 22:32:59 - r - INFO: - Episode: 108/200, Reward: 174.000, Step: 174 +2023-02-04 22:33:01 - r - INFO: - Episode: 109/200, Reward: 121.000, Step: 121 +2023-02-04 22:33:02 - r - INFO: - Episode: 110/200, Reward: 82.000, Step: 82 +2023-02-04 22:33:03 - r - INFO: - Episode: 111/200, Reward: 19.000, Step: 19 +2023-02-04 22:33:03 - r - INFO: - Episode: 112/200, Reward: 36.000, Step: 36 +2023-02-04 22:33:03 - r - INFO: - Episode: 113/200, Reward: 10.000, Step: 10 +2023-02-04 22:33:03 - r - INFO: - Episode: 114/200, Reward: 12.000, Step: 12 +2023-02-04 22:33:05 - r - INFO: - Episode: 115/200, Reward: 83.000, Step: 83 +2023-02-04 22:33:06 - r - INFO: - Episode: 116/200, Reward: 71.000, Step: 71 +2023-02-04 22:33:07 - r - INFO: - Episode: 117/200, Reward: 78.000, Step: 78 +2023-02-04 22:33:09 - r - INFO: - Episode: 118/200, Reward: 102.000, Step: 102 +2023-02-04 22:33:11 - r - INFO: - Episode: 119/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:14 - r - INFO: - Episode: 120/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:16 - r - INFO: - Current episode 120 has the best eval reward: 181.000 +2023-02-04 22:33:18 - r - INFO: - Episode: 121/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:21 - r - INFO: - Episode: 122/200, Reward: 158.000, Step: 158 +2023-02-04 22:33:23 - r - INFO: - Episode: 123/200, Reward: 153.000, Step: 153 +2023-02-04 22:33:24 - r - INFO: - Episode: 124/200, Reward: 141.000, Step: 141 +2023-02-04 22:33:27 - r - INFO: - Episode: 125/200, Reward: 178.000, Step: 178 +2023-02-04 22:33:30 - r - INFO: - Episode: 126/200, Reward: 158.000, Step: 158 +2023-02-04 22:33:32 - r - INFO: - Episode: 127/200, Reward: 147.000, Step: 147 +2023-02-04 22:33:35 - r - INFO: - Episode: 128/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:37 - r - INFO: - Episode: 129/200, Reward: 168.000, Step: 168 +2023-02-04 22:33:40 - r - INFO: - Episode: 130/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:42 - r - INFO: - Current episode 130 has the best eval reward: 200.000 +2023-02-04 22:33:45 - r - INFO: - Episode: 131/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:47 - r - INFO: - Episode: 132/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:50 - r - INFO: - Episode: 133/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:53 - r - INFO: - Episode: 134/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:55 - r - INFO: - Episode: 135/200, Reward: 200.000, Step: 200 +2023-02-04 22:33:58 - r - INFO: - Current episode 135 has the best eval reward: 200.000 +2023-02-04 22:33:58 - r - INFO: - Episode: 136/200, Reward: 30.000, Step: 30 +2023-02-04 22:34:01 - r - INFO: - Episode: 137/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:03 - r - INFO: - Episode: 138/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:06 - r - INFO: - Episode: 139/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:08 - r - INFO: - Episode: 140/200, Reward: 115.000, Step: 115 +2023-02-04 22:34:10 - r - INFO: - Current episode 140 has the best eval reward: 200.000 +2023-02-04 22:34:12 - r - INFO: - Episode: 141/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:15 - r - INFO: - Episode: 142/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:18 - r - INFO: - Episode: 143/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:19 - r - INFO: - Episode: 144/200, Reward: 53.000, Step: 53 +2023-02-04 22:34:19 - r - INFO: - Episode: 145/200, Reward: 58.000, Step: 58 +2023-02-04 22:34:24 - r - INFO: - Episode: 146/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:27 - r - INFO: - Episode: 147/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:28 - r - INFO: - Episode: 148/200, Reward: 113.000, Step: 113 +2023-02-04 22:34:30 - r - INFO: - Episode: 149/200, Reward: 102.000, Step: 102 +2023-02-04 22:34:31 - r - INFO: - Episode: 150/200, Reward: 111.000, Step: 111 +2023-02-04 22:34:33 - r - INFO: - Episode: 151/200, Reward: 54.000, Step: 54 +2023-02-04 22:34:33 - r - INFO: - Episode: 152/200, Reward: 48.000, Step: 48 +2023-02-04 22:34:36 - r - INFO: - Episode: 153/200, Reward: 188.000, Step: 188 +2023-02-04 22:34:39 - r - INFO: - Episode: 154/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:41 - r - INFO: - Episode: 155/200, Reward: 172.000, Step: 172 +2023-02-04 22:34:45 - r - INFO: - Episode: 156/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:48 - r - INFO: - Episode: 157/200, Reward: 200.000, Step: 200 +2023-02-04 22:34:49 - r - INFO: - Episode: 158/200, Reward: 93.000, Step: 93 +2023-02-04 22:34:52 - r - INFO: - Episode: 159/200, Reward: 183.000, Step: 183 +2023-02-04 22:34:53 - r - INFO: - Episode: 160/200, Reward: 81.000, Step: 81 +2023-02-04 22:34:55 - r - INFO: - Current episode 160 has the best eval reward: 200.000 +2023-02-04 22:34:56 - r - INFO: - Episode: 161/200, Reward: 85.000, Step: 85 +2023-02-04 22:34:57 - r - INFO: - Episode: 162/200, Reward: 47.000, Step: 47 +2023-02-04 22:34:59 - r - INFO: - Episode: 163/200, Reward: 136.000, Step: 136 +2023-02-04 22:35:01 - r - INFO: - Episode: 164/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:03 - r - INFO: - Episode: 165/200, Reward: 130.000, Step: 130 +2023-02-04 22:35:07 - r - INFO: - Episode: 166/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:10 - r - INFO: - Episode: 167/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:12 - r - INFO: - Episode: 168/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:15 - r - INFO: - Episode: 169/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:18 - r - INFO: - Episode: 170/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:19 - r - INFO: - Episode: 171/200, Reward: 37.000, Step: 37 +2023-02-04 22:35:20 - r - INFO: - Episode: 172/200, Reward: 129.000, Step: 129 +2023-02-04 22:35:23 - r - INFO: - Episode: 173/200, Reward: 153.000, Step: 153 +2023-02-04 22:35:24 - r - INFO: - Episode: 174/200, Reward: 133.000, Step: 133 +2023-02-04 22:35:26 - r - INFO: - Episode: 175/200, Reward: 106.000, Step: 106 +2023-02-04 22:35:29 - r - INFO: - Episode: 176/200, Reward: 122.000, Step: 122 +2023-02-04 22:35:30 - r - INFO: - Episode: 177/200, Reward: 78.000, Step: 78 +2023-02-04 22:35:32 - r - INFO: - Episode: 178/200, Reward: 147.000, Step: 147 +2023-02-04 22:35:34 - r - INFO: - Episode: 179/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:37 - r - INFO: - Episode: 180/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:39 - r - INFO: - Current episode 180 has the best eval reward: 200.000 +2023-02-04 22:35:42 - r - INFO: - Episode: 181/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:45 - r - INFO: - Episode: 182/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:47 - r - INFO: - Episode: 183/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:50 - r - INFO: - Episode: 184/200, Reward: 163.000, Step: 163 +2023-02-04 22:35:52 - r - INFO: - Episode: 185/200, Reward: 200.000, Step: 200 +2023-02-04 22:35:54 - r - INFO: - Current episode 185 has the best eval reward: 200.000 +2023-02-04 22:35:57 - r - INFO: - Episode: 186/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:00 - r - INFO: - Episode: 187/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:02 - r - INFO: - Episode: 188/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:05 - r - INFO: - Episode: 189/200, Reward: 177.000, Step: 177 +2023-02-04 22:36:07 - r - INFO: - Episode: 190/200, Reward: 159.000, Step: 159 +2023-02-04 22:36:09 - r - INFO: - Current episode 190 has the best eval reward: 200.000 +2023-02-04 22:36:12 - r - INFO: - Episode: 191/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:13 - r - INFO: - Episode: 192/200, Reward: 44.000, Step: 44 +2023-02-04 22:36:15 - r - INFO: - Episode: 193/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:18 - r - INFO: - Episode: 194/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:21 - r - INFO: - Episode: 195/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:26 - r - INFO: - Episode: 196/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:28 - r - INFO: - Episode: 197/200, Reward: 199.000, Step: 199 +2023-02-04 22:36:29 - r - INFO: - Episode: 198/200, Reward: 13.000, Step: 13 +2023-02-04 22:36:29 - r - INFO: - Episode: 199/200, Reward: 15.000, Step: 15 +2023-02-04 22:36:31 - r - INFO: - Episode: 200/200, Reward: 200.000, Step: 200 +2023-02-04 22:36:33 - r - INFO: - Current episode 200 has the best eval reward: 200.000 +2023-02-04 22:36:33 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..98cef9a31285e3db14be205d395e83f61f95b384 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c948973a3263599781dad691c7f1a8a8fbe02ad814fa673bdd551f44536aaf3 +size 137231 diff --git a/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..1f34523fedbb4eea612dd6cb1956eb18a5aba4af Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/results/res.csv b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c4dd22aa80b52a9ffed4deda3cfd5a6070e284d --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DRQN_20230204-223146/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,10.0,10 +1,10.0,10 +2,10.0,10 +3,10.0,10 +4,10.0,10 +5,10.0,10 +6,10.0,10 +7,10.0,10 +8,10.0,10 +9,12.0,12 +10,11.0,11 +11,10.0,10 +12,10.0,10 +13,10.0,10 +14,11.0,11 +15,10.0,10 +16,13.0,13 +17,14.0,14 +18,18.0,18 +19,13.0,13 +20,12.0,12 +21,12.0,12 +22,14.0,14 +23,11.0,11 +24,9.0,9 +25,20.0,20 +26,19.0,19 +27,13.0,13 +28,16.0,16 +29,17.0,17 +30,74.0,74 +31,87.0,87 +32,142.0,142 +33,53.0,53 +34,56.0,56 +35,57.0,57 +36,24.0,24 +37,56.0,56 +38,110.0,110 +39,35.0,35 +40,27.0,27 +41,31.0,31 +42,28.0,28 +43,27.0,27 +44,38.0,38 +45,27.0,27 +46,30.0,30 +47,25.0,25 +48,22.0,22 +49,30.0,30 +50,26.0,26 +51,29.0,29 +52,34.0,34 +53,30.0,30 +54,30.0,30 +55,25.0,25 +56,78.0,78 +57,23.0,23 +58,53.0,53 +59,46.0,46 +60,39.0,39 +61,33.0,33 +62,31.0,31 +63,34.0,34 +64,57.0,57 +65,52.0,52 +66,90.0,90 +67,22.0,22 +68,21.0,21 +69,39.0,39 +70,21.0,21 +71,25.0,25 +72,18.0,18 +73,18.0,18 +74,69.0,69 +75,28.0,28 +76,28.0,28 +77,18.0,18 +78,18.0,18 +79,20.0,20 +80,41.0,41 +81,69.0,69 +82,18.0,18 +83,27.0,27 +84,30.0,30 +85,28.0,28 +86,131.0,131 +87,83.0,83 +88,103.0,103 +89,30.0,30 +90,30.0,30 +91,31.0,31 +92,115.0,115 +93,160.0,160 +94,64.0,64 +95,143.0,143 +96,88.0,88 +97,111.0,111 +98,103.0,103 +99,120.0,120 +100,112.0,112 +101,98.0,98 +102,100.0,100 +103,91.0,91 +104,112.0,112 +105,107.0,107 +106,123.0,123 +107,174.0,174 +108,121.0,121 +109,82.0,82 +110,19.0,19 +111,36.0,36 +112,10.0,10 +113,12.0,12 +114,83.0,83 +115,71.0,71 +116,78.0,78 +117,102.0,102 +118,200.0,200 +119,200.0,200 +120,200.0,200 +121,158.0,158 +122,153.0,153 +123,141.0,141 +124,178.0,178 +125,158.0,158 +126,147.0,147 +127,200.0,200 +128,168.0,168 +129,200.0,200 +130,200.0,200 +131,200.0,200 +132,200.0,200 +133,200.0,200 +134,200.0,200 +135,30.0,30 +136,200.0,200 +137,200.0,200 +138,200.0,200 +139,115.0,115 +140,200.0,200 +141,200.0,200 +142,200.0,200 +143,53.0,53 +144,58.0,58 +145,200.0,200 +146,200.0,200 +147,113.0,113 +148,102.0,102 +149,111.0,111 +150,54.0,54 +151,48.0,48 +152,188.0,188 +153,200.0,200 +154,172.0,172 +155,200.0,200 +156,200.0,200 +157,93.0,93 +158,183.0,183 +159,81.0,81 +160,85.0,85 +161,47.0,47 +162,136.0,136 +163,200.0,200 +164,130.0,130 +165,200.0,200 +166,200.0,200 +167,200.0,200 +168,200.0,200 +169,200.0,200 +170,37.0,37 +171,129.0,129 +172,153.0,153 +173,133.0,133 +174,106.0,106 +175,122.0,122 +176,78.0,78 +177,147.0,147 +178,200.0,200 +179,200.0,200 +180,200.0,200 +181,200.0,200 +182,200.0,200 +183,163.0,163 +184,200.0,200 +185,200.0,200 +186,200.0,200 +187,200.0,200 +188,177.0,177 +189,159.0,159 +190,200.0,200 +191,44.0,44 +192,200.0,200 +193,200.0,200 +194,200.0,200 +195,200.0,200 +196,199.0,199 +197,13.0,13 +198,15.0,15 +199,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/config.yaml b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3847978e720c68cf0fafd5dc6a10e2a39a2bbe79 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/config.yaml @@ -0,0 +1,40 @@ +general_cfg: + algo_name: DoubleDQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/logs/log.txt b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..209a7ba57be2eca83009483d690e916c7257e755 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/logs/log.txt @@ -0,0 +1,116 @@ +2022-11-22 12:55:16 - r - INFO: - n_states: 4, n_actions: 2 +2022-11-22 12:55:19 - r - INFO: - Start training! +2022-11-22 12:55:19 - r - INFO: - Env: CartPole-v1, Algorithm: DoubleDQN, Device: cuda +2022-11-22 12:55:19 - r - INFO: - Episode: 1/100, Reward: 18.000, Step: 18 +2022-11-22 12:55:19 - r - INFO: - Episode: 2/100, Reward: 35.000, Step: 35 +2022-11-22 12:55:19 - r - INFO: - Episode: 3/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:19 - r - INFO: - Episode: 4/100, Reward: 32.000, Step: 32 +2022-11-22 12:55:19 - r - INFO: - Episode: 5/100, Reward: 16.000, Step: 16 +2022-11-22 12:55:19 - r - INFO: - Current episode 5 has the best eval reward: 9.100 +2022-11-22 12:55:19 - r - INFO: - Episode: 6/100, Reward: 9.000, Step: 9 +2022-11-22 12:55:19 - r - INFO: - Episode: 7/100, Reward: 12.000, Step: 12 +2022-11-22 12:55:19 - r - INFO: - Episode: 8/100, Reward: 16.000, Step: 16 +2022-11-22 12:55:19 - r - INFO: - Episode: 9/100, Reward: 14.000, Step: 14 +2022-11-22 12:55:19 - r - INFO: - Episode: 10/100, Reward: 12.000, Step: 12 +2022-11-22 12:55:19 - r - INFO: - Current episode 10 has the best eval reward: 9.200 +2022-11-22 12:55:19 - r - INFO: - Episode: 11/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:19 - r - INFO: - Episode: 12/100, Reward: 14.000, Step: 14 +2022-11-22 12:55:19 - r - INFO: - Episode: 13/100, Reward: 19.000, Step: 19 +2022-11-22 12:55:19 - r - INFO: - Episode: 14/100, Reward: 9.000, Step: 9 +2022-11-22 12:55:19 - r - INFO: - Episode: 15/100, Reward: 15.000, Step: 15 +2022-11-22 12:55:19 - r - INFO: - Current episode 15 has the best eval reward: 9.300 +2022-11-22 12:55:19 - r - INFO: - Episode: 16/100, Reward: 12.000, Step: 12 +2022-11-22 12:55:19 - r - INFO: - Episode: 17/100, Reward: 11.000, Step: 11 +2022-11-22 12:55:19 - r - INFO: - Episode: 18/100, Reward: 9.000, Step: 9 +2022-11-22 12:55:19 - r - INFO: - Episode: 19/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:19 - r - INFO: - Episode: 20/100, Reward: 17.000, Step: 17 +2022-11-22 12:55:19 - r - INFO: - Episode: 21/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:19 - r - INFO: - Episode: 22/100, Reward: 15.000, Step: 15 +2022-11-22 12:55:19 - r - INFO: - Episode: 23/100, Reward: 22.000, Step: 22 +2022-11-22 12:55:20 - r - INFO: - Episode: 24/100, Reward: 26.000, Step: 26 +2022-11-22 12:55:20 - r - INFO: - Episode: 25/100, Reward: 19.000, Step: 19 +2022-11-22 12:55:20 - r - INFO: - Current episode 25 has the best eval reward: 9.800 +2022-11-22 12:55:20 - r - INFO: - Episode: 26/100, Reward: 10.000, Step: 10 +2022-11-22 12:55:20 - r - INFO: - Episode: 27/100, Reward: 10.000, Step: 10 +2022-11-22 12:55:20 - r - INFO: - Episode: 28/100, Reward: 11.000, Step: 11 +2022-11-22 12:55:20 - r - INFO: - Episode: 29/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:20 - r - INFO: - Episode: 30/100, Reward: 16.000, Step: 16 +2022-11-22 12:55:20 - r - INFO: - Episode: 31/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:20 - r - INFO: - Episode: 32/100, Reward: 15.000, Step: 15 +2022-11-22 12:55:20 - r - INFO: - Episode: 33/100, Reward: 12.000, Step: 12 +2022-11-22 12:55:20 - r - INFO: - Episode: 34/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:20 - r - INFO: - Episode: 35/100, Reward: 13.000, Step: 13 +2022-11-22 12:55:20 - r - INFO: - Episode: 36/100, Reward: 11.000, Step: 11 +2022-11-22 12:55:20 - r - INFO: - Episode: 37/100, Reward: 9.000, Step: 9 +2022-11-22 12:55:20 - r - INFO: - Episode: 38/100, Reward: 9.000, Step: 9 +2022-11-22 12:55:20 - r - INFO: - Episode: 39/100, Reward: 10.000, Step: 10 +2022-11-22 12:55:20 - r - INFO: - Episode: 40/100, Reward: 14.000, Step: 14 +2022-11-22 12:55:20 - r - INFO: - Episode: 41/100, Reward: 9.000, Step: 9 +2022-11-22 12:55:20 - r - INFO: - Episode: 42/100, Reward: 10.000, Step: 10 +2022-11-22 12:55:20 - r - INFO: - Episode: 43/100, Reward: 9.000, Step: 9 +2022-11-22 12:55:20 - r - INFO: - Episode: 44/100, Reward: 14.000, Step: 14 +2022-11-22 12:55:20 - r - INFO: - Episode: 45/100, Reward: 10.000, Step: 10 +2022-11-22 12:55:20 - r - INFO: - Episode: 46/100, Reward: 19.000, Step: 19 +2022-11-22 12:55:20 - r - INFO: - Episode: 47/100, Reward: 10.000, Step: 10 +2022-11-22 12:55:20 - r - INFO: - Episode: 48/100, Reward: 14.000, Step: 14 +2022-11-22 12:55:20 - r - INFO: - Episode: 49/100, Reward: 18.000, Step: 18 +2022-11-22 12:55:20 - r - INFO: - Episode: 50/100, Reward: 32.000, Step: 32 +2022-11-22 12:55:20 - r - INFO: - Current episode 50 has the best eval reward: 24.300 +2022-11-22 12:55:21 - r - INFO: - Episode: 51/100, Reward: 17.000, Step: 17 +2022-11-22 12:55:21 - r - INFO: - Episode: 52/100, Reward: 15.000, Step: 15 +2022-11-22 12:55:21 - r - INFO: - Episode: 53/100, Reward: 18.000, Step: 18 +2022-11-22 12:55:21 - r - INFO: - Episode: 54/100, Reward: 14.000, Step: 14 +2022-11-22 12:55:21 - r - INFO: - Episode: 55/100, Reward: 22.000, Step: 22 +2022-11-22 12:55:21 - r - INFO: - Episode: 56/100, Reward: 14.000, Step: 14 +2022-11-22 12:55:21 - r - INFO: - Episode: 57/100, Reward: 21.000, Step: 21 +2022-11-22 12:55:21 - r - INFO: - Episode: 58/100, Reward: 21.000, Step: 21 +2022-11-22 12:55:21 - r - INFO: - Episode: 59/100, Reward: 23.000, Step: 23 +2022-11-22 12:55:21 - r - INFO: - Episode: 60/100, Reward: 21.000, Step: 21 +2022-11-22 12:55:21 - r - INFO: - Episode: 61/100, Reward: 21.000, Step: 21 +2022-11-22 12:55:21 - r - INFO: - Episode: 62/100, Reward: 35.000, Step: 35 +2022-11-22 12:55:21 - r - INFO: - Episode: 63/100, Reward: 23.000, Step: 23 +2022-11-22 12:55:21 - r - INFO: - Episode: 64/100, Reward: 27.000, Step: 27 +2022-11-22 12:55:21 - r - INFO: - Episode: 65/100, Reward: 24.000, Step: 24 +2022-11-22 12:55:21 - r - INFO: - Current episode 65 has the best eval reward: 29.700 +2022-11-22 12:55:21 - r - INFO: - Episode: 66/100, Reward: 28.000, Step: 28 +2022-11-22 12:55:21 - r - INFO: - Episode: 67/100, Reward: 30.000, Step: 30 +2022-11-22 12:55:22 - r - INFO: - Episode: 68/100, Reward: 33.000, Step: 33 +2022-11-22 12:55:22 - r - INFO: - Episode: 69/100, Reward: 33.000, Step: 33 +2022-11-22 12:55:22 - r - INFO: - Episode: 70/100, Reward: 26.000, Step: 26 +2022-11-22 12:55:22 - r - INFO: - Current episode 70 has the best eval reward: 34.400 +2022-11-22 12:55:22 - r - INFO: - Episode: 71/100, Reward: 37.000, Step: 37 +2022-11-22 12:55:22 - r - INFO: - Episode: 72/100, Reward: 28.000, Step: 28 +2022-11-22 12:55:22 - r - INFO: - Episode: 73/100, Reward: 30.000, Step: 30 +2022-11-22 12:55:22 - r - INFO: - Episode: 74/100, Reward: 41.000, Step: 41 +2022-11-22 12:55:22 - r - INFO: - Episode: 75/100, Reward: 45.000, Step: 45 +2022-11-22 12:55:22 - r - INFO: - Current episode 75 has the best eval reward: 35.600 +2022-11-22 12:55:23 - r - INFO: - Episode: 76/100, Reward: 68.000, Step: 68 +2022-11-22 12:55:23 - r - INFO: - Episode: 77/100, Reward: 33.000, Step: 33 +2022-11-22 12:55:23 - r - INFO: - Episode: 78/100, Reward: 46.000, Step: 46 +2022-11-22 12:55:23 - r - INFO: - Episode: 79/100, Reward: 54.000, Step: 54 +2022-11-22 12:55:23 - r - INFO: - Episode: 80/100, Reward: 37.000, Step: 37 +2022-11-22 12:55:23 - r - INFO: - Current episode 80 has the best eval reward: 42.800 +2022-11-22 12:55:23 - r - INFO: - Episode: 81/100, Reward: 43.000, Step: 43 +2022-11-22 12:55:23 - r - INFO: - Episode: 82/100, Reward: 79.000, Step: 79 +2022-11-22 12:55:23 - r - INFO: - Episode: 83/100, Reward: 36.000, Step: 36 +2022-11-22 12:55:24 - r - INFO: - Episode: 84/100, Reward: 58.000, Step: 58 +2022-11-22 12:55:24 - r - INFO: - Episode: 85/100, Reward: 42.000, Step: 42 +2022-11-22 12:55:24 - r - INFO: - Current episode 85 has the best eval reward: 62.100 +2022-11-22 12:55:24 - r - INFO: - Episode: 86/100, Reward: 136.000, Step: 136 +2022-11-22 12:55:24 - r - INFO: - Episode: 87/100, Reward: 57.000, Step: 57 +2022-11-22 12:55:24 - r - INFO: - Episode: 88/100, Reward: 46.000, Step: 46 +2022-11-22 12:55:25 - r - INFO: - Episode: 89/100, Reward: 105.000, Step: 105 +2022-11-22 12:55:25 - r - INFO: - Episode: 90/100, Reward: 63.000, Step: 63 +2022-11-22 12:55:25 - r - INFO: - Current episode 90 has the best eval reward: 76.600 +2022-11-22 12:55:25 - r - INFO: - Episode: 91/100, Reward: 84.000, Step: 84 +2022-11-22 12:55:26 - r - INFO: - Episode: 92/100, Reward: 136.000, Step: 136 +2022-11-22 12:55:26 - r - INFO: - Episode: 93/100, Reward: 121.000, Step: 121 +2022-11-22 12:55:26 - r - INFO: - Episode: 94/100, Reward: 96.000, Step: 96 +2022-11-22 12:55:26 - r - INFO: - Episode: 95/100, Reward: 106.000, Step: 106 +2022-11-22 12:55:27 - r - INFO: - Current episode 95 has the best eval reward: 187.300 +2022-11-22 12:55:27 - r - INFO: - Episode: 96/100, Reward: 200.000, Step: 200 +2022-11-22 12:55:28 - r - INFO: - Episode: 97/100, Reward: 200.000, Step: 200 +2022-11-22 12:55:28 - r - INFO: - Episode: 98/100, Reward: 113.000, Step: 113 +2022-11-22 12:55:28 - r - INFO: - Episode: 99/100, Reward: 113.000, Step: 113 +2022-11-22 12:55:29 - r - INFO: - Episode: 100/100, Reward: 132.000, Step: 132 +2022-11-22 12:55:29 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/models/checkpoint.pth b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/models/checkpoint.pth new file mode 100644 index 0000000000000000000000000000000000000000..27cfa80035673442926e5e8ae4a75214ae1e6aad --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/models/checkpoint.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c80e643e52ee0e109e55ba083247021287455374ec28f27c4f2705e51fee23 +size 272471 diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..960506f50ae56e8eaf3ccf1aaf5eab33a330df7a Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/results/res.csv b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..63e820e8a20e94235a1ba88cf959752718c0d476 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_20221122-125516/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,18.0,18 +1,35.0,35 +2,13.0,13 +3,32.0,32 +4,16.0,16 +5,9.0,9 +6,12.0,12 +7,16.0,16 +8,14.0,14 +9,12.0,12 +10,13.0,13 +11,14.0,14 +12,19.0,19 +13,9.0,9 +14,15.0,15 +15,12.0,12 +16,11.0,11 +17,9.0,9 +18,13.0,13 +19,17.0,17 +20,13.0,13 +21,15.0,15 +22,22.0,22 +23,26.0,26 +24,19.0,19 +25,10.0,10 +26,10.0,10 +27,11.0,11 +28,13.0,13 +29,16.0,16 +30,13.0,13 +31,15.0,15 +32,12.0,12 +33,13.0,13 +34,13.0,13 +35,11.0,11 +36,9.0,9 +37,9.0,9 +38,10.0,10 +39,14.0,14 +40,9.0,9 +41,10.0,10 +42,9.0,9 +43,14.0,14 +44,10.0,10 +45,19.0,19 +46,10.0,10 +47,14.0,14 +48,18.0,18 +49,32.0,32 +50,17.0,17 +51,15.0,15 +52,18.0,18 +53,14.0,14 +54,22.0,22 +55,14.0,14 +56,21.0,21 +57,21.0,21 +58,23.0,23 +59,21.0,21 +60,21.0,21 +61,35.0,35 +62,23.0,23 +63,27.0,27 +64,24.0,24 +65,28.0,28 +66,30.0,30 +67,33.0,33 +68,33.0,33 +69,26.0,26 +70,37.0,37 +71,28.0,28 +72,30.0,30 +73,41.0,41 +74,45.0,45 +75,68.0,68 +76,33.0,33 +77,46.0,46 +78,54.0,54 +79,37.0,37 +80,43.0,43 +81,79.0,79 +82,36.0,36 +83,58.0,58 +84,42.0,42 +85,136.0,136 +86,57.0,57 +87,46.0,46 +88,105.0,105 +89,63.0,63 +90,84.0,84 +91,136.0,136 +92,121.0,121 +93,96.0,96 +94,106.0,106 +95,200.0,200 +96,200.0,200 +97,113.0,113 +98,113.0,113 +99,132.0,132 diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/config.yaml b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..faa408b92309eec46d4404df33100e4a39696f7b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/config.yaml @@ -0,0 +1,46 @@ +general_cfg: + algo_name: DoubleDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + mp_backend: mp + n_workers: 2 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/logs/log.txt b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..be12ec011e5691053604d327e293f0de3e16e75b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/logs/log.txt @@ -0,0 +1,42 @@ +2023-04-06 16:00:28 - r - INFO: - Hyperparameters: +2023-04-06 16:00:28 - r - INFO: - ================================================================================ +2023-04-06 16:00:28 - r - INFO: - Name Value Type +2023-04-06 16:00:28 - r - INFO: - env_name CartPole-v1 +2023-04-06 16:00:28 - r - INFO: - new_step_api 1 +2023-04-06 16:00:28 - r - INFO: - wrapper None +2023-04-06 16:00:28 - r - INFO: - render 0 +2023-04-06 16:00:28 - r - INFO: - render_mode human +2023-04-06 16:00:28 - r - INFO: - algo_name DoubleDQN +2023-04-06 16:00:28 - r - INFO: - mode train +2023-04-06 16:00:28 - r - INFO: - mp_backend mp +2023-04-06 16:00:28 - r - INFO: - seed 1 +2023-04-06 16:00:28 - r - INFO: - device cpu +2023-04-06 16:00:28 - r - INFO: - train_eps 400 +2023-04-06 16:00:28 - r - INFO: - test_eps 10 +2023-04-06 16:00:28 - r - INFO: - eval_eps 10 +2023-04-06 16:00:28 - r - INFO: - eval_per_episode 5 +2023-04-06 16:00:28 - r - INFO: - max_steps 200 +2023-04-06 16:00:28 - r - INFO: - load_checkpoint 0 +2023-04-06 16:00:28 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-06 16:00:28 - r - INFO: - show_fig 0 +2023-04-06 16:00:28 - r - INFO: - save_fig 1 +2023-04-06 16:00:28 - r - INFO: - n_workers 2 +2023-04-06 16:00:28 - r - INFO: - epsilon_start 0.95 +2023-04-06 16:00:28 - r - INFO: - epsilon_end 0.01 +2023-04-06 16:00:28 - r - INFO: - epsilon_decay 500 +2023-04-06 16:00:28 - r - INFO: - gamma 0.95 +2023-04-06 16:00:28 - r - INFO: - lr 0.0001 +2023-04-06 16:00:28 - r - INFO: - buffer_size 100000 +2023-04-06 16:00:28 - r - INFO: - batch_size 64 +2023-04-06 16:00:28 - r - INFO: - target_update 4 +2023-04-06 16:00:28 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-06 16:00:28 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-160028 +2023-04-06 16:00:28 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-160028/results +2023-04-06 16:00:28 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-160028/logs +2023-04-06 16:00:28 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-160028/traj +2023-04-06 16:00:28 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-160028/videos +2023-04-06 16:00:28 - r - INFO: - ================================================================================ +2023-04-06 16:00:28 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-06 16:00:28 - r - INFO: - Start training! +2023-04-06 16:00:28 - r - INFO: - Env: CartPole-v1, Algorithm: DoubleDQN, Device: cpu +2023-04-06 16:01:56 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/models/checkpoint.pth b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/models/checkpoint.pth new file mode 100644 index 0000000000000000000000000000000000000000..a145e71eceffd117546543c32d5e5214414c6eb2 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/models/checkpoint.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cd537f6ab4ac0ff27caa323076685e3a3fff04b064dbdfc509baae76e9a9406 +size 272407 diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..d11ab17669185bc5bd7055fa42c99635af0d424c Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/results/res.csv b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..d40ff05607ca48ad5e4d119a3bb9f3498c8a5440 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_mp_20230406-160028/results/res.csv @@ -0,0 +1,402 @@ +episodes,rewards +0,18.0 +1,39.0 +2,18.0 +3,28.0 +4,15.0 +5,12.0 +6,15.0 +7,40.0 +8,31.0 +9,17.0 +10,17.0 +11,31.0 +12,13.0 +13,14.0 +14,15.0 +15,21.0 +16,9.0 +17,10.0 +18,22.0 +19,19.0 +20,11.0 +21,13.0 +22,15.0 +23,20.0 +24,14.0 +25,12.0 +26,11.0 +27,10.0 +28,14.0 +29,12.0 +30,9.0 +31,10.0 +32,16.0 +33,13.0 +34,12.0 +35,15.0 +36,10.0 +37,17.0 +38,10.0 +39,10.0 +40,13.0 +41,12.0 +42,10.0 +43,12.0 +44,12.0 +45,12.0 +46,9.0 +47,10.0 +48,10.0 +49,11.0 +50,9.0 +51,13.0 +52,9.0 +53,10.0 +54,10.0 +55,11.0 +56,10.0 +57,10.0 +58,14.0 +59,9.0 +60,10.0 +61,10.0 +62,9.0 +63,12.0 +64,10.0 +65,13.0 +66,12.0 +67,15.0 +68,10.0 +69,13.0 +70,14.0 +71,10.0 +72,30.0 +73,11.0 +74,9.0 +75,11.0 +76,9.0 +77,11.0 +78,9.0 +79,11.0 +80,16.0 +81,21.0 +82,10.0 +83,9.0 +84,14.0 +85,9.0 +86,13.0 +87,9.0 +88,13.0 +89,17.0 +90,26.0 +91,32.0 +92,14.0 +93,17.0 +94,11.0 +95,34.0 +96,10.0 +97,23.0 +98,14.0 +99,37.0 +100,27.0 +101,34.0 +102,23.0 +103,59.0 +104,9.0 +105,91.0 +106,61.0 +107,47.0 +108,21.0 +109,27.0 +110,22.0 +111,30.0 +112,22.0 +113,29.0 +114,25.0 +115,68.0 +116,102.0 +117,54.0 +118,46.0 +119,34.0 +120,61.0 +121,81.0 +122,55.0 +123,67.0 +124,71.0 +125,46.0 +126,88.0 +127,90.0 +128,68.0 +129,114.0 +130,66.0 +131,102.0 +132,100.0 +133,88.0 +134,80.0 +135,81.0 +136,49.0 +137,123.0 +138,197.0 +139,146.0 +140,93.0 +141,135.0 +142,117.0 +143,104.0 +144,168.0 +145,114.0 +146,82.0 +147,153.0 +148,106.0 +149,140.0 +150,100.0 +151,120.0 +152,117.0 +153,173.0 +154,200.0 +155,142.0 +156,180.0 +157,156.0 +158,149.0 +159,173.0 +160,187.0 +161,200.0 +162,188.0 +163,156.0 +164,170.0 +165,158.0 +166,200.0 +167,152.0 +168,194.0 +169,196.0 +170,189.0 +171,200.0 +172,173.0 +173,200.0 +174,154.0 +175,200.0 +176,200.0 +177,200.0 +178,189.0 +179,194.0 +180,199.0 +181,200.0 +182,200.0 +183,189.0 +184,200.0 +185,200.0 +186,200.0 +187,200.0 +188,200.0 +189,200.0 +190,200.0 +191,200.0 +192,200.0 +193,200.0 +194,200.0 +195,189.0 +196,198.0 +197,195.0 +198,199.0 +199,200.0 +200,200.0 +201,200.0 +202,198.0 +203,196.0 +204,200.0 +205,200.0 +206,200.0 +207,200.0 +208,200.0 +209,200.0 +210,195.0 +211,198.0 +212,200.0 +213,200.0 +214,200.0 +215,200.0 +216,200.0 +217,194.0 +218,200.0 +219,200.0 +220,200.0 +221,200.0 +222,200.0 +223,200.0 +224,197.0 +225,200.0 +226,200.0 +227,200.0 +228,199.0 +229,200.0 +230,200.0 +231,198.0 +232,200.0 +233,200.0 +234,197.0 +235,200.0 +236,200.0 +237,200.0 +238,200.0 +239,196.0 +240,200.0 +241,200.0 +242,195.0 +243,200.0 +244,200.0 +245,200.0 +246,200.0 +247,200.0 +248,200.0 +249,200.0 +250,200.0 +251,200.0 +252,200.0 +253,200.0 +254,200.0 +255,199.0 +256,200.0 +257,200.0 +258,200.0 +259,200.0 +260,200.0 +261,200.0 +262,200.0 +263,200.0 +264,200.0 +265,200.0 +266,200.0 +267,200.0 +268,200.0 +269,200.0 +270,200.0 +271,200.0 +272,200.0 +273,200.0 +274,200.0 +275,200.0 +276,200.0 +277,200.0 +278,200.0 +279,200.0 +280,200.0 +281,200.0 +282,200.0 +283,200.0 +284,200.0 +285,200.0 +286,200.0 +287,200.0 +288,200.0 +289,200.0 +290,200.0 +291,200.0 +292,200.0 +293,200.0 +294,200.0 +295,200.0 +296,200.0 +297,200.0 +298,200.0 +299,200.0 +300,200.0 +301,200.0 +302,200.0 +303,200.0 +304,200.0 +305,200.0 +306,200.0 +307,200.0 +308,200.0 +309,200.0 +310,200.0 +311,200.0 +312,200.0 +313,200.0 +314,200.0 +315,200.0 +316,200.0 +317,200.0 +318,200.0 +319,200.0 +320,200.0 +321,200.0 +322,200.0 +323,200.0 +324,200.0 +325,200.0 +326,200.0 +327,200.0 +328,200.0 +329,200.0 +330,200.0 +331,200.0 +332,200.0 +333,200.0 +334,200.0 +335,200.0 +336,200.0 +337,200.0 +338,200.0 +339,200.0 +340,200.0 +341,200.0 +342,200.0 +343,200.0 +344,200.0 +345,200.0 +346,200.0 +347,200.0 +348,200.0 +349,200.0 +350,200.0 +351,200.0 +352,200.0 +353,200.0 +354,200.0 +355,200.0 +356,200.0 +357,200.0 +358,200.0 +359,200.0 +360,200.0 +361,200.0 +362,200.0 +363,200.0 +364,200.0 +365,200.0 +366,200.0 +367,200.0 +368,200.0 +369,200.0 +370,200.0 +371,200.0 +372,200.0 +373,200.0 +374,200.0 +375,200.0 +376,200.0 +377,200.0 +378,200.0 +379,200.0 +380,200.0 +381,200.0 +382,200.0 +383,200.0 +384,200.0 +385,200.0 +386,200.0 +387,200.0 +388,200.0 +389,200.0 +390,200.0 +391,200.0 +392,200.0 +393,200.0 +394,200.0 +395,200.0 +396,200.0 +397,200.0 +398,200.0 +399,200.0 +400,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/config.yaml b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..277b45a4dae8725ccfe44c5cddf3dff9276250d7 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/config.yaml @@ -0,0 +1,46 @@ +general_cfg: + algo_name: DoubleDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + mp_backend: ray + n_workers: 2 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/logs/log.txt b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..e526872af487ae5b47029e37d58ad3067e04dccb --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/logs/log.txt @@ -0,0 +1,42 @@ +2023-04-06 16:29:38 - r - INFO: - Hyperparameters: +2023-04-06 16:29:38 - r - INFO: - ================================================================================ +2023-04-06 16:29:38 - r - INFO: - Name Value Type +2023-04-06 16:29:38 - r - INFO: - env_name CartPole-v1 +2023-04-06 16:29:38 - r - INFO: - new_step_api 1 +2023-04-06 16:29:38 - r - INFO: - wrapper None +2023-04-06 16:29:38 - r - INFO: - render 0 +2023-04-06 16:29:38 - r - INFO: - render_mode human +2023-04-06 16:29:38 - r - INFO: - algo_name DoubleDQN +2023-04-06 16:29:38 - r - INFO: - mode train +2023-04-06 16:29:38 - r - INFO: - mp_backend ray +2023-04-06 16:29:38 - r - INFO: - seed 1 +2023-04-06 16:29:38 - r - INFO: - device cpu +2023-04-06 16:29:38 - r - INFO: - train_eps 400 +2023-04-06 16:29:38 - r - INFO: - test_eps 10 +2023-04-06 16:29:38 - r - INFO: - eval_eps 10 +2023-04-06 16:29:38 - r - INFO: - eval_per_episode 5 +2023-04-06 16:29:38 - r - INFO: - max_steps 200 +2023-04-06 16:29:38 - r - INFO: - load_checkpoint 0 +2023-04-06 16:29:38 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-06 16:29:38 - r - INFO: - show_fig 0 +2023-04-06 16:29:38 - r - INFO: - save_fig 1 +2023-04-06 16:29:38 - r - INFO: - n_workers 2 +2023-04-06 16:29:38 - r - INFO: - epsilon_start 0.95 +2023-04-06 16:29:38 - r - INFO: - epsilon_end 0.01 +2023-04-06 16:29:38 - r - INFO: - epsilon_decay 500 +2023-04-06 16:29:38 - r - INFO: - gamma 0.95 +2023-04-06 16:29:38 - r - INFO: - lr 0.0001 +2023-04-06 16:29:38 - r - INFO: - buffer_size 100000 +2023-04-06 16:29:38 - r - INFO: - batch_size 64 +2023-04-06 16:29:38 - r - INFO: - target_update 4 +2023-04-06 16:29:38 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-06 16:29:38 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-162938 +2023-04-06 16:29:38 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-162938/results +2023-04-06 16:29:38 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-162938/logs +2023-04-06 16:29:38 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-162938/traj +2023-04-06 16:29:38 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DoubleDQN_20230406-162938/videos +2023-04-06 16:29:38 - r - INFO: - ================================================================================ +2023-04-06 16:29:40 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-06 16:29:40 - r - INFO: - Start training! +2023-04-06 16:29:40 - r - INFO: - Env: CartPole-v1, Algorithm: DoubleDQN, Device: cpu +2023-04-06 16:37:19 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cfaf6fe119daae13b1e48521ac5f0a6dfd5731c --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3a724152cea263dec5b58d80bee101405e7b3268a34c265d414ebbd771c5ac +size 272407 diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..b8c37adadf6279152bb177b581261610b76abe42 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/results/res.csv b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..041045a06958699a5cb9542d0184a015b947d90b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DoubleDQN_ray_20230406-162938/results/res.csv @@ -0,0 +1,401 @@ +episodes,rewards +0,16.0 +1,18.0 +2,18.0 +3,28.0 +4,16.0 +5,15.0 +6,14.0 +7,15.0 +8,11.0 +9,49.0 +10,31.0 +11,31.0 +12,11.0 +13,14.0 +14,13.0 +15,21.0 +16,9.0 +17,28.0 +18,10.0 +19,19.0 +20,15.0 +21,12.0 +22,13.0 +23,18.0 +24,15.0 +25,14.0 +26,13.0 +27,11.0 +28,12.0 +29,14.0 +30,12.0 +31,10.0 +32,9.0 +33,13.0 +34,16.0 +35,17.0 +36,15.0 +37,12.0 +38,10.0 +39,18.0 +40,10.0 +41,11.0 +42,21.0 +43,16.0 +44,15.0 +45,19.0 +46,15.0 +47,20.0 +48,15.0 +49,14.0 +50,12.0 +51,10.0 +52,12.0 +53,11.0 +54,12.0 +55,10.0 +56,11.0 +57,9.0 +58,15.0 +59,10.0 +60,15.0 +61,11.0 +62,12.0 +63,15.0 +64,13.0 +65,11.0 +66,12.0 +67,12.0 +68,10.0 +69,11.0 +70,11.0 +71,9.0 +72,9.0 +73,11.0 +74,9.0 +75,10.0 +76,9.0 +77,10.0 +78,9.0 +79,10.0 +80,10.0 +81,9.0 +82,15.0 +83,10.0 +84,10.0 +85,11.0 +86,11.0 +87,13.0 +88,13.0 +89,9.0 +90,16.0 +91,12.0 +92,15.0 +93,9.0 +94,10.0 +95,10.0 +96,11.0 +97,11.0 +98,9.0 +99,12.0 +100,16.0 +101,10.0 +102,15.0 +103,9.0 +104,9.0 +105,10.0 +106,11.0 +107,10.0 +108,13.0 +109,11.0 +110,9.0 +111,14.0 +112,10.0 +113,12.0 +114,10.0 +115,10.0 +116,10.0 +117,13.0 +118,10.0 +119,11.0 +120,9.0 +121,11.0 +122,10.0 +123,10.0 +124,12.0 +125,41.0 +126,9.0 +127,31.0 +128,14.0 +129,14.0 +130,14.0 +131,11.0 +132,14.0 +133,12.0 +134,16.0 +135,11.0 +136,12.0 +137,16.0 +138,12.0 +139,14.0 +140,12.0 +141,18.0 +142,15.0 +143,18.0 +144,14.0 +145,14.0 +146,18.0 +147,17.0 +148,24.0 +149,13.0 +150,18.0 +151,15.0 +152,20.0 +153,17.0 +154,16.0 +155,17.0 +156,14.0 +157,26.0 +158,26.0 +159,42.0 +160,25.0 +161,58.0 +162,48.0 +163,48.0 +164,61.0 +165,115.0 +166,156.0 +167,56.0 +168,61.0 +169,83.0 +170,36.0 +171,47.0 +172,31.0 +173,27.0 +174,50.0 +175,34.0 +176,32.0 +177,49.0 +178,30.0 +179,50.0 +180,34.0 +181,27.0 +182,49.0 +183,35.0 +184,52.0 +185,35.0 +186,47.0 +187,50.0 +188,35.0 +189,54.0 +190,33.0 +191,50.0 +192,63.0 +193,121.0 +194,86.0 +195,46.0 +196,54.0 +197,42.0 +198,73.0 +199,45.0 +200,48.0 +201,72.0 +202,60.0 +203,96.0 +204,40.0 +205,46.0 +206,65.0 +207,84.0 +208,115.0 +209,78.0 +210,33.0 +211,40.0 +212,32.0 +213,39.0 +214,47.0 +215,37.0 +216,53.0 +217,37.0 +218,56.0 +219,36.0 +220,101.0 +221,105.0 +222,172.0 +223,116.0 +224,200.0 +225,162.0 +226,200.0 +227,200.0 +228,200.0 +229,200.0 +230,200.0 +231,200.0 +232,200.0 +233,200.0 +234,200.0 +235,200.0 +236,200.0 +237,200.0 +238,200.0 +239,200.0 +240,200.0 +241,200.0 +242,200.0 +243,200.0 +244,200.0 +245,200.0 +246,200.0 +247,200.0 +248,200.0 +249,200.0 +250,200.0 +251,200.0 +252,200.0 +253,200.0 +254,200.0 +255,200.0 +256,200.0 +257,200.0 +258,200.0 +259,200.0 +260,200.0 +261,200.0 +262,200.0 +263,200.0 +264,200.0 +265,200.0 +266,200.0 +267,200.0 +268,200.0 +269,200.0 +270,200.0 +271,200.0 +272,200.0 +273,200.0 +274,200.0 +275,200.0 +276,200.0 +277,200.0 +278,200.0 +279,200.0 +280,200.0 +281,200.0 +282,200.0 +283,200.0 +284,200.0 +285,200.0 +286,200.0 +287,200.0 +288,200.0 +289,199.0 +290,200.0 +291,190.0 +292,179.0 +293,189.0 +294,193.0 +295,200.0 +296,200.0 +297,200.0 +298,195.0 +299,200.0 +300,186.0 +301,175.0 +302,177.0 +303,185.0 +304,167.0 +305,172.0 +306,164.0 +307,146.0 +308,187.0 +309,150.0 +310,146.0 +311,165.0 +312,200.0 +313,200.0 +314,200.0 +315,200.0 +316,200.0 +317,158.0 +318,181.0 +319,174.0 +320,175.0 +321,176.0 +322,170.0 +323,161.0 +324,180.0 +325,200.0 +326,198.0 +327,179.0 +328,192.0 +329,157.0 +330,151.0 +331,198.0 +332,154.0 +333,165.0 +334,200.0 +335,179.0 +336,200.0 +337,191.0 +338,177.0 +339,200.0 +340,171.0 +341,200.0 +342,200.0 +343,200.0 +344,200.0 +345,200.0 +346,200.0 +347,163.0 +348,134.0 +349,200.0 +350,140.0 +351,200.0 +352,200.0 +353,139.0 +354,152.0 +355,136.0 +356,200.0 +357,200.0 +358,173.0 +359,200.0 +360,155.0 +361,134.0 +362,200.0 +363,186.0 +364,142.0 +365,200.0 +366,200.0 +367,128.0 +368,200.0 +369,200.0 +370,200.0 +371,200.0 +372,200.0 +373,200.0 +374,200.0 +375,142.0 +376,162.0 +377,180.0 +378,120.0 +379,190.0 +380,169.0 +381,125.0 +382,189.0 +383,158.0 +384,197.0 +385,200.0 +386,200.0 +387,139.0 +388,158.0 +389,165.0 +390,200.0 +391,200.0 +392,113.0 +393,115.0 +394,117.0 +395,119.0 +396,110.0 +397,119.0 +398,200.0 +399,133.0 diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/config.yaml b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a67fa1f30ac96e885331c757c96522fd98e0a3e --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/config.yaml @@ -0,0 +1,41 @@ +general_cfg: + algo_name: DuelingDQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/logs/log.txt b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..e7f22f9e0180b5a9c7236a7c7a3784c36f7c9cd7 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/logs/log.txt @@ -0,0 +1,119 @@ +2022-11-22 12:54:03 - r - INFO: - n_states: 4, n_actions: 2 +2022-11-22 12:54:06 - r - INFO: - Start training! +2022-11-22 12:54:06 - r - INFO: - Env: CartPole-v1, Algorithm: DuelingDQN, Device: cuda +2022-11-22 12:54:06 - r - INFO: - Episode: 1/100, Reward: 18.000, Step: 18 +2022-11-22 12:54:06 - r - INFO: - Episode: 2/100, Reward: 35.000, Step: 35 +2022-11-22 12:54:06 - r - INFO: - Episode: 3/100, Reward: 13.000, Step: 13 +2022-11-22 12:54:06 - r - INFO: - Episode: 4/100, Reward: 32.000, Step: 32 +2022-11-22 12:54:06 - r - INFO: - Episode: 5/100, Reward: 16.000, Step: 16 +2022-11-22 12:54:06 - r - INFO: - Current episode 5 has the best eval reward: 9.100 +2022-11-22 12:54:06 - r - INFO: - Episode: 6/100, Reward: 9.000, Step: 9 +2022-11-22 12:54:06 - r - INFO: - Episode: 7/100, Reward: 12.000, Step: 12 +2022-11-22 12:54:06 - r - INFO: - Episode: 8/100, Reward: 16.000, Step: 16 +2022-11-22 12:54:06 - r - INFO: - Episode: 9/100, Reward: 14.000, Step: 14 +2022-11-22 12:54:06 - r - INFO: - Episode: 10/100, Reward: 12.000, Step: 12 +2022-11-22 12:54:06 - r - INFO: - Current episode 10 has the best eval reward: 9.200 +2022-11-22 12:54:06 - r - INFO: - Episode: 11/100, Reward: 13.000, Step: 13 +2022-11-22 12:54:06 - r - INFO: - Episode: 12/100, Reward: 14.000, Step: 14 +2022-11-22 12:54:06 - r - INFO: - Episode: 13/100, Reward: 19.000, Step: 19 +2022-11-22 12:54:06 - r - INFO: - Episode: 14/100, Reward: 9.000, Step: 9 +2022-11-22 12:54:06 - r - INFO: - Episode: 15/100, Reward: 15.000, Step: 15 +2022-11-22 12:54:06 - r - INFO: - Current episode 15 has the best eval reward: 9.300 +2022-11-22 12:54:06 - r - INFO: - Episode: 16/100, Reward: 12.000, Step: 12 +2022-11-22 12:54:06 - r - INFO: - Episode: 17/100, Reward: 11.000, Step: 11 +2022-11-22 12:54:06 - r - INFO: - Episode: 18/100, Reward: 9.000, Step: 9 +2022-11-22 12:54:07 - r - INFO: - Episode: 19/100, Reward: 13.000, Step: 13 +2022-11-22 12:54:07 - r - INFO: - Episode: 20/100, Reward: 17.000, Step: 17 +2022-11-22 12:54:07 - r - INFO: - Current episode 20 has the best eval reward: 9.900 +2022-11-22 12:54:07 - r - INFO: - Episode: 21/100, Reward: 14.000, Step: 14 +2022-11-22 12:54:07 - r - INFO: - Episode: 22/100, Reward: 20.000, Step: 20 +2022-11-22 12:54:07 - r - INFO: - Episode: 23/100, Reward: 11.000, Step: 11 +2022-11-22 12:54:07 - r - INFO: - Episode: 24/100, Reward: 24.000, Step: 24 +2022-11-22 12:54:07 - r - INFO: - Episode: 25/100, Reward: 11.000, Step: 11 +2022-11-22 12:54:07 - r - INFO: - Episode: 26/100, Reward: 11.000, Step: 11 +2022-11-22 12:54:07 - r - INFO: - Episode: 27/100, Reward: 11.000, Step: 11 +2022-11-22 12:54:07 - r - INFO: - Episode: 28/100, Reward: 13.000, Step: 13 +2022-11-22 12:54:07 - r - INFO: - Episode: 29/100, Reward: 11.000, Step: 11 +2022-11-22 12:54:07 - r - INFO: - Episode: 30/100, Reward: 8.000, Step: 8 +2022-11-22 12:54:07 - r - INFO: - Episode: 31/100, Reward: 13.000, Step: 13 +2022-11-22 12:54:07 - r - INFO: - Episode: 32/100, Reward: 9.000, Step: 9 +2022-11-22 12:54:07 - r - INFO: - Episode: 33/100, Reward: 34.000, Step: 34 +2022-11-22 12:54:07 - r - INFO: - Episode: 34/100, Reward: 10.000, Step: 10 +2022-11-22 12:54:07 - r - INFO: - Episode: 35/100, Reward: 10.000, Step: 10 +2022-11-22 12:54:07 - r - INFO: - Episode: 36/100, Reward: 10.000, Step: 10 +2022-11-22 12:54:07 - r - INFO: - Episode: 37/100, Reward: 34.000, Step: 34 +2022-11-22 12:54:07 - r - INFO: - Episode: 38/100, Reward: 35.000, Step: 35 +2022-11-22 12:54:07 - r - INFO: - Episode: 39/100, Reward: 32.000, Step: 32 +2022-11-22 12:54:08 - r - INFO: - Episode: 40/100, Reward: 37.000, Step: 37 +2022-11-22 12:54:08 - r - INFO: - Current episode 40 has the best eval reward: 27.500 +2022-11-22 12:54:08 - r - INFO: - Episode: 41/100, Reward: 29.000, Step: 29 +2022-11-22 12:54:08 - r - INFO: - Episode: 42/100, Reward: 52.000, Step: 52 +2022-11-22 12:54:08 - r - INFO: - Episode: 43/100, Reward: 54.000, Step: 54 +2022-11-22 12:54:08 - r - INFO: - Episode: 44/100, Reward: 90.000, Step: 90 +2022-11-22 12:54:08 - r - INFO: - Episode: 45/100, Reward: 91.000, Step: 91 +2022-11-22 12:54:09 - r - INFO: - Current episode 45 has the best eval reward: 87.500 +2022-11-22 12:54:09 - r - INFO: - Episode: 46/100, Reward: 51.000, Step: 51 +2022-11-22 12:54:09 - r - INFO: - Episode: 47/100, Reward: 101.000, Step: 101 +2022-11-22 12:54:09 - r - INFO: - Episode: 48/100, Reward: 67.000, Step: 67 +2022-11-22 12:54:09 - r - INFO: - Episode: 49/100, Reward: 103.000, Step: 103 +2022-11-22 12:54:10 - r - INFO: - Episode: 50/100, Reward: 45.000, Step: 45 +2022-11-22 12:54:10 - r - INFO: - Episode: 51/100, Reward: 137.000, Step: 137 +2022-11-22 12:54:10 - r - INFO: - Episode: 52/100, Reward: 47.000, Step: 47 +2022-11-22 12:54:10 - r - INFO: - Episode: 53/100, Reward: 89.000, Step: 89 +2022-11-22 12:54:11 - r - INFO: - Episode: 54/100, Reward: 95.000, Step: 95 +2022-11-22 12:54:11 - r - INFO: - Episode: 55/100, Reward: 55.000, Step: 55 +2022-11-22 12:54:11 - r - INFO: - Episode: 56/100, Reward: 92.000, Step: 92 +2022-11-22 12:54:12 - r - INFO: - Episode: 57/100, Reward: 155.000, Step: 155 +2022-11-22 12:54:12 - r - INFO: - Episode: 58/100, Reward: 125.000, Step: 125 +2022-11-22 12:54:12 - r - INFO: - Episode: 59/100, Reward: 152.000, Step: 152 +2022-11-22 12:54:13 - r - INFO: - Episode: 60/100, Reward: 199.000, Step: 199 +2022-11-22 12:54:13 - r - INFO: - Current episode 60 has the best eval reward: 179.100 +2022-11-22 12:54:14 - r - INFO: - Episode: 61/100, Reward: 88.000, Step: 88 +2022-11-22 12:54:14 - r - INFO: - Episode: 62/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:14 - r - INFO: - Episode: 63/100, Reward: 176.000, Step: 176 +2022-11-22 12:54:15 - r - INFO: - Episode: 64/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:15 - r - INFO: - Episode: 65/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:16 - r - INFO: - Current episode 65 has the best eval reward: 198.700 +2022-11-22 12:54:16 - r - INFO: - Episode: 66/100, Reward: 193.000, Step: 193 +2022-11-22 12:54:17 - r - INFO: - Episode: 67/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:17 - r - INFO: - Episode: 68/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:18 - r - INFO: - Episode: 69/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:18 - r - INFO: - Episode: 70/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:19 - r - INFO: - Current episode 70 has the best eval reward: 200.000 +2022-11-22 12:54:20 - r - INFO: - Episode: 71/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:20 - r - INFO: - Episode: 72/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:20 - r - INFO: - Episode: 73/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:21 - r - INFO: - Episode: 74/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:21 - r - INFO: - Episode: 75/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:22 - r - INFO: - Current episode 75 has the best eval reward: 200.000 +2022-11-22 12:54:23 - r - INFO: - Episode: 76/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:23 - r - INFO: - Episode: 77/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:24 - r - INFO: - Episode: 78/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:24 - r - INFO: - Episode: 79/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:24 - r - INFO: - Episode: 80/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:25 - r - INFO: - Current episode 80 has the best eval reward: 200.000 +2022-11-22 12:54:26 - r - INFO: - Episode: 81/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:26 - r - INFO: - Episode: 82/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:27 - r - INFO: - Episode: 83/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:27 - r - INFO: - Episode: 84/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:27 - r - INFO: - Episode: 85/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:28 - r - INFO: - Current episode 85 has the best eval reward: 200.000 +2022-11-22 12:54:29 - r - INFO: - Episode: 86/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:29 - r - INFO: - Episode: 87/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:30 - r - INFO: - Episode: 88/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:30 - r - INFO: - Episode: 89/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:30 - r - INFO: - Episode: 90/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:31 - r - INFO: - Current episode 90 has the best eval reward: 200.000 +2022-11-22 12:54:32 - r - INFO: - Episode: 91/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:32 - r - INFO: - Episode: 92/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:33 - r - INFO: - Episode: 93/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:33 - r - INFO: - Episode: 94/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:34 - r - INFO: - Episode: 95/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:34 - r - INFO: - Current episode 95 has the best eval reward: 200.000 +2022-11-22 12:54:35 - r - INFO: - Episode: 96/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:35 - r - INFO: - Episode: 97/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:36 - r - INFO: - Episode: 98/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:36 - r - INFO: - Episode: 99/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:37 - r - INFO: - Episode: 100/100, Reward: 200.000, Step: 200 +2022-11-22 12:54:37 - r - INFO: - Current episode 100 has the best eval reward: 200.000 +2022-11-22 12:54:37 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f8f1bb2765fa7bb055a8236cbc3aa26d8825a98 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39a64f85a8848a1f4c937e359c4370458cf318c2bffbae406915ce7ee449dc50 +size 539719 diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..85e24583ce03fbb4f497cfffaf1cb2eb4637736b Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/results/res.csv b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..0785d1aa07110fa323de7c6ed268527fc42555f3 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_20221122-125403/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,18.0,18 +1,35.0,35 +2,13.0,13 +3,32.0,32 +4,16.0,16 +5,9.0,9 +6,12.0,12 +7,16.0,16 +8,14.0,14 +9,12.0,12 +10,13.0,13 +11,14.0,14 +12,19.0,19 +13,9.0,9 +14,15.0,15 +15,12.0,12 +16,11.0,11 +17,9.0,9 +18,13.0,13 +19,17.0,17 +20,14.0,14 +21,20.0,20 +22,11.0,11 +23,24.0,24 +24,11.0,11 +25,11.0,11 +26,11.0,11 +27,13.0,13 +28,11.0,11 +29,8.0,8 +30,13.0,13 +31,9.0,9 +32,34.0,34 +33,10.0,10 +34,10.0,10 +35,10.0,10 +36,34.0,34 +37,35.0,35 +38,32.0,32 +39,37.0,37 +40,29.0,29 +41,52.0,52 +42,54.0,54 +43,90.0,90 +44,91.0,91 +45,51.0,51 +46,101.0,101 +47,67.0,67 +48,103.0,103 +49,45.0,45 +50,137.0,137 +51,47.0,47 +52,89.0,89 +53,95.0,95 +54,55.0,55 +55,92.0,92 +56,155.0,155 +57,125.0,125 +58,152.0,152 +59,199.0,199 +60,88.0,88 +61,200.0,200 +62,176.0,176 +63,200.0,200 +64,200.0,200 +65,193.0,193 +66,200.0,200 +67,200.0,200 +68,200.0,200 +69,200.0,200 +70,200.0,200 +71,200.0,200 +72,200.0,200 +73,200.0,200 +74,200.0,200 +75,200.0,200 +76,200.0,200 +77,200.0,200 +78,200.0,200 +79,200.0,200 +80,200.0,200 +81,200.0,200 +82,200.0,200 +83,200.0,200 +84,200.0,200 +85,200.0,200 +86,200.0,200 +87,200.0,200 +88,200.0,200 +89,200.0,200 +90,200.0,200 +91,200.0,200 +92,200.0,200 +93,200.0,200 +94,200.0,200 +95,200.0,200 +96,200.0,200 +97,200.0,200 +98,200.0,200 +99,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/config.yaml b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..568a83fd1348cc893db157b052213bb314d1d4a3 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/config.yaml @@ -0,0 +1,47 @@ +general_cfg: + algo_name: DuelingDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + mp_backend: mp + n_workers: 4 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/logs/log.txt b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..48370a2194a38c23fce62f232cc066a9e00cb77c --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/logs/log.txt @@ -0,0 +1,43 @@ +2023-04-07 17:08:53 - r - INFO: - Hyperparameters: +2023-04-07 17:08:53 - r - INFO: - ================================================================================ +2023-04-07 17:08:53 - r - INFO: - Name Value Type +2023-04-07 17:08:53 - r - INFO: - env_name CartPole-v1 +2023-04-07 17:08:53 - r - INFO: - new_step_api 1 +2023-04-07 17:08:53 - r - INFO: - wrapper None +2023-04-07 17:08:53 - r - INFO: - render 0 +2023-04-07 17:08:53 - r - INFO: - render_mode human +2023-04-07 17:08:53 - r - INFO: - algo_name DuelingDQN +2023-04-07 17:08:53 - r - INFO: - mode train +2023-04-07 17:08:53 - r - INFO: - mp_backend mp +2023-04-07 17:08:53 - r - INFO: - seed 1 +2023-04-07 17:08:53 - r - INFO: - device cpu +2023-04-07 17:08:53 - r - INFO: - train_eps 200 +2023-04-07 17:08:53 - r - INFO: - test_eps 10 +2023-04-07 17:08:53 - r - INFO: - eval_eps 10 +2023-04-07 17:08:53 - r - INFO: - eval_per_episode 5 +2023-04-07 17:08:53 - r - INFO: - max_steps 200 +2023-04-07 17:08:53 - r - INFO: - load_checkpoint 0 +2023-04-07 17:08:53 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-07 17:08:53 - r - INFO: - show_fig 0 +2023-04-07 17:08:53 - r - INFO: - save_fig 1 +2023-04-07 17:08:53 - r - INFO: - n_workers 4 +2023-04-07 17:08:53 - r - INFO: - epsilon_start 0.95 +2023-04-07 17:08:53 - r - INFO: - epsilon_end 0.01 +2023-04-07 17:08:53 - r - INFO: - epsilon_decay 500 +2023-04-07 17:08:53 - r - INFO: - gamma 0.99 +2023-04-07 17:08:53 - r - INFO: - lr 0.0001 +2023-04-07 17:08:53 - r - INFO: - buffer_size 100000 +2023-04-07 17:08:53 - r - INFO: - batch_size 64 +2023-04-07 17:08:53 - r - INFO: - target_update 4 +2023-04-07 17:08:53 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-07 17:08:53 - r - INFO: - hidden_dim 256 +2023-04-07 17:08:53 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-170853 +2023-04-07 17:08:53 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-170853/results +2023-04-07 17:08:53 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-170853/logs +2023-04-07 17:08:53 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-170853/traj +2023-04-07 17:08:53 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-170853/videos +2023-04-07 17:08:53 - r - INFO: - ================================================================================ +2023-04-07 17:08:53 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-07 17:08:53 - r - INFO: - Start training! +2023-04-07 17:08:53 - r - INFO: - Env: CartPole-v1, Algorithm: DuelingDQN, Device: cpu +2023-04-07 17:10:11 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f8c7f24c0c9fc6d4a576d479507b204876171ea --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87aab291b33d6423c3c54eced436183398700a290427c1913be9d65f5503b5ae +size 537607 diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..8929d278ea2794d0b5f2a947e6a4611037515262 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/results/res.csv b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..e5886e684e410c0116bce001ca27f99855ce62e7 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_mp_20230407-170853/results/res.csv @@ -0,0 +1,202 @@ +episodes,rewards +0,17.0 +1,16.0 +2,24.0 +3,38.0 +4,18.0 +5,18.0 +6,19.0 +7,17.0 +8,27.0 +9,16.0 +10,15.0 +11,31.0 +12,33.0 +13,13.0 +14,9.0 +15,34.0 +16,32.0 +17,34.0 +18,12.0 +19,13.0 +20,11.0 +21,9.0 +22,9.0 +23,11.0 +24,38.0 +25,13.0 +26,20.0 +27,10.0 +28,33.0 +29,15.0 +30,32.0 +31,11.0 +32,13.0 +33,12.0 +34,9.0 +35,13.0 +36,14.0 +37,21.0 +38,48.0 +39,19.0 +40,65.0 +41,28.0 +42,15.0 +43,11.0 +44,54.0 +45,26.0 +46,37.0 +47,40.0 +48,54.0 +49,54.0 +50,50.0 +51,84.0 +52,55.0 +53,43.0 +54,45.0 +55,48.0 +56,88.0 +57,41.0 +58,46.0 +59,61.0 +60,32.0 +61,53.0 +62,59.0 +63,49.0 +64,60.0 +65,35.0 +66,82.0 +67,50.0 +68,108.0 +69,121.0 +70,113.0 +71,67.0 +72,87.0 +73,96.0 +74,181.0 +75,62.0 +76,137.0 +77,175.0 +78,123.0 +79,149.0 +80,172.0 +81,200.0 +82,156.0 +83,146.0 +84,200.0 +85,200.0 +86,200.0 +87,128.0 +88,188.0 +89,200.0 +90,200.0 +91,200.0 +92,200.0 +93,200.0 +94,200.0 +95,200.0 +96,200.0 +97,200.0 +98,195.0 +99,200.0 +100,200.0 +101,196.0 +102,200.0 +103,200.0 +104,192.0 +105,200.0 +106,190.0 +107,200.0 +108,200.0 +109,200.0 +110,197.0 +111,200.0 +112,200.0 +113,200.0 +114,200.0 +115,200.0 +116,200.0 +117,200.0 +118,200.0 +119,200.0 +120,200.0 +121,200.0 +122,200.0 +123,200.0 +124,200.0 +125,200.0 +126,200.0 +127,200.0 +128,200.0 +129,200.0 +130,200.0 +131,200.0 +132,200.0 +133,200.0 +134,200.0 +135,200.0 +136,200.0 +137,197.0 +138,200.0 +139,200.0 +140,200.0 +141,200.0 +142,200.0 +143,200.0 +144,21.0 +145,193.0 +146,123.0 +147,194.0 +148,9.0 +149,9.0 +150,48.0 +151,200.0 +152,200.0 +153,200.0 +154,200.0 +155,200.0 +156,200.0 +157,200.0 +158,200.0 +159,200.0 +160,200.0 +161,200.0 +162,200.0 +163,200.0 +164,200.0 +165,200.0 +166,200.0 +167,200.0 +168,200.0 +169,200.0 +170,200.0 +171,200.0 +172,200.0 +173,200.0 +174,200.0 +175,200.0 +176,200.0 +177,200.0 +178,200.0 +179,200.0 +180,200.0 +181,200.0 +182,200.0 +183,200.0 +184,200.0 +185,200.0 +186,200.0 +187,200.0 +188,200.0 +189,200.0 +190,200.0 +191,200.0 +192,200.0 +193,200.0 +194,200.0 +195,200.0 +196,200.0 +197,200.0 +198,200.0 +199,200.0 +200,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/config.yaml b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfb5a718011cda7f7c2d8c08ec0563c432ad955d --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/config.yaml @@ -0,0 +1,47 @@ +general_cfg: + algo_name: DuelingDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + mp_backend: ray + n_workers: 2 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/logs/log.txt b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c6331b3bfb85948bc7c3d56ce3596e8c4158760 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/logs/log.txt @@ -0,0 +1,43 @@ +2023-04-07 15:32:36 - r - INFO: - Hyperparameters: +2023-04-07 15:32:36 - r - INFO: - ================================================================================ +2023-04-07 15:32:36 - r - INFO: - Name Value Type +2023-04-07 15:32:36 - r - INFO: - env_name CartPole-v1 +2023-04-07 15:32:36 - r - INFO: - new_step_api 1 +2023-04-07 15:32:36 - r - INFO: - wrapper None +2023-04-07 15:32:36 - r - INFO: - render 0 +2023-04-07 15:32:36 - r - INFO: - render_mode human +2023-04-07 15:32:36 - r - INFO: - algo_name DuelingDQN +2023-04-07 15:32:36 - r - INFO: - mode train +2023-04-07 15:32:36 - r - INFO: - mp_backend ray +2023-04-07 15:32:36 - r - INFO: - seed 1 +2023-04-07 15:32:36 - r - INFO: - device cpu +2023-04-07 15:32:36 - r - INFO: - train_eps 400 +2023-04-07 15:32:36 - r - INFO: - test_eps 10 +2023-04-07 15:32:36 - r - INFO: - eval_eps 10 +2023-04-07 15:32:36 - r - INFO: - eval_per_episode 5 +2023-04-07 15:32:36 - r - INFO: - max_steps 200 +2023-04-07 15:32:36 - r - INFO: - load_checkpoint 0 +2023-04-07 15:32:36 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-07 15:32:36 - r - INFO: - show_fig 0 +2023-04-07 15:32:36 - r - INFO: - save_fig 1 +2023-04-07 15:32:36 - r - INFO: - n_workers 2 +2023-04-07 15:32:36 - r - INFO: - epsilon_start 0.95 +2023-04-07 15:32:36 - r - INFO: - epsilon_end 0.01 +2023-04-07 15:32:36 - r - INFO: - epsilon_decay 500 +2023-04-07 15:32:36 - r - INFO: - gamma 0.99 +2023-04-07 15:32:36 - r - INFO: - lr 0.0001 +2023-04-07 15:32:36 - r - INFO: - buffer_size 100000 +2023-04-07 15:32:36 - r - INFO: - batch_size 64 +2023-04-07 15:32:36 - r - INFO: - target_update 4 +2023-04-07 15:32:36 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-07 15:32:36 - r - INFO: - hidden_dim 256 +2023-04-07 15:32:36 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-153236 +2023-04-07 15:32:36 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-153236/results +2023-04-07 15:32:36 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-153236/logs +2023-04-07 15:32:36 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-153236/traj +2023-04-07 15:32:36 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DuelingDQN_20230407-153236/videos +2023-04-07 15:32:36 - r - INFO: - ================================================================================ +2023-04-07 15:32:39 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-07 15:32:39 - r - INFO: - Start training! +2023-04-07 15:32:39 - r - INFO: - Env: CartPole-v1, Algorithm: DuelingDQN, Device: cpu +2023-04-07 15:40:31 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..d64126c7ac0958c7567fbb3cc692d7d11ed40efc --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f1262598e3d636dd22e3b2fc0dfe52bf7a55348d54f51f02a8410682ec5a18 +size 537607 diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..cfdfafd18636c2948a7b26bfba0f154ba7391cd7 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/results/res.csv b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..7526f7040d7073e2ac8d4fb26a106b41cd50d186 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_DuelingDQN_ray_20230407-153236/results/res.csv @@ -0,0 +1,401 @@ +episodes,rewards +0,18.0 +1,18.0 +2,19.0 +3,28.0 +4,17.0 +5,15.0 +6,13.0 +7,15.0 +8,38.0 +9,31.0 +10,11.0 +11,31.0 +12,10.0 +13,17.0 +14,14.0 +15,13.0 +16,21.0 +17,15.0 +18,9.0 +19,10.0 +20,22.0 +21,19.0 +22,11.0 +23,13.0 +24,20.0 +25,15.0 +26,14.0 +27,12.0 +28,10.0 +29,11.0 +30,12.0 +31,14.0 +32,9.0 +33,10.0 +34,16.0 +35,13.0 +36,15.0 +37,12.0 +38,14.0 +39,10.0 +40,14.0 +41,10.0 +42,11.0 +43,16.0 +44,16.0 +45,12.0 +46,15.0 +47,19.0 +48,15.0 +49,20.0 +50,15.0 +51,11.0 +52,13.0 +53,12.0 +54,12.0 +55,12.0 +56,12.0 +57,12.0 +58,11.0 +59,10.0 +60,13.0 +61,11.0 +62,12.0 +63,9.0 +64,11.0 +65,11.0 +66,10.0 +67,9.0 +68,11.0 +69,11.0 +70,11.0 +71,12.0 +72,10.0 +73,12.0 +74,9.0 +75,10.0 +76,9.0 +77,10.0 +78,9.0 +79,10.0 +80,11.0 +81,9.0 +82,12.0 +83,11.0 +84,12.0 +85,10.0 +86,9.0 +87,11.0 +88,9.0 +89,9.0 +90,10.0 +91,15.0 +92,11.0 +93,9.0 +94,10.0 +95,16.0 +96,13.0 +97,9.0 +98,10.0 +99,10.0 +100,13.0 +101,11.0 +102,10.0 +103,9.0 +104,13.0 +105,16.0 +106,12.0 +107,9.0 +108,11.0 +109,9.0 +110,13.0 +111,11.0 +112,18.0 +113,13.0 +114,9.0 +115,12.0 +116,10.0 +117,10.0 +118,10.0 +119,13.0 +120,10.0 +121,11.0 +122,10.0 +123,10.0 +124,9.0 +125,10.0 +126,11.0 +127,14.0 +128,12.0 +129,9.0 +130,11.0 +131,14.0 +132,11.0 +133,10.0 +134,13.0 +135,9.0 +136,11.0 +137,11.0 +138,11.0 +139,9.0 +140,10.0 +141,9.0 +142,9.0 +143,12.0 +144,9.0 +145,10.0 +146,9.0 +147,10.0 +148,9.0 +149,10.0 +150,9.0 +151,12.0 +152,9.0 +153,9.0 +154,10.0 +155,9.0 +156,10.0 +157,13.0 +158,14.0 +159,10.0 +160,12.0 +161,11.0 +162,10.0 +163,11.0 +164,11.0 +165,9.0 +166,31.0 +167,39.0 +168,18.0 +169,24.0 +170,18.0 +171,18.0 +172,24.0 +173,16.0 +174,25.0 +175,23.0 +176,26.0 +177,23.0 +178,26.0 +179,21.0 +180,28.0 +181,20.0 +182,22.0 +183,30.0 +184,27.0 +185,34.0 +186,31.0 +187,39.0 +188,29.0 +189,29.0 +190,37.0 +191,27.0 +192,36.0 +193,34.0 +194,46.0 +195,35.0 +196,52.0 +197,32.0 +198,30.0 +199,69.0 +200,38.0 +201,39.0 +202,57.0 +203,38.0 +204,68.0 +205,47.0 +206,45.0 +207,63.0 +208,47.0 +209,86.0 +210,67.0 +211,60.0 +212,48.0 +213,55.0 +214,95.0 +215,58.0 +216,70.0 +217,58.0 +218,42.0 +219,69.0 +220,47.0 +221,109.0 +222,70.0 +223,80.0 +224,77.0 +225,61.0 +226,72.0 +227,55.0 +228,77.0 +229,61.0 +230,79.0 +231,66.0 +232,68.0 +233,99.0 +234,143.0 +235,82.0 +236,85.0 +237,103.0 +238,99.0 +239,93.0 +240,100.0 +241,101.0 +242,151.0 +243,195.0 +244,100.0 +245,99.0 +246,127.0 +247,105.0 +248,127.0 +249,142.0 +250,169.0 +251,108.0 +252,128.0 +253,123.0 +254,134.0 +255,126.0 +256,114.0 +257,200.0 +258,123.0 +259,159.0 +260,125.0 +261,142.0 +262,178.0 +263,96.0 +264,200.0 +265,200.0 +266,113.0 +267,90.0 +268,200.0 +269,122.0 +270,140.0 +271,116.0 +272,128.0 +273,190.0 +274,170.0 +275,96.0 +276,126.0 +277,200.0 +278,88.0 +279,76.0 +280,74.0 +281,84.0 +282,130.0 +283,200.0 +284,86.0 +285,153.0 +286,200.0 +287,59.0 +288,135.0 +289,62.0 +290,200.0 +291,182.0 +292,138.0 +293,200.0 +294,118.0 +295,50.0 +296,74.0 +297,62.0 +298,200.0 +299,124.0 +300,111.0 +301,61.0 +302,132.0 +303,200.0 +304,80.0 +305,60.0 +306,77.0 +307,47.0 +308,80.0 +309,64.0 +310,96.0 +311,200.0 +312,200.0 +313,133.0 +314,200.0 +315,188.0 +316,132.0 +317,150.0 +318,135.0 +319,184.0 +320,138.0 +321,176.0 +322,200.0 +323,161.0 +324,158.0 +325,142.0 +326,133.0 +327,151.0 +328,143.0 +329,160.0 +330,150.0 +331,134.0 +332,147.0 +333,132.0 +334,143.0 +335,137.0 +336,155.0 +337,138.0 +338,138.0 +339,130.0 +340,148.0 +341,146.0 +342,152.0 +343,135.0 +344,175.0 +345,153.0 +346,155.0 +347,131.0 +348,156.0 +349,138.0 +350,151.0 +351,162.0 +352,200.0 +353,175.0 +354,156.0 +355,145.0 +356,168.0 +357,200.0 +358,181.0 +359,145.0 +360,189.0 +361,200.0 +362,144.0 +363,200.0 +364,178.0 +365,200.0 +366,179.0 +367,200.0 +368,177.0 +369,200.0 +370,185.0 +371,195.0 +372,200.0 +373,200.0 +374,190.0 +375,200.0 +376,200.0 +377,200.0 +378,200.0 +379,200.0 +380,200.0 +381,200.0 +382,170.0 +383,173.0 +384,162.0 +385,162.0 +386,149.0 +387,173.0 +388,200.0 +389,200.0 +390,200.0 +391,156.0 +392,157.0 +393,169.0 +394,182.0 +395,154.0 +396,200.0 +397,200.0 +398,200.0 +399,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/config.yaml b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bfa05f3ecbd1d8ba44bbf3a9dc6207f2ce39454 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/config.yaml @@ -0,0 +1,39 @@ +general_cfg: + adversarial_batch_size: 128 + adversarial_epochs: 5 + algo_name: GAIL + batch_size: 2048 + device: cuda + discount: 0.99 + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + hidden_dim: 32 + imitation_replay_size: 4 + load_checkpoint: false + load_path: Collect_CartPole-v1_PPO_20221206-173222 + lr: 0.001 + max_steps: 200 + mode: train + new_step_api: true + num_workers: 8 + ppo_epochs: 5 + r1_reg_coeff: 1 + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 1500 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + batch_size: 2048 + critic_hidden_dim: 256 + critic_lr: 0.0003 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + update_freq: 2048 diff --git a/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/logs/log.txt b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..02e1f8293ff637a767f45a8de5948ff4d0f0b180 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/logs/log.txt @@ -0,0 +1,1588 @@ +2022-12-07 16:09:45 - r - INFO: - Hyperparameters: +2022-12-07 16:09:45 - r - INFO: - ================================================================================ +2022-12-07 16:09:45 - r - INFO: - Name Value Type +2022-12-07 16:09:45 - r - INFO: - env_name CartPole-v1 +2022-12-07 16:09:45 - r - INFO: - new_step_api 1 +2022-12-07 16:09:45 - r - INFO: - wrapper None +2022-12-07 16:09:45 - r - INFO: - render 0 +2022-12-07 16:09:45 - r - INFO: - algo_name GAIL +2022-12-07 16:09:45 - r - INFO: - mode train +2022-12-07 16:09:45 - r - INFO: - seed 1 +2022-12-07 16:09:45 - r - INFO: - device cuda +2022-12-07 16:09:45 - r - INFO: - train_eps 1500 +2022-12-07 16:09:45 - r - INFO: - test_eps 10 +2022-12-07 16:09:45 - r - INFO: - eval_eps 10 +2022-12-07 16:09:45 - r - INFO: - eval_per_episode 5 +2022-12-07 16:09:45 - r - INFO: - max_steps 200 +2022-12-07 16:09:45 - r - INFO: - load_checkpoint 0 +2022-12-07 16:09:45 - r - INFO: - load_path Collect_CartPole-v1_PPO_20221206-173222 +2022-12-07 16:09:45 - r - INFO: - show_fig 0 +2022-12-07 16:09:45 - r - INFO: - save_fig 1 +2022-12-07 16:09:45 - r - INFO: - adversarial_batch_size 128 +2022-12-07 16:09:45 - r - INFO: - adversarial_epochs 5 +2022-12-07 16:09:45 - r - INFO: - batch_size 2048 +2022-12-07 16:09:45 - r - INFO: - discount 0.99 +2022-12-07 16:09:45 - r - INFO: - num_workers 8 +2022-12-07 16:09:45 - r - INFO: - hidden_dim 32 +2022-12-07 16:09:45 - r - INFO: - lr 0.001 +2022-12-07 16:09:45 - r - INFO: - imitation_replay_size 4 +2022-12-07 16:09:45 - r - INFO: - r1_reg_coeff 1 +2022-12-07 16:09:45 - r - INFO: - ppo_epochs 5 +2022-12-07 16:09:45 - r - INFO: - gamma 0.99 +2022-12-07 16:09:45 - r - INFO: - k_epochs 4 +2022-12-07 16:09:45 - r - INFO: - actor_lr 0.0003 +2022-12-07 16:09:45 - r - INFO: - critic_lr 0.0003 +2022-12-07 16:09:45 - r - INFO: - eps_clip 0.2 +2022-12-07 16:09:45 - r - INFO: - entropy_coef 0.01 +2022-12-07 16:09:45 - r - INFO: - update_freq 2048 +2022-12-07 16:09:45 - r - INFO: - actor_hidden_dim 256 +2022-12-07 16:09:45 - r - INFO: - critic_hidden_dim 256 +2022-12-07 16:09:45 - r - INFO: - ================================================================================ +2022-12-07 16:09:46 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-07 16:09:46 - r - INFO: - Start training! +2022-12-07 16:09:46 - r - INFO: - Env: CartPole-v1, Algorithm: GAIL, Device: cuda +2022-12-07 16:09:48 - r - INFO: - Episode: 1/1500, Reward: 25.000, Step: 25 +2022-12-07 16:09:48 - r - INFO: - Episode: 2/1500, Reward: 15.000, Step: 15 +2022-12-07 16:09:48 - r - INFO: - Episode: 3/1500, Reward: 13.000, Step: 13 +2022-12-07 16:09:48 - r - INFO: - Episode: 4/1500, Reward: 14.000, Step: 14 +2022-12-07 16:09:48 - r - INFO: - Episode: 5/1500, Reward: 25.000, Step: 25 +2022-12-07 16:09:49 - r - INFO: - Current episode 5 has the best eval reward: 23.900 +2022-12-07 16:09:49 - r - INFO: - Episode: 6/1500, Reward: 41.000, Step: 41 +2022-12-07 16:09:49 - r - INFO: - Episode: 7/1500, Reward: 33.000, Step: 33 +2022-12-07 16:09:49 - r - INFO: - Episode: 8/1500, Reward: 12.000, Step: 12 +2022-12-07 16:09:49 - r - INFO: - Episode: 9/1500, Reward: 20.000, Step: 20 +2022-12-07 16:09:49 - r - INFO: - Episode: 10/1500, Reward: 33.000, Step: 33 +2022-12-07 16:09:49 - r - INFO: - Current episode 10 has the best eval reward: 27.400 +2022-12-07 16:09:49 - r - INFO: - Episode: 11/1500, Reward: 19.000, Step: 19 +2022-12-07 16:09:50 - r - INFO: - Episode: 12/1500, Reward: 35.000, Step: 35 +2022-12-07 16:09:50 - r - INFO: - Episode: 13/1500, Reward: 22.000, Step: 22 +2022-12-07 16:09:50 - r - INFO: - Episode: 14/1500, Reward: 9.000, Step: 9 +2022-12-07 16:09:50 - r - INFO: - Episode: 15/1500, Reward: 17.000, Step: 17 +2022-12-07 16:09:50 - r - INFO: - Episode: 16/1500, Reward: 36.000, Step: 36 +2022-12-07 16:09:50 - r - INFO: - Episode: 17/1500, Reward: 16.000, Step: 16 +2022-12-07 16:09:50 - r - INFO: - Episode: 18/1500, Reward: 18.000, Step: 18 +2022-12-07 16:09:50 - r - INFO: - Episode: 19/1500, Reward: 34.000, Step: 34 +2022-12-07 16:09:50 - r - INFO: - Episode: 20/1500, Reward: 11.000, Step: 11 +2022-12-07 16:09:51 - r - INFO: - Episode: 21/1500, Reward: 21.000, Step: 21 +2022-12-07 16:09:51 - r - INFO: - Episode: 22/1500, Reward: 16.000, Step: 16 +2022-12-07 16:09:51 - r - INFO: - Episode: 23/1500, Reward: 17.000, Step: 17 +2022-12-07 16:09:51 - r - INFO: - Episode: 24/1500, Reward: 13.000, Step: 13 +2022-12-07 16:09:51 - r - INFO: - Episode: 25/1500, Reward: 20.000, Step: 20 +2022-12-07 16:09:51 - r - INFO: - Current episode 25 has the best eval reward: 30.400 +2022-12-07 16:09:51 - r - INFO: - Episode: 26/1500, Reward: 18.000, Step: 18 +2022-12-07 16:09:51 - r - INFO: - Episode: 27/1500, Reward: 18.000, Step: 18 +2022-12-07 16:09:51 - r - INFO: - Episode: 28/1500, Reward: 28.000, Step: 28 +2022-12-07 16:09:51 - r - INFO: - Episode: 29/1500, Reward: 12.000, Step: 12 +2022-12-07 16:09:51 - r - INFO: - Episode: 30/1500, Reward: 17.000, Step: 17 +2022-12-07 16:09:52 - r - INFO: - Episode: 31/1500, Reward: 10.000, Step: 10 +2022-12-07 16:09:52 - r - INFO: - Episode: 32/1500, Reward: 56.000, Step: 56 +2022-12-07 16:09:52 - r - INFO: - Episode: 33/1500, Reward: 45.000, Step: 45 +2022-12-07 16:09:52 - r - INFO: - Episode: 34/1500, Reward: 11.000, Step: 11 +2022-12-07 16:09:52 - r - INFO: - Episode: 35/1500, Reward: 19.000, Step: 19 +2022-12-07 16:09:52 - r - INFO: - Episode: 36/1500, Reward: 13.000, Step: 13 +2022-12-07 16:09:52 - r - INFO: - Episode: 37/1500, Reward: 30.000, Step: 30 +2022-12-07 16:09:52 - r - INFO: - Episode: 38/1500, Reward: 13.000, Step: 13 +2022-12-07 16:09:53 - r - INFO: - Episode: 39/1500, Reward: 21.000, Step: 21 +2022-12-07 16:09:53 - r - INFO: - Episode: 40/1500, Reward: 15.000, Step: 15 +2022-12-07 16:09:53 - r - INFO: - Episode: 41/1500, Reward: 12.000, Step: 12 +2022-12-07 16:09:53 - r - INFO: - Episode: 42/1500, Reward: 21.000, Step: 21 +2022-12-07 16:09:53 - r - INFO: - Episode: 43/1500, Reward: 9.000, Step: 9 +2022-12-07 16:09:53 - r - INFO: - Episode: 44/1500, Reward: 26.000, Step: 26 +2022-12-07 16:09:53 - r - INFO: - Episode: 45/1500, Reward: 19.000, Step: 19 +2022-12-07 16:09:53 - r - INFO: - Episode: 46/1500, Reward: 12.000, Step: 12 +2022-12-07 16:09:54 - r - INFO: - Episode: 47/1500, Reward: 14.000, Step: 14 +2022-12-07 16:09:54 - r - INFO: - Episode: 48/1500, Reward: 9.000, Step: 9 +2022-12-07 16:09:54 - r - INFO: - Episode: 49/1500, Reward: 21.000, Step: 21 +2022-12-07 16:09:54 - r - INFO: - Episode: 50/1500, Reward: 10.000, Step: 10 +2022-12-07 16:09:54 - r - INFO: - Episode: 51/1500, Reward: 20.000, Step: 20 +2022-12-07 16:09:54 - r - INFO: - Episode: 52/1500, Reward: 20.000, Step: 20 +2022-12-07 16:09:54 - r - INFO: - Episode: 53/1500, Reward: 35.000, Step: 35 +2022-12-07 16:09:54 - r - INFO: - Episode: 54/1500, Reward: 28.000, Step: 28 +2022-12-07 16:09:54 - r - INFO: - Episode: 55/1500, Reward: 50.000, Step: 50 +2022-12-07 16:09:54 - r - INFO: - Episode: 56/1500, Reward: 29.000, Step: 29 +2022-12-07 16:09:54 - r - INFO: - Episode: 57/1500, Reward: 10.000, Step: 10 +2022-12-07 16:09:54 - r - INFO: - Episode: 58/1500, Reward: 15.000, Step: 15 +2022-12-07 16:09:55 - r - INFO: - Episode: 59/1500, Reward: 20.000, Step: 20 +2022-12-07 16:09:55 - r - INFO: - Episode: 60/1500, Reward: 66.000, Step: 66 +2022-12-07 16:09:55 - r - INFO: - Episode: 61/1500, Reward: 38.000, Step: 38 +2022-12-07 16:09:55 - r - INFO: - Episode: 62/1500, Reward: 19.000, Step: 19 +2022-12-07 16:09:55 - r - INFO: - Episode: 63/1500, Reward: 39.000, Step: 39 +2022-12-07 16:09:55 - r - INFO: - Episode: 64/1500, Reward: 67.000, Step: 67 +2022-12-07 16:09:55 - r - INFO: - Episode: 65/1500, Reward: 13.000, Step: 13 +2022-12-07 16:09:56 - r - INFO: - Episode: 66/1500, Reward: 18.000, Step: 18 +2022-12-07 16:09:56 - r - INFO: - Episode: 67/1500, Reward: 22.000, Step: 22 +2022-12-07 16:09:56 - r - INFO: - Episode: 68/1500, Reward: 16.000, Step: 16 +2022-12-07 16:09:56 - r - INFO: - Episode: 69/1500, Reward: 21.000, Step: 21 +2022-12-07 16:09:56 - r - INFO: - Episode: 70/1500, Reward: 11.000, Step: 11 +2022-12-07 16:09:56 - r - INFO: - Episode: 71/1500, Reward: 42.000, Step: 42 +2022-12-07 16:09:56 - r - INFO: - Episode: 72/1500, Reward: 15.000, Step: 15 +2022-12-07 16:09:56 - r - INFO: - Episode: 73/1500, Reward: 45.000, Step: 45 +2022-12-07 16:09:56 - r - INFO: - Episode: 74/1500, Reward: 17.000, Step: 17 +2022-12-07 16:09:56 - r - INFO: - Episode: 75/1500, Reward: 35.000, Step: 35 +2022-12-07 16:09:57 - r - INFO: - Episode: 76/1500, Reward: 18.000, Step: 18 +2022-12-07 16:09:57 - r - INFO: - Episode: 77/1500, Reward: 26.000, Step: 26 +2022-12-07 16:09:57 - r - INFO: - Episode: 78/1500, Reward: 19.000, Step: 19 +2022-12-07 16:09:57 - r - INFO: - Episode: 79/1500, Reward: 13.000, Step: 13 +2022-12-07 16:09:57 - r - INFO: - Episode: 80/1500, Reward: 11.000, Step: 11 +2022-12-07 16:09:57 - r - INFO: - Episode: 81/1500, Reward: 19.000, Step: 19 +2022-12-07 16:09:57 - r - INFO: - Episode: 82/1500, Reward: 18.000, Step: 18 +2022-12-07 16:09:57 - r - INFO: - Episode: 83/1500, Reward: 73.000, Step: 73 +2022-12-07 16:09:58 - r - INFO: - Episode: 84/1500, Reward: 35.000, Step: 35 +2022-12-07 16:09:58 - r - INFO: - Episode: 85/1500, Reward: 15.000, Step: 15 +2022-12-07 16:09:58 - r - INFO: - Episode: 86/1500, Reward: 20.000, Step: 20 +2022-12-07 16:09:58 - r - INFO: - Episode: 87/1500, Reward: 19.000, Step: 19 +2022-12-07 16:09:58 - r - INFO: - Episode: 88/1500, Reward: 16.000, Step: 16 +2022-12-07 16:09:58 - r - INFO: - Episode: 89/1500, Reward: 18.000, Step: 18 +2022-12-07 16:10:34 - r - INFO: - Episode: 90/1500, Reward: 17.000, Step: 17 +2022-12-07 16:10:34 - r - INFO: - Episode: 91/1500, Reward: 35.000, Step: 35 +2022-12-07 16:10:34 - r - INFO: - Episode: 92/1500, Reward: 27.000, Step: 27 +2022-12-07 16:10:34 - r - INFO: - Episode: 93/1500, Reward: 9.000, Step: 9 +2022-12-07 16:10:34 - r - INFO: - Episode: 94/1500, Reward: 46.000, Step: 46 +2022-12-07 16:10:34 - r - INFO: - Episode: 95/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:34 - r - INFO: - Episode: 96/1500, Reward: 22.000, Step: 22 +2022-12-07 16:10:34 - r - INFO: - Episode: 97/1500, Reward: 13.000, Step: 13 +2022-12-07 16:10:34 - r - INFO: - Episode: 98/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:34 - r - INFO: - Episode: 99/1500, Reward: 10.000, Step: 10 +2022-12-07 16:10:34 - r - INFO: - Episode: 100/1500, Reward: 18.000, Step: 18 +2022-12-07 16:10:34 - r - INFO: - Episode: 101/1500, Reward: 25.000, Step: 25 +2022-12-07 16:10:34 - r - INFO: - Episode: 102/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:34 - r - INFO: - Episode: 103/1500, Reward: 14.000, Step: 14 +2022-12-07 16:10:35 - r - INFO: - Episode: 104/1500, Reward: 36.000, Step: 36 +2022-12-07 16:10:35 - r - INFO: - Episode: 105/1500, Reward: 21.000, Step: 21 +2022-12-07 16:10:35 - r - INFO: - Episode: 106/1500, Reward: 27.000, Step: 27 +2022-12-07 16:10:35 - r - INFO: - Episode: 107/1500, Reward: 30.000, Step: 30 +2022-12-07 16:10:35 - r - INFO: - Episode: 108/1500, Reward: 23.000, Step: 23 +2022-12-07 16:10:35 - r - INFO: - Episode: 109/1500, Reward: 28.000, Step: 28 +2022-12-07 16:10:35 - r - INFO: - Episode: 110/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:35 - r - INFO: - Episode: 111/1500, Reward: 16.000, Step: 16 +2022-12-07 16:10:35 - r - INFO: - Episode: 112/1500, Reward: 29.000, Step: 29 +2022-12-07 16:10:35 - r - INFO: - Episode: 113/1500, Reward: 37.000, Step: 37 +2022-12-07 16:10:35 - r - INFO: - Episode: 114/1500, Reward: 37.000, Step: 37 +2022-12-07 16:10:35 - r - INFO: - Episode: 115/1500, Reward: 17.000, Step: 17 +2022-12-07 16:10:35 - r - INFO: - Episode: 116/1500, Reward: 28.000, Step: 28 +2022-12-07 16:10:35 - r - INFO: - Episode: 117/1500, Reward: 43.000, Step: 43 +2022-12-07 16:10:35 - r - INFO: - Episode: 118/1500, Reward: 9.000, Step: 9 +2022-12-07 16:10:35 - r - INFO: - Episode: 119/1500, Reward: 16.000, Step: 16 +2022-12-07 16:10:35 - r - INFO: - Episode: 120/1500, Reward: 20.000, Step: 20 +2022-12-07 16:10:36 - r - INFO: - Episode: 121/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:36 - r - INFO: - Episode: 122/1500, Reward: 40.000, Step: 40 +2022-12-07 16:10:36 - r - INFO: - Episode: 123/1500, Reward: 13.000, Step: 13 +2022-12-07 16:10:36 - r - INFO: - Episode: 124/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:36 - r - INFO: - Episode: 125/1500, Reward: 9.000, Step: 9 +2022-12-07 16:10:36 - r - INFO: - Episode: 126/1500, Reward: 18.000, Step: 18 +2022-12-07 16:10:36 - r - INFO: - Episode: 127/1500, Reward: 20.000, Step: 20 +2022-12-07 16:10:36 - r - INFO: - Episode: 128/1500, Reward: 21.000, Step: 21 +2022-12-07 16:10:36 - r - INFO: - Episode: 129/1500, Reward: 13.000, Step: 13 +2022-12-07 16:10:36 - r - INFO: - Episode: 130/1500, Reward: 33.000, Step: 33 +2022-12-07 16:10:36 - r - INFO: - Episode: 131/1500, Reward: 13.000, Step: 13 +2022-12-07 16:10:36 - r - INFO: - Episode: 132/1500, Reward: 17.000, Step: 17 +2022-12-07 16:10:36 - r - INFO: - Episode: 133/1500, Reward: 44.000, Step: 44 +2022-12-07 16:10:36 - r - INFO: - Episode: 134/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:36 - r - INFO: - Episode: 135/1500, Reward: 16.000, Step: 16 +2022-12-07 16:10:36 - r - INFO: - Episode: 136/1500, Reward: 19.000, Step: 19 +2022-12-07 16:10:36 - r - INFO: - Episode: 137/1500, Reward: 14.000, Step: 14 +2022-12-07 16:10:36 - r - INFO: - Episode: 138/1500, Reward: 20.000, Step: 20 +2022-12-07 16:10:36 - r - INFO: - Episode: 139/1500, Reward: 19.000, Step: 19 +2022-12-07 16:10:36 - r - INFO: - Episode: 140/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:37 - r - INFO: - Episode: 141/1500, Reward: 14.000, Step: 14 +2022-12-07 16:10:37 - r - INFO: - Episode: 142/1500, Reward: 22.000, Step: 22 +2022-12-07 16:10:37 - r - INFO: - Episode: 143/1500, Reward: 17.000, Step: 17 +2022-12-07 16:10:37 - r - INFO: - Episode: 144/1500, Reward: 13.000, Step: 13 +2022-12-07 16:10:37 - r - INFO: - Episode: 145/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:37 - r - INFO: - Episode: 146/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:37 - r - INFO: - Episode: 147/1500, Reward: 19.000, Step: 19 +2022-12-07 16:10:37 - r - INFO: - Episode: 148/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:37 - r - INFO: - Episode: 149/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:37 - r - INFO: - Episode: 150/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:37 - r - INFO: - Episode: 151/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:37 - r - INFO: - Episode: 152/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:37 - r - INFO: - Episode: 153/1500, Reward: 30.000, Step: 30 +2022-12-07 16:10:37 - r - INFO: - Episode: 154/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:37 - r - INFO: - Episode: 155/1500, Reward: 22.000, Step: 22 +2022-12-07 16:10:37 - r - INFO: - Episode: 156/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:37 - r - INFO: - Episode: 157/1500, Reward: 21.000, Step: 21 +2022-12-07 16:10:37 - r - INFO: - Episode: 158/1500, Reward: 37.000, Step: 37 +2022-12-07 16:10:37 - r - INFO: - Episode: 159/1500, Reward: 16.000, Step: 16 +2022-12-07 16:10:37 - r - INFO: - Episode: 160/1500, Reward: 17.000, Step: 17 +2022-12-07 16:10:37 - r - INFO: - Episode: 161/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:37 - r - INFO: - Episode: 162/1500, Reward: 16.000, Step: 16 +2022-12-07 16:10:37 - r - INFO: - Episode: 163/1500, Reward: 32.000, Step: 32 +2022-12-07 16:10:38 - r - INFO: - Episode: 164/1500, Reward: 31.000, Step: 31 +2022-12-07 16:10:38 - r - INFO: - Episode: 165/1500, Reward: 16.000, Step: 16 +2022-12-07 16:10:38 - r - INFO: - Episode: 166/1500, Reward: 19.000, Step: 19 +2022-12-07 16:10:38 - r - INFO: - Episode: 167/1500, Reward: 16.000, Step: 16 +2022-12-07 16:10:38 - r - INFO: - Episode: 168/1500, Reward: 19.000, Step: 19 +2022-12-07 16:10:38 - r - INFO: - Episode: 169/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:38 - r - INFO: - Episode: 170/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:38 - r - INFO: - Episode: 171/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:38 - r - INFO: - Episode: 172/1500, Reward: 20.000, Step: 20 +2022-12-07 16:10:38 - r - INFO: - Episode: 173/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:38 - r - INFO: - Episode: 174/1500, Reward: 26.000, Step: 26 +2022-12-07 16:10:38 - r - INFO: - Episode: 175/1500, Reward: 19.000, Step: 19 +2022-12-07 16:10:38 - r - INFO: - Episode: 176/1500, Reward: 30.000, Step: 30 +2022-12-07 16:10:38 - r - INFO: - Episode: 177/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:38 - r - INFO: - Episode: 178/1500, Reward: 17.000, Step: 17 +2022-12-07 16:10:38 - r - INFO: - Episode: 179/1500, Reward: 18.000, Step: 18 +2022-12-07 16:10:38 - r - INFO: - Episode: 180/1500, Reward: 14.000, Step: 14 +2022-12-07 16:10:38 - r - INFO: - Episode: 181/1500, Reward: 19.000, Step: 19 +2022-12-07 16:10:38 - r - INFO: - Episode: 182/1500, Reward: 28.000, Step: 28 +2022-12-07 16:10:38 - r - INFO: - Episode: 183/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:39 - r - INFO: - Episode: 184/1500, Reward: 20.000, Step: 20 +2022-12-07 16:10:39 - r - INFO: - Episode: 185/1500, Reward: 14.000, Step: 14 +2022-12-07 16:10:39 - r - INFO: - Episode: 186/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:39 - r - INFO: - Episode: 187/1500, Reward: 24.000, Step: 24 +2022-12-07 16:10:39 - r - INFO: - Episode: 188/1500, Reward: 9.000, Step: 9 +2022-12-07 16:10:39 - r - INFO: - Episode: 189/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:39 - r - INFO: - Episode: 190/1500, Reward: 11.000, Step: 11 +2022-12-07 16:10:39 - r - INFO: - Episode: 191/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:39 - r - INFO: - Episode: 192/1500, Reward: 12.000, Step: 12 +2022-12-07 16:10:39 - r - INFO: - Episode: 193/1500, Reward: 15.000, Step: 15 +2022-12-07 16:10:39 - r - INFO: - Episode: 194/1500, Reward: 17.000, Step: 17 +2022-12-07 16:10:39 - r - INFO: - Episode: 195/1500, Reward: 32.000, Step: 32 +2022-12-07 16:11:03 - r - INFO: - Episode: 196/1500, Reward: 42.000, Step: 42 +2022-12-07 16:11:03 - r - INFO: - Episode: 197/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:03 - r - INFO: - Episode: 198/1500, Reward: 52.000, Step: 52 +2022-12-07 16:11:03 - r - INFO: - Episode: 199/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:03 - r - INFO: - Episode: 200/1500, Reward: 62.000, Step: 62 +2022-12-07 16:11:03 - r - INFO: - Episode: 201/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:03 - r - INFO: - Episode: 202/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:03 - r - INFO: - Episode: 203/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:04 - r - INFO: - Episode: 204/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:04 - r - INFO: - Episode: 205/1500, Reward: 30.000, Step: 30 +2022-12-07 16:11:04 - r - INFO: - Episode: 206/1500, Reward: 26.000, Step: 26 +2022-12-07 16:11:04 - r - INFO: - Episode: 207/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:04 - r - INFO: - Episode: 208/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:04 - r - INFO: - Episode: 209/1500, Reward: 25.000, Step: 25 +2022-12-07 16:11:04 - r - INFO: - Episode: 210/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:04 - r - INFO: - Episode: 211/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:04 - r - INFO: - Episode: 212/1500, Reward: 29.000, Step: 29 +2022-12-07 16:11:04 - r - INFO: - Episode: 213/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:04 - r - INFO: - Episode: 214/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:04 - r - INFO: - Episode: 215/1500, Reward: 21.000, Step: 21 +2022-12-07 16:11:04 - r - INFO: - Episode: 216/1500, Reward: 9.000, Step: 9 +2022-12-07 16:11:04 - r - INFO: - Episode: 217/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:04 - r - INFO: - Episode: 218/1500, Reward: 27.000, Step: 27 +2022-12-07 16:11:04 - r - INFO: - Episode: 219/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:04 - r - INFO: - Episode: 220/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:04 - r - INFO: - Episode: 221/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:04 - r - INFO: - Episode: 222/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:04 - r - INFO: - Episode: 223/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:04 - r - INFO: - Episode: 224/1500, Reward: 43.000, Step: 43 +2022-12-07 16:11:04 - r - INFO: - Episode: 225/1500, Reward: 10.000, Step: 10 +2022-12-07 16:11:05 - r - INFO: - Episode: 226/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:05 - r - INFO: - Episode: 227/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:05 - r - INFO: - Episode: 228/1500, Reward: 29.000, Step: 29 +2022-12-07 16:11:05 - r - INFO: - Episode: 229/1500, Reward: 21.000, Step: 21 +2022-12-07 16:11:05 - r - INFO: - Episode: 230/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:05 - r - INFO: - Episode: 231/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:05 - r - INFO: - Episode: 232/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:05 - r - INFO: - Episode: 233/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:05 - r - INFO: - Episode: 234/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:05 - r - INFO: - Episode: 235/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:05 - r - INFO: - Episode: 236/1500, Reward: 37.000, Step: 37 +2022-12-07 16:11:05 - r - INFO: - Episode: 237/1500, Reward: 9.000, Step: 9 +2022-12-07 16:11:05 - r - INFO: - Episode: 238/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:05 - r - INFO: - Episode: 239/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:05 - r - INFO: - Episode: 240/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:05 - r - INFO: - Episode: 241/1500, Reward: 9.000, Step: 9 +2022-12-07 16:11:05 - r - INFO: - Episode: 242/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:05 - r - INFO: - Episode: 243/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:05 - r - INFO: - Episode: 244/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:05 - r - INFO: - Episode: 245/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:05 - r - INFO: - Episode: 246/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:06 - r - INFO: - Episode: 247/1500, Reward: 28.000, Step: 28 +2022-12-07 16:11:06 - r - INFO: - Episode: 248/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:06 - r - INFO: - Episode: 249/1500, Reward: 22.000, Step: 22 +2022-12-07 16:11:06 - r - INFO: - Episode: 250/1500, Reward: 37.000, Step: 37 +2022-12-07 16:11:06 - r - INFO: - Episode: 251/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:06 - r - INFO: - Episode: 252/1500, Reward: 64.000, Step: 64 +2022-12-07 16:11:06 - r - INFO: - Episode: 253/1500, Reward: 27.000, Step: 27 +2022-12-07 16:11:06 - r - INFO: - Episode: 254/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:06 - r - INFO: - Episode: 255/1500, Reward: 21.000, Step: 21 +2022-12-07 16:11:06 - r - INFO: - Episode: 256/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:06 - r - INFO: - Episode: 257/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:06 - r - INFO: - Episode: 258/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:06 - r - INFO: - Episode: 259/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:06 - r - INFO: - Episode: 260/1500, Reward: 21.000, Step: 21 +2022-12-07 16:11:06 - r - INFO: - Episode: 261/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:06 - r - INFO: - Episode: 262/1500, Reward: 31.000, Step: 31 +2022-12-07 16:11:06 - r - INFO: - Episode: 263/1500, Reward: 33.000, Step: 33 +2022-12-07 16:11:06 - r - INFO: - Episode: 264/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:06 - r - INFO: - Episode: 265/1500, Reward: 29.000, Step: 29 +2022-12-07 16:11:07 - r - INFO: - Episode: 266/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:07 - r - INFO: - Episode: 267/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:07 - r - INFO: - Episode: 268/1500, Reward: 10.000, Step: 10 +2022-12-07 16:11:07 - r - INFO: - Episode: 269/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:07 - r - INFO: - Episode: 270/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:07 - r - INFO: - Episode: 271/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:07 - r - INFO: - Episode: 272/1500, Reward: 26.000, Step: 26 +2022-12-07 16:11:07 - r - INFO: - Episode: 273/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:07 - r - INFO: - Episode: 274/1500, Reward: 9.000, Step: 9 +2022-12-07 16:11:07 - r - INFO: - Episode: 275/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:07 - r - INFO: - Episode: 276/1500, Reward: 10.000, Step: 10 +2022-12-07 16:11:07 - r - INFO: - Episode: 277/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:07 - r - INFO: - Episode: 278/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:07 - r - INFO: - Episode: 279/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:07 - r - INFO: - Episode: 280/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:07 - r - INFO: - Episode: 281/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:07 - r - INFO: - Episode: 282/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:07 - r - INFO: - Episode: 283/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:07 - r - INFO: - Episode: 284/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:07 - r - INFO: - Episode: 285/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:07 - r - INFO: - Episode: 286/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:08 - r - INFO: - Episode: 287/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:08 - r - INFO: - Episode: 288/1500, Reward: 34.000, Step: 34 +2022-12-07 16:11:08 - r - INFO: - Episode: 289/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:08 - r - INFO: - Episode: 290/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:08 - r - INFO: - Episode: 291/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:08 - r - INFO: - Episode: 292/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:08 - r - INFO: - Episode: 293/1500, Reward: 22.000, Step: 22 +2022-12-07 16:11:08 - r - INFO: - Episode: 294/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:08 - r - INFO: - Episode: 295/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:08 - r - INFO: - Episode: 296/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:08 - r - INFO: - Episode: 297/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:08 - r - INFO: - Episode: 298/1500, Reward: 25.000, Step: 25 +2022-12-07 16:11:08 - r - INFO: - Episode: 299/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:08 - r - INFO: - Episode: 300/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:08 - r - INFO: - Episode: 301/1500, Reward: 10.000, Step: 10 +2022-12-07 16:11:31 - r - INFO: - Episode: 302/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:31 - r - INFO: - Episode: 303/1500, Reward: 27.000, Step: 27 +2022-12-07 16:11:31 - r - INFO: - Episode: 304/1500, Reward: 24.000, Step: 24 +2022-12-07 16:11:31 - r - INFO: - Episode: 305/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:32 - r - INFO: - Episode: 306/1500, Reward: 22.000, Step: 22 +2022-12-07 16:11:32 - r - INFO: - Episode: 307/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:32 - r - INFO: - Episode: 308/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:32 - r - INFO: - Episode: 309/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:32 - r - INFO: - Episode: 310/1500, Reward: 26.000, Step: 26 +2022-12-07 16:11:32 - r - INFO: - Episode: 311/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:32 - r - INFO: - Episode: 312/1500, Reward: 27.000, Step: 27 +2022-12-07 16:11:32 - r - INFO: - Episode: 313/1500, Reward: 21.000, Step: 21 +2022-12-07 16:11:32 - r - INFO: - Episode: 314/1500, Reward: 26.000, Step: 26 +2022-12-07 16:11:32 - r - INFO: - Episode: 315/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:32 - r - INFO: - Episode: 316/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:32 - r - INFO: - Episode: 317/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:32 - r - INFO: - Episode: 318/1500, Reward: 26.000, Step: 26 +2022-12-07 16:11:32 - r - INFO: - Episode: 319/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:32 - r - INFO: - Episode: 320/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:32 - r - INFO: - Episode: 321/1500, Reward: 24.000, Step: 24 +2022-12-07 16:11:32 - r - INFO: - Episode: 322/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:32 - r - INFO: - Episode: 323/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:32 - r - INFO: - Episode: 324/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:32 - r - INFO: - Episode: 325/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:32 - r - INFO: - Episode: 326/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:33 - r - INFO: - Episode: 327/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:33 - r - INFO: - Episode: 328/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:33 - r - INFO: - Episode: 329/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:33 - r - INFO: - Episode: 330/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:33 - r - INFO: - Episode: 331/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:33 - r - INFO: - Episode: 332/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:33 - r - INFO: - Episode: 333/1500, Reward: 29.000, Step: 29 +2022-12-07 16:11:33 - r - INFO: - Episode: 334/1500, Reward: 36.000, Step: 36 +2022-12-07 16:11:33 - r - INFO: - Episode: 335/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:33 - r - INFO: - Episode: 336/1500, Reward: 32.000, Step: 32 +2022-12-07 16:11:33 - r - INFO: - Episode: 337/1500, Reward: 41.000, Step: 41 +2022-12-07 16:11:33 - r - INFO: - Episode: 338/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:33 - r - INFO: - Episode: 339/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:33 - r - INFO: - Episode: 340/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:33 - r - INFO: - Episode: 341/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:33 - r - INFO: - Episode: 342/1500, Reward: 24.000, Step: 24 +2022-12-07 16:11:33 - r - INFO: - Episode: 343/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:33 - r - INFO: - Episode: 344/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:33 - r - INFO: - Episode: 345/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:34 - r - INFO: - Episode: 346/1500, Reward: 26.000, Step: 26 +2022-12-07 16:11:34 - r - INFO: - Episode: 347/1500, Reward: 22.000, Step: 22 +2022-12-07 16:11:34 - r - INFO: - Episode: 348/1500, Reward: 57.000, Step: 57 +2022-12-07 16:11:34 - r - INFO: - Episode: 349/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:34 - r - INFO: - Episode: 350/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:34 - r - INFO: - Episode: 351/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:34 - r - INFO: - Episode: 352/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:34 - r - INFO: - Episode: 353/1500, Reward: 24.000, Step: 24 +2022-12-07 16:11:34 - r - INFO: - Episode: 354/1500, Reward: 26.000, Step: 26 +2022-12-07 16:11:34 - r - INFO: - Episode: 355/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:34 - r - INFO: - Episode: 356/1500, Reward: 36.000, Step: 36 +2022-12-07 16:11:34 - r - INFO: - Episode: 357/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:34 - r - INFO: - Episode: 358/1500, Reward: 22.000, Step: 22 +2022-12-07 16:11:34 - r - INFO: - Episode: 359/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:34 - r - INFO: - Episode: 360/1500, Reward: 47.000, Step: 47 +2022-12-07 16:11:34 - r - INFO: - Episode: 361/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:34 - r - INFO: - Episode: 362/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:35 - r - INFO: - Episode: 363/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:35 - r - INFO: - Episode: 364/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:35 - r - INFO: - Episode: 365/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:35 - r - INFO: - Episode: 366/1500, Reward: 17.000, Step: 17 +2022-12-07 16:11:35 - r - INFO: - Episode: 367/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:35 - r - INFO: - Episode: 368/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:35 - r - INFO: - Episode: 369/1500, Reward: 10.000, Step: 10 +2022-12-07 16:11:35 - r - INFO: - Episode: 370/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:35 - r - INFO: - Episode: 371/1500, Reward: 35.000, Step: 35 +2022-12-07 16:11:35 - r - INFO: - Episode: 372/1500, Reward: 9.000, Step: 9 +2022-12-07 16:11:35 - r - INFO: - Episode: 373/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:35 - r - INFO: - Episode: 374/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:35 - r - INFO: - Episode: 375/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:35 - r - INFO: - Episode: 376/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:35 - r - INFO: - Episode: 377/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:35 - r - INFO: - Episode: 378/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:35 - r - INFO: - Episode: 379/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:35 - r - INFO: - Episode: 380/1500, Reward: 33.000, Step: 33 +2022-12-07 16:11:35 - r - INFO: - Episode: 381/1500, Reward: 9.000, Step: 9 +2022-12-07 16:11:35 - r - INFO: - Episode: 382/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:35 - r - INFO: - Episode: 383/1500, Reward: 27.000, Step: 27 +2022-12-07 16:11:35 - r - INFO: - Episode: 384/1500, Reward: 19.000, Step: 19 +2022-12-07 16:11:36 - r - INFO: - Episode: 385/1500, Reward: 15.000, Step: 15 +2022-12-07 16:11:36 - r - INFO: - Episode: 386/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:36 - r - INFO: - Episode: 387/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:36 - r - INFO: - Episode: 388/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:36 - r - INFO: - Episode: 389/1500, Reward: 24.000, Step: 24 +2022-12-07 16:11:36 - r - INFO: - Episode: 390/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:36 - r - INFO: - Episode: 391/1500, Reward: 11.000, Step: 11 +2022-12-07 16:11:36 - r - INFO: - Episode: 392/1500, Reward: 23.000, Step: 23 +2022-12-07 16:11:36 - r - INFO: - Episode: 393/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:36 - r - INFO: - Episode: 394/1500, Reward: 12.000, Step: 12 +2022-12-07 16:11:36 - r - INFO: - Episode: 395/1500, Reward: 24.000, Step: 24 +2022-12-07 16:11:36 - r - INFO: - Episode: 396/1500, Reward: 32.000, Step: 32 +2022-12-07 16:11:36 - r - INFO: - Episode: 397/1500, Reward: 10.000, Step: 10 +2022-12-07 16:11:36 - r - INFO: - Episode: 398/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:36 - r - INFO: - Episode: 399/1500, Reward: 18.000, Step: 18 +2022-12-07 16:11:36 - r - INFO: - Episode: 400/1500, Reward: 38.000, Step: 38 +2022-12-07 16:11:36 - r - INFO: - Episode: 401/1500, Reward: 14.000, Step: 14 +2022-12-07 16:11:37 - r - INFO: - Episode: 402/1500, Reward: 20.000, Step: 20 +2022-12-07 16:11:37 - r - INFO: - Episode: 403/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:37 - r - INFO: - Episode: 404/1500, Reward: 13.000, Step: 13 +2022-12-07 16:11:37 - r - INFO: - Episode: 405/1500, Reward: 16.000, Step: 16 +2022-12-07 16:11:37 - r - INFO: - Episode: 406/1500, Reward: 35.000, Step: 35 +2022-12-07 16:11:37 - r - INFO: - Episode: 407/1500, Reward: 10.000, Step: 10 +2022-12-07 16:12:00 - r - INFO: - Episode: 408/1500, Reward: 9.000, Step: 9 +2022-12-07 16:12:00 - r - INFO: - Episode: 409/1500, Reward: 26.000, Step: 26 +2022-12-07 16:12:00 - r - INFO: - Episode: 410/1500, Reward: 47.000, Step: 47 +2022-12-07 16:12:01 - r - INFO: - Episode: 411/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:01 - r - INFO: - Episode: 412/1500, Reward: 26.000, Step: 26 +2022-12-07 16:12:01 - r - INFO: - Episode: 413/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:01 - r - INFO: - Episode: 414/1500, Reward: 24.000, Step: 24 +2022-12-07 16:12:01 - r - INFO: - Episode: 415/1500, Reward: 10.000, Step: 10 +2022-12-07 16:12:01 - r - INFO: - Episode: 416/1500, Reward: 25.000, Step: 25 +2022-12-07 16:12:01 - r - INFO: - Episode: 417/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:01 - r - INFO: - Episode: 418/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:01 - r - INFO: - Episode: 419/1500, Reward: 11.000, Step: 11 +2022-12-07 16:12:01 - r - INFO: - Episode: 420/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:01 - r - INFO: - Episode: 421/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:01 - r - INFO: - Episode: 422/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:01 - r - INFO: - Episode: 423/1500, Reward: 27.000, Step: 27 +2022-12-07 16:12:01 - r - INFO: - Episode: 424/1500, Reward: 29.000, Step: 29 +2022-12-07 16:12:01 - r - INFO: - Episode: 425/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:01 - r - INFO: - Episode: 426/1500, Reward: 27.000, Step: 27 +2022-12-07 16:12:01 - r - INFO: - Episode: 427/1500, Reward: 26.000, Step: 26 +2022-12-07 16:12:01 - r - INFO: - Episode: 428/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:01 - r - INFO: - Episode: 429/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:01 - r - INFO: - Episode: 430/1500, Reward: 10.000, Step: 10 +2022-12-07 16:12:02 - r - INFO: - Episode: 431/1500, Reward: 12.000, Step: 12 +2022-12-07 16:12:02 - r - INFO: - Episode: 432/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:02 - r - INFO: - Episode: 433/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:02 - r - INFO: - Episode: 434/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:02 - r - INFO: - Episode: 435/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:02 - r - INFO: - Episode: 436/1500, Reward: 10.000, Step: 10 +2022-12-07 16:12:02 - r - INFO: - Episode: 437/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:02 - r - INFO: - Episode: 438/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:02 - r - INFO: - Episode: 439/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:02 - r - INFO: - Episode: 440/1500, Reward: 35.000, Step: 35 +2022-12-07 16:12:02 - r - INFO: - Episode: 441/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:02 - r - INFO: - Episode: 442/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:02 - r - INFO: - Episode: 443/1500, Reward: 45.000, Step: 45 +2022-12-07 16:12:02 - r - INFO: - Episode: 444/1500, Reward: 19.000, Step: 19 +2022-12-07 16:12:02 - r - INFO: - Episode: 445/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:03 - r - INFO: - Episode: 446/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:03 - r - INFO: - Episode: 447/1500, Reward: 33.000, Step: 33 +2022-12-07 16:12:03 - r - INFO: - Episode: 448/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:03 - r - INFO: - Episode: 449/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:03 - r - INFO: - Episode: 450/1500, Reward: 30.000, Step: 30 +2022-12-07 16:12:03 - r - INFO: - Episode: 451/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:03 - r - INFO: - Episode: 452/1500, Reward: 11.000, Step: 11 +2022-12-07 16:12:03 - r - INFO: - Episode: 453/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:03 - r - INFO: - Episode: 454/1500, Reward: 19.000, Step: 19 +2022-12-07 16:12:03 - r - INFO: - Episode: 455/1500, Reward: 25.000, Step: 25 +2022-12-07 16:12:03 - r - INFO: - Episode: 456/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:03 - r - INFO: - Episode: 457/1500, Reward: 10.000, Step: 10 +2022-12-07 16:12:03 - r - INFO: - Episode: 458/1500, Reward: 12.000, Step: 12 +2022-12-07 16:12:03 - r - INFO: - Episode: 459/1500, Reward: 38.000, Step: 38 +2022-12-07 16:12:03 - r - INFO: - Episode: 460/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:03 - r - INFO: - Episode: 461/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:03 - r - INFO: - Episode: 462/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:03 - r - INFO: - Episode: 463/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:03 - r - INFO: - Episode: 464/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:04 - r - INFO: - Episode: 465/1500, Reward: 62.000, Step: 62 +2022-12-07 16:12:04 - r - INFO: - Episode: 466/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:04 - r - INFO: - Episode: 467/1500, Reward: 27.000, Step: 27 +2022-12-07 16:12:04 - r - INFO: - Episode: 468/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:04 - r - INFO: - Episode: 469/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:04 - r - INFO: - Episode: 470/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:04 - r - INFO: - Episode: 471/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:04 - r - INFO: - Episode: 472/1500, Reward: 34.000, Step: 34 +2022-12-07 16:12:04 - r - INFO: - Episode: 473/1500, Reward: 76.000, Step: 76 +2022-12-07 16:12:04 - r - INFO: - Episode: 474/1500, Reward: 40.000, Step: 40 +2022-12-07 16:12:04 - r - INFO: - Episode: 475/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:04 - r - INFO: - Episode: 476/1500, Reward: 32.000, Step: 32 +2022-12-07 16:12:04 - r - INFO: - Episode: 477/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:04 - r - INFO: - Episode: 478/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:04 - r - INFO: - Episode: 479/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:04 - r - INFO: - Episode: 480/1500, Reward: 25.000, Step: 25 +2022-12-07 16:12:05 - r - INFO: - Episode: 481/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:05 - r - INFO: - Episode: 482/1500, Reward: 25.000, Step: 25 +2022-12-07 16:12:05 - r - INFO: - Episode: 483/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:05 - r - INFO: - Episode: 484/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:05 - r - INFO: - Episode: 485/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:05 - r - INFO: - Episode: 486/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:05 - r - INFO: - Episode: 487/1500, Reward: 11.000, Step: 11 +2022-12-07 16:12:05 - r - INFO: - Episode: 488/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:05 - r - INFO: - Episode: 489/1500, Reward: 35.000, Step: 35 +2022-12-07 16:12:05 - r - INFO: - Episode: 490/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:05 - r - INFO: - Episode: 491/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:05 - r - INFO: - Episode: 492/1500, Reward: 42.000, Step: 42 +2022-12-07 16:12:05 - r - INFO: - Episode: 493/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:05 - r - INFO: - Episode: 494/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:05 - r - INFO: - Episode: 495/1500, Reward: 19.000, Step: 19 +2022-12-07 16:12:06 - r - INFO: - Episode: 496/1500, Reward: 27.000, Step: 27 +2022-12-07 16:12:06 - r - INFO: - Episode: 497/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:06 - r - INFO: - Episode: 498/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:29 - r - INFO: - Episode: 499/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:29 - r - INFO: - Episode: 500/1500, Reward: 9.000, Step: 9 +2022-12-07 16:12:30 - r - INFO: - Episode: 501/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:30 - r - INFO: - Episode: 502/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:30 - r - INFO: - Episode: 503/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:30 - r - INFO: - Episode: 504/1500, Reward: 25.000, Step: 25 +2022-12-07 16:12:30 - r - INFO: - Episode: 505/1500, Reward: 41.000, Step: 41 +2022-12-07 16:12:30 - r - INFO: - Episode: 506/1500, Reward: 31.000, Step: 31 +2022-12-07 16:12:30 - r - INFO: - Episode: 507/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:30 - r - INFO: - Episode: 508/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:30 - r - INFO: - Episode: 509/1500, Reward: 12.000, Step: 12 +2022-12-07 16:12:30 - r - INFO: - Episode: 510/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:30 - r - INFO: - Episode: 511/1500, Reward: 12.000, Step: 12 +2022-12-07 16:12:30 - r - INFO: - Episode: 512/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:30 - r - INFO: - Episode: 513/1500, Reward: 50.000, Step: 50 +2022-12-07 16:12:30 - r - INFO: - Episode: 514/1500, Reward: 11.000, Step: 11 +2022-12-07 16:12:30 - r - INFO: - Episode: 515/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:31 - r - INFO: - Episode: 516/1500, Reward: 19.000, Step: 19 +2022-12-07 16:12:31 - r - INFO: - Episode: 517/1500, Reward: 12.000, Step: 12 +2022-12-07 16:12:31 - r - INFO: - Episode: 518/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:31 - r - INFO: - Episode: 519/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:31 - r - INFO: - Episode: 520/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:31 - r - INFO: - Episode: 521/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:31 - r - INFO: - Episode: 522/1500, Reward: 38.000, Step: 38 +2022-12-07 16:12:31 - r - INFO: - Episode: 523/1500, Reward: 30.000, Step: 30 +2022-12-07 16:12:31 - r - INFO: - Episode: 524/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:31 - r - INFO: - Episode: 525/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:31 - r - INFO: - Episode: 526/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:31 - r - INFO: - Episode: 527/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:31 - r - INFO: - Episode: 528/1500, Reward: 12.000, Step: 12 +2022-12-07 16:12:31 - r - INFO: - Episode: 529/1500, Reward: 46.000, Step: 46 +2022-12-07 16:12:31 - r - INFO: - Episode: 530/1500, Reward: 32.000, Step: 32 +2022-12-07 16:12:31 - r - INFO: - Episode: 531/1500, Reward: 37.000, Step: 37 +2022-12-07 16:12:31 - r - INFO: - Episode: 532/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:32 - r - INFO: - Episode: 533/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:32 - r - INFO: - Episode: 534/1500, Reward: 11.000, Step: 11 +2022-12-07 16:12:32 - r - INFO: - Episode: 535/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:32 - r - INFO: - Episode: 536/1500, Reward: 10.000, Step: 10 +2022-12-07 16:12:32 - r - INFO: - Episode: 537/1500, Reward: 15.000, Step: 15 +2022-12-07 16:12:32 - r - INFO: - Episode: 538/1500, Reward: 39.000, Step: 39 +2022-12-07 16:12:32 - r - INFO: - Episode: 539/1500, Reward: 67.000, Step: 67 +2022-12-07 16:12:32 - r - INFO: - Episode: 540/1500, Reward: 33.000, Step: 33 +2022-12-07 16:12:32 - r - INFO: - Episode: 541/1500, Reward: 25.000, Step: 25 +2022-12-07 16:12:32 - r - INFO: - Episode: 542/1500, Reward: 30.000, Step: 30 +2022-12-07 16:12:32 - r - INFO: - Episode: 543/1500, Reward: 32.000, Step: 32 +2022-12-07 16:12:32 - r - INFO: - Episode: 544/1500, Reward: 43.000, Step: 43 +2022-12-07 16:12:32 - r - INFO: - Episode: 545/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:32 - r - INFO: - Episode: 546/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:32 - r - INFO: - Episode: 547/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:33 - r - INFO: - Episode: 548/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:33 - r - INFO: - Episode: 549/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:33 - r - INFO: - Episode: 550/1500, Reward: 27.000, Step: 27 +2022-12-07 16:12:33 - r - INFO: - Episode: 551/1500, Reward: 37.000, Step: 37 +2022-12-07 16:12:33 - r - INFO: - Episode: 552/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:33 - r - INFO: - Episode: 553/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:33 - r - INFO: - Episode: 554/1500, Reward: 19.000, Step: 19 +2022-12-07 16:12:33 - r - INFO: - Episode: 555/1500, Reward: 36.000, Step: 36 +2022-12-07 16:12:33 - r - INFO: - Episode: 556/1500, Reward: 33.000, Step: 33 +2022-12-07 16:12:33 - r - INFO: - Episode: 557/1500, Reward: 41.000, Step: 41 +2022-12-07 16:12:33 - r - INFO: - Episode: 558/1500, Reward: 41.000, Step: 41 +2022-12-07 16:12:33 - r - INFO: - Episode: 559/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:33 - r - INFO: - Episode: 560/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:33 - r - INFO: - Episode: 561/1500, Reward: 24.000, Step: 24 +2022-12-07 16:12:34 - r - INFO: - Episode: 562/1500, Reward: 26.000, Step: 26 +2022-12-07 16:12:34 - r - INFO: - Episode: 563/1500, Reward: 29.000, Step: 29 +2022-12-07 16:12:34 - r - INFO: - Episode: 564/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:34 - r - INFO: - Episode: 565/1500, Reward: 17.000, Step: 17 +2022-12-07 16:12:34 - r - INFO: - Episode: 566/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:34 - r - INFO: - Episode: 567/1500, Reward: 23.000, Step: 23 +2022-12-07 16:12:34 - r - INFO: - Episode: 568/1500, Reward: 13.000, Step: 13 +2022-12-07 16:12:34 - r - INFO: - Episode: 569/1500, Reward: 30.000, Step: 30 +2022-12-07 16:12:34 - r - INFO: - Episode: 570/1500, Reward: 24.000, Step: 24 +2022-12-07 16:12:34 - r - INFO: - Episode: 571/1500, Reward: 41.000, Step: 41 +2022-12-07 16:12:34 - r - INFO: - Episode: 572/1500, Reward: 14.000, Step: 14 +2022-12-07 16:12:34 - r - INFO: - Episode: 573/1500, Reward: 32.000, Step: 32 +2022-12-07 16:12:34 - r - INFO: - Episode: 574/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:34 - r - INFO: - Episode: 575/1500, Reward: 18.000, Step: 18 +2022-12-07 16:12:34 - r - INFO: - Episode: 576/1500, Reward: 34.000, Step: 34 +2022-12-07 16:12:34 - r - INFO: - Episode: 577/1500, Reward: 39.000, Step: 39 +2022-12-07 16:12:34 - r - INFO: - Episode: 578/1500, Reward: 22.000, Step: 22 +2022-12-07 16:12:35 - r - INFO: - Episode: 579/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:35 - r - INFO: - Episode: 580/1500, Reward: 24.000, Step: 24 +2022-12-07 16:12:35 - r - INFO: - Current episode 580 has the best eval reward: 30.500 +2022-12-07 16:12:35 - r - INFO: - Episode: 581/1500, Reward: 34.000, Step: 34 +2022-12-07 16:12:58 - r - INFO: - Episode: 582/1500, Reward: 64.000, Step: 64 +2022-12-07 16:12:58 - r - INFO: - Episode: 583/1500, Reward: 43.000, Step: 43 +2022-12-07 16:12:58 - r - INFO: - Episode: 584/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:58 - r - INFO: - Episode: 585/1500, Reward: 11.000, Step: 11 +2022-12-07 16:12:59 - r - INFO: - Episode: 586/1500, Reward: 61.000, Step: 61 +2022-12-07 16:12:59 - r - INFO: - Episode: 587/1500, Reward: 20.000, Step: 20 +2022-12-07 16:12:59 - r - INFO: - Episode: 588/1500, Reward: 51.000, Step: 51 +2022-12-07 16:12:59 - r - INFO: - Episode: 589/1500, Reward: 21.000, Step: 21 +2022-12-07 16:12:59 - r - INFO: - Episode: 590/1500, Reward: 11.000, Step: 11 +2022-12-07 16:12:59 - r - INFO: - Current episode 590 has the best eval reward: 31.200 +2022-12-07 16:12:59 - r - INFO: - Episode: 591/1500, Reward: 32.000, Step: 32 +2022-12-07 16:12:59 - r - INFO: - Episode: 592/1500, Reward: 44.000, Step: 44 +2022-12-07 16:12:59 - r - INFO: - Episode: 593/1500, Reward: 26.000, Step: 26 +2022-12-07 16:12:59 - r - INFO: - Episode: 594/1500, Reward: 28.000, Step: 28 +2022-12-07 16:12:59 - r - INFO: - Episode: 595/1500, Reward: 16.000, Step: 16 +2022-12-07 16:12:59 - r - INFO: - Current episode 595 has the best eval reward: 37.000 +2022-12-07 16:12:59 - r - INFO: - Episode: 596/1500, Reward: 60.000, Step: 60 +2022-12-07 16:12:59 - r - INFO: - Episode: 597/1500, Reward: 14.000, Step: 14 +2022-12-07 16:13:00 - r - INFO: - Episode: 598/1500, Reward: 30.000, Step: 30 +2022-12-07 16:13:00 - r - INFO: - Episode: 599/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:00 - r - INFO: - Episode: 600/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:00 - r - INFO: - Episode: 601/1500, Reward: 74.000, Step: 74 +2022-12-07 16:13:00 - r - INFO: - Episode: 602/1500, Reward: 19.000, Step: 19 +2022-12-07 16:13:00 - r - INFO: - Episode: 603/1500, Reward: 30.000, Step: 30 +2022-12-07 16:13:00 - r - INFO: - Episode: 604/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:00 - r - INFO: - Episode: 605/1500, Reward: 37.000, Step: 37 +2022-12-07 16:13:00 - r - INFO: - Episode: 606/1500, Reward: 12.000, Step: 12 +2022-12-07 16:13:00 - r - INFO: - Episode: 607/1500, Reward: 45.000, Step: 45 +2022-12-07 16:13:00 - r - INFO: - Episode: 608/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:00 - r - INFO: - Episode: 609/1500, Reward: 38.000, Step: 38 +2022-12-07 16:13:00 - r - INFO: - Episode: 610/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:01 - r - INFO: - Episode: 611/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:01 - r - INFO: - Episode: 612/1500, Reward: 45.000, Step: 45 +2022-12-07 16:13:01 - r - INFO: - Episode: 613/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:01 - r - INFO: - Episode: 614/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:01 - r - INFO: - Episode: 615/1500, Reward: 59.000, Step: 59 +2022-12-07 16:13:01 - r - INFO: - Episode: 616/1500, Reward: 10.000, Step: 10 +2022-12-07 16:13:01 - r - INFO: - Episode: 617/1500, Reward: 10.000, Step: 10 +2022-12-07 16:13:01 - r - INFO: - Episode: 618/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:01 - r - INFO: - Episode: 619/1500, Reward: 48.000, Step: 48 +2022-12-07 16:13:01 - r - INFO: - Episode: 620/1500, Reward: 12.000, Step: 12 +2022-12-07 16:13:01 - r - INFO: - Episode: 621/1500, Reward: 23.000, Step: 23 +2022-12-07 16:13:01 - r - INFO: - Episode: 622/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:01 - r - INFO: - Episode: 623/1500, Reward: 42.000, Step: 42 +2022-12-07 16:13:01 - r - INFO: - Episode: 624/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:01 - r - INFO: - Episode: 625/1500, Reward: 9.000, Step: 9 +2022-12-07 16:13:02 - r - INFO: - Episode: 626/1500, Reward: 29.000, Step: 29 +2022-12-07 16:13:02 - r - INFO: - Episode: 627/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:02 - r - INFO: - Episode: 628/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:02 - r - INFO: - Episode: 629/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:02 - r - INFO: - Episode: 630/1500, Reward: 14.000, Step: 14 +2022-12-07 16:13:02 - r - INFO: - Episode: 631/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:02 - r - INFO: - Episode: 632/1500, Reward: 33.000, Step: 33 +2022-12-07 16:13:02 - r - INFO: - Episode: 633/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:02 - r - INFO: - Episode: 634/1500, Reward: 9.000, Step: 9 +2022-12-07 16:13:02 - r - INFO: - Episode: 635/1500, Reward: 17.000, Step: 17 +2022-12-07 16:13:02 - r - INFO: - Episode: 636/1500, Reward: 31.000, Step: 31 +2022-12-07 16:13:02 - r - INFO: - Episode: 637/1500, Reward: 73.000, Step: 73 +2022-12-07 16:13:02 - r - INFO: - Episode: 638/1500, Reward: 16.000, Step: 16 +2022-12-07 16:13:02 - r - INFO: - Episode: 639/1500, Reward: 10.000, Step: 10 +2022-12-07 16:13:02 - r - INFO: - Episode: 640/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:03 - r - INFO: - Episode: 641/1500, Reward: 12.000, Step: 12 +2022-12-07 16:13:03 - r - INFO: - Episode: 642/1500, Reward: 34.000, Step: 34 +2022-12-07 16:13:03 - r - INFO: - Episode: 643/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:03 - r - INFO: - Episode: 644/1500, Reward: 71.000, Step: 71 +2022-12-07 16:13:03 - r - INFO: - Episode: 645/1500, Reward: 40.000, Step: 40 +2022-12-07 16:13:03 - r - INFO: - Episode: 646/1500, Reward: 19.000, Step: 19 +2022-12-07 16:13:03 - r - INFO: - Episode: 647/1500, Reward: 46.000, Step: 46 +2022-12-07 16:13:03 - r - INFO: - Episode: 648/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:03 - r - INFO: - Episode: 649/1500, Reward: 19.000, Step: 19 +2022-12-07 16:13:03 - r - INFO: - Episode: 650/1500, Reward: 24.000, Step: 24 +2022-12-07 16:13:03 - r - INFO: - Episode: 651/1500, Reward: 54.000, Step: 54 +2022-12-07 16:13:03 - r - INFO: - Episode: 652/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:03 - r - INFO: - Episode: 653/1500, Reward: 26.000, Step: 26 +2022-12-07 16:13:27 - r - INFO: - Episode: 654/1500, Reward: 42.000, Step: 42 +2022-12-07 16:13:27 - r - INFO: - Episode: 655/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:27 - r - INFO: - Episode: 656/1500, Reward: 26.000, Step: 26 +2022-12-07 16:13:27 - r - INFO: - Episode: 657/1500, Reward: 40.000, Step: 40 +2022-12-07 16:13:27 - r - INFO: - Episode: 658/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:27 - r - INFO: - Episode: 659/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:27 - r - INFO: - Episode: 660/1500, Reward: 40.000, Step: 40 +2022-12-07 16:13:27 - r - INFO: - Episode: 661/1500, Reward: 29.000, Step: 29 +2022-12-07 16:13:27 - r - INFO: - Episode: 662/1500, Reward: 48.000, Step: 48 +2022-12-07 16:13:27 - r - INFO: - Episode: 663/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:27 - r - INFO: - Episode: 664/1500, Reward: 13.000, Step: 13 +2022-12-07 16:13:27 - r - INFO: - Episode: 665/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:28 - r - INFO: - Episode: 666/1500, Reward: 12.000, Step: 12 +2022-12-07 16:13:28 - r - INFO: - Episode: 667/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:28 - r - INFO: - Episode: 668/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:28 - r - INFO: - Episode: 669/1500, Reward: 13.000, Step: 13 +2022-12-07 16:13:28 - r - INFO: - Episode: 670/1500, Reward: 86.000, Step: 86 +2022-12-07 16:13:28 - r - INFO: - Current episode 670 has the best eval reward: 42.300 +2022-12-07 16:13:28 - r - INFO: - Episode: 671/1500, Reward: 72.000, Step: 72 +2022-12-07 16:13:28 - r - INFO: - Episode: 672/1500, Reward: 34.000, Step: 34 +2022-12-07 16:13:28 - r - INFO: - Episode: 673/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:28 - r - INFO: - Episode: 674/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:28 - r - INFO: - Episode: 675/1500, Reward: 27.000, Step: 27 +2022-12-07 16:13:29 - r - INFO: - Episode: 676/1500, Reward: 41.000, Step: 41 +2022-12-07 16:13:29 - r - INFO: - Episode: 677/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:29 - r - INFO: - Episode: 678/1500, Reward: 58.000, Step: 58 +2022-12-07 16:13:29 - r - INFO: - Episode: 679/1500, Reward: 17.000, Step: 17 +2022-12-07 16:13:29 - r - INFO: - Episode: 680/1500, Reward: 33.000, Step: 33 +2022-12-07 16:13:29 - r - INFO: - Episode: 681/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:29 - r - INFO: - Episode: 682/1500, Reward: 36.000, Step: 36 +2022-12-07 16:13:29 - r - INFO: - Episode: 683/1500, Reward: 23.000, Step: 23 +2022-12-07 16:13:29 - r - INFO: - Episode: 684/1500, Reward: 38.000, Step: 38 +2022-12-07 16:13:29 - r - INFO: - Episode: 685/1500, Reward: 45.000, Step: 45 +2022-12-07 16:13:29 - r - INFO: - Episode: 686/1500, Reward: 44.000, Step: 44 +2022-12-07 16:13:29 - r - INFO: - Episode: 687/1500, Reward: 55.000, Step: 55 +2022-12-07 16:13:29 - r - INFO: - Episode: 688/1500, Reward: 26.000, Step: 26 +2022-12-07 16:13:30 - r - INFO: - Episode: 689/1500, Reward: 43.000, Step: 43 +2022-12-07 16:13:30 - r - INFO: - Episode: 690/1500, Reward: 28.000, Step: 28 +2022-12-07 16:13:30 - r - INFO: - Episode: 691/1500, Reward: 21.000, Step: 21 +2022-12-07 16:13:30 - r - INFO: - Episode: 692/1500, Reward: 25.000, Step: 25 +2022-12-07 16:13:30 - r - INFO: - Episode: 693/1500, Reward: 40.000, Step: 40 +2022-12-07 16:13:30 - r - INFO: - Episode: 694/1500, Reward: 16.000, Step: 16 +2022-12-07 16:13:30 - r - INFO: - Episode: 695/1500, Reward: 19.000, Step: 19 +2022-12-07 16:13:30 - r - INFO: - Episode: 696/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:30 - r - INFO: - Episode: 697/1500, Reward: 35.000, Step: 35 +2022-12-07 16:13:30 - r - INFO: - Episode: 698/1500, Reward: 12.000, Step: 12 +2022-12-07 16:13:30 - r - INFO: - Episode: 699/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:30 - r - INFO: - Episode: 700/1500, Reward: 50.000, Step: 50 +2022-12-07 16:13:30 - r - INFO: - Episode: 701/1500, Reward: 46.000, Step: 46 +2022-12-07 16:13:30 - r - INFO: - Episode: 702/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:30 - r - INFO: - Episode: 703/1500, Reward: 17.000, Step: 17 +2022-12-07 16:13:31 - r - INFO: - Episode: 704/1500, Reward: 30.000, Step: 30 +2022-12-07 16:13:31 - r - INFO: - Episode: 705/1500, Reward: 33.000, Step: 33 +2022-12-07 16:13:31 - r - INFO: - Episode: 706/1500, Reward: 24.000, Step: 24 +2022-12-07 16:13:31 - r - INFO: - Episode: 707/1500, Reward: 25.000, Step: 25 +2022-12-07 16:13:31 - r - INFO: - Episode: 708/1500, Reward: 29.000, Step: 29 +2022-12-07 16:13:31 - r - INFO: - Episode: 709/1500, Reward: 28.000, Step: 28 +2022-12-07 16:13:31 - r - INFO: - Episode: 710/1500, Reward: 28.000, Step: 28 +2022-12-07 16:13:31 - r - INFO: - Episode: 711/1500, Reward: 44.000, Step: 44 +2022-12-07 16:13:31 - r - INFO: - Episode: 712/1500, Reward: 34.000, Step: 34 +2022-12-07 16:13:31 - r - INFO: - Episode: 713/1500, Reward: 39.000, Step: 39 +2022-12-07 16:13:31 - r - INFO: - Episode: 714/1500, Reward: 23.000, Step: 23 +2022-12-07 16:13:31 - r - INFO: - Episode: 715/1500, Reward: 53.000, Step: 53 +2022-12-07 16:13:32 - r - INFO: - Episode: 716/1500, Reward: 26.000, Step: 26 +2022-12-07 16:13:32 - r - INFO: - Episode: 717/1500, Reward: 36.000, Step: 36 +2022-12-07 16:13:32 - r - INFO: - Episode: 718/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:32 - r - INFO: - Episode: 719/1500, Reward: 25.000, Step: 25 +2022-12-07 16:13:55 - r - INFO: - Episode: 720/1500, Reward: 18.000, Step: 18 +2022-12-07 16:13:55 - r - INFO: - Episode: 721/1500, Reward: 39.000, Step: 39 +2022-12-07 16:13:55 - r - INFO: - Episode: 722/1500, Reward: 17.000, Step: 17 +2022-12-07 16:13:55 - r - INFO: - Episode: 723/1500, Reward: 34.000, Step: 34 +2022-12-07 16:13:55 - r - INFO: - Episode: 724/1500, Reward: 21.000, Step: 21 +2022-12-07 16:13:55 - r - INFO: - Episode: 725/1500, Reward: 43.000, Step: 43 +2022-12-07 16:13:55 - r - INFO: - Episode: 726/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:55 - r - INFO: - Episode: 727/1500, Reward: 49.000, Step: 49 +2022-12-07 16:13:55 - r - INFO: - Episode: 728/1500, Reward: 21.000, Step: 21 +2022-12-07 16:13:55 - r - INFO: - Episode: 729/1500, Reward: 37.000, Step: 37 +2022-12-07 16:13:55 - r - INFO: - Episode: 730/1500, Reward: 41.000, Step: 41 +2022-12-07 16:13:56 - r - INFO: - Current episode 730 has the best eval reward: 48.200 +2022-12-07 16:13:56 - r - INFO: - Episode: 731/1500, Reward: 43.000, Step: 43 +2022-12-07 16:13:56 - r - INFO: - Episode: 732/1500, Reward: 36.000, Step: 36 +2022-12-07 16:13:56 - r - INFO: - Episode: 733/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:56 - r - INFO: - Episode: 734/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:56 - r - INFO: - Episode: 735/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:56 - r - INFO: - Episode: 736/1500, Reward: 30.000, Step: 30 +2022-12-07 16:13:56 - r - INFO: - Episode: 737/1500, Reward: 46.000, Step: 46 +2022-12-07 16:13:56 - r - INFO: - Episode: 738/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:56 - r - INFO: - Episode: 739/1500, Reward: 29.000, Step: 29 +2022-12-07 16:13:56 - r - INFO: - Episode: 740/1500, Reward: 36.000, Step: 36 +2022-12-07 16:13:57 - r - INFO: - Episode: 741/1500, Reward: 12.000, Step: 12 +2022-12-07 16:13:57 - r - INFO: - Episode: 742/1500, Reward: 74.000, Step: 74 +2022-12-07 16:13:57 - r - INFO: - Episode: 743/1500, Reward: 11.000, Step: 11 +2022-12-07 16:13:57 - r - INFO: - Episode: 744/1500, Reward: 26.000, Step: 26 +2022-12-07 16:13:57 - r - INFO: - Episode: 745/1500, Reward: 21.000, Step: 21 +2022-12-07 16:13:57 - r - INFO: - Episode: 746/1500, Reward: 25.000, Step: 25 +2022-12-07 16:13:57 - r - INFO: - Episode: 747/1500, Reward: 19.000, Step: 19 +2022-12-07 16:13:57 - r - INFO: - Episode: 748/1500, Reward: 47.000, Step: 47 +2022-12-07 16:13:57 - r - INFO: - Episode: 749/1500, Reward: 15.000, Step: 15 +2022-12-07 16:13:57 - r - INFO: - Episode: 750/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:58 - r - INFO: - Episode: 751/1500, Reward: 20.000, Step: 20 +2022-12-07 16:13:58 - r - INFO: - Episode: 752/1500, Reward: 42.000, Step: 42 +2022-12-07 16:13:58 - r - INFO: - Episode: 753/1500, Reward: 26.000, Step: 26 +2022-12-07 16:13:58 - r - INFO: - Episode: 754/1500, Reward: 30.000, Step: 30 +2022-12-07 16:13:58 - r - INFO: - Episode: 755/1500, Reward: 25.000, Step: 25 +2022-12-07 16:13:58 - r - INFO: - Episode: 756/1500, Reward: 46.000, Step: 46 +2022-12-07 16:13:58 - r - INFO: - Episode: 757/1500, Reward: 32.000, Step: 32 +2022-12-07 16:13:58 - r - INFO: - Episode: 758/1500, Reward: 28.000, Step: 28 +2022-12-07 16:13:58 - r - INFO: - Episode: 759/1500, Reward: 34.000, Step: 34 +2022-12-07 16:13:58 - r - INFO: - Episode: 760/1500, Reward: 35.000, Step: 35 +2022-12-07 16:13:58 - r - INFO: - Episode: 761/1500, Reward: 73.000, Step: 73 +2022-12-07 16:13:58 - r - INFO: - Episode: 762/1500, Reward: 25.000, Step: 25 +2022-12-07 16:13:59 - r - INFO: - Episode: 763/1500, Reward: 13.000, Step: 13 +2022-12-07 16:13:59 - r - INFO: - Episode: 764/1500, Reward: 22.000, Step: 22 +2022-12-07 16:13:59 - r - INFO: - Episode: 765/1500, Reward: 39.000, Step: 39 +2022-12-07 16:13:59 - r - INFO: - Episode: 766/1500, Reward: 13.000, Step: 13 +2022-12-07 16:13:59 - r - INFO: - Episode: 767/1500, Reward: 63.000, Step: 63 +2022-12-07 16:13:59 - r - INFO: - Episode: 768/1500, Reward: 25.000, Step: 25 +2022-12-07 16:13:59 - r - INFO: - Episode: 769/1500, Reward: 31.000, Step: 31 +2022-12-07 16:13:59 - r - INFO: - Episode: 770/1500, Reward: 42.000, Step: 42 +2022-12-07 16:13:59 - r - INFO: - Episode: 771/1500, Reward: 21.000, Step: 21 +2022-12-07 16:13:59 - r - INFO: - Episode: 772/1500, Reward: 33.000, Step: 33 +2022-12-07 16:13:59 - r - INFO: - Episode: 773/1500, Reward: 28.000, Step: 28 +2022-12-07 16:13:59 - r - INFO: - Episode: 774/1500, Reward: 36.000, Step: 36 +2022-12-07 16:13:59 - r - INFO: - Episode: 775/1500, Reward: 19.000, Step: 19 +2022-12-07 16:14:00 - r - INFO: - Episode: 776/1500, Reward: 39.000, Step: 39 +2022-12-07 16:14:00 - r - INFO: - Episode: 777/1500, Reward: 19.000, Step: 19 +2022-12-07 16:14:00 - r - INFO: - Episode: 778/1500, Reward: 15.000, Step: 15 +2022-12-07 16:14:00 - r - INFO: - Episode: 779/1500, Reward: 12.000, Step: 12 +2022-12-07 16:14:00 - r - INFO: - Episode: 780/1500, Reward: 33.000, Step: 33 +2022-12-07 16:14:00 - r - INFO: - Episode: 781/1500, Reward: 14.000, Step: 14 +2022-12-07 16:14:00 - r - INFO: - Episode: 782/1500, Reward: 36.000, Step: 36 +2022-12-07 16:14:00 - r - INFO: - Episode: 783/1500, Reward: 20.000, Step: 20 +2022-12-07 16:14:00 - r - INFO: - Episode: 784/1500, Reward: 80.000, Step: 80 +2022-12-07 16:14:00 - r - INFO: - Episode: 785/1500, Reward: 21.000, Step: 21 +2022-12-07 16:14:01 - r - INFO: - Episode: 786/1500, Reward: 44.000, Step: 44 +2022-12-07 16:14:24 - r - INFO: - Episode: 787/1500, Reward: 110.000, Step: 110 +2022-12-07 16:14:24 - r - INFO: - Episode: 788/1500, Reward: 15.000, Step: 15 +2022-12-07 16:14:24 - r - INFO: - Episode: 789/1500, Reward: 20.000, Step: 20 +2022-12-07 16:14:24 - r - INFO: - Episode: 790/1500, Reward: 31.000, Step: 31 +2022-12-07 16:14:24 - r - INFO: - Episode: 791/1500, Reward: 26.000, Step: 26 +2022-12-07 16:14:24 - r - INFO: - Episode: 792/1500, Reward: 41.000, Step: 41 +2022-12-07 16:14:24 - r - INFO: - Episode: 793/1500, Reward: 39.000, Step: 39 +2022-12-07 16:14:24 - r - INFO: - Episode: 794/1500, Reward: 40.000, Step: 40 +2022-12-07 16:14:24 - r - INFO: - Episode: 795/1500, Reward: 42.000, Step: 42 +2022-12-07 16:14:24 - r - INFO: - Episode: 796/1500, Reward: 21.000, Step: 21 +2022-12-07 16:14:25 - r - INFO: - Episode: 797/1500, Reward: 38.000, Step: 38 +2022-12-07 16:14:25 - r - INFO: - Episode: 798/1500, Reward: 19.000, Step: 19 +2022-12-07 16:14:25 - r - INFO: - Episode: 799/1500, Reward: 45.000, Step: 45 +2022-12-07 16:14:25 - r - INFO: - Episode: 800/1500, Reward: 30.000, Step: 30 +2022-12-07 16:14:25 - r - INFO: - Episode: 801/1500, Reward: 11.000, Step: 11 +2022-12-07 16:14:25 - r - INFO: - Episode: 802/1500, Reward: 55.000, Step: 55 +2022-12-07 16:14:25 - r - INFO: - Episode: 803/1500, Reward: 37.000, Step: 37 +2022-12-07 16:14:25 - r - INFO: - Episode: 804/1500, Reward: 114.000, Step: 114 +2022-12-07 16:14:25 - r - INFO: - Episode: 805/1500, Reward: 48.000, Step: 48 +2022-12-07 16:14:25 - r - INFO: - Episode: 806/1500, Reward: 52.000, Step: 52 +2022-12-07 16:14:26 - r - INFO: - Episode: 807/1500, Reward: 29.000, Step: 29 +2022-12-07 16:14:26 - r - INFO: - Episode: 808/1500, Reward: 34.000, Step: 34 +2022-12-07 16:14:26 - r - INFO: - Episode: 809/1500, Reward: 45.000, Step: 45 +2022-12-07 16:14:26 - r - INFO: - Episode: 810/1500, Reward: 24.000, Step: 24 +2022-12-07 16:14:26 - r - INFO: - Episode: 811/1500, Reward: 34.000, Step: 34 +2022-12-07 16:14:26 - r - INFO: - Episode: 812/1500, Reward: 16.000, Step: 16 +2022-12-07 16:14:26 - r - INFO: - Episode: 813/1500, Reward: 28.000, Step: 28 +2022-12-07 16:14:26 - r - INFO: - Episode: 814/1500, Reward: 40.000, Step: 40 +2022-12-07 16:14:26 - r - INFO: - Episode: 815/1500, Reward: 18.000, Step: 18 +2022-12-07 16:14:26 - r - INFO: - Episode: 816/1500, Reward: 24.000, Step: 24 +2022-12-07 16:14:26 - r - INFO: - Episode: 817/1500, Reward: 20.000, Step: 20 +2022-12-07 16:14:26 - r - INFO: - Episode: 818/1500, Reward: 45.000, Step: 45 +2022-12-07 16:14:26 - r - INFO: - Episode: 819/1500, Reward: 102.000, Step: 102 +2022-12-07 16:14:26 - r - INFO: - Episode: 820/1500, Reward: 15.000, Step: 15 +2022-12-07 16:14:27 - r - INFO: - Episode: 821/1500, Reward: 117.000, Step: 117 +2022-12-07 16:14:27 - r - INFO: - Episode: 822/1500, Reward: 21.000, Step: 21 +2022-12-07 16:14:27 - r - INFO: - Episode: 823/1500, Reward: 21.000, Step: 21 +2022-12-07 16:14:27 - r - INFO: - Episode: 824/1500, Reward: 18.000, Step: 18 +2022-12-07 16:14:27 - r - INFO: - Episode: 825/1500, Reward: 61.000, Step: 61 +2022-12-07 16:14:27 - r - INFO: - Episode: 826/1500, Reward: 32.000, Step: 32 +2022-12-07 16:14:27 - r - INFO: - Episode: 827/1500, Reward: 39.000, Step: 39 +2022-12-07 16:14:27 - r - INFO: - Episode: 828/1500, Reward: 22.000, Step: 22 +2022-12-07 16:14:27 - r - INFO: - Episode: 829/1500, Reward: 29.000, Step: 29 +2022-12-07 16:14:27 - r - INFO: - Episode: 830/1500, Reward: 40.000, Step: 40 +2022-12-07 16:14:28 - r - INFO: - Current episode 830 has the best eval reward: 60.400 +2022-12-07 16:14:28 - r - INFO: - Episode: 831/1500, Reward: 27.000, Step: 27 +2022-12-07 16:14:28 - r - INFO: - Episode: 832/1500, Reward: 26.000, Step: 26 +2022-12-07 16:14:28 - r - INFO: - Episode: 833/1500, Reward: 30.000, Step: 30 +2022-12-07 16:14:28 - r - INFO: - Episode: 834/1500, Reward: 39.000, Step: 39 +2022-12-07 16:14:28 - r - INFO: - Episode: 835/1500, Reward: 32.000, Step: 32 +2022-12-07 16:14:28 - r - INFO: - Episode: 836/1500, Reward: 24.000, Step: 24 +2022-12-07 16:14:28 - r - INFO: - Episode: 837/1500, Reward: 23.000, Step: 23 +2022-12-07 16:14:28 - r - INFO: - Episode: 838/1500, Reward: 56.000, Step: 56 +2022-12-07 16:14:28 - r - INFO: - Episode: 839/1500, Reward: 18.000, Step: 18 +2022-12-07 16:14:28 - r - INFO: - Episode: 840/1500, Reward: 25.000, Step: 25 +2022-12-07 16:14:29 - r - INFO: - Episode: 841/1500, Reward: 30.000, Step: 30 +2022-12-07 16:14:52 - r - INFO: - Episode: 842/1500, Reward: 86.000, Step: 86 +2022-12-07 16:14:52 - r - INFO: - Episode: 843/1500, Reward: 60.000, Step: 60 +2022-12-07 16:14:52 - r - INFO: - Episode: 844/1500, Reward: 54.000, Step: 54 +2022-12-07 16:14:52 - r - INFO: - Episode: 845/1500, Reward: 55.000, Step: 55 +2022-12-07 16:14:53 - r - INFO: - Episode: 846/1500, Reward: 31.000, Step: 31 +2022-12-07 16:14:53 - r - INFO: - Episode: 847/1500, Reward: 68.000, Step: 68 +2022-12-07 16:14:53 - r - INFO: - Episode: 848/1500, Reward: 38.000, Step: 38 +2022-12-07 16:14:53 - r - INFO: - Episode: 849/1500, Reward: 19.000, Step: 19 +2022-12-07 16:14:53 - r - INFO: - Episode: 850/1500, Reward: 31.000, Step: 31 +2022-12-07 16:14:53 - r - INFO: - Episode: 851/1500, Reward: 57.000, Step: 57 +2022-12-07 16:14:53 - r - INFO: - Episode: 852/1500, Reward: 78.000, Step: 78 +2022-12-07 16:14:53 - r - INFO: - Episode: 853/1500, Reward: 25.000, Step: 25 +2022-12-07 16:14:53 - r - INFO: - Episode: 854/1500, Reward: 57.000, Step: 57 +2022-12-07 16:14:53 - r - INFO: - Episode: 855/1500, Reward: 29.000, Step: 29 +2022-12-07 16:14:54 - r - INFO: - Episode: 856/1500, Reward: 26.000, Step: 26 +2022-12-07 16:14:54 - r - INFO: - Episode: 857/1500, Reward: 23.000, Step: 23 +2022-12-07 16:14:54 - r - INFO: - Episode: 858/1500, Reward: 18.000, Step: 18 +2022-12-07 16:14:54 - r - INFO: - Episode: 859/1500, Reward: 26.000, Step: 26 +2022-12-07 16:14:54 - r - INFO: - Episode: 860/1500, Reward: 30.000, Step: 30 +2022-12-07 16:14:54 - r - INFO: - Episode: 861/1500, Reward: 59.000, Step: 59 +2022-12-07 16:14:54 - r - INFO: - Episode: 862/1500, Reward: 69.000, Step: 69 +2022-12-07 16:14:54 - r - INFO: - Episode: 863/1500, Reward: 54.000, Step: 54 +2022-12-07 16:14:54 - r - INFO: - Episode: 864/1500, Reward: 68.000, Step: 68 +2022-12-07 16:14:55 - r - INFO: - Episode: 865/1500, Reward: 118.000, Step: 118 +2022-12-07 16:14:55 - r - INFO: - Episode: 866/1500, Reward: 44.000, Step: 44 +2022-12-07 16:14:55 - r - INFO: - Episode: 867/1500, Reward: 74.000, Step: 74 +2022-12-07 16:14:55 - r - INFO: - Episode: 868/1500, Reward: 65.000, Step: 65 +2022-12-07 16:14:55 - r - INFO: - Episode: 869/1500, Reward: 73.000, Step: 73 +2022-12-07 16:14:55 - r - INFO: - Episode: 870/1500, Reward: 42.000, Step: 42 +2022-12-07 16:14:56 - r - INFO: - Episode: 871/1500, Reward: 27.000, Step: 27 +2022-12-07 16:14:56 - r - INFO: - Episode: 872/1500, Reward: 20.000, Step: 20 +2022-12-07 16:14:56 - r - INFO: - Episode: 873/1500, Reward: 59.000, Step: 59 +2022-12-07 16:14:56 - r - INFO: - Episode: 874/1500, Reward: 36.000, Step: 36 +2022-12-07 16:14:56 - r - INFO: - Episode: 875/1500, Reward: 24.000, Step: 24 +2022-12-07 16:14:56 - r - INFO: - Episode: 876/1500, Reward: 71.000, Step: 71 +2022-12-07 16:14:56 - r - INFO: - Episode: 877/1500, Reward: 35.000, Step: 35 +2022-12-07 16:14:56 - r - INFO: - Episode: 878/1500, Reward: 37.000, Step: 37 +2022-12-07 16:14:56 - r - INFO: - Episode: 879/1500, Reward: 73.000, Step: 73 +2022-12-07 16:14:56 - r - INFO: - Episode: 880/1500, Reward: 44.000, Step: 44 +2022-12-07 16:14:57 - r - INFO: - Episode: 881/1500, Reward: 72.000, Step: 72 +2022-12-07 16:14:57 - r - INFO: - Episode: 882/1500, Reward: 26.000, Step: 26 +2022-12-07 16:15:20 - r - INFO: - Episode: 883/1500, Reward: 62.000, Step: 62 +2022-12-07 16:15:20 - r - INFO: - Episode: 884/1500, Reward: 33.000, Step: 33 +2022-12-07 16:15:20 - r - INFO: - Episode: 885/1500, Reward: 21.000, Step: 21 +2022-12-07 16:15:20 - r - INFO: - Episode: 886/1500, Reward: 23.000, Step: 23 +2022-12-07 16:15:20 - r - INFO: - Episode: 887/1500, Reward: 26.000, Step: 26 +2022-12-07 16:15:20 - r - INFO: - Episode: 888/1500, Reward: 33.000, Step: 33 +2022-12-07 16:15:20 - r - INFO: - Episode: 889/1500, Reward: 27.000, Step: 27 +2022-12-07 16:15:20 - r - INFO: - Episode: 890/1500, Reward: 24.000, Step: 24 +2022-12-07 16:15:21 - r - INFO: - Episode: 891/1500, Reward: 63.000, Step: 63 +2022-12-07 16:15:21 - r - INFO: - Episode: 892/1500, Reward: 49.000, Step: 49 +2022-12-07 16:15:21 - r - INFO: - Episode: 893/1500, Reward: 30.000, Step: 30 +2022-12-07 16:15:21 - r - INFO: - Episode: 894/1500, Reward: 20.000, Step: 20 +2022-12-07 16:15:21 - r - INFO: - Episode: 895/1500, Reward: 48.000, Step: 48 +2022-12-07 16:15:21 - r - INFO: - Episode: 896/1500, Reward: 31.000, Step: 31 +2022-12-07 16:15:22 - r - INFO: - Episode: 897/1500, Reward: 96.000, Step: 96 +2022-12-07 16:15:22 - r - INFO: - Episode: 898/1500, Reward: 67.000, Step: 67 +2022-12-07 16:15:22 - r - INFO: - Episode: 899/1500, Reward: 58.000, Step: 58 +2022-12-07 16:15:22 - r - INFO: - Episode: 900/1500, Reward: 40.000, Step: 40 +2022-12-07 16:15:22 - r - INFO: - Episode: 901/1500, Reward: 25.000, Step: 25 +2022-12-07 16:15:22 - r - INFO: - Episode: 902/1500, Reward: 18.000, Step: 18 +2022-12-07 16:15:22 - r - INFO: - Episode: 903/1500, Reward: 22.000, Step: 22 +2022-12-07 16:15:22 - r - INFO: - Episode: 904/1500, Reward: 74.000, Step: 74 +2022-12-07 16:15:22 - r - INFO: - Episode: 905/1500, Reward: 26.000, Step: 26 +2022-12-07 16:15:23 - r - INFO: - Episode: 906/1500, Reward: 18.000, Step: 18 +2022-12-07 16:15:23 - r - INFO: - Episode: 907/1500, Reward: 38.000, Step: 38 +2022-12-07 16:15:23 - r - INFO: - Episode: 908/1500, Reward: 22.000, Step: 22 +2022-12-07 16:15:23 - r - INFO: - Episode: 909/1500, Reward: 50.000, Step: 50 +2022-12-07 16:15:23 - r - INFO: - Episode: 910/1500, Reward: 15.000, Step: 15 +2022-12-07 16:15:23 - r - INFO: - Current episode 910 has the best eval reward: 64.400 +2022-12-07 16:15:23 - r - INFO: - Episode: 911/1500, Reward: 27.000, Step: 27 +2022-12-07 16:15:23 - r - INFO: - Episode: 912/1500, Reward: 39.000, Step: 39 +2022-12-07 16:15:23 - r - INFO: - Episode: 913/1500, Reward: 87.000, Step: 87 +2022-12-07 16:15:24 - r - INFO: - Episode: 914/1500, Reward: 148.000, Step: 148 +2022-12-07 16:15:24 - r - INFO: - Episode: 915/1500, Reward: 13.000, Step: 13 +2022-12-07 16:15:24 - r - INFO: - Episode: 916/1500, Reward: 89.000, Step: 89 +2022-12-07 16:15:24 - r - INFO: - Episode: 917/1500, Reward: 56.000, Step: 56 +2022-12-07 16:15:24 - r - INFO: - Episode: 918/1500, Reward: 37.000, Step: 37 +2022-12-07 16:15:24 - r - INFO: - Episode: 919/1500, Reward: 22.000, Step: 22 +2022-12-07 16:15:24 - r - INFO: - Episode: 920/1500, Reward: 20.000, Step: 20 +2022-12-07 16:15:24 - r - INFO: - Episode: 921/1500, Reward: 80.000, Step: 80 +2022-12-07 16:15:24 - r - INFO: - Episode: 922/1500, Reward: 71.000, Step: 71 +2022-12-07 16:15:25 - r - INFO: - Episode: 923/1500, Reward: 50.000, Step: 50 +2022-12-07 16:15:25 - r - INFO: - Episode: 924/1500, Reward: 33.000, Step: 33 +2022-12-07 16:15:25 - r - INFO: - Episode: 925/1500, Reward: 80.000, Step: 80 +2022-12-07 16:15:25 - r - INFO: - Current episode 925 has the best eval reward: 69.400 +2022-12-07 16:15:25 - r - INFO: - Episode: 926/1500, Reward: 69.000, Step: 69 +2022-12-07 16:15:25 - r - INFO: - Episode: 927/1500, Reward: 28.000, Step: 28 +2022-12-07 16:15:25 - r - INFO: - Episode: 928/1500, Reward: 55.000, Step: 55 +2022-12-07 16:15:49 - r - INFO: - Episode: 929/1500, Reward: 55.000, Step: 55 +2022-12-07 16:15:49 - r - INFO: - Episode: 930/1500, Reward: 94.000, Step: 94 +2022-12-07 16:15:50 - r - INFO: - Current episode 930 has the best eval reward: 73.000 +2022-12-07 16:15:50 - r - INFO: - Episode: 931/1500, Reward: 86.000, Step: 86 +2022-12-07 16:15:50 - r - INFO: - Episode: 932/1500, Reward: 83.000, Step: 83 +2022-12-07 16:15:50 - r - INFO: - Episode: 933/1500, Reward: 64.000, Step: 64 +2022-12-07 16:15:50 - r - INFO: - Episode: 934/1500, Reward: 94.000, Step: 94 +2022-12-07 16:15:50 - r - INFO: - Episode: 935/1500, Reward: 17.000, Step: 17 +2022-12-07 16:15:51 - r - INFO: - Episode: 936/1500, Reward: 104.000, Step: 104 +2022-12-07 16:15:51 - r - INFO: - Episode: 937/1500, Reward: 37.000, Step: 37 +2022-12-07 16:15:51 - r - INFO: - Episode: 938/1500, Reward: 51.000, Step: 51 +2022-12-07 16:15:51 - r - INFO: - Episode: 939/1500, Reward: 33.000, Step: 33 +2022-12-07 16:15:51 - r - INFO: - Episode: 940/1500, Reward: 106.000, Step: 106 +2022-12-07 16:15:51 - r - INFO: - Episode: 941/1500, Reward: 73.000, Step: 73 +2022-12-07 16:15:51 - r - INFO: - Episode: 942/1500, Reward: 54.000, Step: 54 +2022-12-07 16:15:51 - r - INFO: - Episode: 943/1500, Reward: 39.000, Step: 39 +2022-12-07 16:15:51 - r - INFO: - Episode: 944/1500, Reward: 52.000, Step: 52 +2022-12-07 16:15:52 - r - INFO: - Episode: 945/1500, Reward: 127.000, Step: 127 +2022-12-07 16:15:52 - r - INFO: - Episode: 946/1500, Reward: 34.000, Step: 34 +2022-12-07 16:15:52 - r - INFO: - Episode: 947/1500, Reward: 32.000, Step: 32 +2022-12-07 16:15:52 - r - INFO: - Episode: 948/1500, Reward: 32.000, Step: 32 +2022-12-07 16:15:52 - r - INFO: - Episode: 949/1500, Reward: 47.000, Step: 47 +2022-12-07 16:15:52 - r - INFO: - Episode: 950/1500, Reward: 15.000, Step: 15 +2022-12-07 16:15:53 - r - INFO: - Episode: 951/1500, Reward: 34.000, Step: 34 +2022-12-07 16:15:53 - r - INFO: - Episode: 952/1500, Reward: 47.000, Step: 47 +2022-12-07 16:15:53 - r - INFO: - Episode: 953/1500, Reward: 32.000, Step: 32 +2022-12-07 16:15:53 - r - INFO: - Episode: 954/1500, Reward: 29.000, Step: 29 +2022-12-07 16:15:53 - r - INFO: - Episode: 955/1500, Reward: 64.000, Step: 64 +2022-12-07 16:15:53 - r - INFO: - Episode: 956/1500, Reward: 73.000, Step: 73 +2022-12-07 16:15:53 - r - INFO: - Episode: 957/1500, Reward: 39.000, Step: 39 +2022-12-07 16:15:53 - r - INFO: - Episode: 958/1500, Reward: 33.000, Step: 33 +2022-12-07 16:15:53 - r - INFO: - Episode: 959/1500, Reward: 25.000, Step: 25 +2022-12-07 16:15:53 - r - INFO: - Episode: 960/1500, Reward: 56.000, Step: 56 +2022-12-07 16:15:54 - r - INFO: - Episode: 961/1500, Reward: 34.000, Step: 34 +2022-12-07 16:15:54 - r - INFO: - Episode: 962/1500, Reward: 47.000, Step: 47 +2022-12-07 16:15:54 - r - INFO: - Episode: 963/1500, Reward: 36.000, Step: 36 +2022-12-07 16:15:54 - r - INFO: - Episode: 964/1500, Reward: 85.000, Step: 85 +2022-12-07 16:15:54 - r - INFO: - Episode: 965/1500, Reward: 58.000, Step: 58 +2022-12-07 16:15:54 - r - INFO: - Current episode 965 has the best eval reward: 73.300 +2022-12-07 16:15:55 - r - INFO: - Episode: 966/1500, Reward: 20.000, Step: 20 +2022-12-07 16:16:18 - r - INFO: - Episode: 967/1500, Reward: 153.000, Step: 153 +2022-12-07 16:16:18 - r - INFO: - Episode: 968/1500, Reward: 35.000, Step: 35 +2022-12-07 16:16:18 - r - INFO: - Episode: 969/1500, Reward: 55.000, Step: 55 +2022-12-07 16:16:18 - r - INFO: - Episode: 970/1500, Reward: 66.000, Step: 66 +2022-12-07 16:16:19 - r - INFO: - Episode: 971/1500, Reward: 41.000, Step: 41 +2022-12-07 16:16:19 - r - INFO: - Episode: 972/1500, Reward: 76.000, Step: 76 +2022-12-07 16:16:19 - r - INFO: - Episode: 973/1500, Reward: 88.000, Step: 88 +2022-12-07 16:16:19 - r - INFO: - Episode: 974/1500, Reward: 35.000, Step: 35 +2022-12-07 16:16:19 - r - INFO: - Episode: 975/1500, Reward: 35.000, Step: 35 +2022-12-07 16:16:19 - r - INFO: - Episode: 976/1500, Reward: 105.000, Step: 105 +2022-12-07 16:16:20 - r - INFO: - Episode: 977/1500, Reward: 116.000, Step: 116 +2022-12-07 16:16:20 - r - INFO: - Episode: 978/1500, Reward: 58.000, Step: 58 +2022-12-07 16:16:20 - r - INFO: - Episode: 979/1500, Reward: 50.000, Step: 50 +2022-12-07 16:16:20 - r - INFO: - Episode: 980/1500, Reward: 26.000, Step: 26 +2022-12-07 16:16:20 - r - INFO: - Episode: 981/1500, Reward: 76.000, Step: 76 +2022-12-07 16:16:20 - r - INFO: - Episode: 982/1500, Reward: 46.000, Step: 46 +2022-12-07 16:16:20 - r - INFO: - Episode: 983/1500, Reward: 57.000, Step: 57 +2022-12-07 16:16:20 - r - INFO: - Episode: 984/1500, Reward: 39.000, Step: 39 +2022-12-07 16:16:20 - r - INFO: - Episode: 985/1500, Reward: 40.000, Step: 40 +2022-12-07 16:16:21 - r - INFO: - Episode: 986/1500, Reward: 41.000, Step: 41 +2022-12-07 16:16:21 - r - INFO: - Episode: 987/1500, Reward: 80.000, Step: 80 +2022-12-07 16:16:21 - r - INFO: - Episode: 988/1500, Reward: 98.000, Step: 98 +2022-12-07 16:16:21 - r - INFO: - Episode: 989/1500, Reward: 57.000, Step: 57 +2022-12-07 16:16:21 - r - INFO: - Episode: 990/1500, Reward: 89.000, Step: 89 +2022-12-07 16:16:22 - r - INFO: - Episode: 991/1500, Reward: 124.000, Step: 124 +2022-12-07 16:16:22 - r - INFO: - Episode: 992/1500, Reward: 42.000, Step: 42 +2022-12-07 16:16:22 - r - INFO: - Episode: 993/1500, Reward: 36.000, Step: 36 +2022-12-07 16:16:22 - r - INFO: - Episode: 994/1500, Reward: 41.000, Step: 41 +2022-12-07 16:16:22 - r - INFO: - Episode: 995/1500, Reward: 67.000, Step: 67 +2022-12-07 16:16:22 - r - INFO: - Current episode 995 has the best eval reward: 94.400 +2022-12-07 16:16:23 - r - INFO: - Episode: 996/1500, Reward: 130.000, Step: 130 +2022-12-07 16:16:23 - r - INFO: - Episode: 997/1500, Reward: 91.000, Step: 91 +2022-12-07 16:16:46 - r - INFO: - Episode: 998/1500, Reward: 36.000, Step: 36 +2022-12-07 16:16:46 - r - INFO: - Episode: 999/1500, Reward: 76.000, Step: 76 +2022-12-07 16:16:46 - r - INFO: - Episode: 1000/1500, Reward: 37.000, Step: 37 +2022-12-07 16:16:46 - r - INFO: - Episode: 1001/1500, Reward: 28.000, Step: 28 +2022-12-07 16:16:46 - r - INFO: - Episode: 1002/1500, Reward: 79.000, Step: 79 +2022-12-07 16:16:47 - r - INFO: - Episode: 1003/1500, Reward: 25.000, Step: 25 +2022-12-07 16:16:47 - r - INFO: - Episode: 1004/1500, Reward: 41.000, Step: 41 +2022-12-07 16:16:47 - r - INFO: - Episode: 1005/1500, Reward: 28.000, Step: 28 +2022-12-07 16:16:47 - r - INFO: - Episode: 1006/1500, Reward: 35.000, Step: 35 +2022-12-07 16:16:47 - r - INFO: - Episode: 1007/1500, Reward: 85.000, Step: 85 +2022-12-07 16:16:47 - r - INFO: - Episode: 1008/1500, Reward: 34.000, Step: 34 +2022-12-07 16:16:47 - r - INFO: - Episode: 1009/1500, Reward: 39.000, Step: 39 +2022-12-07 16:16:48 - r - INFO: - Episode: 1010/1500, Reward: 200.000, Step: 200 +2022-12-07 16:16:48 - r - INFO: - Episode: 1011/1500, Reward: 39.000, Step: 39 +2022-12-07 16:16:48 - r - INFO: - Episode: 1012/1500, Reward: 41.000, Step: 41 +2022-12-07 16:16:48 - r - INFO: - Episode: 1013/1500, Reward: 64.000, Step: 64 +2022-12-07 16:16:48 - r - INFO: - Episode: 1014/1500, Reward: 56.000, Step: 56 +2022-12-07 16:16:48 - r - INFO: - Episode: 1015/1500, Reward: 39.000, Step: 39 +2022-12-07 16:16:49 - r - INFO: - Episode: 1016/1500, Reward: 87.000, Step: 87 +2022-12-07 16:16:49 - r - INFO: - Episode: 1017/1500, Reward: 48.000, Step: 48 +2022-12-07 16:16:49 - r - INFO: - Episode: 1018/1500, Reward: 82.000, Step: 82 +2022-12-07 16:16:49 - r - INFO: - Episode: 1019/1500, Reward: 70.000, Step: 70 +2022-12-07 16:16:49 - r - INFO: - Episode: 1020/1500, Reward: 129.000, Step: 129 +2022-12-07 16:16:50 - r - INFO: - Episode: 1021/1500, Reward: 101.000, Step: 101 +2022-12-07 16:16:50 - r - INFO: - Episode: 1022/1500, Reward: 49.000, Step: 49 +2022-12-07 16:16:50 - r - INFO: - Episode: 1023/1500, Reward: 199.000, Step: 199 +2022-12-07 16:16:50 - r - INFO: - Episode: 1024/1500, Reward: 102.000, Step: 102 +2022-12-07 16:16:50 - r - INFO: - Episode: 1025/1500, Reward: 80.000, Step: 80 +2022-12-07 16:16:51 - r - INFO: - Episode: 1026/1500, Reward: 30.000, Step: 30 +2022-12-07 16:16:51 - r - INFO: - Episode: 1027/1500, Reward: 37.000, Step: 37 +2022-12-07 16:16:51 - r - INFO: - Episode: 1028/1500, Reward: 53.000, Step: 53 +2022-12-07 16:17:14 - r - INFO: - Episode: 1029/1500, Reward: 147.000, Step: 147 +2022-12-07 16:17:14 - r - INFO: - Episode: 1030/1500, Reward: 59.000, Step: 59 +2022-12-07 16:17:15 - r - INFO: - Episode: 1031/1500, Reward: 151.000, Step: 151 +2022-12-07 16:17:15 - r - INFO: - Episode: 1032/1500, Reward: 87.000, Step: 87 +2022-12-07 16:17:15 - r - INFO: - Episode: 1033/1500, Reward: 74.000, Step: 74 +2022-12-07 16:17:15 - r - INFO: - Episode: 1034/1500, Reward: 46.000, Step: 46 +2022-12-07 16:17:15 - r - INFO: - Episode: 1035/1500, Reward: 78.000, Step: 78 +2022-12-07 16:17:16 - r - INFO: - Episode: 1036/1500, Reward: 69.000, Step: 69 +2022-12-07 16:17:16 - r - INFO: - Episode: 1037/1500, Reward: 91.000, Step: 91 +2022-12-07 16:17:16 - r - INFO: - Episode: 1038/1500, Reward: 79.000, Step: 79 +2022-12-07 16:17:16 - r - INFO: - Episode: 1039/1500, Reward: 77.000, Step: 77 +2022-12-07 16:17:16 - r - INFO: - Episode: 1040/1500, Reward: 35.000, Step: 35 +2022-12-07 16:17:17 - r - INFO: - Episode: 1041/1500, Reward: 91.000, Step: 91 +2022-12-07 16:17:17 - r - INFO: - Episode: 1042/1500, Reward: 154.000, Step: 154 +2022-12-07 16:17:17 - r - INFO: - Episode: 1043/1500, Reward: 129.000, Step: 129 +2022-12-07 16:17:17 - r - INFO: - Episode: 1044/1500, Reward: 67.000, Step: 67 +2022-12-07 16:17:17 - r - INFO: - Episode: 1045/1500, Reward: 66.000, Step: 66 +2022-12-07 16:17:18 - r - INFO: - Episode: 1046/1500, Reward: 182.000, Step: 182 +2022-12-07 16:17:18 - r - INFO: - Episode: 1047/1500, Reward: 43.000, Step: 43 +2022-12-07 16:17:18 - r - INFO: - Episode: 1048/1500, Reward: 62.000, Step: 62 +2022-12-07 16:17:18 - r - INFO: - Episode: 1049/1500, Reward: 153.000, Step: 153 +2022-12-07 16:17:18 - r - INFO: - Episode: 1050/1500, Reward: 25.000, Step: 25 +2022-12-07 16:17:19 - r - INFO: - Episode: 1051/1500, Reward: 57.000, Step: 57 +2022-12-07 16:17:41 - r - INFO: - Episode: 1052/1500, Reward: 113.000, Step: 113 +2022-12-07 16:17:42 - r - INFO: - Episode: 1053/1500, Reward: 78.000, Step: 78 +2022-12-07 16:17:42 - r - INFO: - Episode: 1054/1500, Reward: 69.000, Step: 69 +2022-12-07 16:17:42 - r - INFO: - Episode: 1055/1500, Reward: 113.000, Step: 113 +2022-12-07 16:17:43 - r - INFO: - Current episode 1055 has the best eval reward: 107.100 +2022-12-07 16:17:43 - r - INFO: - Episode: 1056/1500, Reward: 112.000, Step: 112 +2022-12-07 16:17:43 - r - INFO: - Episode: 1057/1500, Reward: 97.000, Step: 97 +2022-12-07 16:17:43 - r - INFO: - Episode: 1058/1500, Reward: 184.000, Step: 184 +2022-12-07 16:17:43 - r - INFO: - Episode: 1059/1500, Reward: 184.000, Step: 184 +2022-12-07 16:17:43 - r - INFO: - Episode: 1060/1500, Reward: 76.000, Step: 76 +2022-12-07 16:17:44 - r - INFO: - Current episode 1060 has the best eval reward: 120.800 +2022-12-07 16:17:44 - r - INFO: - Episode: 1061/1500, Reward: 172.000, Step: 172 +2022-12-07 16:17:44 - r - INFO: - Episode: 1062/1500, Reward: 84.000, Step: 84 +2022-12-07 16:17:44 - r - INFO: - Episode: 1063/1500, Reward: 85.000, Step: 85 +2022-12-07 16:17:45 - r - INFO: - Episode: 1064/1500, Reward: 125.000, Step: 125 +2022-12-07 16:17:45 - r - INFO: - Episode: 1065/1500, Reward: 52.000, Step: 52 +2022-12-07 16:17:45 - r - INFO: - Episode: 1066/1500, Reward: 40.000, Step: 40 +2022-12-07 16:17:45 - r - INFO: - Episode: 1067/1500, Reward: 79.000, Step: 79 +2022-12-07 16:17:45 - r - INFO: - Episode: 1068/1500, Reward: 183.000, Step: 183 +2022-12-07 16:17:46 - r - INFO: - Episode: 1069/1500, Reward: 88.000, Step: 88 +2022-12-07 16:17:46 - r - INFO: - Episode: 1070/1500, Reward: 55.000, Step: 55 +2022-12-07 16:17:47 - r - INFO: - Current episode 1070 has the best eval reward: 127.400 +2022-12-07 16:17:47 - r - INFO: - Episode: 1071/1500, Reward: 53.000, Step: 53 +2022-12-07 16:17:47 - r - INFO: - Episode: 1072/1500, Reward: 34.000, Step: 34 +2022-12-07 16:18:09 - r - INFO: - Episode: 1073/1500, Reward: 77.000, Step: 77 +2022-12-07 16:18:09 - r - INFO: - Episode: 1074/1500, Reward: 68.000, Step: 68 +2022-12-07 16:18:09 - r - INFO: - Episode: 1075/1500, Reward: 31.000, Step: 31 +2022-12-07 16:18:10 - r - INFO: - Episode: 1076/1500, Reward: 75.000, Step: 75 +2022-12-07 16:18:11 - r - INFO: - Episode: 1077/1500, Reward: 200.000, Step: 200 +2022-12-07 16:18:11 - r - INFO: - Episode: 1078/1500, Reward: 41.000, Step: 41 +2022-12-07 16:18:11 - r - INFO: - Episode: 1079/1500, Reward: 149.000, Step: 149 +2022-12-07 16:18:11 - r - INFO: - Episode: 1080/1500, Reward: 113.000, Step: 113 +2022-12-07 16:18:12 - r - INFO: - Episode: 1081/1500, Reward: 57.000, Step: 57 +2022-12-07 16:18:12 - r - INFO: - Episode: 1082/1500, Reward: 79.000, Step: 79 +2022-12-07 16:18:12 - r - INFO: - Episode: 1083/1500, Reward: 38.000, Step: 38 +2022-12-07 16:18:12 - r - INFO: - Episode: 1084/1500, Reward: 44.000, Step: 44 +2022-12-07 16:18:12 - r - INFO: - Episode: 1085/1500, Reward: 200.000, Step: 200 +2022-12-07 16:18:13 - r - INFO: - Current episode 1085 has the best eval reward: 133.400 +2022-12-07 16:18:13 - r - INFO: - Episode: 1086/1500, Reward: 39.000, Step: 39 +2022-12-07 16:18:13 - r - INFO: - Episode: 1087/1500, Reward: 143.000, Step: 143 +2022-12-07 16:18:13 - r - INFO: - Episode: 1088/1500, Reward: 103.000, Step: 103 +2022-12-07 16:18:13 - r - INFO: - Episode: 1089/1500, Reward: 35.000, Step: 35 +2022-12-07 16:18:13 - r - INFO: - Episode: 1090/1500, Reward: 123.000, Step: 123 +2022-12-07 16:18:14 - r - INFO: - Current episode 1090 has the best eval reward: 139.900 +2022-12-07 16:18:14 - r - INFO: - Episode: 1091/1500, Reward: 56.000, Step: 56 +2022-12-07 16:18:15 - r - INFO: - Episode: 1092/1500, Reward: 177.000, Step: 177 +2022-12-07 16:18:15 - r - INFO: - Episode: 1093/1500, Reward: 28.000, Step: 28 +2022-12-07 16:18:15 - r - INFO: - Episode: 1094/1500, Reward: 77.000, Step: 77 +2022-12-07 16:18:38 - r - INFO: - Episode: 1095/1500, Reward: 200.000, Step: 200 +2022-12-07 16:18:39 - r - INFO: - Episode: 1096/1500, Reward: 90.000, Step: 90 +2022-12-07 16:18:39 - r - INFO: - Episode: 1097/1500, Reward: 101.000, Step: 101 +2022-12-07 16:18:39 - r - INFO: - Episode: 1098/1500, Reward: 69.000, Step: 69 +2022-12-07 16:18:39 - r - INFO: - Episode: 1099/1500, Reward: 200.000, Step: 200 +2022-12-07 16:18:39 - r - INFO: - Episode: 1100/1500, Reward: 129.000, Step: 129 +2022-12-07 16:18:40 - r - INFO: - Episode: 1101/1500, Reward: 72.000, Step: 72 +2022-12-07 16:18:41 - r - INFO: - Episode: 1102/1500, Reward: 200.000, Step: 200 +2022-12-07 16:18:41 - r - INFO: - Episode: 1103/1500, Reward: 154.000, Step: 154 +2022-12-07 16:18:41 - r - INFO: - Episode: 1104/1500, Reward: 62.000, Step: 62 +2022-12-07 16:18:41 - r - INFO: - Episode: 1105/1500, Reward: 146.000, Step: 146 +2022-12-07 16:18:42 - r - INFO: - Episode: 1106/1500, Reward: 184.000, Step: 184 +2022-12-07 16:18:42 - r - INFO: - Episode: 1107/1500, Reward: 138.000, Step: 138 +2022-12-07 16:18:42 - r - INFO: - Episode: 1108/1500, Reward: 139.000, Step: 139 +2022-12-07 16:18:43 - r - INFO: - Episode: 1109/1500, Reward: 200.000, Step: 200 +2022-12-07 16:18:43 - r - INFO: - Episode: 1110/1500, Reward: 27.000, Step: 27 +2022-12-07 16:18:44 - r - INFO: - Current episode 1110 has the best eval reward: 150.300 +2022-12-07 16:19:07 - r - INFO: - Episode: 1111/1500, Reward: 124.000, Step: 124 +2022-12-07 16:19:07 - r - INFO: - Episode: 1112/1500, Reward: 98.000, Step: 98 +2022-12-07 16:19:07 - r - INFO: - Episode: 1113/1500, Reward: 114.000, Step: 114 +2022-12-07 16:19:07 - r - INFO: - Episode: 1114/1500, Reward: 194.000, Step: 194 +2022-12-07 16:19:08 - r - INFO: - Episode: 1115/1500, Reward: 140.000, Step: 140 +2022-12-07 16:19:08 - r - INFO: - Episode: 1116/1500, Reward: 52.000, Step: 52 +2022-12-07 16:19:09 - r - INFO: - Episode: 1117/1500, Reward: 140.000, Step: 140 +2022-12-07 16:19:09 - r - INFO: - Episode: 1118/1500, Reward: 110.000, Step: 110 +2022-12-07 16:19:09 - r - INFO: - Episode: 1119/1500, Reward: 125.000, Step: 125 +2022-12-07 16:19:09 - r - INFO: - Episode: 1120/1500, Reward: 55.000, Step: 55 +2022-12-07 16:19:10 - r - INFO: - Episode: 1121/1500, Reward: 97.000, Step: 97 +2022-12-07 16:19:10 - r - INFO: - Episode: 1122/1500, Reward: 148.000, Step: 148 +2022-12-07 16:19:10 - r - INFO: - Episode: 1123/1500, Reward: 115.000, Step: 115 +2022-12-07 16:19:10 - r - INFO: - Episode: 1124/1500, Reward: 121.000, Step: 121 +2022-12-07 16:19:10 - r - INFO: - Episode: 1125/1500, Reward: 123.000, Step: 123 +2022-12-07 16:19:11 - r - INFO: - Episode: 1126/1500, Reward: 43.000, Step: 43 +2022-12-07 16:19:11 - r - INFO: - Episode: 1127/1500, Reward: 96.000, Step: 96 +2022-12-07 16:19:12 - r - INFO: - Episode: 1128/1500, Reward: 111.000, Step: 111 +2022-12-07 16:19:35 - r - INFO: - Episode: 1129/1500, Reward: 172.000, Step: 172 +2022-12-07 16:19:35 - r - INFO: - Episode: 1130/1500, Reward: 87.000, Step: 87 +2022-12-07 16:19:36 - r - INFO: - Episode: 1131/1500, Reward: 116.000, Step: 116 +2022-12-07 16:19:36 - r - INFO: - Episode: 1132/1500, Reward: 103.000, Step: 103 +2022-12-07 16:19:36 - r - INFO: - Episode: 1133/1500, Reward: 111.000, Step: 111 +2022-12-07 16:19:36 - r - INFO: - Episode: 1134/1500, Reward: 81.000, Step: 81 +2022-12-07 16:19:37 - r - INFO: - Episode: 1135/1500, Reward: 162.000, Step: 162 +2022-12-07 16:19:38 - r - INFO: - Episode: 1136/1500, Reward: 200.000, Step: 200 +2022-12-07 16:19:38 - r - INFO: - Episode: 1137/1500, Reward: 69.000, Step: 69 +2022-12-07 16:19:38 - r - INFO: - Episode: 1138/1500, Reward: 94.000, Step: 94 +2022-12-07 16:19:38 - r - INFO: - Episode: 1139/1500, Reward: 125.000, Step: 125 +2022-12-07 16:19:38 - r - INFO: - Episode: 1140/1500, Reward: 82.000, Step: 82 +2022-12-07 16:19:39 - r - INFO: - Current episode 1140 has the best eval reward: 158.900 +2022-12-07 16:19:39 - r - INFO: - Episode: 1141/1500, Reward: 109.000, Step: 109 +2022-12-07 16:19:39 - r - INFO: - Episode: 1142/1500, Reward: 120.000, Step: 120 +2022-12-07 16:19:40 - r - INFO: - Episode: 1143/1500, Reward: 96.000, Step: 96 +2022-12-07 16:19:40 - r - INFO: - Episode: 1144/1500, Reward: 34.000, Step: 34 +2022-12-07 16:19:40 - r - INFO: - Episode: 1145/1500, Reward: 128.000, Step: 128 +2022-12-07 16:19:41 - r - INFO: - Episode: 1146/1500, Reward: 62.000, Step: 62 +2022-12-07 16:19:41 - r - INFO: - Episode: 1147/1500, Reward: 156.000, Step: 156 +2022-12-07 16:20:04 - r - INFO: - Episode: 1148/1500, Reward: 118.000, Step: 118 +2022-12-07 16:20:04 - r - INFO: - Episode: 1149/1500, Reward: 166.000, Step: 166 +2022-12-07 16:20:05 - r - INFO: - Episode: 1150/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:06 - r - INFO: - Episode: 1151/1500, Reward: 53.000, Step: 53 +2022-12-07 16:20:06 - r - INFO: - Episode: 1152/1500, Reward: 89.000, Step: 89 +2022-12-07 16:20:06 - r - INFO: - Episode: 1153/1500, Reward: 161.000, Step: 161 +2022-12-07 16:20:06 - r - INFO: - Episode: 1154/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:06 - r - INFO: - Episode: 1155/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:07 - r - INFO: - Episode: 1156/1500, Reward: 156.000, Step: 156 +2022-12-07 16:20:08 - r - INFO: - Episode: 1157/1500, Reward: 167.000, Step: 167 +2022-12-07 16:20:08 - r - INFO: - Episode: 1158/1500, Reward: 38.000, Step: 38 +2022-12-07 16:20:08 - r - INFO: - Episode: 1159/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:08 - r - INFO: - Episode: 1160/1500, Reward: 63.000, Step: 63 +2022-12-07 16:20:09 - r - INFO: - Episode: 1161/1500, Reward: 73.000, Step: 73 +2022-12-07 16:20:09 - r - INFO: - Episode: 1162/1500, Reward: 161.000, Step: 161 +2022-12-07 16:20:32 - r - INFO: - Episode: 1163/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:33 - r - INFO: - Episode: 1164/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:33 - r - INFO: - Episode: 1165/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:34 - r - INFO: - Current episode 1165 has the best eval reward: 182.500 +2022-12-07 16:20:34 - r - INFO: - Episode: 1166/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:35 - r - INFO: - Episode: 1167/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:35 - r - INFO: - Episode: 1168/1500, Reward: 197.000, Step: 197 +2022-12-07 16:20:35 - r - INFO: - Episode: 1169/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:35 - r - INFO: - Episode: 1170/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:37 - r - INFO: - Current episode 1170 has the best eval reward: 199.600 +2022-12-07 16:20:37 - r - INFO: - Episode: 1171/1500, Reward: 200.000, Step: 200 +2022-12-07 16:20:37 - r - INFO: - Episode: 1172/1500, Reward: 120.000, Step: 120 +2022-12-07 16:20:37 - r - INFO: - Episode: 1173/1500, Reward: 121.000, Step: 121 +2022-12-07 16:21:00 - r - INFO: - Episode: 1174/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:00 - r - INFO: - Episode: 1175/1500, Reward: 162.000, Step: 162 +2022-12-07 16:21:02 - r - INFO: - Episode: 1176/1500, Reward: 157.000, Step: 157 +2022-12-07 16:21:02 - r - INFO: - Episode: 1177/1500, Reward: 151.000, Step: 151 +2022-12-07 16:21:02 - r - INFO: - Episode: 1178/1500, Reward: 161.000, Step: 161 +2022-12-07 16:21:02 - r - INFO: - Episode: 1179/1500, Reward: 115.000, Step: 115 +2022-12-07 16:21:02 - r - INFO: - Episode: 1180/1500, Reward: 176.000, Step: 176 +2022-12-07 16:21:04 - r - INFO: - Episode: 1181/1500, Reward: 148.000, Step: 148 +2022-12-07 16:21:04 - r - INFO: - Episode: 1182/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:04 - r - INFO: - Episode: 1183/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:04 - r - INFO: - Episode: 1184/1500, Reward: 188.000, Step: 188 +2022-12-07 16:21:04 - r - INFO: - Episode: 1185/1500, Reward: 176.000, Step: 176 +2022-12-07 16:21:29 - r - INFO: - Episode: 1186/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:29 - r - INFO: - Episode: 1187/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:29 - r - INFO: - Episode: 1188/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:29 - r - INFO: - Episode: 1189/1500, Reward: 172.000, Step: 172 +2022-12-07 16:21:29 - r - INFO: - Episode: 1190/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:31 - r - INFO: - Episode: 1191/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:31 - r - INFO: - Episode: 1192/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:31 - r - INFO: - Episode: 1193/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:32 - r - INFO: - Episode: 1194/1500, Reward: 180.000, Step: 180 +2022-12-07 16:21:32 - r - INFO: - Episode: 1195/1500, Reward: 109.000, Step: 109 +2022-12-07 16:21:33 - r - INFO: - Episode: 1196/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:56 - r - INFO: - Episode: 1197/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:56 - r - INFO: - Episode: 1198/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:56 - r - INFO: - Episode: 1199/1500, Reward: 118.000, Step: 118 +2022-12-07 16:21:57 - r - INFO: - Episode: 1200/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:58 - r - INFO: - Episode: 1201/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:58 - r - INFO: - Episode: 1202/1500, Reward: 185.000, Step: 185 +2022-12-07 16:21:58 - r - INFO: - Episode: 1203/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:59 - r - INFO: - Episode: 1204/1500, Reward: 200.000, Step: 200 +2022-12-07 16:21:59 - r - INFO: - Episode: 1205/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:00 - r - INFO: - Episode: 1206/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:00 - r - INFO: - Episode: 1207/1500, Reward: 162.000, Step: 162 +2022-12-07 16:22:24 - r - INFO: - Episode: 1208/1500, Reward: 195.000, Step: 195 +2022-12-07 16:22:24 - r - INFO: - Episode: 1209/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:24 - r - INFO: - Episode: 1210/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:26 - r - INFO: - Episode: 1211/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:26 - r - INFO: - Episode: 1212/1500, Reward: 114.000, Step: 114 +2022-12-07 16:22:26 - r - INFO: - Episode: 1213/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:26 - r - INFO: - Episode: 1214/1500, Reward: 194.000, Step: 194 +2022-12-07 16:22:26 - r - INFO: - Episode: 1215/1500, Reward: 132.000, Step: 132 +2022-12-07 16:22:28 - r - INFO: - Episode: 1216/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:28 - r - INFO: - Episode: 1217/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:28 - r - INFO: - Episode: 1218/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:51 - r - INFO: - Episode: 1219/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:52 - r - INFO: - Episode: 1220/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:53 - r - INFO: - Episode: 1221/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:53 - r - INFO: - Episode: 1222/1500, Reward: 179.000, Step: 179 +2022-12-07 16:22:54 - r - INFO: - Episode: 1223/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:54 - r - INFO: - Episode: 1224/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:54 - r - INFO: - Episode: 1225/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:55 - r - INFO: - Episode: 1226/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:56 - r - INFO: - Episode: 1227/1500, Reward: 200.000, Step: 200 +2022-12-07 16:22:56 - r - INFO: - Episode: 1228/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:19 - r - INFO: - Episode: 1229/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:19 - r - INFO: - Episode: 1230/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:21 - r - INFO: - Episode: 1231/1500, Reward: 172.000, Step: 172 +2022-12-07 16:23:21 - r - INFO: - Episode: 1232/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:21 - r - INFO: - Episode: 1233/1500, Reward: 186.000, Step: 186 +2022-12-07 16:23:21 - r - INFO: - Episode: 1234/1500, Reward: 186.000, Step: 186 +2022-12-07 16:23:22 - r - INFO: - Episode: 1235/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:23 - r - INFO: - Episode: 1236/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:23 - r - INFO: - Episode: 1237/1500, Reward: 141.000, Step: 141 +2022-12-07 16:23:24 - r - INFO: - Episode: 1238/1500, Reward: 195.000, Step: 195 +2022-12-07 16:23:24 - r - INFO: - Episode: 1239/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:47 - r - INFO: - Episode: 1240/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:48 - r - INFO: - Episode: 1241/1500, Reward: 117.000, Step: 117 +2022-12-07 16:23:48 - r - INFO: - Episode: 1242/1500, Reward: 179.000, Step: 179 +2022-12-07 16:23:49 - r - INFO: - Episode: 1243/1500, Reward: 164.000, Step: 164 +2022-12-07 16:23:49 - r - INFO: - Episode: 1244/1500, Reward: 72.000, Step: 72 +2022-12-07 16:23:49 - r - INFO: - Episode: 1245/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:50 - r - INFO: - Episode: 1246/1500, Reward: 118.000, Step: 118 +2022-12-07 16:23:50 - r - INFO: - Episode: 1247/1500, Reward: 116.000, Step: 116 +2022-12-07 16:23:50 - r - INFO: - Episode: 1248/1500, Reward: 130.000, Step: 130 +2022-12-07 16:23:50 - r - INFO: - Episode: 1249/1500, Reward: 200.000, Step: 200 +2022-12-07 16:23:51 - r - INFO: - Episode: 1250/1500, Reward: 143.000, Step: 143 +2022-12-07 16:23:52 - r - INFO: - Episode: 1251/1500, Reward: 128.000, Step: 128 +2022-12-07 16:23:52 - r - INFO: - Episode: 1252/1500, Reward: 160.000, Step: 160 +2022-12-07 16:23:52 - r - INFO: - Episode: 1253/1500, Reward: 158.000, Step: 158 +2022-12-07 16:24:15 - r - INFO: - Episode: 1254/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:15 - r - INFO: - Episode: 1255/1500, Reward: 182.000, Step: 182 +2022-12-07 16:24:17 - r - INFO: - Episode: 1256/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:17 - r - INFO: - Episode: 1257/1500, Reward: 175.000, Step: 175 +2022-12-07 16:24:17 - r - INFO: - Episode: 1258/1500, Reward: 194.000, Step: 194 +2022-12-07 16:24:17 - r - INFO: - Episode: 1259/1500, Reward: 162.000, Step: 162 +2022-12-07 16:24:18 - r - INFO: - Episode: 1260/1500, Reward: 149.000, Step: 149 +2022-12-07 16:24:19 - r - INFO: - Episode: 1261/1500, Reward: 157.000, Step: 157 +2022-12-07 16:24:19 - r - INFO: - Episode: 1262/1500, Reward: 173.000, Step: 173 +2022-12-07 16:24:19 - r - INFO: - Episode: 1263/1500, Reward: 140.000, Step: 140 +2022-12-07 16:24:19 - r - INFO: - Episode: 1264/1500, Reward: 132.000, Step: 132 +2022-12-07 16:24:20 - r - INFO: - Episode: 1265/1500, Reward: 173.000, Step: 173 +2022-12-07 16:24:44 - r - INFO: - Episode: 1266/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:44 - r - INFO: - Episode: 1267/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:44 - r - INFO: - Episode: 1268/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:45 - r - INFO: - Episode: 1269/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:45 - r - INFO: - Episode: 1270/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:46 - r - INFO: - Episode: 1271/1500, Reward: 200.000, Step: 200 +2022-12-07 16:24:47 - r - INFO: - Episode: 1272/1500, Reward: 176.000, Step: 176 +2022-12-07 16:24:47 - r - INFO: - Episode: 1273/1500, Reward: 188.000, Step: 188 +2022-12-07 16:24:47 - r - INFO: - Episode: 1274/1500, Reward: 166.000, Step: 166 +2022-12-07 16:24:47 - r - INFO: - Episode: 1275/1500, Reward: 34.000, Step: 34 +2022-12-07 16:24:48 - r - INFO: - Episode: 1276/1500, Reward: 196.000, Step: 196 +2022-12-07 16:25:11 - r - INFO: - Episode: 1277/1500, Reward: 183.000, Step: 183 +2022-12-07 16:25:12 - r - INFO: - Episode: 1278/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:12 - r - INFO: - Episode: 1279/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:12 - r - INFO: - Episode: 1280/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:13 - r - INFO: - Episode: 1281/1500, Reward: 176.000, Step: 176 +2022-12-07 16:25:14 - r - INFO: - Episode: 1282/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:14 - r - INFO: - Episode: 1283/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:14 - r - INFO: - Episode: 1284/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:14 - r - INFO: - Episode: 1285/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:16 - r - INFO: - Current episode 1285 has the best eval reward: 200.000 +2022-12-07 16:25:16 - r - INFO: - Episode: 1286/1500, Reward: 197.000, Step: 197 +2022-12-07 16:25:16 - r - INFO: - Episode: 1287/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:40 - r - INFO: - Episode: 1288/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:40 - r - INFO: - Episode: 1289/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:40 - r - INFO: - Episode: 1290/1500, Reward: 178.000, Step: 178 +2022-12-07 16:25:42 - r - INFO: - Current episode 1290 has the best eval reward: 200.000 +2022-12-07 16:25:42 - r - INFO: - Episode: 1291/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:42 - r - INFO: - Episode: 1292/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:42 - r - INFO: - Episode: 1293/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:42 - r - INFO: - Episode: 1294/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:43 - r - INFO: - Episode: 1295/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:44 - r - INFO: - Current episode 1295 has the best eval reward: 200.000 +2022-12-07 16:25:44 - r - INFO: - Episode: 1296/1500, Reward: 200.000, Step: 200 +2022-12-07 16:25:44 - r - INFO: - Episode: 1297/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:08 - r - INFO: - Episode: 1298/1500, Reward: 143.000, Step: 143 +2022-12-07 16:26:08 - r - INFO: - Episode: 1299/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:08 - r - INFO: - Episode: 1300/1500, Reward: 191.000, Step: 191 +2022-12-07 16:26:10 - r - INFO: - Current episode 1300 has the best eval reward: 200.000 +2022-12-07 16:26:10 - r - INFO: - Episode: 1301/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:10 - r - INFO: - Episode: 1302/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:10 - r - INFO: - Episode: 1303/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:10 - r - INFO: - Episode: 1304/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:11 - r - INFO: - Episode: 1305/1500, Reward: 190.000, Step: 190 +2022-12-07 16:26:12 - r - INFO: - Episode: 1306/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:12 - r - INFO: - Episode: 1307/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:13 - r - INFO: - Episode: 1308/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:36 - r - INFO: - Episode: 1309/1500, Reward: 199.000, Step: 199 +2022-12-07 16:26:36 - r - INFO: - Episode: 1310/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:38 - r - INFO: - Episode: 1311/1500, Reward: 186.000, Step: 186 +2022-12-07 16:26:38 - r - INFO: - Episode: 1312/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:38 - r - INFO: - Episode: 1313/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:38 - r - INFO: - Episode: 1314/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:39 - r - INFO: - Episode: 1315/1500, Reward: 182.000, Step: 182 +2022-12-07 16:26:40 - r - INFO: - Episode: 1316/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:40 - r - INFO: - Episode: 1317/1500, Reward: 200.000, Step: 200 +2022-12-07 16:26:41 - r - INFO: - Episode: 1318/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:04 - r - INFO: - Episode: 1319/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:04 - r - INFO: - Episode: 1320/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:06 - r - INFO: - Current episode 1320 has the best eval reward: 200.000 +2022-12-07 16:27:06 - r - INFO: - Episode: 1321/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:06 - r - INFO: - Episode: 1322/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:06 - r - INFO: - Episode: 1323/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:07 - r - INFO: - Episode: 1324/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:07 - r - INFO: - Episode: 1325/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:08 - r - INFO: - Episode: 1326/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:09 - r - INFO: - Episode: 1327/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:09 - r - INFO: - Episode: 1328/1500, Reward: 185.000, Step: 185 +2022-12-07 16:27:32 - r - INFO: - Episode: 1329/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:32 - r - INFO: - Episode: 1330/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:34 - r - INFO: - Episode: 1331/1500, Reward: 97.000, Step: 97 +2022-12-07 16:27:34 - r - INFO: - Episode: 1332/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:34 - r - INFO: - Episode: 1333/1500, Reward: 188.000, Step: 188 +2022-12-07 16:27:34 - r - INFO: - Episode: 1334/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:34 - r - INFO: - Episode: 1335/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:36 - r - INFO: - Episode: 1336/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:36 - r - INFO: - Episode: 1337/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:36 - r - INFO: - Episode: 1338/1500, Reward: 200.000, Step: 200 +2022-12-07 16:27:37 - r - INFO: - Episode: 1339/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:00 - r - INFO: - Episode: 1340/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:02 - r - INFO: - Episode: 1341/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:02 - r - INFO: - Episode: 1342/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:02 - r - INFO: - Episode: 1343/1500, Reward: 185.000, Step: 185 +2022-12-07 16:28:03 - r - INFO: - Episode: 1344/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:03 - r - INFO: - Episode: 1345/1500, Reward: 80.000, Step: 80 +2022-12-07 16:28:04 - r - INFO: - Episode: 1346/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:04 - r - INFO: - Episode: 1347/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:05 - r - INFO: - Episode: 1348/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:05 - r - INFO: - Episode: 1349/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:05 - r - INFO: - Episode: 1350/1500, Reward: 170.000, Step: 170 +2022-12-07 16:28:30 - r - INFO: - Episode: 1351/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:30 - r - INFO: - Episode: 1352/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:30 - r - INFO: - Episode: 1353/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:30 - r - INFO: - Episode: 1354/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:31 - r - INFO: - Episode: 1355/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:32 - r - INFO: - Episode: 1356/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:32 - r - INFO: - Episode: 1357/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:33 - r - INFO: - Episode: 1358/1500, Reward: 182.000, Step: 182 +2022-12-07 16:28:33 - r - INFO: - Episode: 1359/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:33 - r - INFO: - Episode: 1360/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:34 - r - INFO: - Current episode 1360 has the best eval reward: 200.000 +2022-12-07 16:28:35 - r - INFO: - Episode: 1361/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:58 - r - INFO: - Episode: 1362/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:58 - r - INFO: - Episode: 1363/1500, Reward: 174.000, Step: 174 +2022-12-07 16:28:58 - r - INFO: - Episode: 1364/1500, Reward: 200.000, Step: 200 +2022-12-07 16:28:58 - r - INFO: - Episode: 1365/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:00 - r - INFO: - Current episode 1365 has the best eval reward: 200.000 +2022-12-07 16:29:00 - r - INFO: - Episode: 1366/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:00 - r - INFO: - Episode: 1367/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:01 - r - INFO: - Episode: 1368/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:01 - r - INFO: - Episode: 1369/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:01 - r - INFO: - Episode: 1370/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:02 - r - INFO: - Current episode 1370 has the best eval reward: 200.000 +2022-12-07 16:29:03 - r - INFO: - Episode: 1371/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:26 - r - INFO: - Episode: 1372/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:26 - r - INFO: - Episode: 1373/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:26 - r - INFO: - Episode: 1374/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:27 - r - INFO: - Episode: 1375/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:28 - r - INFO: - Current episode 1375 has the best eval reward: 200.000 +2022-12-07 16:29:28 - r - INFO: - Episode: 1376/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:28 - r - INFO: - Episode: 1377/1500, Reward: 183.000, Step: 183 +2022-12-07 16:29:29 - r - INFO: - Episode: 1378/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:29 - r - INFO: - Episode: 1379/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:29 - r - INFO: - Episode: 1380/1500, Reward: 184.000, Step: 184 +2022-12-07 16:29:30 - r - INFO: - Current episode 1380 has the best eval reward: 200.000 +2022-12-07 16:29:31 - r - INFO: - Episode: 1381/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:54 - r - INFO: - Episode: 1382/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:54 - r - INFO: - Episode: 1383/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:54 - r - INFO: - Episode: 1384/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:55 - r - INFO: - Episode: 1385/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:56 - r - INFO: - Episode: 1386/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:56 - r - INFO: - Episode: 1387/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:57 - r - INFO: - Episode: 1388/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:57 - r - INFO: - Episode: 1389/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:57 - r - INFO: - Episode: 1390/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:58 - r - INFO: - Current episode 1390 has the best eval reward: 200.000 +2022-12-07 16:29:59 - r - INFO: - Episode: 1391/1500, Reward: 200.000, Step: 200 +2022-12-07 16:29:59 - r - INFO: - Episode: 1392/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:22 - r - INFO: - Episode: 1393/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:23 - r - INFO: - Episode: 1394/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:23 - r - INFO: - Episode: 1395/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:24 - r - INFO: - Episode: 1396/1500, Reward: 195.000, Step: 195 +2022-12-07 16:30:25 - r - INFO: - Episode: 1397/1500, Reward: 196.000, Step: 196 +2022-12-07 16:30:25 - r - INFO: - Episode: 1398/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:25 - r - INFO: - Episode: 1399/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:25 - r - INFO: - Episode: 1400/1500, Reward: 152.000, Step: 152 +2022-12-07 16:30:27 - r - INFO: - Episode: 1401/1500, Reward: 175.000, Step: 175 +2022-12-07 16:30:27 - r - INFO: - Episode: 1402/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:50 - r - INFO: - Episode: 1403/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:51 - r - INFO: - Episode: 1404/1500, Reward: 185.000, Step: 185 +2022-12-07 16:30:51 - r - INFO: - Episode: 1405/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:52 - r - INFO: - Episode: 1406/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:53 - r - INFO: - Episode: 1407/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:53 - r - INFO: - Episode: 1408/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:53 - r - INFO: - Episode: 1409/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:53 - r - INFO: - Episode: 1410/1500, Reward: 156.000, Step: 156 +2022-12-07 16:30:55 - r - INFO: - Episode: 1411/1500, Reward: 173.000, Step: 173 +2022-12-07 16:30:55 - r - INFO: - Episode: 1412/1500, Reward: 200.000, Step: 200 +2022-12-07 16:30:55 - r - INFO: - Episode: 1413/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:20 - r - INFO: - Episode: 1414/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:20 - r - INFO: - Episode: 1415/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:22 - r - INFO: - Current episode 1415 has the best eval reward: 200.000 +2022-12-07 16:31:22 - r - INFO: - Episode: 1416/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:22 - r - INFO: - Episode: 1417/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:22 - r - INFO: - Episode: 1418/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:23 - r - INFO: - Episode: 1419/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:23 - r - INFO: - Episode: 1420/1500, Reward: 197.000, Step: 197 +2022-12-07 16:31:24 - r - INFO: - Episode: 1421/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:25 - r - INFO: - Episode: 1422/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:25 - r - INFO: - Episode: 1423/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:48 - r - INFO: - Episode: 1424/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:48 - r - INFO: - Episode: 1425/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:50 - r - INFO: - Current episode 1425 has the best eval reward: 200.000 +2022-12-07 16:31:50 - r - INFO: - Episode: 1426/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:50 - r - INFO: - Episode: 1427/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:50 - r - INFO: - Episode: 1428/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:51 - r - INFO: - Episode: 1429/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:51 - r - INFO: - Episode: 1430/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:52 - r - INFO: - Current episode 1430 has the best eval reward: 200.000 +2022-12-07 16:31:53 - r - INFO: - Episode: 1431/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:53 - r - INFO: - Episode: 1432/1500, Reward: 200.000, Step: 200 +2022-12-07 16:31:53 - r - INFO: - Episode: 1433/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:16 - r - INFO: - Episode: 1434/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:17 - r - INFO: - Episode: 1435/1500, Reward: 191.000, Step: 191 +2022-12-07 16:32:18 - r - INFO: - Current episode 1435 has the best eval reward: 200.000 +2022-12-07 16:32:18 - r - INFO: - Episode: 1436/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:18 - r - INFO: - Episode: 1437/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:19 - r - INFO: - Episode: 1438/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:19 - r - INFO: - Episode: 1439/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:19 - r - INFO: - Episode: 1440/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:20 - r - INFO: - Current episode 1440 has the best eval reward: 200.000 +2022-12-07 16:32:21 - r - INFO: - Episode: 1441/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:21 - r - INFO: - Episode: 1442/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:21 - r - INFO: - Episode: 1443/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:21 - r - INFO: - Episode: 1444/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:45 - r - INFO: - Episode: 1445/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:46 - r - INFO: - Episode: 1446/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:47 - r - INFO: - Episode: 1447/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:47 - r - INFO: - Episode: 1448/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:47 - r - INFO: - Episode: 1449/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:47 - r - INFO: - Episode: 1450/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:49 - r - INFO: - Episode: 1451/1500, Reward: 162.000, Step: 162 +2022-12-07 16:32:49 - r - INFO: - Episode: 1452/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:49 - r - INFO: - Episode: 1453/1500, Reward: 200.000, Step: 200 +2022-12-07 16:32:49 - r - INFO: - Episode: 1454/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:13 - r - INFO: - Episode: 1455/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:15 - r - INFO: - Episode: 1456/1500, Reward: 96.000, Step: 96 +2022-12-07 16:33:15 - r - INFO: - Episode: 1457/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:16 - r - INFO: - Episode: 1458/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:16 - r - INFO: - Episode: 1459/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:16 - r - INFO: - Episode: 1460/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:18 - r - INFO: - Episode: 1461/1500, Reward: 178.000, Step: 178 +2022-12-07 16:33:18 - r - INFO: - Episode: 1462/1500, Reward: 166.000, Step: 166 +2022-12-07 16:33:18 - r - INFO: - Episode: 1463/1500, Reward: 120.000, Step: 120 +2022-12-07 16:33:18 - r - INFO: - Episode: 1464/1500, Reward: 188.000, Step: 188 +2022-12-07 16:33:18 - r - INFO: - Episode: 1465/1500, Reward: 166.000, Step: 166 +2022-12-07 16:33:20 - r - INFO: - Current episode 1465 has the best eval reward: 200.000 +2022-12-07 16:33:20 - r - INFO: - Episode: 1466/1500, Reward: 194.000, Step: 194 +2022-12-07 16:33:43 - r - INFO: - Episode: 1467/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:44 - r - INFO: - Episode: 1468/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:44 - r - INFO: - Episode: 1469/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:44 - r - INFO: - Episode: 1470/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:46 - r - INFO: - Episode: 1471/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:46 - r - INFO: - Episode: 1472/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:46 - r - INFO: - Episode: 1473/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:46 - r - INFO: - Episode: 1474/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:47 - r - INFO: - Episode: 1475/1500, Reward: 200.000, Step: 200 +2022-12-07 16:33:48 - r - INFO: - Current episode 1475 has the best eval reward: 200.000 +2022-12-07 16:33:48 - r - INFO: - Episode: 1476/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:11 - r - INFO: - Episode: 1477/1500, Reward: 159.000, Step: 159 +2022-12-07 16:34:12 - r - INFO: - Episode: 1478/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:12 - r - INFO: - Episode: 1479/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:12 - r - INFO: - Episode: 1480/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:14 - r - INFO: - Current episode 1480 has the best eval reward: 200.000 +2022-12-07 16:34:14 - r - INFO: - Episode: 1481/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:14 - r - INFO: - Episode: 1482/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:14 - r - INFO: - Episode: 1483/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:14 - r - INFO: - Episode: 1484/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:15 - r - INFO: - Episode: 1485/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:16 - r - INFO: - Current episode 1485 has the best eval reward: 200.000 +2022-12-07 16:34:16 - r - INFO: - Episode: 1486/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:40 - r - INFO: - Episode: 1487/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:40 - r - INFO: - Episode: 1488/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:40 - r - INFO: - Episode: 1489/1500, Reward: 151.000, Step: 151 +2022-12-07 16:34:40 - r - INFO: - Episode: 1490/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:42 - r - INFO: - Episode: 1491/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:42 - r - INFO: - Episode: 1492/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:43 - r - INFO: - Episode: 1493/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:43 - r - INFO: - Episode: 1494/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:43 - r - INFO: - Episode: 1495/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:45 - r - INFO: - Episode: 1496/1500, Reward: 200.000, Step: 200 +2022-12-07 16:34:45 - r - INFO: - Episode: 1497/1500, Reward: 200.000, Step: 200 +2022-12-07 16:35:08 - r - INFO: - Episode: 1498/1500, Reward: 200.000, Step: 200 +2022-12-07 16:35:08 - r - INFO: - Episode: 1499/1500, Reward: 200.000, Step: 200 +2022-12-07 16:35:08 - r - INFO: - Episode: 1500/1500, Reward: 200.000, Step: 200 +2022-12-07 16:35:10 - r - INFO: - Current episode 1500 has the best eval reward: 200.000 +2022-12-07 16:35:10 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/models/actor.pth b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..4cce6e4d2cd12fc5524c7db6123b2b8db3eca0d8 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a7cadc94484feb4feafb0b53f7fee416ba201906622cdce8e82bcfe2ab47ca8 +size 272215 diff --git a/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/models/critic.pth b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..8940c8a31d888cf8984afaa02c447d6eca453347 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2003a930fc6c0045c6933875066426140b002a1b6a74735854bae1c04bc5a99 +size 271191 diff --git a/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..f3cbd6528fae9b7123554ab5cff00efa4a2817c5 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/results/res.csv b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..eee56c53dbc80f495c469a7b4b670f3e853091b3 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_GAIL_20221207-160945/results/res.csv @@ -0,0 +1,1501 @@ +episodes,rewards,steps +0,25.0,25 +1,15.0,15 +2,13.0,13 +3,14.0,14 +4,25.0,25 +5,41.0,41 +6,33.0,33 +7,12.0,12 +8,20.0,20 +9,33.0,33 +10,19.0,19 +11,35.0,35 +12,22.0,22 +13,9.0,9 +14,17.0,17 +15,36.0,36 +16,16.0,16 +17,18.0,18 +18,34.0,34 +19,11.0,11 +20,21.0,21 +21,16.0,16 +22,17.0,17 +23,13.0,13 +24,20.0,20 +25,18.0,18 +26,18.0,18 +27,28.0,28 +28,12.0,12 +29,17.0,17 +30,10.0,10 +31,56.0,56 +32,45.0,45 +33,11.0,11 +34,19.0,19 +35,13.0,13 +36,30.0,30 +37,13.0,13 +38,21.0,21 +39,15.0,15 +40,12.0,12 +41,21.0,21 +42,9.0,9 +43,26.0,26 +44,19.0,19 +45,12.0,12 +46,14.0,14 +47,9.0,9 +48,21.0,21 +49,10.0,10 +50,20.0,20 +51,20.0,20 +52,35.0,35 +53,28.0,28 +54,50.0,50 +55,29.0,29 +56,10.0,10 +57,15.0,15 +58,20.0,20 +59,66.0,66 +60,38.0,38 +61,19.0,19 +62,39.0,39 +63,67.0,67 +64,13.0,13 +65,18.0,18 +66,22.0,22 +67,16.0,16 +68,21.0,21 +69,11.0,11 +70,42.0,42 +71,15.0,15 +72,45.0,45 +73,17.0,17 +74,35.0,35 +75,18.0,18 +76,26.0,26 +77,19.0,19 +78,13.0,13 +79,11.0,11 +80,19.0,19 +81,18.0,18 +82,73.0,73 +83,35.0,35 +84,15.0,15 +85,20.0,20 +86,19.0,19 +87,16.0,16 +88,18.0,18 +89,17.0,17 +90,35.0,35 +91,27.0,27 +92,9.0,9 +93,46.0,46 +94,11.0,11 +95,22.0,22 +96,13.0,13 +97,12.0,12 +98,10.0,10 +99,18.0,18 +100,25.0,25 +101,15.0,15 +102,14.0,14 +103,36.0,36 +104,21.0,21 +105,27.0,27 +106,30.0,30 +107,23.0,23 +108,28.0,28 +109,12.0,12 +110,16.0,16 +111,29.0,29 +112,37.0,37 +113,37.0,37 +114,17.0,17 +115,28.0,28 +116,43.0,43 +117,9.0,9 +118,16.0,16 +119,20.0,20 +120,12.0,12 +121,40.0,40 +122,13.0,13 +123,11.0,11 +124,9.0,9 +125,18.0,18 +126,20.0,20 +127,21.0,21 +128,13.0,13 +129,33.0,33 +130,13.0,13 +131,17.0,17 +132,44.0,44 +133,11.0,11 +134,16.0,16 +135,19.0,19 +136,14.0,14 +137,20.0,20 +138,19.0,19 +139,11.0,11 +140,14.0,14 +141,22.0,22 +142,17.0,17 +143,13.0,13 +144,12.0,12 +145,15.0,15 +146,19.0,19 +147,15.0,15 +148,11.0,11 +149,15.0,15 +150,12.0,12 +151,11.0,11 +152,30.0,30 +153,12.0,12 +154,22.0,22 +155,11.0,11 +156,21.0,21 +157,37.0,37 +158,16.0,16 +159,17.0,17 +160,15.0,15 +161,16.0,16 +162,32.0,32 +163,31.0,31 +164,16.0,16 +165,19.0,19 +166,16.0,16 +167,19.0,19 +168,12.0,12 +169,11.0,11 +170,15.0,15 +171,20.0,20 +172,11.0,11 +173,26.0,26 +174,19.0,19 +175,30.0,30 +176,12.0,12 +177,17.0,17 +178,18.0,18 +179,14.0,14 +180,19.0,19 +181,28.0,28 +182,11.0,11 +183,20.0,20 +184,14.0,14 +185,15.0,15 +186,24.0,24 +187,9.0,9 +188,12.0,12 +189,11.0,11 +190,12.0,12 +191,12.0,12 +192,15.0,15 +193,17.0,17 +194,32.0,32 +195,42.0,42 +196,15.0,15 +197,52.0,52 +198,12.0,12 +199,62.0,62 +200,19.0,19 +201,23.0,23 +202,16.0,16 +203,11.0,11 +204,30.0,30 +205,26.0,26 +206,17.0,17 +207,16.0,16 +208,25.0,25 +209,15.0,15 +210,12.0,12 +211,29.0,29 +212,17.0,17 +213,15.0,15 +214,21.0,21 +215,9.0,9 +216,12.0,12 +217,27.0,27 +218,20.0,20 +219,13.0,13 +220,12.0,12 +221,12.0,12 +222,14.0,14 +223,43.0,43 +224,10.0,10 +225,16.0,16 +226,17.0,17 +227,29.0,29 +228,21.0,21 +229,18.0,18 +230,17.0,17 +231,12.0,12 +232,15.0,15 +233,11.0,11 +234,23.0,23 +235,37.0,37 +236,9.0,9 +237,16.0,16 +238,15.0,15 +239,17.0,17 +240,9.0,9 +241,12.0,12 +242,17.0,17 +243,13.0,13 +244,13.0,13 +245,15.0,15 +246,28.0,28 +247,15.0,15 +248,22.0,22 +249,37.0,37 +250,17.0,17 +251,64.0,64 +252,27.0,27 +253,14.0,14 +254,21.0,21 +255,19.0,19 +256,23.0,23 +257,23.0,23 +258,13.0,13 +259,21.0,21 +260,17.0,17 +261,31.0,31 +262,33.0,33 +263,20.0,20 +264,29.0,29 +265,17.0,17 +266,12.0,12 +267,10.0,10 +268,14.0,14 +269,12.0,12 +270,11.0,11 +271,26.0,26 +272,16.0,16 +273,9.0,9 +274,23.0,23 +275,10.0,10 +276,13.0,13 +277,20.0,20 +278,15.0,15 +279,18.0,18 +280,20.0,20 +281,19.0,19 +282,15.0,15 +283,20.0,20 +284,13.0,13 +285,11.0,11 +286,12.0,12 +287,34.0,34 +288,11.0,11 +289,17.0,17 +290,14.0,14 +291,11.0,11 +292,22.0,22 +293,23.0,23 +294,15.0,15 +295,13.0,13 +296,14.0,14 +297,25.0,25 +298,20.0,20 +299,17.0,17 +300,10.0,10 +301,17.0,17 +302,27.0,27 +303,24.0,24 +304,15.0,15 +305,22.0,22 +306,23.0,23 +307,20.0,20 +308,12.0,12 +309,26.0,26 +310,17.0,17 +311,27.0,27 +312,21.0,21 +313,26.0,26 +314,23.0,23 +315,13.0,13 +316,12.0,12 +317,26.0,26 +318,15.0,15 +319,13.0,13 +320,24.0,24 +321,11.0,11 +322,19.0,19 +323,12.0,12 +324,14.0,14 +325,19.0,19 +326,15.0,15 +327,14.0,14 +328,16.0,16 +329,11.0,11 +330,20.0,20 +331,14.0,14 +332,29.0,29 +333,36.0,36 +334,23.0,23 +335,32.0,32 +336,41.0,41 +337,15.0,15 +338,12.0,12 +339,16.0,16 +340,11.0,11 +341,24.0,24 +342,11.0,11 +343,16.0,16 +344,18.0,18 +345,26.0,26 +346,22.0,22 +347,57.0,57 +348,15.0,15 +349,13.0,13 +350,12.0,12 +351,23.0,23 +352,24.0,24 +353,26.0,26 +354,17.0,17 +355,36.0,36 +356,19.0,19 +357,22.0,22 +358,16.0,16 +359,47.0,47 +360,15.0,15 +361,15.0,15 +362,11.0,11 +363,16.0,16 +364,19.0,19 +365,17.0,17 +366,11.0,11 +367,14.0,14 +368,10.0,10 +369,18.0,18 +370,35.0,35 +371,9.0,9 +372,20.0,20 +373,11.0,11 +374,18.0,18 +375,18.0,18 +376,16.0,16 +377,11.0,11 +378,19.0,19 +379,33.0,33 +380,9.0,9 +381,12.0,12 +382,27.0,27 +383,19.0,19 +384,15.0,15 +385,11.0,11 +386,13.0,13 +387,12.0,12 +388,24.0,24 +389,18.0,18 +390,11.0,11 +391,23.0,23 +392,20.0,20 +393,12.0,12 +394,24.0,24 +395,32.0,32 +396,10.0,10 +397,14.0,14 +398,18.0,18 +399,38.0,38 +400,14.0,14 +401,20.0,20 +402,16.0,16 +403,13.0,13 +404,16.0,16 +405,35.0,35 +406,10.0,10 +407,9.0,9 +408,26.0,26 +409,47.0,47 +410,21.0,21 +411,26.0,26 +412,15.0,15 +413,24.0,24 +414,10.0,10 +415,25.0,25 +416,28.0,28 +417,18.0,18 +418,11.0,11 +419,22.0,22 +420,22.0,22 +421,18.0,18 +422,27.0,27 +423,29.0,29 +424,15.0,15 +425,27.0,27 +426,26.0,26 +427,21.0,21 +428,13.0,13 +429,10.0,10 +430,12.0,12 +431,16.0,16 +432,21.0,21 +433,14.0,14 +434,17.0,17 +435,10.0,10 +436,20.0,20 +437,16.0,16 +438,15.0,15 +439,35.0,35 +440,28.0,28 +441,21.0,21 +442,45.0,45 +443,19.0,19 +444,23.0,23 +445,21.0,21 +446,33.0,33 +447,20.0,20 +448,15.0,15 +449,30.0,30 +450,14.0,14 +451,11.0,11 +452,14.0,14 +453,19.0,19 +454,25.0,25 +455,28.0,28 +456,10.0,10 +457,12.0,12 +458,38.0,38 +459,15.0,15 +460,15.0,15 +461,18.0,18 +462,13.0,13 +463,17.0,17 +464,62.0,62 +465,15.0,15 +466,27.0,27 +467,22.0,22 +468,28.0,28 +469,20.0,20 +470,23.0,23 +471,34.0,34 +472,76.0,76 +473,40.0,40 +474,16.0,16 +475,32.0,32 +476,18.0,18 +477,13.0,13 +478,23.0,23 +479,25.0,25 +480,23.0,23 +481,25.0,25 +482,20.0,20 +483,16.0,16 +484,22.0,22 +485,28.0,28 +486,11.0,11 +487,17.0,17 +488,35.0,35 +489,20.0,20 +490,20.0,20 +491,42.0,42 +492,21.0,21 +493,23.0,23 +494,19.0,19 +495,27.0,27 +496,13.0,13 +497,18.0,18 +498,14.0,14 +499,9.0,9 +500,14.0,14 +501,21.0,21 +502,20.0,20 +503,25.0,25 +504,41.0,41 +505,31.0,31 +506,13.0,13 +507,17.0,17 +508,12.0,12 +509,13.0,13 +510,12.0,12 +511,16.0,16 +512,50.0,50 +513,11.0,11 +514,17.0,17 +515,19.0,19 +516,12.0,12 +517,13.0,13 +518,17.0,17 +519,23.0,23 +520,18.0,18 +521,38.0,38 +522,30.0,30 +523,28.0,28 +524,28.0,28 +525,22.0,22 +526,14.0,14 +527,12.0,12 +528,46.0,46 +529,32.0,32 +530,37.0,37 +531,17.0,17 +532,21.0,21 +533,11.0,11 +534,16.0,16 +535,10.0,10 +536,15.0,15 +537,39.0,39 +538,67.0,67 +539,33.0,33 +540,25.0,25 +541,30.0,30 +542,32.0,32 +543,43.0,43 +544,21.0,21 +545,23.0,23 +546,14.0,14 +547,22.0,22 +548,28.0,28 +549,27.0,27 +550,37.0,37 +551,21.0,21 +552,18.0,18 +553,19.0,19 +554,36.0,36 +555,33.0,33 +556,41.0,41 +557,41.0,41 +558,28.0,28 +559,16.0,16 +560,24.0,24 +561,26.0,26 +562,29.0,29 +563,21.0,21 +564,17.0,17 +565,22.0,22 +566,23.0,23 +567,13.0,13 +568,30.0,30 +569,24.0,24 +570,41.0,41 +571,14.0,14 +572,32.0,32 +573,28.0,28 +574,18.0,18 +575,34.0,34 +576,39.0,39 +577,22.0,22 +578,16.0,16 +579,24.0,24 +580,34.0,34 +581,64.0,64 +582,43.0,43 +583,20.0,20 +584,11.0,11 +585,61.0,61 +586,20.0,20 +587,51.0,51 +588,21.0,21 +589,11.0,11 +590,32.0,32 +591,44.0,44 +592,26.0,26 +593,28.0,28 +594,16.0,16 +595,60.0,60 +596,14.0,14 +597,30.0,30 +598,18.0,18 +599,32.0,32 +600,74.0,74 +601,19.0,19 +602,30.0,30 +603,22.0,22 +604,37.0,37 +605,12.0,12 +606,45.0,45 +607,15.0,15 +608,38.0,38 +609,32.0,32 +610,15.0,15 +611,45.0,45 +612,15.0,15 +613,18.0,18 +614,59.0,59 +615,10.0,10 +616,10.0,10 +617,15.0,15 +618,48.0,48 +619,12.0,12 +620,23.0,23 +621,20.0,20 +622,42.0,42 +623,18.0,18 +624,9.0,9 +625,29.0,29 +626,15.0,15 +627,20.0,20 +628,18.0,18 +629,14.0,14 +630,22.0,22 +631,33.0,33 +632,20.0,20 +633,9.0,9 +634,17.0,17 +635,31.0,31 +636,73.0,73 +637,16.0,16 +638,10.0,10 +639,22.0,22 +640,12.0,12 +641,34.0,34 +642,15.0,15 +643,71.0,71 +644,40.0,40 +645,19.0,19 +646,46.0,46 +647,32.0,32 +648,19.0,19 +649,24.0,24 +650,54.0,54 +651,18.0,18 +652,26.0,26 +653,42.0,42 +654,15.0,15 +655,26.0,26 +656,40.0,40 +657,15.0,15 +658,22.0,22 +659,40.0,40 +660,29.0,29 +661,48.0,48 +662,15.0,15 +663,13.0,13 +664,20.0,20 +665,12.0,12 +666,32.0,32 +667,18.0,18 +668,13.0,13 +669,86.0,86 +670,72.0,72 +671,34.0,34 +672,22.0,22 +673,22.0,22 +674,27.0,27 +675,41.0,41 +676,18.0,18 +677,58.0,58 +678,17.0,17 +679,33.0,33 +680,32.0,32 +681,36.0,36 +682,23.0,23 +683,38.0,38 +684,45.0,45 +685,44.0,44 +686,55.0,55 +687,26.0,26 +688,43.0,43 +689,28.0,28 +690,21.0,21 +691,25.0,25 +692,40.0,40 +693,16.0,16 +694,19.0,19 +695,15.0,15 +696,35.0,35 +697,12.0,12 +698,32.0,32 +699,50.0,50 +700,46.0,46 +701,32.0,32 +702,17.0,17 +703,30.0,30 +704,33.0,33 +705,24.0,24 +706,25.0,25 +707,29.0,29 +708,28.0,28 +709,28.0,28 +710,44.0,44 +711,34.0,34 +712,39.0,39 +713,23.0,23 +714,53.0,53 +715,26.0,26 +716,36.0,36 +717,15.0,15 +718,25.0,25 +719,18.0,18 +720,39.0,39 +721,17.0,17 +722,34.0,34 +723,21.0,21 +724,43.0,43 +725,20.0,20 +726,49.0,49 +727,21.0,21 +728,37.0,37 +729,41.0,41 +730,43.0,43 +731,36.0,36 +732,20.0,20 +733,20.0,20 +734,22.0,22 +735,30.0,30 +736,46.0,46 +737,32.0,32 +738,29.0,29 +739,36.0,36 +740,12.0,12 +741,74.0,74 +742,11.0,11 +743,26.0,26 +744,21.0,21 +745,25.0,25 +746,19.0,19 +747,47.0,47 +748,15.0,15 +749,22.0,22 +750,20.0,20 +751,42.0,42 +752,26.0,26 +753,30.0,30 +754,25.0,25 +755,46.0,46 +756,32.0,32 +757,28.0,28 +758,34.0,34 +759,35.0,35 +760,73.0,73 +761,25.0,25 +762,13.0,13 +763,22.0,22 +764,39.0,39 +765,13.0,13 +766,63.0,63 +767,25.0,25 +768,31.0,31 +769,42.0,42 +770,21.0,21 +771,33.0,33 +772,28.0,28 +773,36.0,36 +774,19.0,19 +775,39.0,39 +776,19.0,19 +777,15.0,15 +778,12.0,12 +779,33.0,33 +780,14.0,14 +781,36.0,36 +782,20.0,20 +783,80.0,80 +784,21.0,21 +785,44.0,44 +786,110.0,110 +787,15.0,15 +788,20.0,20 +789,31.0,31 +790,26.0,26 +791,41.0,41 +792,39.0,39 +793,40.0,40 +794,42.0,42 +795,21.0,21 +796,38.0,38 +797,19.0,19 +798,45.0,45 +799,30.0,30 +800,11.0,11 +801,55.0,55 +802,37.0,37 +803,114.0,114 +804,48.0,48 +805,52.0,52 +806,29.0,29 +807,34.0,34 +808,45.0,45 +809,24.0,24 +810,34.0,34 +811,16.0,16 +812,28.0,28 +813,40.0,40 +814,18.0,18 +815,24.0,24 +816,20.0,20 +817,45.0,45 +818,102.0,102 +819,15.0,15 +820,117.0,117 +821,21.0,21 +822,21.0,21 +823,18.0,18 +824,61.0,61 +825,32.0,32 +826,39.0,39 +827,22.0,22 +828,29.0,29 +829,40.0,40 +830,27.0,27 +831,26.0,26 +832,30.0,30 +833,39.0,39 +834,32.0,32 +835,24.0,24 +836,23.0,23 +837,56.0,56 +838,18.0,18 +839,25.0,25 +840,30.0,30 +841,86.0,86 +842,60.0,60 +843,54.0,54 +844,55.0,55 +845,31.0,31 +846,68.0,68 +847,38.0,38 +848,19.0,19 +849,31.0,31 +850,57.0,57 +851,78.0,78 +852,25.0,25 +853,57.0,57 +854,29.0,29 +855,26.0,26 +856,23.0,23 +857,18.0,18 +858,26.0,26 +859,30.0,30 +860,59.0,59 +861,69.0,69 +862,54.0,54 +863,68.0,68 +864,118.0,118 +865,44.0,44 +866,74.0,74 +867,65.0,65 +868,73.0,73 +869,42.0,42 +870,27.0,27 +871,20.0,20 +872,59.0,59 +873,36.0,36 +874,24.0,24 +875,71.0,71 +876,35.0,35 +877,37.0,37 +878,73.0,73 +879,44.0,44 +880,72.0,72 +881,26.0,26 +882,62.0,62 +883,33.0,33 +884,21.0,21 +885,23.0,23 +886,26.0,26 +887,33.0,33 +888,27.0,27 +889,24.0,24 +890,63.0,63 +891,49.0,49 +892,30.0,30 +893,20.0,20 +894,48.0,48 +895,31.0,31 +896,96.0,96 +897,67.0,67 +898,58.0,58 +899,40.0,40 +900,25.0,25 +901,18.0,18 +902,22.0,22 +903,74.0,74 +904,26.0,26 +905,18.0,18 +906,38.0,38 +907,22.0,22 +908,50.0,50 +909,15.0,15 +910,27.0,27 +911,39.0,39 +912,87.0,87 +913,148.0,148 +914,13.0,13 +915,89.0,89 +916,56.0,56 +917,37.0,37 +918,22.0,22 +919,20.0,20 +920,80.0,80 +921,71.0,71 +922,50.0,50 +923,33.0,33 +924,80.0,80 +925,69.0,69 +926,28.0,28 +927,55.0,55 +928,55.0,55 +929,94.0,94 +930,86.0,86 +931,83.0,83 +932,64.0,64 +933,94.0,94 +934,17.0,17 +935,104.0,104 +936,37.0,37 +937,51.0,51 +938,33.0,33 +939,106.0,106 +940,73.0,73 +941,54.0,54 +942,39.0,39 +943,52.0,52 +944,127.0,127 +945,34.0,34 +946,32.0,32 +947,32.0,32 +948,47.0,47 +949,15.0,15 +950,34.0,34 +951,47.0,47 +952,32.0,32 +953,29.0,29 +954,64.0,64 +955,73.0,73 +956,39.0,39 +957,33.0,33 +958,25.0,25 +959,56.0,56 +960,34.0,34 +961,47.0,47 +962,36.0,36 +963,85.0,85 +964,58.0,58 +965,20.0,20 +966,153.0,153 +967,35.0,35 +968,55.0,55 +969,66.0,66 +970,41.0,41 +971,76.0,76 +972,88.0,88 +973,35.0,35 +974,35.0,35 +975,105.0,105 +976,116.0,116 +977,58.0,58 +978,50.0,50 +979,26.0,26 +980,76.0,76 +981,46.0,46 +982,57.0,57 +983,39.0,39 +984,40.0,40 +985,41.0,41 +986,80.0,80 +987,98.0,98 +988,57.0,57 +989,89.0,89 +990,124.0,124 +991,42.0,42 +992,36.0,36 +993,41.0,41 +994,67.0,67 +995,130.0,130 +996,91.0,91 +997,36.0,36 +998,76.0,76 +999,37.0,37 +1000,28.0,28 +1001,79.0,79 +1002,25.0,25 +1003,41.0,41 +1004,28.0,28 +1005,35.0,35 +1006,85.0,85 +1007,34.0,34 +1008,39.0,39 +1009,200.0,200 +1010,39.0,39 +1011,41.0,41 +1012,64.0,64 +1013,56.0,56 +1014,39.0,39 +1015,87.0,87 +1016,48.0,48 +1017,82.0,82 +1018,70.0,70 +1019,129.0,129 +1020,101.0,101 +1021,49.0,49 +1022,199.0,199 +1023,102.0,102 +1024,80.0,80 +1025,30.0,30 +1026,37.0,37 +1027,53.0,53 +1028,147.0,147 +1029,59.0,59 +1030,151.0,151 +1031,87.0,87 +1032,74.0,74 +1033,46.0,46 +1034,78.0,78 +1035,69.0,69 +1036,91.0,91 +1037,79.0,79 +1038,77.0,77 +1039,35.0,35 +1040,91.0,91 +1041,154.0,154 +1042,129.0,129 +1043,67.0,67 +1044,66.0,66 +1045,182.0,182 +1046,43.0,43 +1047,62.0,62 +1048,153.0,153 +1049,25.0,25 +1050,57.0,57 +1051,113.0,113 +1052,78.0,78 +1053,69.0,69 +1054,113.0,113 +1055,112.0,112 +1056,97.0,97 +1057,184.0,184 +1058,184.0,184 +1059,76.0,76 +1060,172.0,172 +1061,84.0,84 +1062,85.0,85 +1063,125.0,125 +1064,52.0,52 +1065,40.0,40 +1066,79.0,79 +1067,183.0,183 +1068,88.0,88 +1069,55.0,55 +1070,53.0,53 +1071,34.0,34 +1072,77.0,77 +1073,68.0,68 +1074,31.0,31 +1075,75.0,75 +1076,200.0,200 +1077,41.0,41 +1078,149.0,149 +1079,113.0,113 +1080,57.0,57 +1081,79.0,79 +1082,38.0,38 +1083,44.0,44 +1084,200.0,200 +1085,39.0,39 +1086,143.0,143 +1087,103.0,103 +1088,35.0,35 +1089,123.0,123 +1090,56.0,56 +1091,177.0,177 +1092,28.0,28 +1093,77.0,77 +1094,200.0,200 +1095,90.0,90 +1096,101.0,101 +1097,69.0,69 +1098,200.0,200 +1099,129.0,129 +1100,72.0,72 +1101,200.0,200 +1102,154.0,154 +1103,62.0,62 +1104,146.0,146 +1105,184.0,184 +1106,138.0,138 +1107,139.0,139 +1108,200.0,200 +1109,27.0,27 +1110,124.0,124 +1111,98.0,98 +1112,114.0,114 +1113,194.0,194 +1114,140.0,140 +1115,52.0,52 +1116,140.0,140 +1117,110.0,110 +1118,125.0,125 +1119,55.0,55 +1120,97.0,97 +1121,148.0,148 +1122,115.0,115 +1123,121.0,121 +1124,123.0,123 +1125,43.0,43 +1126,96.0,96 +1127,111.0,111 +1128,172.0,172 +1129,87.0,87 +1130,116.0,116 +1131,103.0,103 +1132,111.0,111 +1133,81.0,81 +1134,162.0,162 +1135,200.0,200 +1136,69.0,69 +1137,94.0,94 +1138,125.0,125 +1139,82.0,82 +1140,109.0,109 +1141,120.0,120 +1142,96.0,96 +1143,34.0,34 +1144,128.0,128 +1145,62.0,62 +1146,156.0,156 +1147,118.0,118 +1148,166.0,166 +1149,200.0,200 +1150,53.0,53 +1151,89.0,89 +1152,161.0,161 +1153,200.0,200 +1154,200.0,200 +1155,156.0,156 +1156,167.0,167 +1157,38.0,38 +1158,200.0,200 +1159,63.0,63 +1160,73.0,73 +1161,161.0,161 +1162,200.0,200 +1163,200.0,200 +1164,200.0,200 +1165,200.0,200 +1166,200.0,200 +1167,197.0,197 +1168,200.0,200 +1169,200.0,200 +1170,200.0,200 +1171,120.0,120 +1172,121.0,121 +1173,200.0,200 +1174,162.0,162 +1175,157.0,157 +1176,151.0,151 +1177,161.0,161 +1178,115.0,115 +1179,176.0,176 +1180,148.0,148 +1181,200.0,200 +1182,200.0,200 +1183,188.0,188 +1184,176.0,176 +1185,200.0,200 +1186,200.0,200 +1187,200.0,200 +1188,172.0,172 +1189,200.0,200 +1190,200.0,200 +1191,200.0,200 +1192,200.0,200 +1193,180.0,180 +1194,109.0,109 +1195,200.0,200 +1196,200.0,200 +1197,200.0,200 +1198,118.0,118 +1199,200.0,200 +1200,200.0,200 +1201,185.0,185 +1202,200.0,200 +1203,200.0,200 +1204,200.0,200 +1205,200.0,200 +1206,162.0,162 +1207,195.0,195 +1208,200.0,200 +1209,200.0,200 +1210,200.0,200 +1211,114.0,114 +1212,200.0,200 +1213,194.0,194 +1214,132.0,132 +1215,200.0,200 +1216,200.0,200 +1217,200.0,200 +1218,200.0,200 +1219,200.0,200 +1220,200.0,200 +1221,179.0,179 +1222,200.0,200 +1223,200.0,200 +1224,200.0,200 +1225,200.0,200 +1226,200.0,200 +1227,200.0,200 +1228,200.0,200 +1229,200.0,200 +1230,172.0,172 +1231,200.0,200 +1232,186.0,186 +1233,186.0,186 +1234,200.0,200 +1235,200.0,200 +1236,141.0,141 +1237,195.0,195 +1238,200.0,200 +1239,200.0,200 +1240,117.0,117 +1241,179.0,179 +1242,164.0,164 +1243,72.0,72 +1244,200.0,200 +1245,118.0,118 +1246,116.0,116 +1247,130.0,130 +1248,200.0,200 +1249,143.0,143 +1250,128.0,128 +1251,160.0,160 +1252,158.0,158 +1253,200.0,200 +1254,182.0,182 +1255,200.0,200 +1256,175.0,175 +1257,194.0,194 +1258,162.0,162 +1259,149.0,149 +1260,157.0,157 +1261,173.0,173 +1262,140.0,140 +1263,132.0,132 +1264,173.0,173 +1265,200.0,200 +1266,200.0,200 +1267,200.0,200 +1268,200.0,200 +1269,200.0,200 +1270,200.0,200 +1271,176.0,176 +1272,188.0,188 +1273,166.0,166 +1274,34.0,34 +1275,196.0,196 +1276,183.0,183 +1277,200.0,200 +1278,200.0,200 +1279,200.0,200 +1280,176.0,176 +1281,200.0,200 +1282,200.0,200 +1283,200.0,200 +1284,200.0,200 +1285,197.0,197 +1286,200.0,200 +1287,200.0,200 +1288,200.0,200 +1289,178.0,178 +1290,200.0,200 +1291,200.0,200 +1292,200.0,200 +1293,200.0,200 +1294,200.0,200 +1295,200.0,200 +1296,200.0,200 +1297,143.0,143 +1298,200.0,200 +1299,191.0,191 +1300,200.0,200 +1301,200.0,200 +1302,200.0,200 +1303,200.0,200 +1304,190.0,190 +1305,200.0,200 +1306,200.0,200 +1307,200.0,200 +1308,199.0,199 +1309,200.0,200 +1310,186.0,186 +1311,200.0,200 +1312,200.0,200 +1313,200.0,200 +1314,182.0,182 +1315,200.0,200 +1316,200.0,200 +1317,200.0,200 +1318,200.0,200 +1319,200.0,200 +1320,200.0,200 +1321,200.0,200 +1322,200.0,200 +1323,200.0,200 +1324,200.0,200 +1325,200.0,200 +1326,200.0,200 +1327,185.0,185 +1328,200.0,200 +1329,200.0,200 +1330,97.0,97 +1331,200.0,200 +1332,188.0,188 +1333,200.0,200 +1334,200.0,200 +1335,200.0,200 +1336,200.0,200 +1337,200.0,200 +1338,200.0,200 +1339,200.0,200 +1340,200.0,200 +1341,200.0,200 +1342,185.0,185 +1343,200.0,200 +1344,80.0,80 +1345,200.0,200 +1346,200.0,200 +1347,200.0,200 +1348,200.0,200 +1349,170.0,170 +1350,200.0,200 +1351,200.0,200 +1352,200.0,200 +1353,200.0,200 +1354,200.0,200 +1355,200.0,200 +1356,200.0,200 +1357,182.0,182 +1358,200.0,200 +1359,200.0,200 +1360,200.0,200 +1361,200.0,200 +1362,174.0,174 +1363,200.0,200 +1364,200.0,200 +1365,200.0,200 +1366,200.0,200 +1367,200.0,200 +1368,200.0,200 +1369,200.0,200 +1370,200.0,200 +1371,200.0,200 +1372,200.0,200 +1373,200.0,200 +1374,200.0,200 +1375,200.0,200 +1376,183.0,183 +1377,200.0,200 +1378,200.0,200 +1379,184.0,184 +1380,200.0,200 +1381,200.0,200 +1382,200.0,200 +1383,200.0,200 +1384,200.0,200 +1385,200.0,200 +1386,200.0,200 +1387,200.0,200 +1388,200.0,200 +1389,200.0,200 +1390,200.0,200 +1391,200.0,200 +1392,200.0,200 +1393,200.0,200 +1394,200.0,200 +1395,195.0,195 +1396,196.0,196 +1397,200.0,200 +1398,200.0,200 +1399,152.0,152 +1400,175.0,175 +1401,200.0,200 +1402,200.0,200 +1403,185.0,185 +1404,200.0,200 +1405,200.0,200 +1406,200.0,200 +1407,200.0,200 +1408,200.0,200 +1409,156.0,156 +1410,173.0,173 +1411,200.0,200 +1412,200.0,200 +1413,200.0,200 +1414,200.0,200 +1415,200.0,200 +1416,200.0,200 +1417,200.0,200 +1418,200.0,200 +1419,197.0,197 +1420,200.0,200 +1421,200.0,200 +1422,200.0,200 +1423,200.0,200 +1424,200.0,200 +1425,200.0,200 +1426,200.0,200 +1427,200.0,200 +1428,200.0,200 +1429,200.0,200 +1430,200.0,200 +1431,200.0,200 +1432,200.0,200 +1433,200.0,200 +1434,191.0,191 +1435,200.0,200 +1436,200.0,200 +1437,200.0,200 +1438,200.0,200 +1439,200.0,200 +1440,200.0,200 +1441,200.0,200 +1442,200.0,200 +1443,200.0,200 +1444,200.0,200 +1445,200.0,200 +1446,200.0,200 +1447,200.0,200 +1448,200.0,200 +1449,200.0,200 +1450,162.0,162 +1451,200.0,200 +1452,200.0,200 +1453,200.0,200 +1454,200.0,200 +1455,96.0,96 +1456,200.0,200 +1457,200.0,200 +1458,200.0,200 +1459,200.0,200 +1460,178.0,178 +1461,166.0,166 +1462,120.0,120 +1463,188.0,188 +1464,166.0,166 +1465,194.0,194 +1466,200.0,200 +1467,200.0,200 +1468,200.0,200 +1469,200.0,200 +1470,200.0,200 +1471,200.0,200 +1472,200.0,200 +1473,200.0,200 +1474,200.0,200 +1475,200.0,200 +1476,159.0,159 +1477,200.0,200 +1478,200.0,200 +1479,200.0,200 +1480,200.0,200 +1481,200.0,200 +1482,200.0,200 +1483,200.0,200 +1484,200.0,200 +1485,200.0,200 +1486,200.0,200 +1487,200.0,200 +1488,151.0,151 +1489,200.0,200 +1490,200.0,200 +1491,200.0,200 +1492,200.0,200 +1493,200.0,200 +1494,200.0,200 +1495,200.0,200 +1496,200.0,200 +1497,200.0,200 +1498,200.0,200 +1499,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/config.yaml b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16cfc077edc93491b75e80245132c3c924df26e1 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/config.yaml @@ -0,0 +1,29 @@ +general_cfg: + algo_name: NoisyDQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_NoisyDQN + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + tau: 1.0 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/logs/log.txt b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..8863b77f4b7bc26281e0cd1cbc340f537a31c11b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/logs/log.txt @@ -0,0 +1,153 @@ +2023-03-18 16:21:39 - r - INFO: - Hyperparameters: +2023-03-18 16:21:39 - r - INFO: - ================================================================================ +2023-03-18 16:21:39 - r - INFO: - Name Value Type +2023-03-18 16:21:39 - r - INFO: - env_name CartPole-v1 +2023-03-18 16:21:39 - r - INFO: - new_step_api 1 +2023-03-18 16:21:39 - r - INFO: - wrapper None +2023-03-18 16:21:39 - r - INFO: - render 0 +2023-03-18 16:21:39 - r - INFO: - algo_name NoisyDQN +2023-03-18 16:21:39 - r - INFO: - mode train +2023-03-18 16:21:39 - r - INFO: - seed 1 +2023-03-18 16:21:39 - r - INFO: - device cpu +2023-03-18 16:21:39 - r - INFO: - train_eps 100 +2023-03-18 16:21:39 - r - INFO: - test_eps 10 +2023-03-18 16:21:39 - r - INFO: - eval_eps 10 +2023-03-18 16:21:39 - r - INFO: - eval_per_episode 5 +2023-03-18 16:21:39 - r - INFO: - max_steps 200 +2023-03-18 16:21:39 - r - INFO: - load_checkpoint 0 +2023-03-18 16:21:39 - r - INFO: - load_path Train_CartPole-v1_NoisyDQN +2023-03-18 16:21:39 - r - INFO: - show_fig 0 +2023-03-18 16:21:39 - r - INFO: - save_fig 1 +2023-03-18 16:21:39 - r - INFO: - epsilon_start 0.95 +2023-03-18 16:21:39 - r - INFO: - tau 1.0 +2023-03-18 16:21:39 - r - INFO: - epsilon_end 0.01 +2023-03-18 16:21:39 - r - INFO: - epsilon_decay 500 +2023-03-18 16:21:39 - r - INFO: - hidden_dim 256 +2023-03-18 16:21:39 - r - INFO: - gamma 0.95 +2023-03-18 16:21:39 - r - INFO: - lr 0.0001 +2023-03-18 16:21:39 - r - INFO: - buffer_size 100000 +2023-03-18 16:21:39 - r - INFO: - batch_size 64 +2023-03-18 16:21:39 - r - INFO: - target_update 4 +2023-03-18 16:21:39 - r - INFO: - task_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_NoisyDQN_20230318-162139 +2023-03-18 16:21:39 - r - INFO: - res_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_NoisyDQN_20230318-162139/results +2023-03-18 16:21:39 - r - INFO: - log_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_NoisyDQN_20230318-162139/logs +2023-03-18 16:21:39 - r - INFO: - traj_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_NoisyDQN_20230318-162139/traj +2023-03-18 16:21:39 - r - INFO: - tb_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_NoisyDQN_20230318-162139/tb_logs +2023-03-18 16:21:39 - r - INFO: - ================================================================================ +2023-03-18 16:21:39 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-18 16:21:39 - r - INFO: - Start training! +2023-03-18 16:21:39 - r - INFO: - Env: CartPole-v1, Algorithm: NoisyDQN, Device: cpu +2023-03-18 16:21:39 - r - INFO: - Episode: 1/100, Reward: 16.000, Step: 16 +2023-03-18 16:21:39 - r - INFO: - Episode: 2/100, Reward: 16.000, Step: 16 +2023-03-18 16:21:39 - r - INFO: - Episode: 3/100, Reward: 18.000, Step: 18 +2023-03-18 16:21:39 - r - INFO: - Episode: 4/100, Reward: 14.000, Step: 14 +2023-03-18 16:21:39 - r - INFO: - Episode: 5/100, Reward: 22.000, Step: 22 +2023-03-18 16:21:39 - r - INFO: - Current episode 5 has the best eval reward: 9.300 +2023-03-18 16:21:39 - r - INFO: - Episode: 6/100, Reward: 27.000, Step: 27 +2023-03-18 16:21:40 - r - INFO: - Episode: 7/100, Reward: 9.000, Step: 9 +2023-03-18 16:21:40 - r - INFO: - Episode: 8/100, Reward: 13.000, Step: 13 +2023-03-18 16:21:40 - r - INFO: - Episode: 9/100, Reward: 17.000, Step: 17 +2023-03-18 16:21:40 - r - INFO: - Episode: 10/100, Reward: 37.000, Step: 37 +2023-03-18 16:21:40 - r - INFO: - Current episode 10 has the best eval reward: 9.500 +2023-03-18 16:21:40 - r - INFO: - Episode: 11/100, Reward: 15.000, Step: 15 +2023-03-18 16:21:40 - r - INFO: - Episode: 12/100, Reward: 22.000, Step: 22 +2023-03-18 16:21:40 - r - INFO: - Episode: 13/100, Reward: 9.000, Step: 9 +2023-03-18 16:21:40 - r - INFO: - Episode: 14/100, Reward: 14.000, Step: 14 +2023-03-18 16:21:40 - r - INFO: - Episode: 15/100, Reward: 12.000, Step: 12 +2023-03-18 16:21:40 - r - INFO: - Current episode 15 has the best eval reward: 9.700 +2023-03-18 16:21:40 - r - INFO: - Episode: 16/100, Reward: 16.000, Step: 16 +2023-03-18 16:21:40 - r - INFO: - Episode: 17/100, Reward: 16.000, Step: 16 +2023-03-18 16:21:40 - r - INFO: - Episode: 18/100, Reward: 14.000, Step: 14 +2023-03-18 16:21:40 - r - INFO: - Episode: 19/100, Reward: 11.000, Step: 11 +2023-03-18 16:21:40 - r - INFO: - Episode: 20/100, Reward: 13.000, Step: 13 +2023-03-18 16:21:40 - r - INFO: - Current episode 20 has the best eval reward: 9.700 +2023-03-18 16:21:40 - r - INFO: - Episode: 21/100, Reward: 13.000, Step: 13 +2023-03-18 16:21:42 - r - INFO: - Episode: 22/100, Reward: 14.000, Step: 14 +2023-03-18 16:21:42 - r - INFO: - Episode: 23/100, Reward: 14.000, Step: 14 +2023-03-18 16:21:42 - r - INFO: - Episode: 24/100, Reward: 37.000, Step: 37 +2023-03-18 16:21:42 - r - INFO: - Episode: 25/100, Reward: 12.000, Step: 12 +2023-03-18 16:21:42 - r - INFO: - Episode: 26/100, Reward: 18.000, Step: 18 +2023-03-18 16:21:42 - r - INFO: - Episode: 27/100, Reward: 13.000, Step: 13 +2023-03-18 16:21:42 - r - INFO: - Episode: 28/100, Reward: 20.000, Step: 20 +2023-03-18 16:21:43 - r - INFO: - Episode: 29/100, Reward: 17.000, Step: 17 +2023-03-18 16:21:43 - r - INFO: - Episode: 30/100, Reward: 10.000, Step: 10 +2023-03-18 16:21:43 - r - INFO: - Current episode 30 has the best eval reward: 13.700 +2023-03-18 16:21:43 - r - INFO: - Episode: 31/100, Reward: 10.000, Step: 10 +2023-03-18 16:21:43 - r - INFO: - Episode: 32/100, Reward: 12.000, Step: 12 +2023-03-18 16:21:43 - r - INFO: - Episode: 33/100, Reward: 11.000, Step: 11 +2023-03-18 16:21:43 - r - INFO: - Episode: 34/100, Reward: 12.000, Step: 12 +2023-03-18 16:21:43 - r - INFO: - Episode: 35/100, Reward: 17.000, Step: 17 +2023-03-18 16:21:43 - r - INFO: - Current episode 35 has the best eval reward: 32.500 +2023-03-18 16:21:43 - r - INFO: - Episode: 36/100, Reward: 17.000, Step: 17 +2023-03-18 16:21:43 - r - INFO: - Episode: 37/100, Reward: 17.000, Step: 17 +2023-03-18 16:21:43 - r - INFO: - Episode: 38/100, Reward: 23.000, Step: 23 +2023-03-18 16:21:43 - r - INFO: - Episode: 39/100, Reward: 35.000, Step: 35 +2023-03-18 16:21:43 - r - INFO: - Episode: 40/100, Reward: 46.000, Step: 46 +2023-03-18 16:21:44 - r - INFO: - Episode: 41/100, Reward: 10.000, Step: 10 +2023-03-18 16:21:44 - r - INFO: - Episode: 42/100, Reward: 13.000, Step: 13 +2023-03-18 16:21:44 - r - INFO: - Episode: 43/100, Reward: 27.000, Step: 27 +2023-03-18 16:21:44 - r - INFO: - Episode: 44/100, Reward: 43.000, Step: 43 +2023-03-18 16:21:44 - r - INFO: - Episode: 45/100, Reward: 23.000, Step: 23 +2023-03-18 16:21:44 - r - INFO: - Episode: 46/100, Reward: 31.000, Step: 31 +2023-03-18 16:21:44 - r - INFO: - Episode: 47/100, Reward: 36.000, Step: 36 +2023-03-18 16:21:44 - r - INFO: - Episode: 48/100, Reward: 27.000, Step: 27 +2023-03-18 16:21:44 - r - INFO: - Episode: 49/100, Reward: 27.000, Step: 27 +2023-03-18 16:21:44 - r - INFO: - Episode: 50/100, Reward: 40.000, Step: 40 +2023-03-18 16:21:44 - r - INFO: - Current episode 50 has the best eval reward: 36.900 +2023-03-18 16:21:45 - r - INFO: - Episode: 51/100, Reward: 47.000, Step: 47 +2023-03-18 16:21:45 - r - INFO: - Episode: 52/100, Reward: 60.000, Step: 60 +2023-03-18 16:21:45 - r - INFO: - Episode: 53/100, Reward: 104.000, Step: 104 +2023-03-18 16:21:45 - r - INFO: - Episode: 54/100, Reward: 70.000, Step: 70 +2023-03-18 16:21:45 - r - INFO: - Episode: 55/100, Reward: 65.000, Step: 65 +2023-03-18 16:21:46 - r - INFO: - Episode: 56/100, Reward: 96.000, Step: 96 +2023-03-18 16:21:46 - r - INFO: - Episode: 57/100, Reward: 34.000, Step: 34 +2023-03-18 16:21:46 - r - INFO: - Episode: 58/100, Reward: 30.000, Step: 30 +2023-03-18 16:21:46 - r - INFO: - Episode: 59/100, Reward: 63.000, Step: 63 +2023-03-18 16:21:46 - r - INFO: - Episode: 60/100, Reward: 32.000, Step: 32 +2023-03-18 16:21:46 - r - INFO: - Current episode 60 has the best eval reward: 104.900 +2023-03-18 16:21:47 - r - INFO: - Episode: 61/100, Reward: 36.000, Step: 36 +2023-03-18 16:21:47 - r - INFO: - Episode: 62/100, Reward: 26.000, Step: 26 +2023-03-18 16:21:47 - r - INFO: - Episode: 63/100, Reward: 29.000, Step: 29 +2023-03-18 16:21:47 - r - INFO: - Episode: 64/100, Reward: 58.000, Step: 58 +2023-03-18 16:21:47 - r - INFO: - Episode: 65/100, Reward: 123.000, Step: 123 +2023-03-18 16:21:47 - r - INFO: - Episode: 66/100, Reward: 74.000, Step: 74 +2023-03-18 16:21:48 - r - INFO: - Episode: 67/100, Reward: 56.000, Step: 56 +2023-03-18 16:21:48 - r - INFO: - Episode: 68/100, Reward: 76.000, Step: 76 +2023-03-18 16:21:48 - r - INFO: - Episode: 69/100, Reward: 63.000, Step: 63 +2023-03-18 16:21:48 - r - INFO: - Episode: 70/100, Reward: 55.000, Step: 55 +2023-03-18 16:21:48 - r - INFO: - Episode: 71/100, Reward: 76.000, Step: 76 +2023-03-18 16:21:49 - r - INFO: - Episode: 72/100, Reward: 59.000, Step: 59 +2023-03-18 16:21:49 - r - INFO: - Episode: 73/100, Reward: 70.000, Step: 70 +2023-03-18 16:21:49 - r - INFO: - Episode: 74/100, Reward: 98.000, Step: 98 +2023-03-18 16:21:49 - r - INFO: - Episode: 75/100, Reward: 60.000, Step: 60 +2023-03-18 16:21:50 - r - INFO: - Episode: 76/100, Reward: 114.000, Step: 114 +2023-03-18 16:21:50 - r - INFO: - Episode: 77/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:51 - r - INFO: - Episode: 78/100, Reward: 199.000, Step: 199 +2023-03-18 16:21:51 - r - INFO: - Episode: 79/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:52 - r - INFO: - Episode: 80/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:52 - r - INFO: - Current episode 80 has the best eval reward: 200.000 +2023-03-18 16:21:53 - r - INFO: - Episode: 81/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:53 - r - INFO: - Episode: 82/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:54 - r - INFO: - Episode: 83/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:55 - r - INFO: - Episode: 84/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:55 - r - INFO: - Episode: 85/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:56 - r - INFO: - Current episode 85 has the best eval reward: 200.000 +2023-03-18 16:21:56 - r - INFO: - Episode: 86/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:57 - r - INFO: - Episode: 87/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:57 - r - INFO: - Episode: 88/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:58 - r - INFO: - Episode: 89/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:58 - r - INFO: - Episode: 90/100, Reward: 200.000, Step: 200 +2023-03-18 16:21:59 - r - INFO: - Current episode 90 has the best eval reward: 200.000 +2023-03-18 16:21:59 - r - INFO: - Episode: 91/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:00 - r - INFO: - Episode: 92/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:01 - r - INFO: - Episode: 93/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:01 - r - INFO: - Episode: 94/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:02 - r - INFO: - Episode: 95/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:02 - r - INFO: - Current episode 95 has the best eval reward: 200.000 +2023-03-18 16:22:03 - r - INFO: - Episode: 96/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:03 - r - INFO: - Episode: 97/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:04 - r - INFO: - Episode: 98/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:04 - r - INFO: - Episode: 99/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:05 - r - INFO: - Episode: 100/100, Reward: 200.000, Step: 200 +2023-03-18 16:22:05 - r - INFO: - Current episode 100 has the best eval reward: 200.000 +2023-03-18 16:22:05 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..3aa536480797ae052c46c05f6e9c38201887cdad --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c018ce117a1834aec6f585e6be780865631d49a8a81fb42ee4db9602b49793f5 +size 804667 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..f050a608bfa5d1d52a397b2b8099325e2c9bf96c Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/results/res.csv b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ffbb62b37c834f897d291eeff3794e3f1380d56 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,16.0,16 +1,16.0,16 +2,18.0,18 +3,14.0,14 +4,22.0,22 +5,27.0,27 +6,9.0,9 +7,13.0,13 +8,17.0,17 +9,37.0,37 +10,15.0,15 +11,22.0,22 +12,9.0,9 +13,14.0,14 +14,12.0,12 +15,16.0,16 +16,16.0,16 +17,14.0,14 +18,11.0,11 +19,13.0,13 +20,13.0,13 +21,14.0,14 +22,14.0,14 +23,37.0,37 +24,12.0,12 +25,18.0,18 +26,13.0,13 +27,20.0,20 +28,17.0,17 +29,10.0,10 +30,10.0,10 +31,12.0,12 +32,11.0,11 +33,12.0,12 +34,17.0,17 +35,17.0,17 +36,17.0,17 +37,23.0,23 +38,35.0,35 +39,46.0,46 +40,10.0,10 +41,13.0,13 +42,27.0,27 +43,43.0,43 +44,23.0,23 +45,31.0,31 +46,36.0,36 +47,27.0,27 +48,27.0,27 +49,40.0,40 +50,47.0,47 +51,60.0,60 +52,104.0,104 +53,70.0,70 +54,65.0,65 +55,96.0,96 +56,34.0,34 +57,30.0,30 +58,63.0,63 +59,32.0,32 +60,36.0,36 +61,26.0,26 +62,29.0,29 +63,58.0,58 +64,123.0,123 +65,74.0,74 +66,56.0,56 +67,76.0,76 +68,63.0,63 +69,55.0,55 +70,76.0,76 +71,59.0,59 +72,70.0,70 +73,98.0,98 +74,60.0,60 +75,114.0,114 +76,200.0,200 +77,199.0,199 +78,200.0,200 +79,200.0,200 +80,200.0,200 +81,200.0,200 +82,200.0,200 +83,200.0,200 +84,200.0,200 +85,200.0,200 +86,200.0,200 +87,200.0,200 +88,200.0,200 +89,200.0,200 +90,200.0,200 +91,200.0,200 +92,200.0,200 +93,200.0,200 +94,200.0,200 +95,200.0,200 +96,200.0,200 +97,200.0,200 +98,200.0,200 +99,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/tb_logs/events.out.tfevents.1679127699.DESKTOP-H34HQIQ.2016.0 b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/tb_logs/events.out.tfevents.1679127699.DESKTOP-H34HQIQ.2016.0 new file mode 100644 index 0000000000000000000000000000000000000000..0905fc999961a6353897b236a0b111aa006d273e --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_20230318-162139/tb_logs/events.out.tfevents.1679127699.DESKTOP-H34HQIQ.2016.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df11f028f48b8106ff87f12754360e62af378e6795428875376d4cc9c70baca9 +size 40 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/config.yaml b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7c11e4943aca50494a910ce9146adeeb34aaee4 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/config.yaml @@ -0,0 +1,36 @@ +general_cfg: + algo_name: NoisyDQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_NoisyDQN + max_steps: 200 + mode: train + mp_backend: mp + n_workers: 2 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + tau: 1.0 +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/logs/log.txt b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012a04afc32f9b02dd787d0b3c12139b314f0e5 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/logs/log.txt @@ -0,0 +1,44 @@ +2023-04-13 22:06:39 - r - INFO: - Hyperparameters: +2023-04-13 22:06:39 - r - INFO: - ================================================================================ +2023-04-13 22:06:39 - r - INFO: - Name Value Type +2023-04-13 22:06:39 - r - INFO: - env_name gym +2023-04-13 22:06:39 - r - INFO: - new_step_api 1 +2023-04-13 22:06:39 - r - INFO: - wrapper None +2023-04-13 22:06:39 - r - INFO: - render 0 +2023-04-13 22:06:39 - r - INFO: - render_mode None +2023-04-13 22:06:39 - r - INFO: - algo_name NoisyDQN +2023-04-13 22:06:39 - r - INFO: - mode train +2023-04-13 22:06:39 - r - INFO: - mp_backend mp +2023-04-13 22:06:39 - r - INFO: - seed 1 +2023-04-13 22:06:39 - r - INFO: - device cpu +2023-04-13 22:06:39 - r - INFO: - train_eps 200 +2023-04-13 22:06:39 - r - INFO: - test_eps 10 +2023-04-13 22:06:39 - r - INFO: - eval_eps 10 +2023-04-13 22:06:39 - r - INFO: - eval_per_episode 5 +2023-04-13 22:06:39 - r - INFO: - max_steps 200 +2023-04-13 22:06:39 - r - INFO: - load_checkpoint 0 +2023-04-13 22:06:39 - r - INFO: - load_path Train_CartPole-v1_NoisyDQN +2023-04-13 22:06:39 - r - INFO: - show_fig 0 +2023-04-13 22:06:39 - r - INFO: - save_fig 1 +2023-04-13 22:06:39 - r - INFO: - n_workers 2 +2023-04-13 22:06:39 - r - INFO: - epsilon_start 0.95 +2023-04-13 22:06:39 - r - INFO: - tau 1.0 +2023-04-13 22:06:39 - r - INFO: - epsilon_end 0.01 +2023-04-13 22:06:39 - r - INFO: - epsilon_decay 500 +2023-04-13 22:06:39 - r - INFO: - hidden_dim 256 +2023-04-13 22:06:39 - r - INFO: - gamma 0.95 +2023-04-13 22:06:39 - r - INFO: - lr 0.0001 +2023-04-13 22:06:39 - r - INFO: - buffer_size 100000 +2023-04-13 22:06:39 - r - INFO: - batch_size 64 +2023-04-13 22:06:39 - r - INFO: - target_update 4 +2023-04-13 22:06:39 - r - INFO: - id CartPole-v1 +2023-04-13 22:06:39 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230413-220639 +2023-04-13 22:06:39 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230413-220639/results +2023-04-13 22:06:39 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230413-220639/logs +2023-04-13 22:06:39 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230413-220639/traj +2023-04-13 22:06:39 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230413-220639/videos +2023-04-13 22:06:39 - r - INFO: - ================================================================================ +2023-04-13 22:06:39 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-13 22:06:39 - r - INFO: - Start training! +2023-04-13 22:06:39 - r - INFO: - Env: gym, Algorithm: NoisyDQN, Device: cpu +2023-04-13 22:07:32 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee646c7afec5e202c73770cd5fc016b932f986e6 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab846435c3ac88c2507622e5f1050d4be6bfc94e33845dc5d7de2fac6df2c92f +size 804667 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..9b9e3adae071ea42b12fffa1d293327af1c652f4 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/results/res.csv b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..509ec3519cb0b22651eed902fc76a81985c0b141 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_mp_20230413-220639/results/res.csv @@ -0,0 +1,202 @@ +episodes,rewards +0,24.0 +1,18.0 +2,18.0 +3,19.0 +4,27.0 +5,17.0 +6,15.0 +7,13.0 +8,21.0 +9,29.0 +10,16.0 +11,33.0 +12,9.0 +13,13.0 +14,11.0 +15,17.0 +16,10.0 +17,25.0 +18,13.0 +19,19.0 +20,11.0 +21,11.0 +22,10.0 +23,12.0 +24,14.0 +25,11.0 +26,17.0 +27,15.0 +28,10.0 +29,17.0 +30,10.0 +31,11.0 +32,10.0 +33,18.0 +34,12.0 +35,11.0 +36,18.0 +37,13.0 +38,11.0 +39,35.0 +40,9.0 +41,12.0 +42,13.0 +43,9.0 +44,9.0 +45,28.0 +46,10.0 +47,10.0 +48,9.0 +49,10.0 +50,10.0 +51,9.0 +52,11.0 +53,13.0 +54,13.0 +55,14.0 +56,9.0 +57,31.0 +58,64.0 +59,55.0 +60,25.0 +61,56.0 +62,40.0 +63,91.0 +64,39.0 +65,33.0 +66,53.0 +67,55.0 +68,43.0 +69,96.0 +70,50.0 +71,56.0 +72,55.0 +73,50.0 +74,64.0 +75,102.0 +76,63.0 +77,41.0 +78,88.0 +79,69.0 +80,186.0 +81,161.0 +82,106.0 +83,129.0 +84,200.0 +85,200.0 +86,200.0 +87,200.0 +88,200.0 +89,200.0 +90,200.0 +91,200.0 +92,200.0 +93,200.0 +94,200.0 +95,200.0 +96,200.0 +97,200.0 +98,200.0 +99,200.0 +100,200.0 +101,200.0 +102,200.0 +103,200.0 +104,200.0 +105,200.0 +106,200.0 +107,200.0 +108,200.0 +109,200.0 +110,200.0 +111,200.0 +112,200.0 +113,200.0 +114,200.0 +115,200.0 +116,200.0 +117,200.0 +118,200.0 +119,200.0 +120,200.0 +121,200.0 +122,200.0 +123,200.0 +124,200.0 +125,200.0 +126,200.0 +127,200.0 +128,200.0 +129,200.0 +130,200.0 +131,200.0 +132,200.0 +133,200.0 +134,200.0 +135,200.0 +136,200.0 +137,200.0 +138,200.0 +139,200.0 +140,200.0 +141,200.0 +142,200.0 +143,200.0 +144,200.0 +145,200.0 +146,200.0 +147,200.0 +148,200.0 +149,200.0 +150,200.0 +151,200.0 +152,200.0 +153,200.0 +154,200.0 +155,200.0 +156,200.0 +157,200.0 +158,200.0 +159,200.0 +160,200.0 +161,200.0 +162,200.0 +163,200.0 +164,200.0 +165,200.0 +166,200.0 +167,200.0 +168,200.0 +169,200.0 +170,200.0 +171,200.0 +172,200.0 +173,200.0 +174,200.0 +175,200.0 +176,200.0 +177,200.0 +178,200.0 +179,200.0 +180,200.0 +181,200.0 +182,200.0 +183,200.0 +184,200.0 +185,200.0 +186,200.0 +187,200.0 +188,200.0 +189,200.0 +190,200.0 +191,200.0 +192,200.0 +193,200.0 +194,200.0 +195,200.0 +196,200.0 +197,200.0 +198,200.0 +199,200.0 +200,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/config.yaml b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29c9bb73e7473ca189ef2725d9366fe76b087d2 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/config.yaml @@ -0,0 +1,36 @@ +general_cfg: + algo_name: NoisyDQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_NoisyDQN + max_steps: 200 + mode: train + mp_backend: ray + n_workers: 2 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 220 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + target_update: 4 + tau: 1.0 +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/logs/log.txt b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b3da269d53cf665f2b39c0724abf72a91d151aa --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/logs/log.txt @@ -0,0 +1,44 @@ +2023-04-14 10:13:51 - r - INFO: - Hyperparameters: +2023-04-14 10:13:51 - r - INFO: - ================================================================================ +2023-04-14 10:13:51 - r - INFO: - Name Value Type +2023-04-14 10:13:51 - r - INFO: - env_name gym +2023-04-14 10:13:51 - r - INFO: - new_step_api 1 +2023-04-14 10:13:51 - r - INFO: - wrapper None +2023-04-14 10:13:51 - r - INFO: - render 0 +2023-04-14 10:13:51 - r - INFO: - render_mode None +2023-04-14 10:13:51 - r - INFO: - algo_name NoisyDQN +2023-04-14 10:13:51 - r - INFO: - mode train +2023-04-14 10:13:51 - r - INFO: - mp_backend ray +2023-04-14 10:13:51 - r - INFO: - seed 1 +2023-04-14 10:13:51 - r - INFO: - device cpu +2023-04-14 10:13:51 - r - INFO: - train_eps 220 +2023-04-14 10:13:51 - r - INFO: - test_eps 10 +2023-04-14 10:13:51 - r - INFO: - eval_eps 10 +2023-04-14 10:13:51 - r - INFO: - eval_per_episode 5 +2023-04-14 10:13:51 - r - INFO: - max_steps 200 +2023-04-14 10:13:51 - r - INFO: - load_checkpoint 0 +2023-04-14 10:13:51 - r - INFO: - load_path Train_CartPole-v1_NoisyDQN +2023-04-14 10:13:51 - r - INFO: - show_fig 0 +2023-04-14 10:13:51 - r - INFO: - save_fig 1 +2023-04-14 10:13:51 - r - INFO: - n_workers 2 +2023-04-14 10:13:51 - r - INFO: - epsilon_start 0.95 +2023-04-14 10:13:51 - r - INFO: - tau 1.0 +2023-04-14 10:13:51 - r - INFO: - epsilon_end 0.01 +2023-04-14 10:13:51 - r - INFO: - epsilon_decay 500 +2023-04-14 10:13:51 - r - INFO: - hidden_dim 256 +2023-04-14 10:13:51 - r - INFO: - gamma 0.95 +2023-04-14 10:13:51 - r - INFO: - lr 0.0001 +2023-04-14 10:13:51 - r - INFO: - buffer_size 100000 +2023-04-14 10:13:51 - r - INFO: - batch_size 64 +2023-04-14 10:13:51 - r - INFO: - target_update 4 +2023-04-14 10:13:51 - r - INFO: - id CartPole-v1 +2023-04-14 10:13:51 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230414-101351 +2023-04-14 10:13:51 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230414-101351/results +2023-04-14 10:13:51 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230414-101351/logs +2023-04-14 10:13:51 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230414-101351/traj +2023-04-14 10:13:51 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_NoisyDQN_20230414-101351/videos +2023-04-14 10:13:51 - r - INFO: - ================================================================================ +2023-04-14 10:13:54 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-14 10:13:54 - r - INFO: - Start training! +2023-04-14 10:13:54 - r - INFO: - Env: gym, Algorithm: NoisyDQN, Device: cpu +2023-04-14 10:18:26 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd194c325cd70fff6bcfdbd0f6494201e76888ae --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5596bf84dd1a6b891f4a5790c45ae7c9f937bf6471c2218350adb67e6787b2d +size 804667 diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..2dfd5c93cc214f70ae2cfea6470b44c8d6c347cb Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/results/res.csv b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..db5417b7f2c86455779b86188e2aee2581baad15 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_NoisyDQN_ray_20230414-101351/results/res.csv @@ -0,0 +1,221 @@ +episodes,rewards +0,18.0 +1,18.0 +2,19.0 +3,28.0 +4,17.0 +5,15.0 +6,13.0 +7,15.0 +8,38.0 +9,31.0 +10,11.0 +11,31.0 +12,10.0 +13,17.0 +14,14.0 +15,13.0 +16,21.0 +17,10.0 +18,10.0 +19,10.0 +20,11.0 +21,13.0 +22,19.0 +23,12.0 +24,13.0 +25,18.0 +26,15.0 +27,13.0 +28,14.0 +29,12.0 +30,11.0 +31,12.0 +32,14.0 +33,9.0 +34,10.0 +35,16.0 +36,13.0 +37,15.0 +38,12.0 +39,11.0 +40,14.0 +41,28.0 +42,16.0 +43,11.0 +44,12.0 +45,11.0 +46,10.0 +47,13.0 +48,10.0 +49,9.0 +50,13.0 +51,11.0 +52,12.0 +53,10.0 +54,9.0 +55,12.0 +56,10.0 +57,15.0 +58,12.0 +59,12.0 +60,9.0 +61,16.0 +62,15.0 +63,16.0 +64,11.0 +65,18.0 +66,14.0 +67,12.0 +68,17.0 +69,13.0 +70,18.0 +71,15.0 +72,19.0 +73,26.0 +74,39.0 +75,28.0 +76,25.0 +77,66.0 +78,24.0 +79,29.0 +80,24.0 +81,25.0 +82,24.0 +83,20.0 +84,46.0 +85,37.0 +86,16.0 +87,24.0 +88,29.0 +89,86.0 +90,128.0 +91,85.0 +92,79.0 +93,58.0 +94,94.0 +95,57.0 +96,53.0 +97,36.0 +98,65.0 +99,64.0 +100,56.0 +101,58.0 +102,200.0 +103,123.0 +104,109.0 +105,67.0 +106,107.0 +107,165.0 +108,200.0 +109,200.0 +110,200.0 +111,170.0 +112,146.0 +113,200.0 +114,200.0 +115,200.0 +116,200.0 +117,200.0 +118,200.0 +119,200.0 +120,200.0 +121,200.0 +122,200.0 +123,200.0 +124,200.0 +125,200.0 +126,200.0 +127,200.0 +128,200.0 +129,200.0 +130,200.0 +131,200.0 +132,200.0 +133,200.0 +134,200.0 +135,200.0 +136,200.0 +137,200.0 +138,200.0 +139,200.0 +140,200.0 +141,200.0 +142,200.0 +143,200.0 +144,200.0 +145,200.0 +146,200.0 +147,200.0 +148,200.0 +149,200.0 +150,200.0 +151,200.0 +152,200.0 +153,200.0 +154,200.0 +155,200.0 +156,200.0 +157,200.0 +158,200.0 +159,191.0 +160,191.0 +161,190.0 +162,192.0 +163,200.0 +164,199.0 +165,200.0 +166,200.0 +167,200.0 +168,200.0 +169,200.0 +170,200.0 +171,200.0 +172,200.0 +173,200.0 +174,200.0 +175,200.0 +176,200.0 +177,200.0 +178,200.0 +179,200.0 +180,200.0 +181,200.0 +182,200.0 +183,200.0 +184,200.0 +185,200.0 +186,200.0 +187,200.0 +188,200.0 +189,200.0 +190,200.0 +191,200.0 +192,200.0 +193,200.0 +194,200.0 +195,200.0 +196,200.0 +197,200.0 +198,200.0 +199,200.0 +200,200.0 +201,200.0 +202,200.0 +203,200.0 +204,200.0 +205,200.0 +206,200.0 +207,200.0 +208,200.0 +209,200.0 +210,200.0 +211,200.0 +212,200.0 +213,200.0 +214,200.0 +215,200.0 +216,200.0 +217,200.0 +218,200.0 +219,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/config.yaml b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9959e23bb897e5ce0aa6bc94d1c20dc032aed88 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/config.yaml @@ -0,0 +1,48 @@ +general_cfg: + algo_name: PER_DQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_PER_DQN + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + per_alpha: 0.6 + per_beta: 0.4 + per_beta_annealing: 0.001 + per_epsilon: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/logs/log.txt b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..03b684d4aa892c1fea6f70f2282235b147994d89 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/logs/log.txt @@ -0,0 +1,260 @@ +2023-03-31 23:37:49 - r - INFO: - Hyperparameters: +2023-03-31 23:37:49 - r - INFO: - ================================================================================ +2023-03-31 23:37:49 - r - INFO: - Name Value Type +2023-03-31 23:37:49 - r - INFO: - env_name CartPole-v1 +2023-03-31 23:37:49 - r - INFO: - new_step_api 1 +2023-03-31 23:37:49 - r - INFO: - wrapper None +2023-03-31 23:37:49 - r - INFO: - render 0 +2023-03-31 23:37:49 - r - INFO: - algo_name PER_DQN +2023-03-31 23:37:49 - r - INFO: - mode train +2023-03-31 23:37:49 - r - INFO: - seed 1 +2023-03-31 23:37:49 - r - INFO: - device cuda +2023-03-31 23:37:49 - r - INFO: - train_eps 200 +2023-03-31 23:37:49 - r - INFO: - test_eps 10 +2023-03-31 23:37:49 - r - INFO: - eval_eps 10 +2023-03-31 23:37:49 - r - INFO: - eval_per_episode 5 +2023-03-31 23:37:49 - r - INFO: - max_steps 200 +2023-03-31 23:37:49 - r - INFO: - load_checkpoint 0 +2023-03-31 23:37:49 - r - INFO: - load_path Train_CartPole-v1_PER_DQN +2023-03-31 23:37:49 - r - INFO: - show_fig 0 +2023-03-31 23:37:49 - r - INFO: - save_fig 1 +2023-03-31 23:37:49 - r - INFO: - epsilon_start 0.95 +2023-03-31 23:37:49 - r - INFO: - epsilon_end 0.01 +2023-03-31 23:37:49 - r - INFO: - epsilon_decay 500 +2023-03-31 23:37:49 - r - INFO: - hidden_dim 256 +2023-03-31 23:37:49 - r - INFO: - gamma 0.99 +2023-03-31 23:37:49 - r - INFO: - lr 0.0001 +2023-03-31 23:37:49 - r - INFO: - buffer_size 100000 +2023-03-31 23:37:49 - r - INFO: - per_alpha 0.6 +2023-03-31 23:37:49 - r - INFO: - per_beta 0.4 +2023-03-31 23:37:49 - r - INFO: - per_beta_annealing 0.001 +2023-03-31 23:37:49 - r - INFO: - per_epsilon 0.01 +2023-03-31 23:37:49 - r - INFO: - batch_size 64 +2023-03-31 23:37:49 - r - INFO: - target_update 4 +2023-03-31 23:37:49 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-03-31 23:37:49 - r - INFO: - task_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-233749 +2023-03-31 23:37:49 - r - INFO: - res_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-233749/results +2023-03-31 23:37:49 - r - INFO: - log_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-233749/logs +2023-03-31 23:37:49 - r - INFO: - traj_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-233749/traj +2023-03-31 23:37:49 - r - INFO: - tb_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-233749/tb_logs +2023-03-31 23:37:49 - r - INFO: - ================================================================================ +2023-03-31 23:37:49 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-31 23:37:50 - r - INFO: - Start training! +2023-03-31 23:37:50 - r - INFO: - Env: CartPole-v1, Algorithm: PER_DQN, Device: cuda +2023-03-31 23:37:51 - r - INFO: - Episode: 1/200, Reward: 16.000, Step: 16 +2023-03-31 23:37:51 - r - INFO: - Episode: 2/200, Reward: 15.000, Step: 15 +2023-03-31 23:37:51 - r - INFO: - Episode: 3/200, Reward: 25.000, Step: 25 +2023-03-31 23:37:51 - r - INFO: - Episode: 4/200, Reward: 16.000, Step: 16 +2023-03-31 23:37:51 - r - INFO: - Episode: 5/200, Reward: 20.000, Step: 20 +2023-03-31 23:37:51 - r - INFO: - Current episode 5 has the best eval reward: 9.000 +2023-03-31 23:37:51 - r - INFO: - Episode: 6/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:51 - r - INFO: - Episode: 7/200, Reward: 24.000, Step: 24 +2023-03-31 23:37:51 - r - INFO: - Episode: 8/200, Reward: 20.000, Step: 20 +2023-03-31 23:37:51 - r - INFO: - Episode: 9/200, Reward: 20.000, Step: 20 +2023-03-31 23:37:51 - r - INFO: - Episode: 10/200, Reward: 25.000, Step: 25 +2023-03-31 23:37:51 - r - INFO: - Current episode 10 has the best eval reward: 9.100 +2023-03-31 23:37:51 - r - INFO: - Episode: 11/200, Reward: 9.000, Step: 9 +2023-03-31 23:37:51 - r - INFO: - Episode: 12/200, Reward: 23.000, Step: 23 +2023-03-31 23:37:51 - r - INFO: - Episode: 13/200, Reward: 14.000, Step: 14 +2023-03-31 23:37:51 - r - INFO: - Episode: 14/200, Reward: 12.000, Step: 12 +2023-03-31 23:37:51 - r - INFO: - Episode: 15/200, Reward: 11.000, Step: 11 +2023-03-31 23:37:51 - r - INFO: - Episode: 16/200, Reward: 17.000, Step: 17 +2023-03-31 23:37:51 - r - INFO: - Episode: 17/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:51 - r - INFO: - Episode: 18/200, Reward: 17.000, Step: 17 +2023-03-31 23:37:51 - r - INFO: - Episode: 19/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:51 - r - INFO: - Episode: 20/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:52 - r - INFO: - Episode: 21/200, Reward: 22.000, Step: 22 +2023-03-31 23:37:52 - r - INFO: - Episode: 22/200, Reward: 18.000, Step: 18 +2023-03-31 23:37:52 - r - INFO: - Episode: 23/200, Reward: 13.000, Step: 13 +2023-03-31 23:37:52 - r - INFO: - Episode: 24/200, Reward: 13.000, Step: 13 +2023-03-31 23:37:52 - r - INFO: - Episode: 25/200, Reward: 9.000, Step: 9 +2023-03-31 23:37:52 - r - INFO: - Current episode 25 has the best eval reward: 9.600 +2023-03-31 23:37:52 - r - INFO: - Episode: 26/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:52 - r - INFO: - Episode: 27/200, Reward: 13.000, Step: 13 +2023-03-31 23:37:52 - r - INFO: - Episode: 28/200, Reward: 11.000, Step: 11 +2023-03-31 23:37:52 - r - INFO: - Episode: 29/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:52 - r - INFO: - Episode: 30/200, Reward: 12.000, Step: 12 +2023-03-31 23:37:52 - r - INFO: - Episode: 31/200, Reward: 14.000, Step: 14 +2023-03-31 23:37:52 - r - INFO: - Episode: 32/200, Reward: 11.000, Step: 11 +2023-03-31 23:37:52 - r - INFO: - Episode: 33/200, Reward: 18.000, Step: 18 +2023-03-31 23:37:52 - r - INFO: - Episode: 34/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:52 - r - INFO: - Episode: 35/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:52 - r - INFO: - Episode: 36/200, Reward: 8.000, Step: 8 +2023-03-31 23:37:52 - r - INFO: - Episode: 37/200, Reward: 12.000, Step: 12 +2023-03-31 23:37:52 - r - INFO: - Episode: 38/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:52 - r - INFO: - Episode: 39/200, Reward: 11.000, Step: 11 +2023-03-31 23:37:52 - r - INFO: - Episode: 40/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:53 - r - INFO: - Episode: 41/200, Reward: 9.000, Step: 9 +2023-03-31 23:37:53 - r - INFO: - Episode: 42/200, Reward: 12.000, Step: 12 +2023-03-31 23:37:53 - r - INFO: - Episode: 43/200, Reward: 9.000, Step: 9 +2023-03-31 23:37:53 - r - INFO: - Episode: 44/200, Reward: 13.000, Step: 13 +2023-03-31 23:37:53 - r - INFO: - Episode: 45/200, Reward: 13.000, Step: 13 +2023-03-31 23:37:53 - r - INFO: - Episode: 46/200, Reward: 12.000, Step: 12 +2023-03-31 23:37:53 - r - INFO: - Episode: 47/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:53 - r - INFO: - Episode: 48/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:53 - r - INFO: - Episode: 49/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:53 - r - INFO: - Episode: 50/200, Reward: 13.000, Step: 13 +2023-03-31 23:37:53 - r - INFO: - Episode: 51/200, Reward: 10.000, Step: 10 +2023-03-31 23:37:53 - r - INFO: - Episode: 52/200, Reward: 15.000, Step: 15 +2023-03-31 23:37:53 - r - INFO: - Episode: 53/200, Reward: 18.000, Step: 18 +2023-03-31 23:37:53 - r - INFO: - Episode: 54/200, Reward: 18.000, Step: 18 +2023-03-31 23:37:53 - r - INFO: - Episode: 55/200, Reward: 16.000, Step: 16 +2023-03-31 23:37:53 - r - INFO: - Current episode 55 has the best eval reward: 28.000 +2023-03-31 23:37:53 - r - INFO: - Episode: 56/200, Reward: 47.000, Step: 47 +2023-03-31 23:37:54 - r - INFO: - Episode: 57/200, Reward: 87.000, Step: 87 +2023-03-31 23:37:54 - r - INFO: - Episode: 58/200, Reward: 20.000, Step: 20 +2023-03-31 23:37:54 - r - INFO: - Episode: 59/200, Reward: 47.000, Step: 47 +2023-03-31 23:37:54 - r - INFO: - Episode: 60/200, Reward: 17.000, Step: 17 +2023-03-31 23:37:54 - r - INFO: - Episode: 61/200, Reward: 37.000, Step: 37 +2023-03-31 23:37:54 - r - INFO: - Episode: 62/200, Reward: 43.000, Step: 43 +2023-03-31 23:37:54 - r - INFO: - Episode: 63/200, Reward: 33.000, Step: 33 +2023-03-31 23:37:55 - r - INFO: - Episode: 64/200, Reward: 18.000, Step: 18 +2023-03-31 23:37:55 - r - INFO: - Episode: 65/200, Reward: 29.000, Step: 29 +2023-03-31 23:37:55 - r - INFO: - Current episode 65 has the best eval reward: 30.700 +2023-03-31 23:37:55 - r - INFO: - Episode: 66/200, Reward: 30.000, Step: 30 +2023-03-31 23:37:55 - r - INFO: - Episode: 67/200, Reward: 23.000, Step: 23 +2023-03-31 23:37:55 - r - INFO: - Episode: 68/200, Reward: 26.000, Step: 26 +2023-03-31 23:37:55 - r - INFO: - Episode: 69/200, Reward: 18.000, Step: 18 +2023-03-31 23:37:55 - r - INFO: - Episode: 70/200, Reward: 20.000, Step: 20 +2023-03-31 23:37:55 - r - INFO: - Episode: 71/200, Reward: 26.000, Step: 26 +2023-03-31 23:37:55 - r - INFO: - Episode: 72/200, Reward: 16.000, Step: 16 +2023-03-31 23:37:55 - r - INFO: - Episode: 73/200, Reward: 23.000, Step: 23 +2023-03-31 23:37:56 - r - INFO: - Episode: 74/200, Reward: 30.000, Step: 30 +2023-03-31 23:37:56 - r - INFO: - Episode: 75/200, Reward: 23.000, Step: 23 +2023-03-31 23:37:56 - r - INFO: - Episode: 76/200, Reward: 26.000, Step: 26 +2023-03-31 23:37:56 - r - INFO: - Episode: 77/200, Reward: 34.000, Step: 34 +2023-03-31 23:37:56 - r - INFO: - Episode: 78/200, Reward: 29.000, Step: 29 +2023-03-31 23:37:56 - r - INFO: - Episode: 79/200, Reward: 32.000, Step: 32 +2023-03-31 23:37:56 - r - INFO: - Episode: 80/200, Reward: 23.000, Step: 23 +2023-03-31 23:37:57 - r - INFO: - Episode: 81/200, Reward: 32.000, Step: 32 +2023-03-31 23:37:57 - r - INFO: - Episode: 82/200, Reward: 72.000, Step: 72 +2023-03-31 23:37:57 - r - INFO: - Episode: 83/200, Reward: 105.000, Step: 105 +2023-03-31 23:37:58 - r - INFO: - Episode: 84/200, Reward: 63.000, Step: 63 +2023-03-31 23:37:58 - r - INFO: - Episode: 85/200, Reward: 119.000, Step: 119 +2023-03-31 23:37:59 - r - INFO: - Current episode 85 has the best eval reward: 86.500 +2023-03-31 23:37:59 - r - INFO: - Episode: 86/200, Reward: 52.000, Step: 52 +2023-03-31 23:37:59 - r - INFO: - Episode: 87/200, Reward: 155.000, Step: 155 +2023-03-31 23:38:00 - r - INFO: - Episode: 88/200, Reward: 79.000, Step: 79 +2023-03-31 23:38:00 - r - INFO: - Episode: 89/200, Reward: 44.000, Step: 44 +2023-03-31 23:38:00 - r - INFO: - Episode: 90/200, Reward: 140.000, Step: 140 +2023-03-31 23:38:01 - r - INFO: - Episode: 91/200, Reward: 86.000, Step: 86 +2023-03-31 23:38:01 - r - INFO: - Episode: 92/200, Reward: 183.000, Step: 183 +2023-03-31 23:38:02 - r - INFO: - Episode: 93/200, Reward: 112.000, Step: 112 +2023-03-31 23:38:03 - r - INFO: - Episode: 94/200, Reward: 190.000, Step: 190 +2023-03-31 23:38:03 - r - INFO: - Episode: 95/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:04 - r - INFO: - Current episode 95 has the best eval reward: 164.200 +2023-03-31 23:38:05 - r - INFO: - Episode: 96/200, Reward: 157.000, Step: 157 +2023-03-31 23:38:05 - r - INFO: - Episode: 97/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:06 - r - INFO: - Episode: 98/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:07 - r - INFO: - Episode: 99/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:08 - r - INFO: - Episode: 100/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:08 - r - INFO: - Current episode 100 has the best eval reward: 200.000 +2023-03-31 23:38:09 - r - INFO: - Episode: 101/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:10 - r - INFO: - Episode: 102/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:11 - r - INFO: - Episode: 103/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:12 - r - INFO: - Episode: 104/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:12 - r - INFO: - Episode: 105/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:13 - r - INFO: - Current episode 105 has the best eval reward: 200.000 +2023-03-31 23:38:14 - r - INFO: - Episode: 106/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:15 - r - INFO: - Episode: 107/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:16 - r - INFO: - Episode: 108/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:17 - r - INFO: - Episode: 109/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:17 - r - INFO: - Episode: 110/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:19 - r - INFO: - Episode: 111/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:20 - r - INFO: - Episode: 112/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:21 - r - INFO: - Episode: 113/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:22 - r - INFO: - Episode: 114/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:23 - r - INFO: - Episode: 115/200, Reward: 190.000, Step: 190 +2023-03-31 23:38:24 - r - INFO: - Episode: 116/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:25 - r - INFO: - Episode: 117/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:26 - r - INFO: - Episode: 118/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:27 - r - INFO: - Episode: 119/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:28 - r - INFO: - Episode: 120/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:28 - r - INFO: - Current episode 120 has the best eval reward: 200.000 +2023-03-31 23:38:29 - r - INFO: - Episode: 121/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:30 - r - INFO: - Episode: 122/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:31 - r - INFO: - Episode: 123/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:32 - r - INFO: - Episode: 124/200, Reward: 198.000, Step: 198 +2023-03-31 23:38:33 - r - INFO: - Episode: 125/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:35 - r - INFO: - Episode: 126/200, Reward: 188.000, Step: 188 +2023-03-31 23:38:36 - r - INFO: - Episode: 127/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:37 - r - INFO: - Episode: 128/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:38 - r - INFO: - Episode: 129/200, Reward: 175.000, Step: 175 +2023-03-31 23:38:39 - r - INFO: - Episode: 130/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:41 - r - INFO: - Episode: 131/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:42 - r - INFO: - Episode: 132/200, Reward: 172.000, Step: 172 +2023-03-31 23:38:43 - r - INFO: - Episode: 133/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:44 - r - INFO: - Episode: 134/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:45 - r - INFO: - Episode: 135/200, Reward: 179.000, Step: 179 +2023-03-31 23:38:46 - r - INFO: - Episode: 136/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:47 - r - INFO: - Episode: 137/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:49 - r - INFO: - Episode: 138/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:49 - r - INFO: - Episode: 139/200, Reward: 161.000, Step: 161 +2023-03-31 23:38:51 - r - INFO: - Episode: 140/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:52 - r - INFO: - Episode: 141/200, Reward: 150.000, Step: 150 +2023-03-31 23:38:53 - r - INFO: - Episode: 142/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:54 - r - INFO: - Episode: 143/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:55 - r - INFO: - Episode: 144/200, Reward: 170.000, Step: 170 +2023-03-31 23:38:56 - r - INFO: - Episode: 145/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:58 - r - INFO: - Episode: 146/200, Reward: 200.000, Step: 200 +2023-03-31 23:38:59 - r - INFO: - Episode: 147/200, Reward: 160.000, Step: 160 +2023-03-31 23:39:00 - r - INFO: - Episode: 148/200, Reward: 160.000, Step: 160 +2023-03-31 23:39:01 - r - INFO: - Episode: 149/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:02 - r - INFO: - Episode: 150/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:04 - r - INFO: - Episode: 151/200, Reward: 177.000, Step: 177 +2023-03-31 23:39:05 - r - INFO: - Episode: 152/200, Reward: 193.000, Step: 193 +2023-03-31 23:39:06 - r - INFO: - Episode: 153/200, Reward: 182.000, Step: 182 +2023-03-31 23:39:08 - r - INFO: - Episode: 154/200, Reward: 176.000, Step: 176 +2023-03-31 23:39:09 - r - INFO: - Episode: 155/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:11 - r - INFO: - Episode: 156/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:12 - r - INFO: - Episode: 157/200, Reward: 171.000, Step: 171 +2023-03-31 23:39:13 - r - INFO: - Episode: 158/200, Reward: 192.000, Step: 192 +2023-03-31 23:39:14 - r - INFO: - Episode: 159/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:15 - r - INFO: - Episode: 160/200, Reward: 179.000, Step: 179 +2023-03-31 23:39:17 - r - INFO: - Episode: 161/200, Reward: 177.000, Step: 177 +2023-03-31 23:39:18 - r - INFO: - Episode: 162/200, Reward: 199.000, Step: 199 +2023-03-31 23:39:19 - r - INFO: - Episode: 163/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:21 - r - INFO: - Episode: 164/200, Reward: 186.000, Step: 186 +2023-03-31 23:39:22 - r - INFO: - Episode: 165/200, Reward: 178.000, Step: 178 +2023-03-31 23:39:23 - r - INFO: - Episode: 166/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:25 - r - INFO: - Episode: 167/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:26 - r - INFO: - Episode: 168/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:27 - r - INFO: - Episode: 169/200, Reward: 179.000, Step: 179 +2023-03-31 23:39:29 - r - INFO: - Episode: 170/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:31 - r - INFO: - Episode: 171/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:32 - r - INFO: - Episode: 172/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:34 - r - INFO: - Episode: 173/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:35 - r - INFO: - Episode: 174/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:36 - r - INFO: - Episode: 175/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:37 - r - INFO: - Current episode 175 has the best eval reward: 200.000 +2023-03-31 23:39:38 - r - INFO: - Episode: 176/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:40 - r - INFO: - Episode: 177/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:41 - r - INFO: - Episode: 178/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:43 - r - INFO: - Episode: 179/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:44 - r - INFO: - Episode: 180/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:45 - r - INFO: - Current episode 180 has the best eval reward: 200.000 +2023-03-31 23:39:46 - r - INFO: - Episode: 181/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:47 - r - INFO: - Episode: 182/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:49 - r - INFO: - Episode: 183/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:50 - r - INFO: - Episode: 184/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:52 - r - INFO: - Episode: 185/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:52 - r - INFO: - Current episode 185 has the best eval reward: 200.000 +2023-03-31 23:39:54 - r - INFO: - Episode: 186/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:55 - r - INFO: - Episode: 187/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:57 - r - INFO: - Episode: 188/200, Reward: 200.000, Step: 200 +2023-03-31 23:39:58 - r - INFO: - Episode: 189/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:00 - r - INFO: - Episode: 190/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:00 - r - INFO: - Current episode 190 has the best eval reward: 200.000 +2023-03-31 23:40:02 - r - INFO: - Episode: 191/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:03 - r - INFO: - Episode: 192/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:05 - r - INFO: - Episode: 193/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:06 - r - INFO: - Episode: 194/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:08 - r - INFO: - Episode: 195/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:09 - r - INFO: - Current episode 195 has the best eval reward: 200.000 +2023-03-31 23:40:10 - r - INFO: - Episode: 196/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:13 - r - INFO: - Episode: 197/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:17 - r - INFO: - Episode: 198/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:24 - r - INFO: - Episode: 199/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:29 - r - INFO: - Episode: 200/200, Reward: 200.000, Step: 200 +2023-03-31 23:40:32 - r - INFO: - Current episode 200 has the best eval reward: 200.000 +2023-03-31 23:40:32 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cc0fd9a643ca074843f2f5fb68af45d3fb79a03 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ba22dbfbe3211e48c45027f9c4efb9981cdf6ddbd972b57201fb68ca90d2fd +size 272471 diff --git a/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..a135ef2b38b5e2b08e174562b1fc70a5b1f7600b Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/results/res.csv b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..5fb086b610314270fadc3d50b77c9d9493b2abd2 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,16.0,16 +1,15.0,15 +2,25.0,25 +3,16.0,16 +4,20.0,20 +5,10.0,10 +6,24.0,24 +7,20.0,20 +8,20.0,20 +9,25.0,25 +10,9.0,9 +11,23.0,23 +12,14.0,14 +13,12.0,12 +14,11.0,11 +15,17.0,17 +16,10.0,10 +17,17.0,17 +18,10.0,10 +19,10.0,10 +20,22.0,22 +21,18.0,18 +22,13.0,13 +23,13.0,13 +24,9.0,9 +25,10.0,10 +26,13.0,13 +27,11.0,11 +28,10.0,10 +29,12.0,12 +30,14.0,14 +31,11.0,11 +32,18.0,18 +33,10.0,10 +34,10.0,10 +35,8.0,8 +36,12.0,12 +37,10.0,10 +38,11.0,11 +39,10.0,10 +40,9.0,9 +41,12.0,12 +42,9.0,9 +43,13.0,13 +44,13.0,13 +45,12.0,12 +46,10.0,10 +47,10.0,10 +48,10.0,10 +49,13.0,13 +50,10.0,10 +51,15.0,15 +52,18.0,18 +53,18.0,18 +54,16.0,16 +55,47.0,47 +56,87.0,87 +57,20.0,20 +58,47.0,47 +59,17.0,17 +60,37.0,37 +61,43.0,43 +62,33.0,33 +63,18.0,18 +64,29.0,29 +65,30.0,30 +66,23.0,23 +67,26.0,26 +68,18.0,18 +69,20.0,20 +70,26.0,26 +71,16.0,16 +72,23.0,23 +73,30.0,30 +74,23.0,23 +75,26.0,26 +76,34.0,34 +77,29.0,29 +78,32.0,32 +79,23.0,23 +80,32.0,32 +81,72.0,72 +82,105.0,105 +83,63.0,63 +84,119.0,119 +85,52.0,52 +86,155.0,155 +87,79.0,79 +88,44.0,44 +89,140.0,140 +90,86.0,86 +91,183.0,183 +92,112.0,112 +93,190.0,190 +94,200.0,200 +95,157.0,157 +96,200.0,200 +97,200.0,200 +98,200.0,200 +99,200.0,200 +100,200.0,200 +101,200.0,200 +102,200.0,200 +103,200.0,200 +104,200.0,200 +105,200.0,200 +106,200.0,200 +107,200.0,200 +108,200.0,200 +109,200.0,200 +110,200.0,200 +111,200.0,200 +112,200.0,200 +113,200.0,200 +114,190.0,190 +115,200.0,200 +116,200.0,200 +117,200.0,200 +118,200.0,200 +119,200.0,200 +120,200.0,200 +121,200.0,200 +122,200.0,200 +123,198.0,198 +124,200.0,200 +125,188.0,188 +126,200.0,200 +127,200.0,200 +128,175.0,175 +129,200.0,200 +130,200.0,200 +131,172.0,172 +132,200.0,200 +133,200.0,200 +134,179.0,179 +135,200.0,200 +136,200.0,200 +137,200.0,200 +138,161.0,161 +139,200.0,200 +140,150.0,150 +141,200.0,200 +142,200.0,200 +143,170.0,170 +144,200.0,200 +145,200.0,200 +146,160.0,160 +147,160.0,160 +148,200.0,200 +149,200.0,200 +150,177.0,177 +151,193.0,193 +152,182.0,182 +153,176.0,176 +154,200.0,200 +155,200.0,200 +156,171.0,171 +157,192.0,192 +158,200.0,200 +159,179.0,179 +160,177.0,177 +161,199.0,199 +162,200.0,200 +163,186.0,186 +164,178.0,178 +165,200.0,200 +166,200.0,200 +167,200.0,200 +168,179.0,179 +169,200.0,200 +170,200.0,200 +171,200.0,200 +172,200.0,200 +173,200.0,200 +174,200.0,200 +175,200.0,200 +176,200.0,200 +177,200.0,200 +178,200.0,200 +179,200.0,200 +180,200.0,200 +181,200.0,200 +182,200.0,200 +183,200.0,200 +184,200.0,200 +185,200.0,200 +186,200.0,200 +187,200.0,200 +188,200.0,200 +189,200.0,200 +190,200.0,200 +191,200.0,200 +192,200.0,200 +193,200.0,200 +194,200.0,200 +195,200.0,200 +196,200.0,200 +197,200.0,200 +198,200.0,200 +199,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/tb_logs/events.out.tfevents.1680277069.DESKTOP-H34HQIQ.305216.0 b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/tb_logs/events.out.tfevents.1680277069.DESKTOP-H34HQIQ.305216.0 new file mode 100644 index 0000000000000000000000000000000000000000..87c8dfa061d1e18c0ff30c2e3765c1547c4018bd --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PERQue_DQN_20230331-233749/tb_logs/events.out.tfevents.1680277069.DESKTOP-H34HQIQ.305216.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ead3d7b1b3efd92eecfb7f314b1922f372c92614db7819dbfa6e06770b12d37 +size 40 diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/config.yaml b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9959e23bb897e5ce0aa6bc94d1c20dc032aed88 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/config.yaml @@ -0,0 +1,48 @@ +general_cfg: + algo_name: PER_DQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_PER_DQN + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + per_alpha: 0.6 + per_beta: 0.4 + per_beta_annealing: 0.001 + per_epsilon: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/logs/log.txt b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..ebe5dc335588441dce4e01cb0981373f17a82a12 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/logs/log.txt @@ -0,0 +1,267 @@ +2023-03-31 22:58:15 - r - INFO: - Hyperparameters: +2023-03-31 22:58:15 - r - INFO: - ================================================================================ +2023-03-31 22:58:15 - r - INFO: - Name Value Type +2023-03-31 22:58:15 - r - INFO: - env_name CartPole-v1 +2023-03-31 22:58:15 - r - INFO: - new_step_api 1 +2023-03-31 22:58:15 - r - INFO: - wrapper None +2023-03-31 22:58:15 - r - INFO: - render 0 +2023-03-31 22:58:15 - r - INFO: - algo_name PER_DQN +2023-03-31 22:58:15 - r - INFO: - mode train +2023-03-31 22:58:15 - r - INFO: - seed 1 +2023-03-31 22:58:15 - r - INFO: - device cuda +2023-03-31 22:58:15 - r - INFO: - train_eps 200 +2023-03-31 22:58:15 - r - INFO: - test_eps 10 +2023-03-31 22:58:15 - r - INFO: - eval_eps 10 +2023-03-31 22:58:15 - r - INFO: - eval_per_episode 5 +2023-03-31 22:58:15 - r - INFO: - max_steps 200 +2023-03-31 22:58:15 - r - INFO: - load_checkpoint 0 +2023-03-31 22:58:15 - r - INFO: - load_path Train_CartPole-v1_PER_DQN +2023-03-31 22:58:15 - r - INFO: - show_fig 0 +2023-03-31 22:58:15 - r - INFO: - save_fig 1 +2023-03-31 22:58:15 - r - INFO: - epsilon_start 0.95 +2023-03-31 22:58:15 - r - INFO: - epsilon_end 0.01 +2023-03-31 22:58:15 - r - INFO: - epsilon_decay 500 +2023-03-31 22:58:15 - r - INFO: - hidden_dim 256 +2023-03-31 22:58:15 - r - INFO: - gamma 0.99 +2023-03-31 22:58:15 - r - INFO: - lr 0.0001 +2023-03-31 22:58:15 - r - INFO: - buffer_size 100000 +2023-03-31 22:58:15 - r - INFO: - per_alpha 0.6 +2023-03-31 22:58:15 - r - INFO: - per_beta 0.4 +2023-03-31 22:58:15 - r - INFO: - per_beta_annealing 0.001 +2023-03-31 22:58:15 - r - INFO: - per_epsilon 0.01 +2023-03-31 22:58:15 - r - INFO: - batch_size 64 +2023-03-31 22:58:15 - r - INFO: - target_update 4 +2023-03-31 22:58:15 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-03-31 22:58:15 - r - INFO: - task_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-225815 +2023-03-31 22:58:15 - r - INFO: - res_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-225815/results +2023-03-31 22:58:15 - r - INFO: - log_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-225815/logs +2023-03-31 22:58:15 - r - INFO: - traj_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-225815/traj +2023-03-31 22:58:15 - r - INFO: - tb_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CartPole-v1_PER_DQN_20230331-225815/tb_logs +2023-03-31 22:58:15 - r - INFO: - ================================================================================ +2023-03-31 22:58:15 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-31 22:58:16 - r - INFO: - Start training! +2023-03-31 22:58:16 - r - INFO: - Env: CartPole-v1, Algorithm: PER_DQN, Device: cuda +2023-03-31 22:58:17 - r - INFO: - Episode: 1/200, Reward: 15.000, Step: 15 +2023-03-31 22:58:17 - r - INFO: - Episode: 2/200, Reward: 29.000, Step: 29 +2023-03-31 22:58:17 - r - INFO: - Episode: 3/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:17 - r - INFO: - Episode: 4/200, Reward: 14.000, Step: 14 +2023-03-31 22:58:17 - r - INFO: - Episode: 5/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:17 - r - INFO: - Current episode 5 has the best eval reward: 9.600 +2023-03-31 22:58:17 - r - INFO: - Episode: 6/200, Reward: 39.000, Step: 39 +2023-03-31 22:58:18 - r - INFO: - Episode: 7/200, Reward: 35.000, Step: 35 +2023-03-31 22:58:18 - r - INFO: - Episode: 8/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:18 - r - INFO: - Episode: 9/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:18 - r - INFO: - Episode: 10/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:18 - r - INFO: - Episode: 11/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:18 - r - INFO: - Episode: 12/200, Reward: 34.000, Step: 34 +2023-03-31 22:58:18 - r - INFO: - Episode: 13/200, Reward: 15.000, Step: 15 +2023-03-31 22:58:18 - r - INFO: - Episode: 14/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:18 - r - INFO: - Episode: 15/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:19 - r - INFO: - Episode: 16/200, Reward: 19.000, Step: 19 +2023-03-31 22:58:19 - r - INFO: - Episode: 17/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:19 - r - INFO: - Episode: 18/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:19 - r - INFO: - Episode: 19/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:19 - r - INFO: - Episode: 20/200, Reward: 19.000, Step: 19 +2023-03-31 22:58:19 - r - INFO: - Current episode 20 has the best eval reward: 9.700 +2023-03-31 22:58:19 - r - INFO: - Episode: 21/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:19 - r - INFO: - Episode: 22/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:19 - r - INFO: - Episode: 23/200, Reward: 14.000, Step: 14 +2023-03-31 22:58:19 - r - INFO: - Episode: 24/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:19 - r - INFO: - Episode: 25/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:19 - r - INFO: - Episode: 26/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:19 - r - INFO: - Episode: 27/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:19 - r - INFO: - Episode: 28/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:19 - r - INFO: - Episode: 29/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:20 - r - INFO: - Episode: 30/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:20 - r - INFO: - Episode: 31/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:20 - r - INFO: - Episode: 32/200, Reward: 8.000, Step: 8 +2023-03-31 22:58:20 - r - INFO: - Episode: 33/200, Reward: 8.000, Step: 8 +2023-03-31 22:58:20 - r - INFO: - Episode: 34/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:20 - r - INFO: - Episode: 35/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:20 - r - INFO: - Episode: 36/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:20 - r - INFO: - Episode: 37/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:20 - r - INFO: - Episode: 38/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:20 - r - INFO: - Episode: 39/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:20 - r - INFO: - Episode: 40/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:20 - r - INFO: - Episode: 41/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:20 - r - INFO: - Episode: 42/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:20 - r - INFO: - Episode: 43/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:20 - r - INFO: - Episode: 44/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:20 - r - INFO: - Episode: 45/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:21 - r - INFO: - Current episode 45 has the best eval reward: 10.600 +2023-03-31 22:58:21 - r - INFO: - Episode: 46/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:21 - r - INFO: - Episode: 47/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:21 - r - INFO: - Episode: 48/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:21 - r - INFO: - Episode: 49/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:21 - r - INFO: - Episode: 50/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:21 - r - INFO: - Episode: 51/200, Reward: 18.000, Step: 18 +2023-03-31 22:58:21 - r - INFO: - Episode: 52/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:21 - r - INFO: - Episode: 53/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:21 - r - INFO: - Episode: 54/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:21 - r - INFO: - Episode: 55/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:21 - r - INFO: - Episode: 56/200, Reward: 8.000, Step: 8 +2023-03-31 22:58:21 - r - INFO: - Episode: 57/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:21 - r - INFO: - Episode: 58/200, Reward: 11.000, Step: 11 +2023-03-31 22:58:21 - r - INFO: - Episode: 59/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:22 - r - INFO: - Episode: 60/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:22 - r - INFO: - Episode: 61/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:22 - r - INFO: - Episode: 62/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:22 - r - INFO: - Episode: 63/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:22 - r - INFO: - Episode: 64/200, Reward: 8.000, Step: 8 +2023-03-31 22:58:22 - r - INFO: - Episode: 65/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:22 - r - INFO: - Episode: 66/200, Reward: 9.000, Step: 9 +2023-03-31 22:58:22 - r - INFO: - Episode: 67/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:22 - r - INFO: - Episode: 68/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:22 - r - INFO: - Episode: 69/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:22 - r - INFO: - Episode: 70/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:22 - r - INFO: - Current episode 70 has the best eval reward: 12.500 +2023-03-31 22:58:22 - r - INFO: - Episode: 71/200, Reward: 10.000, Step: 10 +2023-03-31 22:58:22 - r - INFO: - Episode: 72/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:22 - r - INFO: - Episode: 73/200, Reward: 20.000, Step: 20 +2023-03-31 22:58:23 - r - INFO: - Episode: 74/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:23 - r - INFO: - Episode: 75/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:23 - r - INFO: - Current episode 75 has the best eval reward: 13.000 +2023-03-31 22:58:23 - r - INFO: - Episode: 76/200, Reward: 15.000, Step: 15 +2023-03-31 22:58:23 - r - INFO: - Episode: 77/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:23 - r - INFO: - Episode: 78/200, Reward: 19.000, Step: 19 +2023-03-31 22:58:23 - r - INFO: - Episode: 79/200, Reward: 14.000, Step: 14 +2023-03-31 22:58:23 - r - INFO: - Episode: 80/200, Reward: 12.000, Step: 12 +2023-03-31 22:58:23 - r - INFO: - Current episode 80 has the best eval reward: 15.400 +2023-03-31 22:58:23 - r - INFO: - Episode: 81/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:24 - r - INFO: - Episode: 82/200, Reward: 14.000, Step: 14 +2023-03-31 22:58:24 - r - INFO: - Episode: 83/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:24 - r - INFO: - Episode: 84/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:24 - r - INFO: - Episode: 85/200, Reward: 14.000, Step: 14 +2023-03-31 22:58:24 - r - INFO: - Current episode 85 has the best eval reward: 16.000 +2023-03-31 22:58:24 - r - INFO: - Episode: 86/200, Reward: 18.000, Step: 18 +2023-03-31 22:58:24 - r - INFO: - Episode: 87/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:24 - r - INFO: - Episode: 88/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:24 - r - INFO: - Episode: 89/200, Reward: 13.000, Step: 13 +2023-03-31 22:58:25 - r - INFO: - Episode: 90/200, Reward: 21.000, Step: 21 +2023-03-31 22:58:25 - r - INFO: - Current episode 90 has the best eval reward: 18.800 +2023-03-31 22:58:25 - r - INFO: - Episode: 91/200, Reward: 17.000, Step: 17 +2023-03-31 22:58:25 - r - INFO: - Episode: 92/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:25 - r - INFO: - Episode: 93/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:25 - r - INFO: - Episode: 94/200, Reward: 22.000, Step: 22 +2023-03-31 22:58:25 - r - INFO: - Episode: 95/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:26 - r - INFO: - Current episode 95 has the best eval reward: 22.000 +2023-03-31 22:58:26 - r - INFO: - Episode: 96/200, Reward: 14.000, Step: 14 +2023-03-31 22:58:26 - r - INFO: - Episode: 97/200, Reward: 20.000, Step: 20 +2023-03-31 22:58:26 - r - INFO: - Episode: 98/200, Reward: 24.000, Step: 24 +2023-03-31 22:58:26 - r - INFO: - Episode: 99/200, Reward: 21.000, Step: 21 +2023-03-31 22:58:26 - r - INFO: - Episode: 100/200, Reward: 22.000, Step: 22 +2023-03-31 22:58:27 - r - INFO: - Episode: 101/200, Reward: 21.000, Step: 21 +2023-03-31 22:58:27 - r - INFO: - Episode: 102/200, Reward: 19.000, Step: 19 +2023-03-31 22:58:27 - r - INFO: - Episode: 103/200, Reward: 18.000, Step: 18 +2023-03-31 22:58:27 - r - INFO: - Episode: 104/200, Reward: 18.000, Step: 18 +2023-03-31 22:58:27 - r - INFO: - Episode: 105/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:27 - r - INFO: - Episode: 106/200, Reward: 16.000, Step: 16 +2023-03-31 22:58:27 - r - INFO: - Episode: 107/200, Reward: 19.000, Step: 19 +2023-03-31 22:58:28 - r - INFO: - Episode: 108/200, Reward: 18.000, Step: 18 +2023-03-31 22:58:28 - r - INFO: - Episode: 109/200, Reward: 21.000, Step: 21 +2023-03-31 22:58:28 - r - INFO: - Episode: 110/200, Reward: 24.000, Step: 24 +2023-03-31 22:58:28 - r - INFO: - Current episode 110 has the best eval reward: 23.300 +2023-03-31 22:58:28 - r - INFO: - Episode: 111/200, Reward: 24.000, Step: 24 +2023-03-31 22:58:28 - r - INFO: - Episode: 112/200, Reward: 27.000, Step: 27 +2023-03-31 22:58:29 - r - INFO: - Episode: 113/200, Reward: 35.000, Step: 35 +2023-03-31 22:58:29 - r - INFO: - Episode: 114/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:29 - r - INFO: - Episode: 115/200, Reward: 29.000, Step: 29 +2023-03-31 22:58:29 - r - INFO: - Current episode 115 has the best eval reward: 24.100 +2023-03-31 22:58:29 - r - INFO: - Episode: 116/200, Reward: 25.000, Step: 25 +2023-03-31 22:58:29 - r - INFO: - Episode: 117/200, Reward: 20.000, Step: 20 +2023-03-31 22:58:30 - r - INFO: - Episode: 118/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:30 - r - INFO: - Episode: 119/200, Reward: 21.000, Step: 21 +2023-03-31 22:58:30 - r - INFO: - Episode: 120/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:30 - r - INFO: - Current episode 120 has the best eval reward: 24.500 +2023-03-31 22:58:30 - r - INFO: - Episode: 121/200, Reward: 17.000, Step: 17 +2023-03-31 22:58:30 - r - INFO: - Episode: 122/200, Reward: 19.000, Step: 19 +2023-03-31 22:58:30 - r - INFO: - Episode: 123/200, Reward: 19.000, Step: 19 +2023-03-31 22:58:31 - r - INFO: - Episode: 124/200, Reward: 21.000, Step: 21 +2023-03-31 22:58:31 - r - INFO: - Episode: 125/200, Reward: 24.000, Step: 24 +2023-03-31 22:58:31 - r - INFO: - Current episode 125 has the best eval reward: 25.600 +2023-03-31 22:58:31 - r - INFO: - Episode: 126/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:31 - r - INFO: - Episode: 127/200, Reward: 22.000, Step: 22 +2023-03-31 22:58:31 - r - INFO: - Episode: 128/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:31 - r - INFO: - Episode: 129/200, Reward: 22.000, Step: 22 +2023-03-31 22:58:32 - r - INFO: - Episode: 130/200, Reward: 28.000, Step: 28 +2023-03-31 22:58:32 - r - INFO: - Current episode 130 has the best eval reward: 29.800 +2023-03-31 22:58:32 - r - INFO: - Episode: 131/200, Reward: 32.000, Step: 32 +2023-03-31 22:58:32 - r - INFO: - Episode: 132/200, Reward: 35.000, Step: 35 +2023-03-31 22:58:32 - r - INFO: - Episode: 133/200, Reward: 27.000, Step: 27 +2023-03-31 22:58:33 - r - INFO: - Episode: 134/200, Reward: 24.000, Step: 24 +2023-03-31 22:58:33 - r - INFO: - Episode: 135/200, Reward: 37.000, Step: 37 +2023-03-31 22:58:33 - r - INFO: - Current episode 135 has the best eval reward: 35.700 +2023-03-31 22:58:33 - r - INFO: - Episode: 136/200, Reward: 33.000, Step: 33 +2023-03-31 22:58:34 - r - INFO: - Episode: 137/200, Reward: 39.000, Step: 39 +2023-03-31 22:58:34 - r - INFO: - Episode: 138/200, Reward: 24.000, Step: 24 +2023-03-31 22:58:34 - r - INFO: - Episode: 139/200, Reward: 24.000, Step: 24 +2023-03-31 22:58:34 - r - INFO: - Episode: 140/200, Reward: 40.000, Step: 40 +2023-03-31 22:58:35 - r - INFO: - Current episode 140 has the best eval reward: 40.200 +2023-03-31 22:58:35 - r - INFO: - Episode: 141/200, Reward: 31.000, Step: 31 +2023-03-31 22:58:35 - r - INFO: - Episode: 142/200, Reward: 30.000, Step: 30 +2023-03-31 22:58:35 - r - INFO: - Episode: 143/200, Reward: 25.000, Step: 25 +2023-03-31 22:58:35 - r - INFO: - Episode: 144/200, Reward: 23.000, Step: 23 +2023-03-31 22:58:35 - r - INFO: - Episode: 145/200, Reward: 29.000, Step: 29 +2023-03-31 22:58:36 - r - INFO: - Current episode 145 has the best eval reward: 58.500 +2023-03-31 22:58:36 - r - INFO: - Episode: 146/200, Reward: 51.000, Step: 51 +2023-03-31 22:58:37 - r - INFO: - Episode: 147/200, Reward: 73.000, Step: 73 +2023-03-31 22:58:37 - r - INFO: - Episode: 148/200, Reward: 38.000, Step: 38 +2023-03-31 22:58:37 - r - INFO: - Episode: 149/200, Reward: 37.000, Step: 37 +2023-03-31 22:58:37 - r - INFO: - Episode: 150/200, Reward: 32.000, Step: 32 +2023-03-31 22:58:38 - r - INFO: - Episode: 151/200, Reward: 43.000, Step: 43 +2023-03-31 22:58:38 - r - INFO: - Episode: 152/200, Reward: 29.000, Step: 29 +2023-03-31 22:58:38 - r - INFO: - Episode: 153/200, Reward: 33.000, Step: 33 +2023-03-31 22:58:38 - r - INFO: - Episode: 154/200, Reward: 31.000, Step: 31 +2023-03-31 22:58:39 - r - INFO: - Episode: 155/200, Reward: 41.000, Step: 41 +2023-03-31 22:58:39 - r - INFO: - Episode: 156/200, Reward: 79.000, Step: 79 +2023-03-31 22:58:40 - r - INFO: - Episode: 157/200, Reward: 47.000, Step: 47 +2023-03-31 22:58:40 - r - INFO: - Episode: 158/200, Reward: 32.000, Step: 32 +2023-03-31 22:58:40 - r - INFO: - Episode: 159/200, Reward: 36.000, Step: 36 +2023-03-31 22:58:41 - r - INFO: - Episode: 160/200, Reward: 76.000, Step: 76 +2023-03-31 22:58:41 - r - INFO: - Current episode 160 has the best eval reward: 75.000 +2023-03-31 22:58:41 - r - INFO: - Episode: 161/200, Reward: 73.000, Step: 73 +2023-03-31 22:58:42 - r - INFO: - Episode: 162/200, Reward: 59.000, Step: 59 +2023-03-31 22:58:42 - r - INFO: - Episode: 163/200, Reward: 102.000, Step: 102 +2023-03-31 22:58:43 - r - INFO: - Episode: 164/200, Reward: 87.000, Step: 87 +2023-03-31 22:58:44 - r - INFO: - Episode: 165/200, Reward: 94.000, Step: 94 +2023-03-31 22:58:44 - r - INFO: - Current episode 165 has the best eval reward: 143.300 +2023-03-31 22:58:45 - r - INFO: - Episode: 166/200, Reward: 116.000, Step: 116 +2023-03-31 22:58:46 - r - INFO: - Episode: 167/200, Reward: 135.000, Step: 135 +2023-03-31 22:58:47 - r - INFO: - Episode: 168/200, Reward: 140.000, Step: 140 +2023-03-31 22:58:48 - r - INFO: - Episode: 169/200, Reward: 167.000, Step: 167 +2023-03-31 22:58:49 - r - INFO: - Episode: 170/200, Reward: 128.000, Step: 128 +2023-03-31 22:58:50 - r - INFO: - Current episode 170 has the best eval reward: 157.400 +2023-03-31 22:58:51 - r - INFO: - Episode: 171/200, Reward: 200.000, Step: 200 +2023-03-31 22:58:52 - r - INFO: - Episode: 172/200, Reward: 135.000, Step: 135 +2023-03-31 22:58:53 - r - INFO: - Episode: 173/200, Reward: 163.000, Step: 163 +2023-03-31 22:58:54 - r - INFO: - Episode: 174/200, Reward: 180.000, Step: 180 +2023-03-31 22:58:56 - r - INFO: - Episode: 175/200, Reward: 185.000, Step: 185 +2023-03-31 22:58:56 - r - INFO: - Current episode 175 has the best eval reward: 165.700 +2023-03-31 22:58:57 - r - INFO: - Episode: 176/200, Reward: 200.000, Step: 200 +2023-03-31 22:58:59 - r - INFO: - Episode: 177/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:00 - r - INFO: - Episode: 178/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:01 - r - INFO: - Episode: 179/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:03 - r - INFO: - Episode: 180/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:04 - r - INFO: - Current episode 180 has the best eval reward: 200.000 +2023-03-31 22:59:05 - r - INFO: - Episode: 181/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:06 - r - INFO: - Episode: 182/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:08 - r - INFO: - Episode: 183/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:09 - r - INFO: - Episode: 184/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:10 - r - INFO: - Episode: 185/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:12 - r - INFO: - Episode: 186/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:18 - r - INFO: - Episode: 187/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:22 - r - INFO: - Episode: 188/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:24 - r - INFO: - Episode: 189/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:26 - r - INFO: - Episode: 190/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:28 - r - INFO: - Episode: 191/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:29 - r - INFO: - Episode: 192/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:30 - r - INFO: - Episode: 193/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:32 - r - INFO: - Episode: 194/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:33 - r - INFO: - Episode: 195/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:35 - r - INFO: - Episode: 196/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:37 - r - INFO: - Episode: 197/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:38 - r - INFO: - Episode: 198/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:39 - r - INFO: - Episode: 199/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:40 - r - INFO: - Episode: 200/200, Reward: 200.000, Step: 200 +2023-03-31 22:59:41 - r - INFO: - Current episode 200 has the best eval reward: 200.000 +2023-03-31 22:59:41 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..663b12f22263098d243896df93ba28f600d9a5fe --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c438616b97ca890557a9e9b1cd42decfc5decc64e5aee660d89158290e92683d +size 272471 diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..451d068a4636d034ba0343c28a90c5da7ba0b748 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/results/res.csv b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..fbca00787d5bb2ba30327255354cba17eeb53d1b --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,15.0,15 +1,29.0,29 +2,13.0,13 +3,14.0,14 +4,11.0,11 +5,39.0,39 +6,35.0,35 +7,16.0,16 +8,13.0,13 +9,12.0,12 +10,11.0,11 +11,34.0,34 +12,15.0,15 +13,23.0,23 +14,9.0,9 +15,19.0,19 +16,9.0,9 +17,10.0,10 +18,16.0,16 +19,19.0,19 +20,11.0,11 +21,10.0,10 +22,14.0,14 +23,12.0,12 +24,16.0,16 +25,11.0,11 +26,10.0,10 +27,16.0,16 +28,12.0,12 +29,16.0,16 +30,11.0,11 +31,8.0,8 +32,8.0,8 +33,12.0,12 +34,10.0,10 +35,9.0,9 +36,11.0,11 +37,10.0,10 +38,11.0,11 +39,10.0,10 +40,10.0,10 +41,10.0,10 +42,10.0,10 +43,9.0,9 +44,11.0,11 +45,10.0,10 +46,10.0,10 +47,11.0,11 +48,10.0,10 +49,13.0,13 +50,18.0,18 +51,12.0,12 +52,10.0,10 +53,10.0,10 +54,11.0,11 +55,8.0,8 +56,16.0,16 +57,11.0,11 +58,9.0,9 +59,9.0,9 +60,10.0,10 +61,10.0,10 +62,9.0,9 +63,8.0,8 +64,10.0,10 +65,9.0,9 +66,10.0,10 +67,12.0,12 +68,12.0,12 +69,12.0,12 +70,10.0,10 +71,13.0,13 +72,20.0,20 +73,12.0,12 +74,13.0,13 +75,15.0,15 +76,13.0,13 +77,19.0,19 +78,14.0,14 +79,12.0,12 +80,13.0,13 +81,14.0,14 +82,13.0,13 +83,13.0,13 +84,14.0,14 +85,18.0,18 +86,23.0,23 +87,13.0,13 +88,13.0,13 +89,21.0,21 +90,17.0,17 +91,23.0,23 +92,16.0,16 +93,22.0,22 +94,23.0,23 +95,14.0,14 +96,20.0,20 +97,24.0,24 +98,21.0,21 +99,22.0,22 +100,21.0,21 +101,19.0,19 +102,18.0,18 +103,18.0,18 +104,23.0,23 +105,16.0,16 +106,19.0,19 +107,18.0,18 +108,21.0,21 +109,24.0,24 +110,24.0,24 +111,27.0,27 +112,35.0,35 +113,23.0,23 +114,29.0,29 +115,25.0,25 +116,20.0,20 +117,23.0,23 +118,21.0,21 +119,23.0,23 +120,17.0,17 +121,19.0,19 +122,19.0,19 +123,21.0,21 +124,24.0,24 +125,23.0,23 +126,22.0,22 +127,23.0,23 +128,22.0,22 +129,28.0,28 +130,32.0,32 +131,35.0,35 +132,27.0,27 +133,24.0,24 +134,37.0,37 +135,33.0,33 +136,39.0,39 +137,24.0,24 +138,24.0,24 +139,40.0,40 +140,31.0,31 +141,30.0,30 +142,25.0,25 +143,23.0,23 +144,29.0,29 +145,51.0,51 +146,73.0,73 +147,38.0,38 +148,37.0,37 +149,32.0,32 +150,43.0,43 +151,29.0,29 +152,33.0,33 +153,31.0,31 +154,41.0,41 +155,79.0,79 +156,47.0,47 +157,32.0,32 +158,36.0,36 +159,76.0,76 +160,73.0,73 +161,59.0,59 +162,102.0,102 +163,87.0,87 +164,94.0,94 +165,116.0,116 +166,135.0,135 +167,140.0,140 +168,167.0,167 +169,128.0,128 +170,200.0,200 +171,135.0,135 +172,163.0,163 +173,180.0,180 +174,185.0,185 +175,200.0,200 +176,200.0,200 +177,200.0,200 +178,200.0,200 +179,200.0,200 +180,200.0,200 +181,200.0,200 +182,200.0,200 +183,200.0,200 +184,200.0,200 +185,200.0,200 +186,200.0,200 +187,200.0,200 +188,200.0,200 +189,200.0,200 +190,200.0,200 +191,200.0,200 +192,200.0,200 +193,200.0,200 +194,200.0,200 +195,200.0,200 +196,200.0,200 +197,200.0,200 +198,200.0,200 +199,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/tb_logs/events.out.tfevents.1680274695.DESKTOP-H34HQIQ.317208.0 b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/tb_logs/events.out.tfevents.1680274695.DESKTOP-H34HQIQ.317208.0 new file mode 100644 index 0000000000000000000000000000000000000000..a33114a37f26100b5d347c05fc48bcd6650e4f4f --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_20230331-225815/tb_logs/events.out.tfevents.1680274695.DESKTOP-H34HQIQ.317208.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4daaaaabe093b8f9d6baf9504a0c5b9e14d2ea89477d20323c5eacbf5942b64 +size 40 diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/config.yaml b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecd5ec863df5c9deb6872d3d93f996a4b2c4471a --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/config.yaml @@ -0,0 +1,55 @@ +general_cfg: + algo_name: PER_DQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_PER_DQN + max_steps: 200 + mode: train + mp_backend: mp + n_workers: 2 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + per_alpha: 0.6 + per_beta: 0.4 + per_beta_annealing: 0.001 + per_epsilon: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/logs/log.txt b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a23796be2fdbbf755f36361d92924558c4d83299 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/logs/log.txt @@ -0,0 +1,48 @@ +2023-04-15 21:50:02 - r - INFO: - Hyperparameters: +2023-04-15 21:50:02 - r - INFO: - ================================================================================ +2023-04-15 21:50:02 - r - INFO: - Name Value Type +2023-04-15 21:50:02 - r - INFO: - env_name gym +2023-04-15 21:50:02 - r - INFO: - new_step_api 1 +2023-04-15 21:50:02 - r - INFO: - wrapper None +2023-04-15 21:50:02 - r - INFO: - render 0 +2023-04-15 21:50:02 - r - INFO: - render_mode None +2023-04-15 21:50:02 - r - INFO: - algo_name PER_DQN +2023-04-15 21:50:02 - r - INFO: - mode train +2023-04-15 21:50:02 - r - INFO: - mp_backend mp +2023-04-15 21:50:02 - r - INFO: - seed 1 +2023-04-15 21:50:02 - r - INFO: - device cpu +2023-04-15 21:50:02 - r - INFO: - train_eps 200 +2023-04-15 21:50:02 - r - INFO: - test_eps 10 +2023-04-15 21:50:02 - r - INFO: - eval_eps 10 +2023-04-15 21:50:02 - r - INFO: - eval_per_episode 5 +2023-04-15 21:50:02 - r - INFO: - max_steps 200 +2023-04-15 21:50:02 - r - INFO: - load_checkpoint 0 +2023-04-15 21:50:02 - r - INFO: - load_path Train_CartPole-v1_PER_DQN +2023-04-15 21:50:02 - r - INFO: - show_fig 0 +2023-04-15 21:50:02 - r - INFO: - save_fig 1 +2023-04-15 21:50:02 - r - INFO: - n_workers 2 +2023-04-15 21:50:02 - r - INFO: - epsilon_start 0.95 +2023-04-15 21:50:02 - r - INFO: - epsilon_end 0.01 +2023-04-15 21:50:02 - r - INFO: - epsilon_decay 500 +2023-04-15 21:50:02 - r - INFO: - hidden_dim 256 +2023-04-15 21:50:02 - r - INFO: - gamma 0.99 +2023-04-15 21:50:02 - r - INFO: - lr 0.0001 +2023-04-15 21:50:02 - r - INFO: - buffer_size 100000 +2023-04-15 21:50:02 - r - INFO: - per_alpha 0.6 +2023-04-15 21:50:02 - r - INFO: - per_beta 0.4 +2023-04-15 21:50:02 - r - INFO: - per_beta_annealing 0.001 +2023-04-15 21:50:02 - r - INFO: - per_epsilon 0.01 +2023-04-15 21:50:02 - r - INFO: - batch_size 64 +2023-04-15 21:50:02 - r - INFO: - target_update 4 +2023-04-15 21:50:02 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-15 21:50:02 - r - INFO: - id CartPole-v1 +2023-04-15 21:50:02 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215002 +2023-04-15 21:50:02 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215002/results +2023-04-15 21:50:02 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215002/logs +2023-04-15 21:50:02 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215002/traj +2023-04-15 21:50:02 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215002/videos +2023-04-15 21:50:02 - r - INFO: - ================================================================================ +2023-04-15 21:50:02 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-15 21:50:02 - r - INFO: - Start training! +2023-04-15 21:50:02 - r - INFO: - Env: gym, Algorithm: PER_DQN, Device: cpu +2023-04-15 21:51:00 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f68746694a6ae30f06b2351e23e70ef9deafc34 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4251c7f141686d5391c5c933b493b27a184102ccf1596bead1dccaa6cc0bd9a +size 272407 diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..aca4e3fc511d1ddaf613b80d9d6972bbe7ae11bf Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/results/res.csv b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..a99e05cb22519b9c31c2af389db33fa7c6a7c771 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_mp__20230415-215002/results/res.csv @@ -0,0 +1,202 @@ +episodes,rewards +0,18.0 +1,19.0 +2,17.0 +3,24.0 +4,18.0 +5,17.0 +6,13.0 +7,16.0 +8,21.0 +9,30.0 +10,17.0 +11,18.0 +12,11.0 +13,13.0 +14,16.0 +15,14.0 +16,28.0 +17,12.0 +18,14.0 +19,19.0 +20,11.0 +21,10.0 +22,31.0 +23,23.0 +24,22.0 +25,11.0 +26,16.0 +27,12.0 +28,12.0 +29,16.0 +30,12.0 +31,16.0 +32,14.0 +33,21.0 +34,12.0 +35,9.0 +36,9.0 +37,26.0 +38,11.0 +39,22.0 +40,17.0 +41,21.0 +42,16.0 +43,27.0 +44,13.0 +45,18.0 +46,19.0 +47,11.0 +48,11.0 +49,16.0 +50,10.0 +51,9.0 +52,9.0 +53,16.0 +54,9.0 +55,12.0 +56,11.0 +57,11.0 +58,10.0 +59,12.0 +60,10.0 +61,14.0 +62,11.0 +63,12.0 +64,12.0 +65,18.0 +66,12.0 +67,16.0 +68,14.0 +69,23.0 +70,20.0 +71,23.0 +72,17.0 +73,18.0 +74,22.0 +75,22.0 +76,49.0 +77,24.0 +78,60.0 +79,35.0 +80,51.0 +81,78.0 +82,49.0 +83,75.0 +84,100.0 +85,78.0 +86,61.0 +87,65.0 +88,86.0 +89,105.0 +90,54.0 +91,60.0 +92,37.0 +93,149.0 +94,44.0 +95,104.0 +96,200.0 +97,112.0 +98,163.0 +99,167.0 +100,113.0 +101,152.0 +102,200.0 +103,200.0 +104,200.0 +105,200.0 +106,200.0 +107,200.0 +108,200.0 +109,200.0 +110,200.0 +111,200.0 +112,200.0 +113,200.0 +114,200.0 +115,200.0 +116,200.0 +117,200.0 +118,200.0 +119,200.0 +120,200.0 +121,200.0 +122,200.0 +123,200.0 +124,200.0 +125,200.0 +126,200.0 +127,200.0 +128,200.0 +129,200.0 +130,191.0 +131,200.0 +132,189.0 +133,200.0 +134,200.0 +135,200.0 +136,185.0 +137,200.0 +138,197.0 +139,200.0 +140,188.0 +141,200.0 +142,199.0 +143,200.0 +144,200.0 +145,200.0 +146,200.0 +147,200.0 +148,200.0 +149,200.0 +150,200.0 +151,200.0 +152,200.0 +153,200.0 +154,200.0 +155,200.0 +156,200.0 +157,200.0 +158,200.0 +159,200.0 +160,200.0 +161,200.0 +162,200.0 +163,200.0 +164,200.0 +165,200.0 +166,200.0 +167,200.0 +168,200.0 +169,200.0 +170,200.0 +171,200.0 +172,200.0 +173,200.0 +174,200.0 +175,200.0 +176,200.0 +177,200.0 +178,200.0 +179,200.0 +180,200.0 +181,200.0 +182,200.0 +183,200.0 +184,200.0 +185,200.0 +186,200.0 +187,200.0 +188,200.0 +189,200.0 +190,200.0 +191,200.0 +192,200.0 +193,200.0 +194,200.0 +195,200.0 +196,200.0 +197,200.0 +198,200.0 +199,200.0 +200,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/config.yaml b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c952bbbc81e0dbca9a835dedbcbe370b4b2c56c --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/config.yaml @@ -0,0 +1,55 @@ +general_cfg: + algo_name: PER_DQN + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_PER_DQN + max_steps: 200 + mode: train + mp_backend: ray + n_workers: 2 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 250 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + hidden_dim: 256 + lr: 0.0001 + per_alpha: 0.6 + per_beta: 0.4 + per_beta_annealing: 0.001 + per_epsilon: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear +env_cfg: + id: CartPole-v1 + new_step_api: true + render_mode: null diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/logs/log.txt b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..5ceecb4d4d23771092ae836d2623fb13d16df89c --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/logs/log.txt @@ -0,0 +1,48 @@ +2023-04-15 21:57:38 - r - INFO: - Hyperparameters: +2023-04-15 21:57:38 - r - INFO: - ================================================================================ +2023-04-15 21:57:38 - r - INFO: - Name Value Type +2023-04-15 21:57:38 - r - INFO: - env_name gym +2023-04-15 21:57:38 - r - INFO: - new_step_api 1 +2023-04-15 21:57:38 - r - INFO: - wrapper None +2023-04-15 21:57:38 - r - INFO: - render 0 +2023-04-15 21:57:38 - r - INFO: - render_mode None +2023-04-15 21:57:38 - r - INFO: - algo_name PER_DQN +2023-04-15 21:57:38 - r - INFO: - mode train +2023-04-15 21:57:38 - r - INFO: - mp_backend ray +2023-04-15 21:57:38 - r - INFO: - seed 1 +2023-04-15 21:57:38 - r - INFO: - device cpu +2023-04-15 21:57:38 - r - INFO: - train_eps 250 +2023-04-15 21:57:38 - r - INFO: - test_eps 10 +2023-04-15 21:57:38 - r - INFO: - eval_eps 10 +2023-04-15 21:57:38 - r - INFO: - eval_per_episode 5 +2023-04-15 21:57:38 - r - INFO: - max_steps 200 +2023-04-15 21:57:38 - r - INFO: - load_checkpoint 0 +2023-04-15 21:57:38 - r - INFO: - load_path Train_CartPole-v1_PER_DQN +2023-04-15 21:57:38 - r - INFO: - show_fig 0 +2023-04-15 21:57:38 - r - INFO: - save_fig 1 +2023-04-15 21:57:38 - r - INFO: - n_workers 2 +2023-04-15 21:57:38 - r - INFO: - epsilon_start 0.95 +2023-04-15 21:57:38 - r - INFO: - epsilon_end 0.01 +2023-04-15 21:57:38 - r - INFO: - epsilon_decay 500 +2023-04-15 21:57:38 - r - INFO: - hidden_dim 256 +2023-04-15 21:57:38 - r - INFO: - gamma 0.99 +2023-04-15 21:57:38 - r - INFO: - lr 0.0001 +2023-04-15 21:57:38 - r - INFO: - buffer_size 100000 +2023-04-15 21:57:38 - r - INFO: - per_alpha 0.6 +2023-04-15 21:57:38 - r - INFO: - per_beta 0.4 +2023-04-15 21:57:38 - r - INFO: - per_beta_annealing 0.001 +2023-04-15 21:57:38 - r - INFO: - per_epsilon 0.01 +2023-04-15 21:57:38 - r - INFO: - batch_size 64 +2023-04-15 21:57:38 - r - INFO: - target_update 4 +2023-04-15 21:57:38 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-15 21:57:38 - r - INFO: - id CartPole-v1 +2023-04-15 21:57:38 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215738 +2023-04-15 21:57:38 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215738/results +2023-04-15 21:57:38 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215738/logs +2023-04-15 21:57:38 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215738/traj +2023-04-15 21:57:38 - r - INFO: - video_dir /media/disk/gsc/joyrl-offline/tasks/Train_gym_PER_DQN_20230415-215738/videos +2023-04-15 21:57:38 - r - INFO: - ================================================================================ +2023-04-15 21:57:40 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-15 21:57:40 - r - INFO: - Start training! +2023-04-15 21:57:40 - r - INFO: - Env: gym, Algorithm: PER_DQN, Device: cpu +2023-04-15 22:00:44 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..75946dbb92769075d2500d272c9ae471ca3675eb --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0efe3ec576afef2311748067e61af0fe6c939f7a2c2a1500001987a5d0092ce3 +size 272407 diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..c88da98b353de91b7405ae49eddcf81756f6b0df Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/results/res.csv b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..1fb6ea0f87d54588fe26438bf1a48f163836131a --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PER_DQN_ray_20230415-215738/results/res.csv @@ -0,0 +1,251 @@ +episodes,rewards +0,18.0 +1,18.0 +2,39.0 +3,28.0 +4,15.0 +5,14.0 +6,39.0 +7,52.0 +8,36.0 +9,28.0 +10,13.0 +11,16.0 +12,20.0 +13,19.0 +14,31.0 +15,11.0 +16,10.0 +17,22.0 +18,23.0 +19,16.0 +20,11.0 +21,12.0 +22,12.0 +23,12.0 +24,16.0 +25,14.0 +26,16.0 +27,12.0 +28,21.0 +29,25.0 +30,9.0 +31,10.0 +32,9.0 +33,41.0 +34,22.0 +35,19.0 +36,13.0 +37,12.0 +38,16.0 +39,13.0 +40,13.0 +41,9.0 +42,11.0 +43,13.0 +44,11.0 +45,11.0 +46,11.0 +47,11.0 +48,10.0 +49,11.0 +50,10.0 +51,14.0 +52,12.0 +53,9.0 +54,10.0 +55,9.0 +56,10.0 +57,10.0 +58,12.0 +59,9.0 +60,10.0 +61,9.0 +62,11.0 +63,13.0 +64,10.0 +65,12.0 +66,15.0 +67,9.0 +68,11.0 +69,10.0 +70,10.0 +71,9.0 +72,10.0 +73,9.0 +74,11.0 +75,9.0 +76,10.0 +77,9.0 +78,9.0 +79,11.0 +80,11.0 +81,10.0 +82,12.0 +83,29.0 +84,14.0 +85,11.0 +86,14.0 +87,10.0 +88,10.0 +89,15.0 +90,18.0 +91,16.0 +92,15.0 +93,17.0 +94,12.0 +95,70.0 +96,27.0 +97,23.0 +98,115.0 +99,77.0 +100,34.0 +101,25.0 +102,18.0 +103,24.0 +104,19.0 +105,29.0 +106,33.0 +107,77.0 +108,44.0 +109,35.0 +110,51.0 +111,31.0 +112,53.0 +113,28.0 +114,33.0 +115,47.0 +116,69.0 +117,30.0 +118,30.0 +119,59.0 +120,41.0 +121,33.0 +122,82.0 +123,58.0 +124,31.0 +125,40.0 +126,38.0 +127,57.0 +128,34.0 +129,47.0 +130,36.0 +131,32.0 +132,38.0 +133,37.0 +134,57.0 +135,33.0 +136,52.0 +137,72.0 +138,55.0 +139,88.0 +140,50.0 +141,35.0 +142,49.0 +143,35.0 +144,54.0 +145,39.0 +146,34.0 +147,47.0 +148,34.0 +149,61.0 +150,39.0 +151,54.0 +152,69.0 +153,72.0 +154,65.0 +155,51.0 +156,101.0 +157,40.0 +158,49.0 +159,65.0 +160,43.0 +161,47.0 +162,154.0 +163,88.0 +164,99.0 +165,72.0 +166,152.0 +167,53.0 +168,74.0 +169,87.0 +170,62.0 +171,104.0 +172,80.0 +173,113.0 +174,75.0 +175,200.0 +176,69.0 +177,200.0 +178,200.0 +179,200.0 +180,130.0 +181,200.0 +182,150.0 +183,191.0 +184,200.0 +185,200.0 +186,200.0 +187,196.0 +188,175.0 +189,200.0 +190,200.0 +191,200.0 +192,200.0 +193,200.0 +194,200.0 +195,200.0 +196,200.0 +197,200.0 +198,200.0 +199,200.0 +200,197.0 +201,200.0 +202,200.0 +203,200.0 +204,200.0 +205,200.0 +206,200.0 +207,200.0 +208,200.0 +209,200.0 +210,200.0 +211,200.0 +212,200.0 +213,200.0 +214,200.0 +215,200.0 +216,200.0 +217,200.0 +218,200.0 +219,200.0 +220,200.0 +221,200.0 +222,200.0 +223,200.0 +224,200.0 +225,200.0 +226,200.0 +227,200.0 +228,200.0 +229,200.0 +230,200.0 +231,200.0 +232,200.0 +233,200.0 +234,200.0 +235,200.0 +236,200.0 +237,200.0 +238,200.0 +239,200.0 +240,200.0 +241,200.0 +242,200.0 +243,200.0 +244,200.0 +245,200.0 +246,200.0 +247,200.0 +248,200.0 +249,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/config.yaml b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f2895376e146417153ff7922ae852a1b2b9808f --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/config.yaml @@ -0,0 +1,35 @@ +general_cfg: + algo_name: PPO + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_PPO-KL_20221206-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + kl_alpha: 2 + kl_beta: 1.5 + kl_lambda: 0.5 + kl_target: 0.01 + ppo_type: kl + sgd_batch_size: 64 + train_batch_size: 100 diff --git a/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/logs/log.txt b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a78ce24cfdd5996a4ef607eff6b210c707cf1687 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/logs/log.txt @@ -0,0 +1,257 @@ +2022-12-17 20:40:03 - r - INFO: - Hyperparameters: +2022-12-17 20:40:03 - r - INFO: - ================================================================================ +2022-12-17 20:40:03 - r - INFO: - Name Value Type +2022-12-17 20:40:03 - r - INFO: - env_name CartPole-v1 +2022-12-17 20:40:03 - r - INFO: - new_step_api 1 +2022-12-17 20:40:03 - r - INFO: - wrapper None +2022-12-17 20:40:03 - r - INFO: - render 0 +2022-12-17 20:40:03 - r - INFO: - algo_name PPO +2022-12-17 20:40:03 - r - INFO: - mode train +2022-12-17 20:40:03 - r - INFO: - seed 1 +2022-12-17 20:40:03 - r - INFO: - device cuda +2022-12-17 20:40:03 - r - INFO: - train_eps 200 +2022-12-17 20:40:03 - r - INFO: - test_eps 10 +2022-12-17 20:40:03 - r - INFO: - eval_eps 10 +2022-12-17 20:40:03 - r - INFO: - eval_per_episode 5 +2022-12-17 20:40:03 - r - INFO: - max_steps 200 +2022-12-17 20:40:03 - r - INFO: - load_checkpoint 0 +2022-12-17 20:40:03 - r - INFO: - load_path Train_CartPole-v1_PPO-KL_20221206-054757 +2022-12-17 20:40:03 - r - INFO: - show_fig 0 +2022-12-17 20:40:03 - r - INFO: - save_fig 1 +2022-12-17 20:40:03 - r - INFO: - ppo_type kl +2022-12-17 20:40:03 - r - INFO: - continuous 0 +2022-12-17 20:40:03 - r - INFO: - gamma 0.99 +2022-12-17 20:40:03 - r - INFO: - k_epochs 4 +2022-12-17 20:40:03 - r - INFO: - actor_lr 0.0003 +2022-12-17 20:40:03 - r - INFO: - critic_lr 0.001 +2022-12-17 20:40:03 - r - INFO: - eps_clip 0.2 +2022-12-17 20:40:03 - r - INFO: - entropy_coef 0.01 +2022-12-17 20:40:03 - r - INFO: - train_batch_size 100 +2022-12-17 20:40:03 - r - INFO: - sgd_batch_size 64 +2022-12-17 20:40:03 - r - INFO: - actor_hidden_dim 256 +2022-12-17 20:40:03 - r - INFO: - critic_hidden_dim 256 +2022-12-17 20:40:03 - r - INFO: - kl_alpha 2 +2022-12-17 20:40:03 - r - INFO: - kl_beta 1.5 +2022-12-17 20:40:03 - r - INFO: - kl_lambda 0.5 +2022-12-17 20:40:03 - r - INFO: - kl_target 0.01 +2022-12-17 20:40:03 - r - INFO: - ================================================================================ +2022-12-17 20:40:03 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-17 20:40:03 - r - INFO: - Start training! +2022-12-17 20:40:03 - r - INFO: - Env: CartPole-v1, Algorithm: PPO, Device: cuda +2022-12-17 20:40:06 - r - INFO: - Episode: 1/200, Reward: 25.000, Step: 25 +2022-12-17 20:40:06 - r - INFO: - Episode: 2/200, Reward: 15.000, Step: 15 +2022-12-17 20:40:06 - r - INFO: - Episode: 3/200, Reward: 13.000, Step: 13 +2022-12-17 20:40:06 - r - INFO: - Episode: 4/200, Reward: 14.000, Step: 14 +2022-12-17 20:40:06 - r - INFO: - Episode: 5/200, Reward: 25.000, Step: 25 +2022-12-17 20:40:06 - r - INFO: - Current episode 5 has the best eval reward: 23.900 +2022-12-17 20:40:06 - r - INFO: - Episode: 6/200, Reward: 41.000, Step: 41 +2022-12-17 20:40:06 - r - INFO: - Episode: 7/200, Reward: 25.000, Step: 25 +2022-12-17 20:40:06 - r - INFO: - Episode: 8/200, Reward: 12.000, Step: 12 +2022-12-17 20:40:06 - r - INFO: - Episode: 9/200, Reward: 37.000, Step: 37 +2022-12-17 20:40:06 - r - INFO: - Episode: 10/200, Reward: 26.000, Step: 26 +2022-12-17 20:40:07 - r - INFO: - Episode: 11/200, Reward: 12.000, Step: 12 +2022-12-17 20:40:07 - r - INFO: - Episode: 12/200, Reward: 23.000, Step: 23 +2022-12-17 20:40:07 - r - INFO: - Episode: 13/200, Reward: 27.000, Step: 27 +2022-12-17 20:40:07 - r - INFO: - Episode: 14/200, Reward: 62.000, Step: 62 +2022-12-17 20:40:07 - r - INFO: - Episode: 15/200, Reward: 13.000, Step: 13 +2022-12-17 20:40:07 - r - INFO: - Current episode 15 has the best eval reward: 28.000 +2022-12-17 20:40:07 - r - INFO: - Episode: 16/200, Reward: 81.000, Step: 81 +2022-12-17 20:40:07 - r - INFO: - Episode: 17/200, Reward: 90.000, Step: 90 +2022-12-17 20:40:07 - r - INFO: - Episode: 18/200, Reward: 41.000, Step: 41 +2022-12-17 20:40:08 - r - INFO: - Episode: 19/200, Reward: 54.000, Step: 54 +2022-12-17 20:40:08 - r - INFO: - Episode: 20/200, Reward: 100.000, Step: 100 +2022-12-17 20:40:08 - r - INFO: - Current episode 20 has the best eval reward: 49.200 +2022-12-17 20:40:08 - r - INFO: - Episode: 21/200, Reward: 19.000, Step: 19 +2022-12-17 20:40:08 - r - INFO: - Episode: 22/200, Reward: 27.000, Step: 27 +2022-12-17 20:40:08 - r - INFO: - Episode: 23/200, Reward: 61.000, Step: 61 +2022-12-17 20:40:08 - r - INFO: - Episode: 24/200, Reward: 45.000, Step: 45 +2022-12-17 20:40:09 - r - INFO: - Episode: 25/200, Reward: 36.000, Step: 36 +2022-12-17 20:40:09 - r - INFO: - Current episode 25 has the best eval reward: 49.400 +2022-12-17 20:40:09 - r - INFO: - Episode: 26/200, Reward: 90.000, Step: 90 +2022-12-17 20:40:09 - r - INFO: - Episode: 27/200, Reward: 23.000, Step: 23 +2022-12-17 20:40:09 - r - INFO: - Episode: 28/200, Reward: 42.000, Step: 42 +2022-12-17 20:40:09 - r - INFO: - Episode: 29/200, Reward: 44.000, Step: 44 +2022-12-17 20:40:10 - r - INFO: - Episode: 30/200, Reward: 110.000, Step: 110 +2022-12-17 20:40:10 - r - INFO: - Current episode 30 has the best eval reward: 51.600 +2022-12-17 20:40:10 - r - INFO: - Episode: 31/200, Reward: 21.000, Step: 21 +2022-12-17 20:40:10 - r - INFO: - Episode: 32/200, Reward: 28.000, Step: 28 +2022-12-17 20:40:10 - r - INFO: - Episode: 33/200, Reward: 44.000, Step: 44 +2022-12-17 20:40:10 - r - INFO: - Episode: 34/200, Reward: 27.000, Step: 27 +2022-12-17 20:40:11 - r - INFO: - Episode: 35/200, Reward: 45.000, Step: 45 +2022-12-17 20:40:11 - r - INFO: - Current episode 35 has the best eval reward: 69.100 +2022-12-17 20:40:11 - r - INFO: - Episode: 36/200, Reward: 29.000, Step: 29 +2022-12-17 20:40:11 - r - INFO: - Episode: 37/200, Reward: 40.000, Step: 40 +2022-12-17 20:40:12 - r - INFO: - Episode: 38/200, Reward: 66.000, Step: 66 +2022-12-17 20:40:12 - r - INFO: - Episode: 39/200, Reward: 27.000, Step: 27 +2022-12-17 20:40:12 - r - INFO: - Episode: 40/200, Reward: 75.000, Step: 75 +2022-12-17 20:40:12 - r - INFO: - Episode: 41/200, Reward: 34.000, Step: 34 +2022-12-17 20:40:12 - r - INFO: - Episode: 42/200, Reward: 153.000, Step: 153 +2022-12-17 20:40:13 - r - INFO: - Episode: 43/200, Reward: 35.000, Step: 35 +2022-12-17 20:40:13 - r - INFO: - Episode: 44/200, Reward: 86.000, Step: 86 +2022-12-17 20:40:13 - r - INFO: - Episode: 45/200, Reward: 27.000, Step: 27 +2022-12-17 20:40:14 - r - INFO: - Episode: 46/200, Reward: 141.000, Step: 141 +2022-12-17 20:40:14 - r - INFO: - Episode: 47/200, Reward: 42.000, Step: 42 +2022-12-17 20:40:14 - r - INFO: - Episode: 48/200, Reward: 53.000, Step: 53 +2022-12-17 20:40:14 - r - INFO: - Episode: 49/200, Reward: 61.000, Step: 61 +2022-12-17 20:40:14 - r - INFO: - Episode: 50/200, Reward: 178.000, Step: 178 +2022-12-17 20:40:15 - r - INFO: - Current episode 50 has the best eval reward: 93.100 +2022-12-17 20:40:16 - r - INFO: - Episode: 51/200, Reward: 81.000, Step: 81 +2022-12-17 20:40:16 - r - INFO: - Episode: 52/200, Reward: 53.000, Step: 53 +2022-12-17 20:40:16 - r - INFO: - Episode: 53/200, Reward: 100.000, Step: 100 +2022-12-17 20:40:16 - r - INFO: - Episode: 54/200, Reward: 113.000, Step: 113 +2022-12-17 20:40:16 - r - INFO: - Episode: 55/200, Reward: 143.000, Step: 143 +2022-12-17 20:40:18 - r - INFO: - Current episode 55 has the best eval reward: 116.500 +2022-12-17 20:40:18 - r - INFO: - Episode: 56/200, Reward: 38.000, Step: 38 +2022-12-17 20:40:18 - r - INFO: - Episode: 57/200, Reward: 66.000, Step: 66 +2022-12-17 20:40:18 - r - INFO: - Episode: 58/200, Reward: 68.000, Step: 68 +2022-12-17 20:40:18 - r - INFO: - Episode: 59/200, Reward: 129.000, Step: 129 +2022-12-17 20:40:18 - r - INFO: - Episode: 60/200, Reward: 32.000, Step: 32 +2022-12-17 20:40:19 - r - INFO: - Episode: 61/200, Reward: 42.000, Step: 42 +2022-12-17 20:40:19 - r - INFO: - Episode: 62/200, Reward: 121.000, Step: 121 +2022-12-17 20:40:19 - r - INFO: - Episode: 63/200, Reward: 30.000, Step: 30 +2022-12-17 20:40:20 - r - INFO: - Episode: 64/200, Reward: 93.000, Step: 93 +2022-12-17 20:40:20 - r - INFO: - Episode: 65/200, Reward: 30.000, Step: 30 +2022-12-17 20:40:21 - r - INFO: - Episode: 66/200, Reward: 49.000, Step: 49 +2022-12-17 20:40:21 - r - INFO: - Episode: 67/200, Reward: 102.000, Step: 102 +2022-12-17 20:40:21 - r - INFO: - Episode: 68/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:21 - r - INFO: - Episode: 69/200, Reward: 72.000, Step: 72 +2022-12-17 20:40:21 - r - INFO: - Episode: 70/200, Reward: 164.000, Step: 164 +2022-12-17 20:40:23 - r - INFO: - Current episode 70 has the best eval reward: 147.000 +2022-12-17 20:40:23 - r - INFO: - Episode: 71/200, Reward: 97.000, Step: 97 +2022-12-17 20:40:23 - r - INFO: - Episode: 72/200, Reward: 16.000, Step: 16 +2022-12-17 20:40:24 - r - INFO: - Episode: 73/200, Reward: 184.000, Step: 184 +2022-12-17 20:40:24 - r - INFO: - Episode: 74/200, Reward: 103.000, Step: 103 +2022-12-17 20:40:24 - r - INFO: - Episode: 75/200, Reward: 157.000, Step: 157 +2022-12-17 20:40:26 - r - INFO: - Current episode 75 has the best eval reward: 152.700 +2022-12-17 20:40:26 - r - INFO: - Episode: 76/200, Reward: 142.000, Step: 142 +2022-12-17 20:40:26 - r - INFO: - Episode: 77/200, Reward: 46.000, Step: 46 +2022-12-17 20:40:26 - r - INFO: - Episode: 78/200, Reward: 123.000, Step: 123 +2022-12-17 20:40:26 - r - INFO: - Episode: 79/200, Reward: 128.000, Step: 128 +2022-12-17 20:40:27 - r - INFO: - Episode: 80/200, Reward: 121.000, Step: 121 +2022-12-17 20:40:28 - r - INFO: - Episode: 81/200, Reward: 150.000, Step: 150 +2022-12-17 20:40:28 - r - INFO: - Episode: 82/200, Reward: 134.000, Step: 134 +2022-12-17 20:40:28 - r - INFO: - Episode: 83/200, Reward: 132.000, Step: 132 +2022-12-17 20:40:29 - r - INFO: - Episode: 84/200, Reward: 127.000, Step: 127 +2022-12-17 20:40:29 - r - INFO: - Episode: 85/200, Reward: 199.000, Step: 199 +2022-12-17 20:40:31 - r - INFO: - Current episode 85 has the best eval reward: 174.800 +2022-12-17 20:40:31 - r - INFO: - Episode: 86/200, Reward: 164.000, Step: 164 +2022-12-17 20:40:32 - r - INFO: - Episode: 87/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:32 - r - INFO: - Episode: 88/200, Reward: 97.000, Step: 97 +2022-12-17 20:40:32 - r - INFO: - Episode: 89/200, Reward: 177.000, Step: 177 +2022-12-17 20:40:32 - r - INFO: - Episode: 90/200, Reward: 77.000, Step: 77 +2022-12-17 20:40:33 - r - INFO: - Episode: 91/200, Reward: 62.000, Step: 62 +2022-12-17 20:40:33 - r - INFO: - Episode: 92/200, Reward: 48.000, Step: 48 +2022-12-17 20:40:33 - r - INFO: - Episode: 93/200, Reward: 98.000, Step: 98 +2022-12-17 20:40:33 - r - INFO: - Episode: 94/200, Reward: 25.000, Step: 25 +2022-12-17 20:40:33 - r - INFO: - Episode: 95/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:38 - r - INFO: - Current episode 95 has the best eval reward: 178.200 +2022-12-17 20:40:38 - r - INFO: - Episode: 96/200, Reward: 144.000, Step: 144 +2022-12-17 20:40:38 - r - INFO: - Episode: 97/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:38 - r - INFO: - Episode: 98/200, Reward: 171.000, Step: 171 +2022-12-17 20:40:39 - r - INFO: - Episode: 99/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:39 - r - INFO: - Episode: 100/200, Reward: 159.000, Step: 159 +2022-12-17 20:40:42 - r - INFO: - Episode: 101/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:42 - r - INFO: - Episode: 102/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:42 - r - INFO: - Episode: 103/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:42 - r - INFO: - Episode: 104/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:43 - r - INFO: - Episode: 105/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:45 - r - INFO: - Current episode 105 has the best eval reward: 199.000 +2022-12-17 20:40:46 - r - INFO: - Episode: 106/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:46 - r - INFO: - Episode: 107/200, Reward: 115.000, Step: 115 +2022-12-17 20:40:46 - r - INFO: - Episode: 108/200, Reward: 145.000, Step: 145 +2022-12-17 20:40:46 - r - INFO: - Episode: 109/200, Reward: 142.000, Step: 142 +2022-12-17 20:40:46 - r - INFO: - Episode: 110/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:49 - r - INFO: - Episode: 111/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:49 - r - INFO: - Episode: 112/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:49 - r - INFO: - Episode: 113/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:50 - r - INFO: - Episode: 114/200, Reward: 72.000, Step: 72 +2022-12-17 20:40:50 - r - INFO: - Episode: 115/200, Reward: 198.000, Step: 198 +2022-12-17 20:40:51 - r - INFO: - Episode: 116/200, Reward: 26.000, Step: 26 +2022-12-17 20:40:52 - r - INFO: - Episode: 117/200, Reward: 166.000, Step: 166 +2022-12-17 20:40:52 - r - INFO: - Episode: 118/200, Reward: 18.000, Step: 18 +2022-12-17 20:40:52 - r - INFO: - Episode: 119/200, Reward: 18.000, Step: 18 +2022-12-17 20:40:52 - r - INFO: - Episode: 120/200, Reward: 35.000, Step: 35 +2022-12-17 20:40:53 - r - INFO: - Episode: 121/200, Reward: 37.000, Step: 37 +2022-12-17 20:40:53 - r - INFO: - Episode: 122/200, Reward: 123.000, Step: 123 +2022-12-17 20:40:54 - r - INFO: - Episode: 123/200, Reward: 108.000, Step: 108 +2022-12-17 20:40:54 - r - INFO: - Episode: 124/200, Reward: 108.000, Step: 108 +2022-12-17 20:40:54 - r - INFO: - Episode: 125/200, Reward: 131.000, Step: 131 +2022-12-17 20:40:55 - r - INFO: - Episode: 126/200, Reward: 106.000, Step: 106 +2022-12-17 20:40:55 - r - INFO: - Episode: 127/200, Reward: 43.000, Step: 43 +2022-12-17 20:40:56 - r - INFO: - Episode: 128/200, Reward: 68.000, Step: 68 +2022-12-17 20:40:56 - r - INFO: - Episode: 129/200, Reward: 83.000, Step: 83 +2022-12-17 20:40:56 - r - INFO: - Episode: 130/200, Reward: 178.000, Step: 178 +2022-12-17 20:40:58 - r - INFO: - Episode: 131/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:59 - r - INFO: - Episode: 132/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:59 - r - INFO: - Episode: 133/200, Reward: 200.000, Step: 200 +2022-12-17 20:40:59 - r - INFO: - Episode: 134/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:00 - r - INFO: - Episode: 135/200, Reward: 163.000, Step: 163 +2022-12-17 20:41:02 - r - INFO: - Episode: 136/200, Reward: 141.000, Step: 141 +2022-12-17 20:41:02 - r - INFO: - Episode: 137/200, Reward: 194.000, Step: 194 +2022-12-17 20:41:03 - r - INFO: - Episode: 138/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:03 - r - INFO: - Episode: 139/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:03 - r - INFO: - Episode: 140/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:06 - r - INFO: - Current episode 140 has the best eval reward: 200.000 +2022-12-17 20:41:06 - r - INFO: - Episode: 141/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:06 - r - INFO: - Episode: 142/200, Reward: 177.000, Step: 177 +2022-12-17 20:41:07 - r - INFO: - Episode: 143/200, Reward: 105.000, Step: 105 +2022-12-17 20:41:07 - r - INFO: - Episode: 144/200, Reward: 101.000, Step: 101 +2022-12-17 20:41:07 - r - INFO: - Episode: 145/200, Reward: 158.000, Step: 158 +2022-12-17 20:41:09 - r - INFO: - Episode: 146/200, Reward: 148.000, Step: 148 +2022-12-17 20:41:09 - r - INFO: - Episode: 147/200, Reward: 177.000, Step: 177 +2022-12-17 20:41:10 - r - INFO: - Episode: 148/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:10 - r - INFO: - Episode: 149/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:10 - r - INFO: - Episode: 150/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:13 - r - INFO: - Current episode 150 has the best eval reward: 200.000 +2022-12-17 20:41:13 - r - INFO: - Episode: 151/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:13 - r - INFO: - Episode: 152/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:14 - r - INFO: - Episode: 153/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:14 - r - INFO: - Episode: 154/200, Reward: 157.000, Step: 157 +2022-12-17 20:41:14 - r - INFO: - Episode: 155/200, Reward: 96.000, Step: 96 +2022-12-17 20:41:16 - r - INFO: - Episode: 156/200, Reward: 150.000, Step: 150 +2022-12-17 20:41:17 - r - INFO: - Episode: 157/200, Reward: 171.000, Step: 171 +2022-12-17 20:41:17 - r - INFO: - Episode: 158/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:17 - r - INFO: - Episode: 159/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:18 - r - INFO: - Episode: 160/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:20 - r - INFO: - Current episode 160 has the best eval reward: 200.000 +2022-12-17 20:41:20 - r - INFO: - Episode: 161/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:21 - r - INFO: - Episode: 162/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:21 - r - INFO: - Episode: 163/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:21 - r - INFO: - Episode: 164/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:22 - r - INFO: - Episode: 165/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:24 - r - INFO: - Episode: 166/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:25 - r - INFO: - Episode: 167/200, Reward: 169.000, Step: 169 +2022-12-17 20:41:25 - r - INFO: - Episode: 168/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:25 - r - INFO: - Episode: 169/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:26 - r - INFO: - Episode: 170/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:28 - r - INFO: - Episode: 171/200, Reward: 195.000, Step: 195 +2022-12-17 20:41:28 - r - INFO: - Episode: 172/200, Reward: 69.000, Step: 69 +2022-12-17 20:41:29 - r - INFO: - Episode: 173/200, Reward: 194.000, Step: 194 +2022-12-17 20:41:29 - r - INFO: - Episode: 174/200, Reward: 179.000, Step: 179 +2022-12-17 20:41:29 - r - INFO: - Episode: 175/200, Reward: 137.000, Step: 137 +2022-12-17 20:41:31 - r - INFO: - Episode: 176/200, Reward: 139.000, Step: 139 +2022-12-17 20:41:31 - r - INFO: - Episode: 177/200, Reward: 142.000, Step: 142 +2022-12-17 20:41:31 - r - INFO: - Episode: 178/200, Reward: 149.000, Step: 149 +2022-12-17 20:41:32 - r - INFO: - Episode: 179/200, Reward: 186.000, Step: 186 +2022-12-17 20:41:32 - r - INFO: - Episode: 180/200, Reward: 154.000, Step: 154 +2022-12-17 20:41:34 - r - INFO: - Episode: 181/200, Reward: 172.000, Step: 172 +2022-12-17 20:41:35 - r - INFO: - Episode: 182/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:35 - r - INFO: - Episode: 183/200, Reward: 172.000, Step: 172 +2022-12-17 20:41:35 - r - INFO: - Episode: 184/200, Reward: 197.000, Step: 197 +2022-12-17 20:41:36 - r - INFO: - Episode: 185/200, Reward: 167.000, Step: 167 +2022-12-17 20:41:38 - r - INFO: - Episode: 186/200, Reward: 171.000, Step: 171 +2022-12-17 20:41:38 - r - INFO: - Episode: 187/200, Reward: 181.000, Step: 181 +2022-12-17 20:41:38 - r - INFO: - Episode: 188/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:39 - r - INFO: - Episode: 189/200, Reward: 145.000, Step: 145 +2022-12-17 20:41:39 - r - INFO: - Episode: 190/200, Reward: 200.000, Step: 200 +2022-12-17 20:41:42 - r - INFO: - Episode: 191/200, Reward: 189.000, Step: 189 +2022-12-17 20:41:42 - r - INFO: - Episode: 192/200, Reward: 164.000, Step: 164 +2022-12-17 20:41:42 - r - INFO: - Episode: 193/200, Reward: 160.000, Step: 160 +2022-12-17 20:41:42 - r - INFO: - Episode: 194/200, Reward: 121.000, Step: 121 +2022-12-17 20:41:42 - r - INFO: - Episode: 195/200, Reward: 63.000, Step: 63 +2022-12-17 20:41:44 - r - INFO: - Episode: 196/200, Reward: 100.000, Step: 100 +2022-12-17 20:41:44 - r - INFO: - Episode: 197/200, Reward: 119.000, Step: 119 +2022-12-17 20:41:44 - r - INFO: - Episode: 198/200, Reward: 138.000, Step: 138 +2022-12-17 20:41:44 - r - INFO: - Episode: 199/200, Reward: 123.000, Step: 123 +2022-12-17 20:41:44 - r - INFO: - Episode: 200/200, Reward: 113.000, Step: 113 +2022-12-17 20:41:46 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/models/actor.pth b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..1fd56fde4dab4d2521f264a2a50ec4fb8f0cbf39 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffb5b77fd6cfb12b05d0565beee7740f4fa416356f845f856f5af472262d6726 +size 272215 diff --git a/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/models/critic.pth b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..02a05142e10e68007354dd5be7e7429fb112339e --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e913c00d51a429d08ba5956d4ee79c8a4c4ad029072c539bb53b684c874cdd +size 271191 diff --git a/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..2b5116d1d8d4396c4407a3c5307c8a153b0fa94b Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/results/res.csv b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e7b68ec42061be8d3f76e408421d90b49ac9017 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO-KL_20221217-204003/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,25.0,25 +1,15.0,15 +2,13.0,13 +3,14.0,14 +4,25.0,25 +5,41.0,41 +6,25.0,25 +7,12.0,12 +8,37.0,37 +9,26.0,26 +10,12.0,12 +11,23.0,23 +12,27.0,27 +13,62.0,62 +14,13.0,13 +15,81.0,81 +16,90.0,90 +17,41.0,41 +18,54.0,54 +19,100.0,100 +20,19.0,19 +21,27.0,27 +22,61.0,61 +23,45.0,45 +24,36.0,36 +25,90.0,90 +26,23.0,23 +27,42.0,42 +28,44.0,44 +29,110.0,110 +30,21.0,21 +31,28.0,28 +32,44.0,44 +33,27.0,27 +34,45.0,45 +35,29.0,29 +36,40.0,40 +37,66.0,66 +38,27.0,27 +39,75.0,75 +40,34.0,34 +41,153.0,153 +42,35.0,35 +43,86.0,86 +44,27.0,27 +45,141.0,141 +46,42.0,42 +47,53.0,53 +48,61.0,61 +49,178.0,178 +50,81.0,81 +51,53.0,53 +52,100.0,100 +53,113.0,113 +54,143.0,143 +55,38.0,38 +56,66.0,66 +57,68.0,68 +58,129.0,129 +59,32.0,32 +60,42.0,42 +61,121.0,121 +62,30.0,30 +63,93.0,93 +64,30.0,30 +65,49.0,49 +66,102.0,102 +67,200.0,200 +68,72.0,72 +69,164.0,164 +70,97.0,97 +71,16.0,16 +72,184.0,184 +73,103.0,103 +74,157.0,157 +75,142.0,142 +76,46.0,46 +77,123.0,123 +78,128.0,128 +79,121.0,121 +80,150.0,150 +81,134.0,134 +82,132.0,132 +83,127.0,127 +84,199.0,199 +85,164.0,164 +86,200.0,200 +87,97.0,97 +88,177.0,177 +89,77.0,77 +90,62.0,62 +91,48.0,48 +92,98.0,98 +93,25.0,25 +94,200.0,200 +95,144.0,144 +96,200.0,200 +97,171.0,171 +98,200.0,200 +99,159.0,159 +100,200.0,200 +101,200.0,200 +102,200.0,200 +103,200.0,200 +104,200.0,200 +105,200.0,200 +106,115.0,115 +107,145.0,145 +108,142.0,142 +109,200.0,200 +110,200.0,200 +111,200.0,200 +112,200.0,200 +113,72.0,72 +114,198.0,198 +115,26.0,26 +116,166.0,166 +117,18.0,18 +118,18.0,18 +119,35.0,35 +120,37.0,37 +121,123.0,123 +122,108.0,108 +123,108.0,108 +124,131.0,131 +125,106.0,106 +126,43.0,43 +127,68.0,68 +128,83.0,83 +129,178.0,178 +130,200.0,200 +131,200.0,200 +132,200.0,200 +133,200.0,200 +134,163.0,163 +135,141.0,141 +136,194.0,194 +137,200.0,200 +138,200.0,200 +139,200.0,200 +140,200.0,200 +141,177.0,177 +142,105.0,105 +143,101.0,101 +144,158.0,158 +145,148.0,148 +146,177.0,177 +147,200.0,200 +148,200.0,200 +149,200.0,200 +150,200.0,200 +151,200.0,200 +152,200.0,200 +153,157.0,157 +154,96.0,96 +155,150.0,150 +156,171.0,171 +157,200.0,200 +158,200.0,200 +159,200.0,200 +160,200.0,200 +161,200.0,200 +162,200.0,200 +163,200.0,200 +164,200.0,200 +165,200.0,200 +166,169.0,169 +167,200.0,200 +168,200.0,200 +169,200.0,200 +170,195.0,195 +171,69.0,69 +172,194.0,194 +173,179.0,179 +174,137.0,137 +175,139.0,139 +176,142.0,142 +177,149.0,149 +178,186.0,186 +179,154.0,154 +180,172.0,172 +181,200.0,200 +182,172.0,172 +183,197.0,197 +184,167.0,167 +185,171.0,171 +186,181.0,181 +187,200.0,200 +188,145.0,145 +189,200.0,200 +190,189.0,189 +191,164.0,164 +192,160.0,160 +193,121.0,121 +194,63.0,63 +195,100.0,100 +196,119.0,119 +197,138.0,138 +198,123.0,123 +199,113.0,113 diff --git a/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/config.yaml b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02a4776cec82aff82c3055087b92007fdddc4c91 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/config.yaml @@ -0,0 +1,31 @@ +general_cfg: + algo_name: PPO + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + ppo_type: clip + sgd_batch_size: 128 + train_batch_size: 256 diff --git a/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/logs/log.txt b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..f210637ce2baf0e58cf8e9e35edf42e49d526182 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/logs/log.txt @@ -0,0 +1,252 @@ +2023-02-20 21:29:59 - r - INFO: - Hyperparameters: +2023-02-20 21:29:59 - r - INFO: - ================================================================================ +2023-02-20 21:29:59 - r - INFO: - Name Value Type +2023-02-20 21:29:59 - r - INFO: - env_name CartPole-v1 +2023-02-20 21:29:59 - r - INFO: - new_step_api 1 +2023-02-20 21:29:59 - r - INFO: - wrapper None +2023-02-20 21:29:59 - r - INFO: - render 0 +2023-02-20 21:29:59 - r - INFO: - algo_name PPO +2023-02-20 21:29:59 - r - INFO: - mode train +2023-02-20 21:29:59 - r - INFO: - seed 1 +2023-02-20 21:29:59 - r - INFO: - device cuda +2023-02-20 21:29:59 - r - INFO: - train_eps 200 +2023-02-20 21:29:59 - r - INFO: - test_eps 10 +2023-02-20 21:29:59 - r - INFO: - eval_eps 10 +2023-02-20 21:29:59 - r - INFO: - eval_per_episode 5 +2023-02-20 21:29:59 - r - INFO: - max_steps 200 +2023-02-20 21:29:59 - r - INFO: - load_checkpoint 0 +2023-02-20 21:29:59 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-02-20 21:29:59 - r - INFO: - show_fig 0 +2023-02-20 21:29:59 - r - INFO: - save_fig 1 +2023-02-20 21:29:59 - r - INFO: - ppo_type clip +2023-02-20 21:29:59 - r - INFO: - continuous 0 +2023-02-20 21:29:59 - r - INFO: - gamma 0.99 +2023-02-20 21:29:59 - r - INFO: - k_epochs 4 +2023-02-20 21:29:59 - r - INFO: - actor_lr 0.0003 +2023-02-20 21:29:59 - r - INFO: - critic_lr 0.001 +2023-02-20 21:29:59 - r - INFO: - eps_clip 0.2 +2023-02-20 21:29:59 - r - INFO: - entropy_coef 0.01 +2023-02-20 21:29:59 - r - INFO: - train_batch_size 256 +2023-02-20 21:29:59 - r - INFO: - sgd_batch_size 128 +2023-02-20 21:29:59 - r - INFO: - actor_hidden_dim 256 +2023-02-20 21:29:59 - r - INFO: - critic_hidden_dim 256 +2023-02-20 21:29:59 - r - INFO: - task_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Train_CartPole-v1_PPO_20230220-212959 +2023-02-20 21:29:59 - r - INFO: - model_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Train_CartPole-v1_PPO_20230220-212959/models +2023-02-20 21:29:59 - r - INFO: - res_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Train_CartPole-v1_PPO_20230220-212959/results +2023-02-20 21:29:59 - r - INFO: - log_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Train_CartPole-v1_PPO_20230220-212959/logs +2023-02-20 21:29:59 - r - INFO: - traj_dir C:\Users\24438\Desktop\rl-tutorials\joyrl/tasks/Train_CartPole-v1_PPO_20230220-212959/traj +2023-02-20 21:29:59 - r - INFO: - ================================================================================ +2023-02-20 21:29:59 - r - INFO: - n_states: 4, n_actions: 2 +2023-02-20 21:29:59 - r - INFO: - Start training! +2023-02-20 21:29:59 - r - INFO: - Env: CartPole-v1, Algorithm: PPO, Device: cuda +2023-02-20 21:30:01 - r - INFO: - Episode: 1/200, Reward: 25.000, Step: 25 +2023-02-20 21:30:01 - r - INFO: - Episode: 2/200, Reward: 15.000, Step: 15 +2023-02-20 21:30:01 - r - INFO: - Episode: 3/200, Reward: 13.000, Step: 13 +2023-02-20 21:30:01 - r - INFO: - Episode: 4/200, Reward: 14.000, Step: 14 +2023-02-20 21:30:01 - r - INFO: - Episode: 5/200, Reward: 25.000, Step: 25 +2023-02-20 21:30:01 - r - INFO: - Current episode 5 has the best eval reward: 23.900 +2023-02-20 21:30:01 - r - INFO: - Episode: 6/200, Reward: 41.000, Step: 41 +2023-02-20 21:30:01 - r - INFO: - Episode: 7/200, Reward: 33.000, Step: 33 +2023-02-20 21:30:01 - r - INFO: - Episode: 8/200, Reward: 12.000, Step: 12 +2023-02-20 21:30:01 - r - INFO: - Episode: 9/200, Reward: 20.000, Step: 20 +2023-02-20 21:30:01 - r - INFO: - Episode: 10/200, Reward: 33.000, Step: 33 +2023-02-20 21:30:02 - r - INFO: - Current episode 10 has the best eval reward: 27.400 +2023-02-20 21:30:02 - r - INFO: - Episode: 11/200, Reward: 19.000, Step: 19 +2023-02-20 21:30:02 - r - INFO: - Episode: 12/200, Reward: 35.000, Step: 35 +2023-02-20 21:30:02 - r - INFO: - Episode: 13/200, Reward: 35.000, Step: 35 +2023-02-20 21:30:02 - r - INFO: - Episode: 14/200, Reward: 9.000, Step: 9 +2023-02-20 21:30:02 - r - INFO: - Episode: 15/200, Reward: 32.000, Step: 32 +2023-02-20 21:30:02 - r - INFO: - Current episode 15 has the best eval reward: 37.600 +2023-02-20 21:30:02 - r - INFO: - Episode: 16/200, Reward: 29.000, Step: 29 +2023-02-20 21:30:02 - r - INFO: - Episode: 17/200, Reward: 17.000, Step: 17 +2023-02-20 21:30:02 - r - INFO: - Episode: 18/200, Reward: 17.000, Step: 17 +2023-02-20 21:30:02 - r - INFO: - Episode: 19/200, Reward: 17.000, Step: 17 +2023-02-20 21:30:02 - r - INFO: - Episode: 20/200, Reward: 20.000, Step: 20 +2023-02-20 21:30:03 - r - INFO: - Episode: 21/200, Reward: 24.000, Step: 24 +2023-02-20 21:30:03 - r - INFO: - Episode: 22/200, Reward: 44.000, Step: 44 +2023-02-20 21:30:03 - r - INFO: - Episode: 23/200, Reward: 39.000, Step: 39 +2023-02-20 21:30:03 - r - INFO: - Episode: 24/200, Reward: 48.000, Step: 48 +2023-02-20 21:30:03 - r - INFO: - Episode: 25/200, Reward: 52.000, Step: 52 +2023-02-20 21:30:03 - r - INFO: - Episode: 26/200, Reward: 32.000, Step: 32 +2023-02-20 21:30:03 - r - INFO: - Episode: 27/200, Reward: 45.000, Step: 45 +2023-02-20 21:30:04 - r - INFO: - Episode: 28/200, Reward: 68.000, Step: 68 +2023-02-20 21:30:04 - r - INFO: - Episode: 29/200, Reward: 45.000, Step: 45 +2023-02-20 21:30:04 - r - INFO: - Episode: 30/200, Reward: 16.000, Step: 16 +2023-02-20 21:30:04 - r - INFO: - Current episode 30 has the best eval reward: 45.300 +2023-02-20 21:30:04 - r - INFO: - Episode: 31/200, Reward: 38.000, Step: 38 +2023-02-20 21:30:04 - r - INFO: - Episode: 32/200, Reward: 17.000, Step: 17 +2023-02-20 21:30:04 - r - INFO: - Episode: 33/200, Reward: 35.000, Step: 35 +2023-02-20 21:30:04 - r - INFO: - Episode: 34/200, Reward: 12.000, Step: 12 +2023-02-20 21:30:04 - r - INFO: - Episode: 35/200, Reward: 56.000, Step: 56 +2023-02-20 21:30:05 - r - INFO: - Episode: 36/200, Reward: 36.000, Step: 36 +2023-02-20 21:30:05 - r - INFO: - Episode: 37/200, Reward: 15.000, Step: 15 +2023-02-20 21:30:05 - r - INFO: - Episode: 38/200, Reward: 25.000, Step: 25 +2023-02-20 21:30:05 - r - INFO: - Episode: 39/200, Reward: 28.000, Step: 28 +2023-02-20 21:30:05 - r - INFO: - Episode: 40/200, Reward: 56.000, Step: 56 +2023-02-20 21:30:05 - r - INFO: - Episode: 41/200, Reward: 18.000, Step: 18 +2023-02-20 21:30:05 - r - INFO: - Episode: 42/200, Reward: 33.000, Step: 33 +2023-02-20 21:30:05 - r - INFO: - Episode: 43/200, Reward: 30.000, Step: 30 +2023-02-20 21:30:05 - r - INFO: - Episode: 44/200, Reward: 30.000, Step: 30 +2023-02-20 21:30:06 - r - INFO: - Episode: 45/200, Reward: 28.000, Step: 28 +2023-02-20 21:30:06 - r - INFO: - Episode: 46/200, Reward: 38.000, Step: 38 +2023-02-20 21:30:06 - r - INFO: - Episode: 47/200, Reward: 70.000, Step: 70 +2023-02-20 21:30:06 - r - INFO: - Episode: 48/200, Reward: 18.000, Step: 18 +2023-02-20 21:30:06 - r - INFO: - Episode: 49/200, Reward: 16.000, Step: 16 +2023-02-20 21:30:06 - r - INFO: - Episode: 50/200, Reward: 36.000, Step: 36 +2023-02-20 21:30:07 - r - INFO: - Current episode 50 has the best eval reward: 48.700 +2023-02-20 21:30:07 - r - INFO: - Episode: 51/200, Reward: 26.000, Step: 26 +2023-02-20 21:30:07 - r - INFO: - Episode: 52/200, Reward: 34.000, Step: 34 +2023-02-20 21:30:07 - r - INFO: - Episode: 53/200, Reward: 70.000, Step: 70 +2023-02-20 21:30:07 - r - INFO: - Episode: 54/200, Reward: 39.000, Step: 39 +2023-02-20 21:30:07 - r - INFO: - Episode: 55/200, Reward: 87.000, Step: 87 +2023-02-20 21:30:08 - r - INFO: - Episode: 56/200, Reward: 75.000, Step: 75 +2023-02-20 21:30:08 - r - INFO: - Episode: 57/200, Reward: 21.000, Step: 21 +2023-02-20 21:30:08 - r - INFO: - Episode: 58/200, Reward: 72.000, Step: 72 +2023-02-20 21:30:08 - r - INFO: - Episode: 59/200, Reward: 43.000, Step: 43 +2023-02-20 21:30:08 - r - INFO: - Episode: 60/200, Reward: 48.000, Step: 48 +2023-02-20 21:30:09 - r - INFO: - Episode: 61/200, Reward: 64.000, Step: 64 +2023-02-20 21:30:09 - r - INFO: - Episode: 62/200, Reward: 135.000, Step: 135 +2023-02-20 21:30:09 - r - INFO: - Episode: 63/200, Reward: 108.000, Step: 108 +2023-02-20 21:30:09 - r - INFO: - Episode: 64/200, Reward: 38.000, Step: 38 +2023-02-20 21:30:09 - r - INFO: - Episode: 65/200, Reward: 22.000, Step: 22 +2023-02-20 21:30:10 - r - INFO: - Current episode 65 has the best eval reward: 68.000 +2023-02-20 21:30:10 - r - INFO: - Episode: 66/200, Reward: 60.000, Step: 60 +2023-02-20 21:30:10 - r - INFO: - Episode: 67/200, Reward: 74.000, Step: 74 +2023-02-20 21:30:10 - r - INFO: - Episode: 68/200, Reward: 93.000, Step: 93 +2023-02-20 21:30:10 - r - INFO: - Episode: 69/200, Reward: 55.000, Step: 55 +2023-02-20 21:30:10 - r - INFO: - Episode: 70/200, Reward: 48.000, Step: 48 +2023-02-20 21:30:11 - r - INFO: - Episode: 71/200, Reward: 29.000, Step: 29 +2023-02-20 21:30:11 - r - INFO: - Episode: 72/200, Reward: 59.000, Step: 59 +2023-02-20 21:30:11 - r - INFO: - Episode: 73/200, Reward: 35.000, Step: 35 +2023-02-20 21:30:11 - r - INFO: - Episode: 74/200, Reward: 40.000, Step: 40 +2023-02-20 21:30:11 - r - INFO: - Episode: 75/200, Reward: 113.000, Step: 113 +2023-02-20 21:30:12 - r - INFO: - Episode: 76/200, Reward: 114.000, Step: 114 +2023-02-20 21:30:12 - r - INFO: - Episode: 77/200, Reward: 52.000, Step: 52 +2023-02-20 21:30:13 - r - INFO: - Episode: 78/200, Reward: 139.000, Step: 139 +2023-02-20 21:30:13 - r - INFO: - Episode: 79/200, Reward: 138.000, Step: 138 +2023-02-20 21:30:13 - r - INFO: - Episode: 80/200, Reward: 54.000, Step: 54 +2023-02-20 21:30:14 - r - INFO: - Current episode 80 has the best eval reward: 110.900 +2023-02-20 21:30:14 - r - INFO: - Episode: 81/200, Reward: 156.000, Step: 156 +2023-02-20 21:30:15 - r - INFO: - Episode: 82/200, Reward: 140.000, Step: 140 +2023-02-20 21:30:15 - r - INFO: - Episode: 83/200, Reward: 144.000, Step: 144 +2023-02-20 21:30:15 - r - INFO: - Episode: 84/200, Reward: 118.000, Step: 118 +2023-02-20 21:30:15 - r - INFO: - Episode: 85/200, Reward: 156.000, Step: 156 +2023-02-20 21:30:16 - r - INFO: - Episode: 86/200, Reward: 135.000, Step: 135 +2023-02-20 21:30:17 - r - INFO: - Episode: 87/200, Reward: 144.000, Step: 144 +2023-02-20 21:30:17 - r - INFO: - Episode: 88/200, Reward: 160.000, Step: 160 +2023-02-20 21:30:17 - r - INFO: - Episode: 89/200, Reward: 30.000, Step: 30 +2023-02-20 21:30:17 - r - INFO: - Episode: 90/200, Reward: 194.000, Step: 194 +2023-02-20 21:30:19 - r - INFO: - Current episode 90 has the best eval reward: 169.300 +2023-02-20 21:30:20 - r - INFO: - Episode: 91/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:20 - r - INFO: - Episode: 92/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:20 - r - INFO: - Episode: 93/200, Reward: 160.000, Step: 160 +2023-02-20 21:30:21 - r - INFO: - Episode: 94/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:21 - r - INFO: - Episode: 95/200, Reward: 59.000, Step: 59 +2023-02-20 21:30:23 - r - INFO: - Episode: 96/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:23 - r - INFO: - Episode: 97/200, Reward: 182.000, Step: 182 +2023-02-20 21:30:23 - r - INFO: - Episode: 98/200, Reward: 125.000, Step: 125 +2023-02-20 21:30:23 - r - INFO: - Episode: 99/200, Reward: 140.000, Step: 140 +2023-02-20 21:30:24 - r - INFO: - Episode: 100/200, Reward: 146.000, Step: 146 +2023-02-20 21:30:25 - r - INFO: - Episode: 101/200, Reward: 130.000, Step: 130 +2023-02-20 21:30:26 - r - INFO: - Episode: 102/200, Reward: 74.000, Step: 74 +2023-02-20 21:30:26 - r - INFO: - Episode: 103/200, Reward: 167.000, Step: 167 +2023-02-20 21:30:26 - r - INFO: - Episode: 104/200, Reward: 171.000, Step: 171 +2023-02-20 21:30:26 - r - INFO: - Episode: 105/200, Reward: 150.000, Step: 150 +2023-02-20 21:30:28 - r - INFO: - Episode: 106/200, Reward: 105.000, Step: 105 +2023-02-20 21:30:28 - r - INFO: - Episode: 107/200, Reward: 65.000, Step: 65 +2023-02-20 21:30:28 - r - INFO: - Episode: 108/200, Reward: 170.000, Step: 170 +2023-02-20 21:30:29 - r - INFO: - Episode: 109/200, Reward: 172.000, Step: 172 +2023-02-20 21:30:29 - r - INFO: - Episode: 110/200, Reward: 164.000, Step: 164 +2023-02-20 21:30:30 - r - INFO: - Current episode 110 has the best eval reward: 180.000 +2023-02-20 21:30:31 - r - INFO: - Episode: 111/200, Reward: 148.000, Step: 148 +2023-02-20 21:30:31 - r - INFO: - Episode: 112/200, Reward: 116.000, Step: 116 +2023-02-20 21:30:31 - r - INFO: - Episode: 113/200, Reward: 59.000, Step: 59 +2023-02-20 21:30:31 - r - INFO: - Episode: 114/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:31 - r - INFO: - Episode: 115/200, Reward: 36.000, Step: 36 +2023-02-20 21:30:33 - r - INFO: - Episode: 116/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:34 - r - INFO: - Episode: 117/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:34 - r - INFO: - Episode: 118/200, Reward: 158.000, Step: 158 +2023-02-20 21:30:34 - r - INFO: - Episode: 119/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:34 - r - INFO: - Episode: 120/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:37 - r - INFO: - Current episode 120 has the best eval reward: 200.000 +2023-02-20 21:30:37 - r - INFO: - Episode: 121/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:38 - r - INFO: - Episode: 122/200, Reward: 172.000, Step: 172 +2023-02-20 21:30:38 - r - INFO: - Episode: 123/200, Reward: 137.000, Step: 137 +2023-02-20 21:30:38 - r - INFO: - Episode: 124/200, Reward: 189.000, Step: 189 +2023-02-20 21:30:38 - r - INFO: - Episode: 125/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:40 - r - INFO: - Episode: 126/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:41 - r - INFO: - Episode: 127/200, Reward: 197.000, Step: 197 +2023-02-20 21:30:41 - r - INFO: - Episode: 128/200, Reward: 125.000, Step: 125 +2023-02-20 21:30:41 - r - INFO: - Episode: 129/200, Reward: 194.000, Step: 194 +2023-02-20 21:30:41 - r - INFO: - Episode: 130/200, Reward: 167.000, Step: 167 +2023-02-20 21:30:43 - r - INFO: - Episode: 131/200, Reward: 135.000, Step: 135 +2023-02-20 21:30:43 - r - INFO: - Episode: 132/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:44 - r - INFO: - Episode: 133/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:44 - r - INFO: - Episode: 134/200, Reward: 170.000, Step: 170 +2023-02-20 21:30:44 - r - INFO: - Episode: 135/200, Reward: 195.000, Step: 195 +2023-02-20 21:30:47 - r - INFO: - Episode: 136/200, Reward: 150.000, Step: 150 +2023-02-20 21:30:47 - r - INFO: - Episode: 137/200, Reward: 187.000, Step: 187 +2023-02-20 21:30:47 - r - INFO: - Episode: 138/200, Reward: 172.000, Step: 172 +2023-02-20 21:30:47 - r - INFO: - Episode: 139/200, Reward: 124.000, Step: 124 +2023-02-20 21:30:47 - r - INFO: - Episode: 140/200, Reward: 105.000, Step: 105 +2023-02-20 21:30:49 - r - INFO: - Episode: 141/200, Reward: 49.000, Step: 49 +2023-02-20 21:30:49 - r - INFO: - Episode: 142/200, Reward: 108.000, Step: 108 +2023-02-20 21:30:49 - r - INFO: - Episode: 143/200, Reward: 117.000, Step: 117 +2023-02-20 21:30:50 - r - INFO: - Episode: 144/200, Reward: 136.000, Step: 136 +2023-02-20 21:30:50 - r - INFO: - Episode: 145/200, Reward: 120.000, Step: 120 +2023-02-20 21:30:52 - r - INFO: - Episode: 146/200, Reward: 172.000, Step: 172 +2023-02-20 21:30:52 - r - INFO: - Episode: 147/200, Reward: 134.000, Step: 134 +2023-02-20 21:30:52 - r - INFO: - Episode: 148/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:53 - r - INFO: - Episode: 149/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:53 - r - INFO: - Episode: 150/200, Reward: 150.000, Step: 150 +2023-02-20 21:30:55 - r - INFO: - Episode: 151/200, Reward: 190.000, Step: 190 +2023-02-20 21:30:55 - r - INFO: - Episode: 152/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:56 - r - INFO: - Episode: 153/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:56 - r - INFO: - Episode: 154/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:56 - r - INFO: - Episode: 155/200, Reward: 179.000, Step: 179 +2023-02-20 21:30:59 - r - INFO: - Episode: 156/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:59 - r - INFO: - Episode: 157/200, Reward: 200.000, Step: 200 +2023-02-20 21:30:59 - r - INFO: - Episode: 158/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:00 - r - INFO: - Episode: 159/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:00 - r - INFO: - Episode: 160/200, Reward: 195.000, Step: 195 +2023-02-20 21:31:02 - r - INFO: - Episode: 161/200, Reward: 195.000, Step: 195 +2023-02-20 21:31:02 - r - INFO: - Episode: 162/200, Reward: 142.000, Step: 142 +2023-02-20 21:31:03 - r - INFO: - Episode: 163/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:03 - r - INFO: - Episode: 164/200, Reward: 108.000, Step: 108 +2023-02-20 21:31:03 - r - INFO: - Episode: 165/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:05 - r - INFO: - Episode: 166/200, Reward: 165.000, Step: 165 +2023-02-20 21:31:05 - r - INFO: - Episode: 167/200, Reward: 153.000, Step: 153 +2023-02-20 21:31:05 - r - INFO: - Episode: 168/200, Reward: 85.000, Step: 85 +2023-02-20 21:31:05 - r - INFO: - Episode: 169/200, Reward: 139.000, Step: 139 +2023-02-20 21:31:06 - r - INFO: - Episode: 170/200, Reward: 155.000, Step: 155 +2023-02-20 21:31:08 - r - INFO: - Episode: 171/200, Reward: 166.000, Step: 166 +2023-02-20 21:31:08 - r - INFO: - Episode: 172/200, Reward: 182.000, Step: 182 +2023-02-20 21:31:08 - r - INFO: - Episode: 173/200, Reward: 190.000, Step: 190 +2023-02-20 21:31:08 - r - INFO: - Episode: 174/200, Reward: 35.000, Step: 35 +2023-02-20 21:31:09 - r - INFO: - Episode: 175/200, Reward: 124.000, Step: 124 +2023-02-20 21:31:11 - r - INFO: - Episode: 176/200, Reward: 114.000, Step: 114 +2023-02-20 21:31:11 - r - INFO: - Episode: 177/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:11 - r - INFO: - Episode: 178/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:12 - r - INFO: - Episode: 179/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:12 - r - INFO: - Episode: 180/200, Reward: 200.000, Step: 200 +2023-02-20 21:31:14 - r - INFO: - Episode: 181/200, Reward: 110.000, Step: 110 +2023-02-20 21:31:14 - r - INFO: - Episode: 182/200, Reward: 128.000, Step: 128 +2023-02-20 21:31:14 - r - INFO: - Episode: 183/200, Reward: 107.000, Step: 107 +2023-02-20 21:31:15 - r - INFO: - Episode: 184/200, Reward: 192.000, Step: 192 +2023-02-20 21:31:15 - r - INFO: - Episode: 185/200, Reward: 106.000, Step: 106 +2023-02-20 21:31:16 - r - INFO: - Episode: 186/200, Reward: 32.000, Step: 32 +2023-02-20 21:31:16 - r - INFO: - Episode: 187/200, Reward: 107.000, Step: 107 +2023-02-20 21:31:17 - r - INFO: - Episode: 188/200, Reward: 129.000, Step: 129 +2023-02-20 21:31:17 - r - INFO: - Episode: 189/200, Reward: 122.000, Step: 122 +2023-02-20 21:31:17 - r - INFO: - Episode: 190/200, Reward: 126.000, Step: 126 +2023-02-20 21:31:18 - r - INFO: - Episode: 191/200, Reward: 120.000, Step: 120 +2023-02-20 21:31:19 - r - INFO: - Episode: 192/200, Reward: 127.000, Step: 127 +2023-02-20 21:31:19 - r - INFO: - Episode: 193/200, Reward: 132.000, Step: 132 +2023-02-20 21:31:19 - r - INFO: - Episode: 194/200, Reward: 128.000, Step: 128 +2023-02-20 21:31:19 - r - INFO: - Episode: 195/200, Reward: 142.000, Step: 142 +2023-02-20 21:31:21 - r - INFO: - Episode: 196/200, Reward: 137.000, Step: 137 +2023-02-20 21:31:21 - r - INFO: - Episode: 197/200, Reward: 125.000, Step: 125 +2023-02-20 21:31:21 - r - INFO: - Episode: 198/200, Reward: 118.000, Step: 118 +2023-02-20 21:31:22 - r - INFO: - Episode: 199/200, Reward: 158.000, Step: 158 +2023-02-20 21:31:22 - r - INFO: - Episode: 200/200, Reward: 144.000, Step: 144 +2023-02-20 21:31:23 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/models/actor.pth b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..22cd1f4300c6f64826192a7b43471008d9044122 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d07e4388597f766e04099380da27cb55fd877e8d26cdd14eab48bc097525216 +size 272215 diff --git a/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/models/critic.pth b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..373c9387a423d3545d43deac3722143e84a8b455 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6743877f9534c272b5e6d8bae3cbc87b1fa32bb21af2935c28e503122e042c2d +size 271191 diff --git a/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..9933ad634f57b9d9e92ec18f7f119676e29f22f2 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/results/res.csv b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff9811416221339b2f250b3b054b924f27c1a7fe --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_PPO_20230220-212959/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,25.0,25 +1,15.0,15 +2,13.0,13 +3,14.0,14 +4,25.0,25 +5,41.0,41 +6,33.0,33 +7,12.0,12 +8,20.0,20 +9,33.0,33 +10,19.0,19 +11,35.0,35 +12,35.0,35 +13,9.0,9 +14,32.0,32 +15,29.0,29 +16,17.0,17 +17,17.0,17 +18,17.0,17 +19,20.0,20 +20,24.0,24 +21,44.0,44 +22,39.0,39 +23,48.0,48 +24,52.0,52 +25,32.0,32 +26,45.0,45 +27,68.0,68 +28,45.0,45 +29,16.0,16 +30,38.0,38 +31,17.0,17 +32,35.0,35 +33,12.0,12 +34,56.0,56 +35,36.0,36 +36,15.0,15 +37,25.0,25 +38,28.0,28 +39,56.0,56 +40,18.0,18 +41,33.0,33 +42,30.0,30 +43,30.0,30 +44,28.0,28 +45,38.0,38 +46,70.0,70 +47,18.0,18 +48,16.0,16 +49,36.0,36 +50,26.0,26 +51,34.0,34 +52,70.0,70 +53,39.0,39 +54,87.0,87 +55,75.0,75 +56,21.0,21 +57,72.0,72 +58,43.0,43 +59,48.0,48 +60,64.0,64 +61,135.0,135 +62,108.0,108 +63,38.0,38 +64,22.0,22 +65,60.0,60 +66,74.0,74 +67,93.0,93 +68,55.0,55 +69,48.0,48 +70,29.0,29 +71,59.0,59 +72,35.0,35 +73,40.0,40 +74,113.0,113 +75,114.0,114 +76,52.0,52 +77,139.0,139 +78,138.0,138 +79,54.0,54 +80,156.0,156 +81,140.0,140 +82,144.0,144 +83,118.0,118 +84,156.0,156 +85,135.0,135 +86,144.0,144 +87,160.0,160 +88,30.0,30 +89,194.0,194 +90,200.0,200 +91,200.0,200 +92,160.0,160 +93,200.0,200 +94,59.0,59 +95,200.0,200 +96,182.0,182 +97,125.0,125 +98,140.0,140 +99,146.0,146 +100,130.0,130 +101,74.0,74 +102,167.0,167 +103,171.0,171 +104,150.0,150 +105,105.0,105 +106,65.0,65 +107,170.0,170 +108,172.0,172 +109,164.0,164 +110,148.0,148 +111,116.0,116 +112,59.0,59 +113,200.0,200 +114,36.0,36 +115,200.0,200 +116,200.0,200 +117,158.0,158 +118,200.0,200 +119,200.0,200 +120,200.0,200 +121,172.0,172 +122,137.0,137 +123,189.0,189 +124,200.0,200 +125,200.0,200 +126,197.0,197 +127,125.0,125 +128,194.0,194 +129,167.0,167 +130,135.0,135 +131,200.0,200 +132,200.0,200 +133,170.0,170 +134,195.0,195 +135,150.0,150 +136,187.0,187 +137,172.0,172 +138,124.0,124 +139,105.0,105 +140,49.0,49 +141,108.0,108 +142,117.0,117 +143,136.0,136 +144,120.0,120 +145,172.0,172 +146,134.0,134 +147,200.0,200 +148,200.0,200 +149,150.0,150 +150,190.0,190 +151,200.0,200 +152,200.0,200 +153,200.0,200 +154,179.0,179 +155,200.0,200 +156,200.0,200 +157,200.0,200 +158,200.0,200 +159,195.0,195 +160,195.0,195 +161,142.0,142 +162,200.0,200 +163,108.0,108 +164,200.0,200 +165,165.0,165 +166,153.0,153 +167,85.0,85 +168,139.0,139 +169,155.0,155 +170,166.0,166 +171,182.0,182 +172,190.0,190 +173,35.0,35 +174,124.0,124 +175,114.0,114 +176,200.0,200 +177,200.0,200 +178,200.0,200 +179,200.0,200 +180,110.0,110 +181,128.0,128 +182,107.0,107 +183,192.0,192 +184,106.0,106 +185,32.0,32 +186,107.0,107 +187,129.0,129 +188,122.0,122 +189,126.0,126 +190,120.0,120 +191,127.0,127 +192,132.0,132 +193,128.0,128 +194,142.0,142 +195,137.0,137 +196,125.0,125 +197,118.0,118 +198,158.0,158 +199,144.0,144 diff --git a/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/config.yaml b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03c30887d9fed11150600df1909df1293a84d2c0 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/config.yaml @@ -0,0 +1,23 @@ +general_cfg: + algo_name: REINFORCE + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + gamma: 0.99 + hidden_dim: 36 + lr: 0.01 + update_freq: 200 diff --git a/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/logs/log.txt b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..c3227a8249321d939d3f9ae89a1c1738efa3d130 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/logs/log.txt @@ -0,0 +1,240 @@ +2022-12-03 14:33:07 - r - INFO: - Hyperparameters: +2022-12-03 14:33:07 - r - INFO: - ================================================================================ +2022-12-03 14:33:07 - r - INFO: - Name Value Type +2022-12-03 14:33:07 - r - INFO: - env_name CartPole-v1 +2022-12-03 14:33:07 - r - INFO: - new_step_api 1 +2022-12-03 14:33:07 - r - INFO: - wrapper None +2022-12-03 14:33:07 - r - INFO: - render 0 +2022-12-03 14:33:07 - r - INFO: - algo_name REINFORCE +2022-12-03 14:33:07 - r - INFO: - mode train +2022-12-03 14:33:07 - r - INFO: - seed 1 +2022-12-03 14:33:07 - r - INFO: - device cpu +2022-12-03 14:33:07 - r - INFO: - train_eps 200 +2022-12-03 14:33:07 - r - INFO: - test_eps 10 +2022-12-03 14:33:07 - r - INFO: - eval_eps 10 +2022-12-03 14:33:07 - r - INFO: - eval_per_episode 5 +2022-12-03 14:33:07 - r - INFO: - max_steps 200 +2022-12-03 14:33:07 - r - INFO: - load_checkpoint 0 +2022-12-03 14:33:07 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2022-12-03 14:33:07 - r - INFO: - show_fig 0 +2022-12-03 14:33:07 - r - INFO: - save_fig 1 +2022-12-03 14:33:07 - r - INFO: - lr 0.01 +2022-12-03 14:33:07 - r - INFO: - gamma 0.99 +2022-12-03 14:33:07 - r - INFO: - hidden_dim 36 +2022-12-03 14:33:07 - r - INFO: - update_freq 200 +2022-12-03 14:33:07 - r - INFO: - ================================================================================ +2022-12-03 14:33:07 - r - INFO: - n_states: 4, n_actions: 2 +2022-12-03 14:33:07 - r - INFO: - Start training! +2022-12-03 14:33:07 - r - INFO: - Env: CartPole-v1, Algorithm: REINFORCE, Device: cpu +2022-12-03 14:33:07 - r - INFO: - Episode: 1/200, Reward: 22.000, Step: 23 +2022-12-03 14:33:08 - r - INFO: - Episode: 2/200, Reward: 21.000, Step: 22 +2022-12-03 14:33:08 - r - INFO: - Episode: 3/200, Reward: 19.000, Step: 20 +2022-12-03 14:33:08 - r - INFO: - Episode: 4/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 5/200, Reward: 19.000, Step: 20 +2022-12-03 14:33:08 - r - INFO: - Current episode 5 has the best eval reward: 23.400 +2022-12-03 14:33:08 - r - INFO: - Episode: 6/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 7/200, Reward: 15.000, Step: 16 +2022-12-03 14:33:08 - r - INFO: - Episode: 8/200, Reward: 11.000, Step: 12 +2022-12-03 14:33:08 - r - INFO: - Episode: 9/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 10/200, Reward: 14.000, Step: 15 +2022-12-03 14:33:08 - r - INFO: - Episode: 11/200, Reward: 16.000, Step: 17 +2022-12-03 14:33:08 - r - INFO: - Episode: 12/200, Reward: 14.000, Step: 15 +2022-12-03 14:33:08 - r - INFO: - Episode: 13/200, Reward: 17.000, Step: 18 +2022-12-03 14:33:08 - r - INFO: - Episode: 14/200, Reward: 22.000, Step: 23 +2022-12-03 14:33:08 - r - INFO: - Episode: 15/200, Reward: 50.000, Step: 51 +2022-12-03 14:33:08 - r - INFO: - Episode: 16/200, Reward: 18.000, Step: 19 +2022-12-03 14:33:08 - r - INFO: - Episode: 17/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 18/200, Reward: 28.000, Step: 29 +2022-12-03 14:33:08 - r - INFO: - Episode: 19/200, Reward: 23.000, Step: 24 +2022-12-03 14:33:08 - r - INFO: - Episode: 20/200, Reward: 20.000, Step: 21 +2022-12-03 14:33:08 - r - INFO: - Current episode 20 has the best eval reward: 23.400 +2022-12-03 14:33:08 - r - INFO: - Episode: 21/200, Reward: 35.000, Step: 36 +2022-12-03 14:33:08 - r - INFO: - Episode: 22/200, Reward: 10.000, Step: 11 +2022-12-03 14:33:08 - r - INFO: - Episode: 23/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 24/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 25/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 26/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 27/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 28/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 29/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 30/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 31/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 32/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 33/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 34/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 35/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 36/200, Reward: 9.000, Step: 10 +2022-12-03 14:33:08 - r - INFO: - Episode: 37/200, Reward: 11.000, Step: 12 +2022-12-03 14:33:08 - r - INFO: - Episode: 38/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 39/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 40/200, Reward: 8.000, Step: 9 +2022-12-03 14:33:08 - r - INFO: - Episode: 41/200, Reward: 13.000, Step: 14 +2022-12-03 14:33:08 - r - INFO: - Episode: 42/200, Reward: 14.000, Step: 15 +2022-12-03 14:33:08 - r - INFO: - Episode: 43/200, Reward: 16.000, Step: 17 +2022-12-03 14:33:08 - r - INFO: - Episode: 44/200, Reward: 17.000, Step: 18 +2022-12-03 14:33:08 - r - INFO: - Episode: 45/200, Reward: 30.000, Step: 31 +2022-12-03 14:33:08 - r - INFO: - Episode: 46/200, Reward: 20.000, Step: 21 +2022-12-03 14:33:08 - r - INFO: - Episode: 47/200, Reward: 15.000, Step: 16 +2022-12-03 14:33:08 - r - INFO: - Episode: 48/200, Reward: 17.000, Step: 18 +2022-12-03 14:33:08 - r - INFO: - Episode: 49/200, Reward: 20.000, Step: 21 +2022-12-03 14:33:08 - r - INFO: - Episode: 50/200, Reward: 20.000, Step: 21 +2022-12-03 14:33:08 - r - INFO: - Episode: 51/200, Reward: 45.000, Step: 46 +2022-12-03 14:33:08 - r - INFO: - Episode: 52/200, Reward: 57.000, Step: 58 +2022-12-03 14:33:08 - r - INFO: - Episode: 53/200, Reward: 52.000, Step: 53 +2022-12-03 14:33:08 - r - INFO: - Episode: 54/200, Reward: 66.000, Step: 67 +2022-12-03 14:33:08 - r - INFO: - Episode: 55/200, Reward: 68.000, Step: 69 +2022-12-03 14:33:08 - r - INFO: - Current episode 55 has the best eval reward: 55.100 +2022-12-03 14:33:08 - r - INFO: - Episode: 56/200, Reward: 48.000, Step: 49 +2022-12-03 14:33:08 - r - INFO: - Episode: 57/200, Reward: 49.000, Step: 50 +2022-12-03 14:33:08 - r - INFO: - Episode: 58/200, Reward: 68.000, Step: 69 +2022-12-03 14:33:08 - r - INFO: - Episode: 59/200, Reward: 113.000, Step: 114 +2022-12-03 14:33:09 - r - INFO: - Episode: 60/200, Reward: 51.000, Step: 52 +2022-12-03 14:33:09 - r - INFO: - Episode: 61/200, Reward: 46.000, Step: 47 +2022-12-03 14:33:09 - r - INFO: - Episode: 62/200, Reward: 30.000, Step: 31 +2022-12-03 14:33:09 - r - INFO: - Episode: 63/200, Reward: 40.000, Step: 41 +2022-12-03 14:33:09 - r - INFO: - Episode: 64/200, Reward: 28.000, Step: 29 +2022-12-03 14:33:09 - r - INFO: - Episode: 65/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:09 - r - INFO: - Current episode 65 has the best eval reward: 76.600 +2022-12-03 14:33:09 - r - INFO: - Episode: 66/200, Reward: 42.000, Step: 43 +2022-12-03 14:33:09 - r - INFO: - Episode: 67/200, Reward: 74.000, Step: 75 +2022-12-03 14:33:09 - r - INFO: - Episode: 68/200, Reward: 83.000, Step: 84 +2022-12-03 14:33:09 - r - INFO: - Episode: 69/200, Reward: 50.000, Step: 51 +2022-12-03 14:33:09 - r - INFO: - Episode: 70/200, Reward: 43.000, Step: 44 +2022-12-03 14:33:09 - r - INFO: - Episode: 71/200, Reward: 61.000, Step: 62 +2022-12-03 14:33:09 - r - INFO: - Episode: 72/200, Reward: 80.000, Step: 81 +2022-12-03 14:33:09 - r - INFO: - Episode: 73/200, Reward: 91.000, Step: 92 +2022-12-03 14:33:09 - r - INFO: - Episode: 74/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:09 - r - INFO: - Episode: 75/200, Reward: 99.000, Step: 100 +2022-12-03 14:33:09 - r - INFO: - Episode: 76/200, Reward: 52.000, Step: 53 +2022-12-03 14:33:09 - r - INFO: - Episode: 77/200, Reward: 126.000, Step: 127 +2022-12-03 14:33:10 - r - INFO: - Episode: 78/200, Reward: 164.000, Step: 165 +2022-12-03 14:33:10 - r - INFO: - Episode: 79/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:10 - r - INFO: - Episode: 80/200, Reward: 82.000, Step: 83 +2022-12-03 14:33:10 - r - INFO: - Current episode 80 has the best eval reward: 155.100 +2022-12-03 14:33:10 - r - INFO: - Episode: 81/200, Reward: 117.000, Step: 118 +2022-12-03 14:33:10 - r - INFO: - Episode: 82/200, Reward: 106.000, Step: 107 +2022-12-03 14:33:10 - r - INFO: - Episode: 83/200, Reward: 82.000, Step: 83 +2022-12-03 14:33:10 - r - INFO: - Episode: 84/200, Reward: 54.000, Step: 55 +2022-12-03 14:33:10 - r - INFO: - Episode: 85/200, Reward: 48.000, Step: 49 +2022-12-03 14:33:10 - r - INFO: - Episode: 86/200, Reward: 15.000, Step: 16 +2022-12-03 14:33:10 - r - INFO: - Episode: 87/200, Reward: 12.000, Step: 13 +2022-12-03 14:33:10 - r - INFO: - Episode: 88/200, Reward: 14.000, Step: 15 +2022-12-03 14:33:10 - r - INFO: - Episode: 89/200, Reward: 38.000, Step: 39 +2022-12-03 14:33:10 - r - INFO: - Episode: 90/200, Reward: 175.000, Step: 176 +2022-12-03 14:33:10 - r - INFO: - Episode: 91/200, Reward: 25.000, Step: 26 +2022-12-03 14:33:10 - r - INFO: - Episode: 92/200, Reward: 163.000, Step: 164 +2022-12-03 14:33:10 - r - INFO: - Episode: 93/200, Reward: 91.000, Step: 92 +2022-12-03 14:33:11 - r - INFO: - Episode: 94/200, Reward: 59.000, Step: 60 +2022-12-03 14:33:11 - r - INFO: - Episode: 95/200, Reward: 39.000, Step: 40 +2022-12-03 14:33:11 - r - INFO: - Episode: 96/200, Reward: 38.000, Step: 39 +2022-12-03 14:33:11 - r - INFO: - Episode: 97/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:11 - r - INFO: - Episode: 98/200, Reward: 121.000, Step: 122 +2022-12-03 14:33:11 - r - INFO: - Episode: 99/200, Reward: 91.000, Step: 92 +2022-12-03 14:33:11 - r - INFO: - Episode: 100/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:11 - r - INFO: - Episode: 101/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:11 - r - INFO: - Episode: 102/200, Reward: 78.000, Step: 79 +2022-12-03 14:33:11 - r - INFO: - Episode: 103/200, Reward: 11.000, Step: 12 +2022-12-03 14:33:11 - r - INFO: - Episode: 104/200, Reward: 19.000, Step: 20 +2022-12-03 14:33:11 - r - INFO: - Episode: 105/200, Reward: 15.000, Step: 16 +2022-12-03 14:33:11 - r - INFO: - Episode: 106/200, Reward: 23.000, Step: 24 +2022-12-03 14:33:11 - r - INFO: - Episode: 107/200, Reward: 13.000, Step: 14 +2022-12-03 14:33:11 - r - INFO: - Episode: 108/200, Reward: 17.000, Step: 18 +2022-12-03 14:33:11 - r - INFO: - Episode: 109/200, Reward: 19.000, Step: 20 +2022-12-03 14:33:11 - r - INFO: - Episode: 110/200, Reward: 14.000, Step: 15 +2022-12-03 14:33:11 - r - INFO: - Episode: 111/200, Reward: 24.000, Step: 25 +2022-12-03 14:33:11 - r - INFO: - Episode: 112/200, Reward: 82.000, Step: 83 +2022-12-03 14:33:11 - r - INFO: - Episode: 113/200, Reward: 94.000, Step: 95 +2022-12-03 14:33:11 - r - INFO: - Episode: 114/200, Reward: 74.000, Step: 75 +2022-12-03 14:33:11 - r - INFO: - Episode: 115/200, Reward: 83.000, Step: 84 +2022-12-03 14:33:11 - r - INFO: - Episode: 116/200, Reward: 78.000, Step: 79 +2022-12-03 14:33:11 - r - INFO: - Episode: 117/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:11 - r - INFO: - Episode: 118/200, Reward: 76.000, Step: 77 +2022-12-03 14:33:11 - r - INFO: - Episode: 119/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:11 - r - INFO: - Episode: 120/200, Reward: 76.000, Step: 77 +2022-12-03 14:33:12 - r - INFO: - Episode: 121/200, Reward: 75.000, Step: 76 +2022-12-03 14:33:12 - r - INFO: - Episode: 122/200, Reward: 29.000, Step: 30 +2022-12-03 14:33:12 - r - INFO: - Episode: 123/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:12 - r - INFO: - Episode: 124/200, Reward: 68.000, Step: 69 +2022-12-03 14:33:12 - r - INFO: - Episode: 125/200, Reward: 67.000, Step: 68 +2022-12-03 14:33:12 - r - INFO: - Episode: 126/200, Reward: 63.000, Step: 64 +2022-12-03 14:33:12 - r - INFO: - Episode: 127/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:12 - r - INFO: - Episode: 128/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:12 - r - INFO: - Episode: 129/200, Reward: 89.000, Step: 90 +2022-12-03 14:33:12 - r - INFO: - Episode: 130/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:12 - r - INFO: - Episode: 131/200, Reward: 90.000, Step: 91 +2022-12-03 14:33:12 - r - INFO: - Episode: 132/200, Reward: 82.000, Step: 83 +2022-12-03 14:33:12 - r - INFO: - Episode: 133/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:12 - r - INFO: - Episode: 134/200, Reward: 97.000, Step: 98 +2022-12-03 14:33:12 - r - INFO: - Episode: 135/200, Reward: 130.000, Step: 131 +2022-12-03 14:33:13 - r - INFO: - Current episode 135 has the best eval reward: 176.900 +2022-12-03 14:33:13 - r - INFO: - Episode: 136/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:13 - r - INFO: - Episode: 137/200, Reward: 81.000, Step: 82 +2022-12-03 14:33:13 - r - INFO: - Episode: 138/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:13 - r - INFO: - Episode: 139/200, Reward: 195.000, Step: 196 +2022-12-03 14:33:13 - r - INFO: - Episode: 140/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:13 - r - INFO: - Current episode 140 has the best eval reward: 200.000 +2022-12-03 14:33:13 - r - INFO: - Episode: 141/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:13 - r - INFO: - Episode: 142/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:13 - r - INFO: - Episode: 143/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:13 - r - INFO: - Episode: 144/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:13 - r - INFO: - Episode: 145/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:14 - r - INFO: - Current episode 145 has the best eval reward: 200.000 +2022-12-03 14:33:14 - r - INFO: - Episode: 146/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:14 - r - INFO: - Episode: 147/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:14 - r - INFO: - Episode: 148/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:14 - r - INFO: - Episode: 149/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:14 - r - INFO: - Episode: 150/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:14 - r - INFO: - Current episode 150 has the best eval reward: 200.000 +2022-12-03 14:33:15 - r - INFO: - Episode: 151/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 152/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 153/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 154/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 155/200, Reward: 187.000, Step: 188 +2022-12-03 14:33:15 - r - INFO: - Current episode 155 has the best eval reward: 200.000 +2022-12-03 14:33:15 - r - INFO: - Episode: 156/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 157/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 158/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 159/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:15 - r - INFO: - Episode: 160/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Current episode 160 has the best eval reward: 200.000 +2022-12-03 14:33:16 - r - INFO: - Episode: 161/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 162/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 163/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 164/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 165/200, Reward: 75.000, Step: 76 +2022-12-03 14:33:16 - r - INFO: - Episode: 166/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 167/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 168/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 169/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:16 - r - INFO: - Episode: 170/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:17 - r - INFO: - Episode: 171/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:17 - r - INFO: - Episode: 172/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:17 - r - INFO: - Episode: 173/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:17 - r - INFO: - Episode: 174/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:17 - r - INFO: - Episode: 175/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:17 - r - INFO: - Episode: 176/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:17 - r - INFO: - Episode: 177/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:18 - r - INFO: - Episode: 178/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:18 - r - INFO: - Episode: 179/200, Reward: 162.000, Step: 163 +2022-12-03 14:33:18 - r - INFO: - Episode: 180/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:18 - r - INFO: - Episode: 181/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:18 - r - INFO: - Episode: 182/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:18 - r - INFO: - Episode: 183/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:18 - r - INFO: - Episode: 184/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:18 - r - INFO: - Episode: 185/200, Reward: 197.000, Step: 198 +2022-12-03 14:33:19 - r - INFO: - Episode: 186/200, Reward: 172.000, Step: 173 +2022-12-03 14:33:19 - r - INFO: - Episode: 187/200, Reward: 190.000, Step: 191 +2022-12-03 14:33:19 - r - INFO: - Episode: 188/200, Reward: 189.000, Step: 190 +2022-12-03 14:33:19 - r - INFO: - Episode: 189/200, Reward: 170.000, Step: 171 +2022-12-03 14:33:19 - r - INFO: - Episode: 190/200, Reward: 143.000, Step: 144 +2022-12-03 14:33:19 - r - INFO: - Episode: 191/200, Reward: 181.000, Step: 182 +2022-12-03 14:33:19 - r - INFO: - Episode: 192/200, Reward: 73.000, Step: 74 +2022-12-03 14:33:19 - r - INFO: - Episode: 193/200, Reward: 145.000, Step: 146 +2022-12-03 14:33:19 - r - INFO: - Episode: 194/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:19 - r - INFO: - Episode: 195/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:20 - r - INFO: - Episode: 196/200, Reward: 200.000, Step: 200 +2022-12-03 14:33:20 - r - INFO: - Episode: 197/200, Reward: 137.000, Step: 138 +2022-12-03 14:33:20 - r - INFO: - Episode: 198/200, Reward: 171.000, Step: 172 +2022-12-03 14:33:20 - r - INFO: - Episode: 199/200, Reward: 177.000, Step: 178 +2022-12-03 14:33:20 - r - INFO: - Episode: 200/200, Reward: 168.000, Step: 169 +2022-12-03 14:33:20 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2c9ea3eed609bf1b815f0f560ea071eec183e17 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9965ae728a6d4aa17b0a11c659cefd12037acb45bcf73265de391a87c69698c8 +size 8279 diff --git a/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..ac17adc10e8ed52cfc8bf13d89de7f9bceef3bae Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/results/res.csv b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..70d65277829d3bfc153507b5df9372697b0fbbe1 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_REINFORCE_20221203-143307/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,22.0,23 +1,21.0,22 +2,19.0,20 +3,9.0,10 +4,19.0,20 +5,8.0,9 +6,15.0,16 +7,11.0,12 +8,9.0,10 +9,14.0,15 +10,16.0,17 +11,14.0,15 +12,17.0,18 +13,22.0,23 +14,50.0,51 +15,18.0,19 +16,9.0,10 +17,28.0,29 +18,23.0,24 +19,20.0,21 +20,35.0,36 +21,10.0,11 +22,9.0,10 +23,8.0,9 +24,8.0,9 +25,8.0,9 +26,8.0,9 +27,8.0,9 +28,8.0,9 +29,9.0,10 +30,8.0,9 +31,8.0,9 +32,9.0,10 +33,8.0,9 +34,9.0,10 +35,9.0,10 +36,11.0,12 +37,8.0,9 +38,8.0,9 +39,8.0,9 +40,13.0,14 +41,14.0,15 +42,16.0,17 +43,17.0,18 +44,30.0,31 +45,20.0,21 +46,15.0,16 +47,17.0,18 +48,20.0,21 +49,20.0,21 +50,45.0,46 +51,57.0,58 +52,52.0,53 +53,66.0,67 +54,68.0,69 +55,48.0,49 +56,49.0,50 +57,68.0,69 +58,113.0,114 +59,51.0,52 +60,46.0,47 +61,30.0,31 +62,40.0,41 +63,28.0,29 +64,73.0,74 +65,42.0,43 +66,74.0,75 +67,83.0,84 +68,50.0,51 +69,43.0,44 +70,61.0,62 +71,80.0,81 +72,91.0,92 +73,200.0,200 +74,99.0,100 +75,52.0,53 +76,126.0,127 +77,164.0,165 +78,200.0,200 +79,82.0,83 +80,117.0,118 +81,106.0,107 +82,82.0,83 +83,54.0,55 +84,48.0,49 +85,15.0,16 +86,12.0,13 +87,14.0,15 +88,38.0,39 +89,175.0,176 +90,25.0,26 +91,163.0,164 +92,91.0,92 +93,59.0,60 +94,39.0,40 +95,38.0,39 +96,200.0,200 +97,121.0,122 +98,91.0,92 +99,73.0,74 +100,73.0,74 +101,78.0,79 +102,11.0,12 +103,19.0,20 +104,15.0,16 +105,23.0,24 +106,13.0,14 +107,17.0,18 +108,19.0,20 +109,14.0,15 +110,24.0,25 +111,82.0,83 +112,94.0,95 +113,74.0,75 +114,83.0,84 +115,78.0,79 +116,73.0,74 +117,76.0,77 +118,73.0,74 +119,76.0,77 +120,75.0,76 +121,29.0,30 +122,73.0,74 +123,68.0,69 +124,67.0,68 +125,63.0,64 +126,73.0,74 +127,73.0,74 +128,89.0,90 +129,200.0,200 +130,90.0,91 +131,82.0,83 +132,200.0,200 +133,97.0,98 +134,130.0,131 +135,200.0,200 +136,81.0,82 +137,200.0,200 +138,195.0,196 +139,200.0,200 +140,200.0,200 +141,200.0,200 +142,200.0,200 +143,200.0,200 +144,200.0,200 +145,200.0,200 +146,200.0,200 +147,200.0,200 +148,200.0,200 +149,200.0,200 +150,200.0,200 +151,200.0,200 +152,200.0,200 +153,200.0,200 +154,187.0,188 +155,200.0,200 +156,200.0,200 +157,200.0,200 +158,200.0,200 +159,200.0,200 +160,200.0,200 +161,200.0,200 +162,200.0,200 +163,200.0,200 +164,75.0,76 +165,200.0,200 +166,200.0,200 +167,200.0,200 +168,200.0,200 +169,200.0,200 +170,200.0,200 +171,200.0,200 +172,200.0,200 +173,200.0,200 +174,200.0,200 +175,200.0,200 +176,200.0,200 +177,200.0,200 +178,162.0,163 +179,200.0,200 +180,200.0,200 +181,200.0,200 +182,200.0,200 +183,200.0,200 +184,197.0,198 +185,172.0,173 +186,190.0,191 +187,189.0,190 +188,170.0,171 +189,143.0,144 +190,181.0,182 +191,73.0,74 +192,145.0,146 +193,200.0,200 +194,200.0,200 +195,200.0,200 +196,137.0,138 +197,171.0,172 +198,177.0,178 +199,168.0,169 diff --git a/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/config.yaml b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b760acf33e6cc536c85d7f80b78b0f17412ca64 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/config.yaml @@ -0,0 +1,73 @@ +general_cfg: + Vmax: 200 + Vmin: 0 + algo_name: RainbowDQN + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_ + max_steps: 200 + mode: train + n_atoms: 51 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + Vmax: 200.0 + Vmin: 0.0 + batch_size: 64 + buffer_size: 100000 + delta_z: 4.0 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + memory_capacity: 10000 + n_step: 1 + num_atoms: 51 + support: !!python/object/apply:torch._utils._rebuild_tensor_v2 + - !!python/object/apply:torch.storage._load_from_bytes + - !!binary | + gAKKCmz8nEb5IGqoUBkugAJN6QMugAJ9cQAoWBAAAABwcm90b2NvbF92ZXJzaW9ucQFN6QNYDQAA + AGxpdHRsZV9lbmRpYW5xAohYCgAAAHR5cGVfc2l6ZXNxA31xBChYBQAAAHNob3J0cQVLAlgDAAAA + aW50cQZLBFgEAAAAbG9uZ3EHSwR1dS6AAihYBwAAAHN0b3JhZ2VxAGN0b3JjaApGbG9hdFN0b3Jh + Z2UKcQFYCQAAADEwOTE5NDU5MnECWAMAAABjcHVxA0szTnRxBFEugAJdcQBYCQAAADEwOTE5NDU5 + MnEBYS4zAAAAAAAAAAAAAAAAAIBAAAAAQQAAQEEAAIBBAACgQQAAwEEAAOBBAAAAQgAAEEIAACBC + AAAwQgAAQEIAAFBCAABgQgAAcEIAAIBCAACIQgAAkEIAAJhCAACgQgAAqEIAALBCAAC4QgAAwEIA + AMhCAADQQgAA2EIAAOBCAADoQgAA8EIAAPhCAAAAQwAABEMAAAhDAAAMQwAAEEMAABRDAAAYQwAA + HEMAACBDAAAkQwAAKEMAACxDAAAwQwAANEMAADhDAAA8QwAAQEMAAERDAABIQw== + - 0 + - !!python/tuple + - 51 + - !!python/tuple + - 1 + - false + - !!python/object/apply:collections.OrderedDict + - [] + target_update: 4 + tau: 1.0 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/logs/log.txt b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..f6e81f58fb3f6578a987f16782dad0df09972dd8 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/logs/log.txt @@ -0,0 +1,164 @@ +2023-01-14 22:20:12 - r - INFO: - Hyperparameters: +2023-01-14 22:20:12 - r - INFO: - ================================================================================ +2023-01-14 22:20:12 - r - INFO: - Name Value Type +2023-01-14 22:20:12 - r - INFO: - env_name CartPole-v1 +2023-01-14 22:20:12 - r - INFO: - new_step_api 1 +2023-01-14 22:20:12 - r - INFO: - wrapper None +2023-01-14 22:20:12 - r - INFO: - render 0 +2023-01-14 22:20:12 - r - INFO: - algo_name RainbowDQN +2023-01-14 22:20:12 - r - INFO: - mode train +2023-01-14 22:20:12 - r - INFO: - seed 1 +2023-01-14 22:20:12 - r - INFO: - device cuda +2023-01-14 22:20:12 - r - INFO: - train_eps 100 +2023-01-14 22:20:12 - r - INFO: - test_eps 10 +2023-01-14 22:20:12 - r - INFO: - eval_eps 10 +2023-01-14 22:20:12 - r - INFO: - eval_per_episode 5 +2023-01-14 22:20:12 - r - INFO: - max_steps 200 +2023-01-14 22:20:12 - r - INFO: - load_checkpoint 0 +2023-01-14 22:20:12 - r - INFO: - load_path Train_CartPole-v1_ +2023-01-14 22:20:12 - r - INFO: - show_fig 0 +2023-01-14 22:20:12 - r - INFO: - save_fig 1 +2023-01-14 22:20:12 - r - INFO: - Vmin 0.0 +2023-01-14 22:20:12 - r - INFO: - Vmax 200.0 +2023-01-14 22:20:12 - r - INFO: - n_atoms 51 +2023-01-14 22:20:12 - r - INFO: - gamma 0.95 +2023-01-14 22:20:12 - r - INFO: - tau 1.0 +2023-01-14 22:20:12 - r - INFO: - hidden_dim 256 +2023-01-14 22:20:12 - r - INFO: - num_atoms 51 +2023-01-14 22:20:12 - r - INFO: - support tensor(0.) +2023-01-14 22:20:12 - r - INFO: - delta_z 4.0 +2023-01-14 22:20:12 - r - INFO: - n_step 1 +2023-01-14 22:20:12 - r - INFO: - batch_size 64 +2023-01-14 22:20:12 - r - INFO: - lr 0.0001 +2023-01-14 22:20:12 - r - INFO: - target_update 4 +2023-01-14 22:20:12 - r - INFO: - memory_capacity 10000 +2023-01-14 22:20:12 - r - INFO: - epsilon_start 0.95 +2023-01-14 22:20:12 - r - INFO: - epsilon_end 0.01 +2023-01-14 22:20:12 - r - INFO: - epsilon_decay 500 +2023-01-14 22:20:12 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-01-14 22:20:12 - r - INFO: - buffer_size 100000 +2023-01-14 22:20:12 - r - INFO: - task_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_RainbowDQN_20230114-222012 +2023-01-14 22:20:12 - r - INFO: - model_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_RainbowDQN_20230114-222012/models +2023-01-14 22:20:12 - r - INFO: - res_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_RainbowDQN_20230114-222012/results +2023-01-14 22:20:12 - r - INFO: - log_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_RainbowDQN_20230114-222012/logs +2023-01-14 22:20:12 - r - INFO: - traj_dir /home/dingli/rl-tutorials/joyrl/tasks/Train_CartPole-v1_RainbowDQN_20230114-222012/traj +2023-01-14 22:20:12 - r - INFO: - ================================================================================ +2023-01-14 22:20:12 - r - INFO: - n_states: 4, n_actions: 2 +2023-01-14 22:20:14 - r - INFO: - Start training! +2023-01-14 22:20:14 - r - INFO: - Env: CartPole-v1, Algorithm: RainbowDQN, Device: cuda +2023-01-14 22:20:14 - r - INFO: - Episode: 1/100, Reward: 16.000, Step: 16 +2023-01-14 22:20:15 - r - INFO: - Episode: 2/100, Reward: 50.000, Step: 50 +2023-01-14 22:20:15 - r - INFO: - Episode: 3/100, Reward: 23.000, Step: 23 +2023-01-14 22:20:15 - r - INFO: - Episode: 4/100, Reward: 11.000, Step: 11 +2023-01-14 22:20:15 - r - INFO: - Episode: 5/100, Reward: 10.000, Step: 10 +2023-01-14 22:20:15 - r - INFO: - Current episode 5 has the best eval reward: 9.200 +2023-01-14 22:20:16 - r - INFO: - Episode: 6/100, Reward: 79.000, Step: 79 +2023-01-14 22:20:16 - r - INFO: - Episode: 7/100, Reward: 11.000, Step: 11 +2023-01-14 22:20:17 - r - INFO: - Episode: 8/100, Reward: 53.000, Step: 53 +2023-01-14 22:20:17 - r - INFO: - Episode: 9/100, Reward: 37.000, Step: 37 +2023-01-14 22:20:18 - r - INFO: - Episode: 10/100, Reward: 38.000, Step: 38 +2023-01-14 22:20:18 - r - INFO: - Current episode 10 has the best eval reward: 46.400 +2023-01-14 22:20:19 - r - INFO: - Episode: 11/100, Reward: 51.000, Step: 51 +2023-01-14 22:20:19 - r - INFO: - Episode: 12/100, Reward: 42.000, Step: 42 +2023-01-14 22:20:20 - r - INFO: - Episode: 13/100, Reward: 43.000, Step: 43 +2023-01-14 22:20:20 - r - INFO: - Episode: 14/100, Reward: 34.000, Step: 34 +2023-01-14 22:20:20 - r - INFO: - Episode: 15/100, Reward: 21.000, Step: 21 +2023-01-14 22:20:21 - r - INFO: - Current episode 15 has the best eval reward: 54.400 +2023-01-14 22:20:21 - r - INFO: - Episode: 16/100, Reward: 44.000, Step: 44 +2023-01-14 22:20:22 - r - INFO: - Episode: 17/100, Reward: 42.000, Step: 42 +2023-01-14 22:20:23 - r - INFO: - Episode: 18/100, Reward: 109.000, Step: 109 +2023-01-14 22:20:24 - r - INFO: - Episode: 19/100, Reward: 33.000, Step: 33 +2023-01-14 22:20:24 - r - INFO: - Episode: 20/100, Reward: 43.000, Step: 43 +2023-01-14 22:20:25 - r - INFO: - Episode: 21/100, Reward: 44.000, Step: 44 +2023-01-14 22:20:26 - r - INFO: - Episode: 22/100, Reward: 84.000, Step: 84 +2023-01-14 22:20:27 - r - INFO: - Episode: 23/100, Reward: 48.000, Step: 48 +2023-01-14 22:20:27 - r - INFO: - Episode: 24/100, Reward: 63.000, Step: 63 +2023-01-14 22:20:30 - r - INFO: - Episode: 25/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:31 - r - INFO: - Current episode 25 has the best eval reward: 122.800 +2023-01-14 22:20:33 - r - INFO: - Episode: 26/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:36 - r - INFO: - Episode: 27/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:38 - r - INFO: - Episode: 28/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:41 - r - INFO: - Episode: 29/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:43 - r - INFO: - Episode: 30/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:44 - r - INFO: - Current episode 30 has the best eval reward: 200.000 +2023-01-14 22:20:47 - r - INFO: - Episode: 31/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:49 - r - INFO: - Episode: 32/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:52 - r - INFO: - Episode: 33/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:54 - r - INFO: - Episode: 34/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:57 - r - INFO: - Episode: 35/100, Reward: 200.000, Step: 200 +2023-01-14 22:20:58 - r - INFO: - Current episode 35 has the best eval reward: 200.000 +2023-01-14 22:21:01 - r - INFO: - Episode: 36/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:03 - r - INFO: - Episode: 37/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:06 - r - INFO: - Episode: 38/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:08 - r - INFO: - Episode: 39/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:11 - r - INFO: - Episode: 40/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:12 - r - INFO: - Current episode 40 has the best eval reward: 200.000 +2023-01-14 22:21:15 - r - INFO: - Episode: 41/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:17 - r - INFO: - Episode: 42/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:20 - r - INFO: - Episode: 43/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:22 - r - INFO: - Episode: 44/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:24 - r - INFO: - Episode: 45/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:26 - r - INFO: - Current episode 45 has the best eval reward: 200.000 +2023-01-14 22:21:28 - r - INFO: - Episode: 46/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:31 - r - INFO: - Episode: 47/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:33 - r - INFO: - Episode: 48/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:36 - r - INFO: - Episode: 49/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:38 - r - INFO: - Episode: 50/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:40 - r - INFO: - Current episode 50 has the best eval reward: 200.000 +2023-01-14 22:21:42 - r - INFO: - Episode: 51/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:45 - r - INFO: - Episode: 52/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:47 - r - INFO: - Episode: 53/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:49 - r - INFO: - Episode: 54/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:52 - r - INFO: - Episode: 55/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:56 - r - INFO: - Episode: 56/100, Reward: 200.000, Step: 200 +2023-01-14 22:21:58 - r - INFO: - Episode: 57/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:01 - r - INFO: - Episode: 58/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:03 - r - INFO: - Episode: 59/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:05 - r - INFO: - Episode: 60/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:07 - r - INFO: - Current episode 60 has the best eval reward: 200.000 +2023-01-14 22:22:09 - r - INFO: - Episode: 61/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:12 - r - INFO: - Episode: 62/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:14 - r - INFO: - Episode: 63/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:17 - r - INFO: - Episode: 64/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:19 - r - INFO: - Episode: 65/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:21 - r - INFO: - Current episode 65 has the best eval reward: 200.000 +2023-01-14 22:22:24 - r - INFO: - Episode: 66/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:26 - r - INFO: - Episode: 67/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:29 - r - INFO: - Episode: 68/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:31 - r - INFO: - Episode: 69/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:34 - r - INFO: - Episode: 70/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:38 - r - INFO: - Episode: 71/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:40 - r - INFO: - Episode: 72/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:42 - r - INFO: - Episode: 73/100, Reward: 174.000, Step: 174 +2023-01-14 22:22:45 - r - INFO: - Episode: 74/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:47 - r - INFO: - Episode: 75/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:51 - r - INFO: - Episode: 76/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:53 - r - INFO: - Episode: 77/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:56 - r - INFO: - Episode: 78/100, Reward: 200.000, Step: 200 +2023-01-14 22:22:58 - r - INFO: - Episode: 79/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:01 - r - INFO: - Episode: 80/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:04 - r - INFO: - Episode: 81/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:07 - r - INFO: - Episode: 82/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:10 - r - INFO: - Episode: 83/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:12 - r - INFO: - Episode: 84/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:15 - r - INFO: - Episode: 85/100, Reward: 193.000, Step: 193 +2023-01-14 22:23:16 - r - INFO: - Current episode 85 has the best eval reward: 200.000 +2023-01-14 22:23:19 - r - INFO: - Episode: 86/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:21 - r - INFO: - Episode: 87/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:24 - r - INFO: - Episode: 88/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:26 - r - INFO: - Episode: 89/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:28 - r - INFO: - Episode: 90/100, Reward: 177.000, Step: 177 +2023-01-14 22:23:30 - r - INFO: - Current episode 90 has the best eval reward: 200.000 +2023-01-14 22:23:32 - r - INFO: - Episode: 91/100, Reward: 191.000, Step: 191 +2023-01-14 22:23:35 - r - INFO: - Episode: 92/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:37 - r - INFO: - Episode: 93/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:40 - r - INFO: - Episode: 94/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:42 - r - INFO: - Episode: 95/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:44 - r - INFO: - Current episode 95 has the best eval reward: 200.000 +2023-01-14 22:23:46 - r - INFO: - Episode: 96/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:49 - r - INFO: - Episode: 97/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:51 - r - INFO: - Episode: 98/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:54 - r - INFO: - Episode: 99/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:56 - r - INFO: - Episode: 100/100, Reward: 200.000, Step: 200 +2023-01-14 22:23:58 - r - INFO: - Current episode 100 has the best eval reward: 200.000 +2023-01-14 22:23:58 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c388dd5bae6b0ccde7d78dd46998d02b7a96fac --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:510ec8e389f33f05d770df84af4fdfca682c7e83dd02da6c1374a578b9ce9ffd +size 2064020 diff --git a/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..fc69e1b72da1be8af785d830feacc6dfb0e31f80 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/results/res.csv b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..efedcb569d2f86fffb65f79f1e6cc894c71db9d4 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_RainbowDQN_20230114-222012/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,16.0,16 +1,50.0,50 +2,23.0,23 +3,11.0,11 +4,10.0,10 +5,79.0,79 +6,11.0,11 +7,53.0,53 +8,37.0,37 +9,38.0,38 +10,51.0,51 +11,42.0,42 +12,43.0,43 +13,34.0,34 +14,21.0,21 +15,44.0,44 +16,42.0,42 +17,109.0,109 +18,33.0,33 +19,43.0,43 +20,44.0,44 +21,84.0,84 +22,48.0,48 +23,63.0,63 +24,200.0,200 +25,200.0,200 +26,200.0,200 +27,200.0,200 +28,200.0,200 +29,200.0,200 +30,200.0,200 +31,200.0,200 +32,200.0,200 +33,200.0,200 +34,200.0,200 +35,200.0,200 +36,200.0,200 +37,200.0,200 +38,200.0,200 +39,200.0,200 +40,200.0,200 +41,200.0,200 +42,200.0,200 +43,200.0,200 +44,200.0,200 +45,200.0,200 +46,200.0,200 +47,200.0,200 +48,200.0,200 +49,200.0,200 +50,200.0,200 +51,200.0,200 +52,200.0,200 +53,200.0,200 +54,200.0,200 +55,200.0,200 +56,200.0,200 +57,200.0,200 +58,200.0,200 +59,200.0,200 +60,200.0,200 +61,200.0,200 +62,200.0,200 +63,200.0,200 +64,200.0,200 +65,200.0,200 +66,200.0,200 +67,200.0,200 +68,200.0,200 +69,200.0,200 +70,200.0,200 +71,200.0,200 +72,174.0,174 +73,200.0,200 +74,200.0,200 +75,200.0,200 +76,200.0,200 +77,200.0,200 +78,200.0,200 +79,200.0,200 +80,200.0,200 +81,200.0,200 +82,200.0,200 +83,200.0,200 +84,193.0,193 +85,200.0,200 +86,200.0,200 +87,200.0,200 +88,200.0,200 +89,177.0,177 +90,191.0,191 +91,200.0,200 +92,200.0,200 +93,200.0,200 +94,200.0,200 +95,200.0,200 +96,200.0,200 +97,200.0,200 +98,200.0,200 +99,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/config.yaml b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69c452878898790b01cb254bb10edeade3379e48 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: SAC_D + device: cuda + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 0 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + alpha: 0.2 + automatic_entropy_tuning: false + batch_size: 64 + buffer_size: 1000000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + hidden_dim: 256 + lr: 0.0001 + n_epochs: 1 + target_update: 1 + tau: 0.005 diff --git a/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/logs/log.txt b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..40829a5b16ac316d6af8b34eff61b42e105f6fa8 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/logs/log.txt @@ -0,0 +1,251 @@ +2023-03-05 11:28:49 - r - INFO: - Hyperparameters: +2023-03-05 11:28:49 - r - INFO: - ================================================================================ +2023-03-05 11:28:49 - r - INFO: - Name Value Type +2023-03-05 11:28:49 - r - INFO: - env_name CartPole-v1 +2023-03-05 11:28:49 - r - INFO: - new_step_api 1 +2023-03-05 11:28:49 - r - INFO: - wrapper None +2023-03-05 11:28:49 - r - INFO: - render 0 +2023-03-05 11:28:49 - r - INFO: - algo_name SAC_D +2023-03-05 11:28:49 - r - INFO: - mode train +2023-03-05 11:28:49 - r - INFO: - seed 0 +2023-03-05 11:28:49 - r - INFO: - device cuda +2023-03-05 11:28:49 - r - INFO: - train_eps 200 +2023-03-05 11:28:49 - r - INFO: - test_eps 10 +2023-03-05 11:28:49 - r - INFO: - eval_eps 10 +2023-03-05 11:28:49 - r - INFO: - eval_per_episode 5 +2023-03-05 11:28:49 - r - INFO: - max_steps 200 +2023-03-05 11:28:49 - r - INFO: - load_checkpoint 0 +2023-03-05 11:28:49 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-03-05 11:28:49 - r - INFO: - show_fig 0 +2023-03-05 11:28:49 - r - INFO: - save_fig 1 +2023-03-05 11:28:49 - r - INFO: - epsilon_start 0.95 +2023-03-05 11:28:49 - r - INFO: - epsilon_end 0.01 +2023-03-05 11:28:49 - r - INFO: - epsilon_decay 500 +2023-03-05 11:28:49 - r - INFO: - lr 0.0001 +2023-03-05 11:28:49 - r - INFO: - gamma 0.95 +2023-03-05 11:28:49 - r - INFO: - tau 0.005 +2023-03-05 11:28:49 - r - INFO: - alpha 0.2 +2023-03-05 11:28:49 - r - INFO: - automatic_entropy_tuning 0 +2023-03-05 11:28:49 - r - INFO: - batch_size 64 +2023-03-05 11:28:49 - r - INFO: - hidden_dim 256 +2023-03-05 11:28:49 - r - INFO: - n_epochs 1 +2023-03-05 11:28:49 - r - INFO: - target_update 1 +2023-03-05 11:28:49 - r - INFO: - buffer_size 1000000 +2023-03-05 11:28:49 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_SAC_D_20230305-112849 +2023-03-05 11:28:49 - r - INFO: - model_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_SAC_D_20230305-112849/models +2023-03-05 11:28:49 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_SAC_D_20230305-112849/results +2023-03-05 11:28:49 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_SAC_D_20230305-112849/logs +2023-03-05 11:28:49 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_SAC_D_20230305-112849/traj +2023-03-05 11:28:49 - r - INFO: - ================================================================================ +2023-03-05 11:28:49 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-05 11:28:51 - r - INFO: - Start training! +2023-03-05 11:28:51 - r - INFO: - Env: CartPole-v1, Algorithm: SAC_D, Device: cuda +2023-03-05 11:28:51 - r - INFO: - Episode: 1/200, Reward: 17.000, Step: 17 +2023-03-05 11:28:52 - r - INFO: - Episode: 2/200, Reward: 77.000, Step: 77 +2023-03-05 11:28:52 - r - INFO: - Episode: 3/200, Reward: 33.000, Step: 33 +2023-03-05 11:28:52 - r - INFO: - Episode: 4/200, Reward: 25.000, Step: 25 +2023-03-05 11:28:53 - r - INFO: - Episode: 5/200, Reward: 49.000, Step: 49 +2023-03-05 11:28:53 - r - INFO: - Current episode 5 has the best eval reward: 63.300 +2023-03-05 11:28:53 - r - INFO: - Episode: 6/200, Reward: 31.000, Step: 31 +2023-03-05 11:28:53 - r - INFO: - Episode: 7/200, Reward: 35.000, Step: 35 +2023-03-05 11:28:54 - r - INFO: - Episode: 8/200, Reward: 15.000, Step: 15 +2023-03-05 11:28:54 - r - INFO: - Episode: 9/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:54 - r - INFO: - Episode: 10/200, Reward: 11.000, Step: 11 +2023-03-05 11:28:54 - r - INFO: - Episode: 11/200, Reward: 16.000, Step: 16 +2023-03-05 11:28:54 - r - INFO: - Episode: 12/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:54 - r - INFO: - Episode: 13/200, Reward: 16.000, Step: 16 +2023-03-05 11:28:54 - r - INFO: - Episode: 14/200, Reward: 8.000, Step: 8 +2023-03-05 11:28:54 - r - INFO: - Episode: 15/200, Reward: 11.000, Step: 11 +2023-03-05 11:28:54 - r - INFO: - Episode: 16/200, Reward: 14.000, Step: 14 +2023-03-05 11:28:54 - r - INFO: - Episode: 17/200, Reward: 11.000, Step: 11 +2023-03-05 11:28:55 - r - INFO: - Episode: 18/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:55 - r - INFO: - Episode: 19/200, Reward: 14.000, Step: 14 +2023-03-05 11:28:55 - r - INFO: - Episode: 20/200, Reward: 14.000, Step: 14 +2023-03-05 11:28:55 - r - INFO: - Episode: 21/200, Reward: 15.000, Step: 15 +2023-03-05 11:28:55 - r - INFO: - Episode: 22/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:55 - r - INFO: - Episode: 23/200, Reward: 13.000, Step: 13 +2023-03-05 11:28:55 - r - INFO: - Episode: 24/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:55 - r - INFO: - Episode: 25/200, Reward: 14.000, Step: 14 +2023-03-05 11:28:55 - r - INFO: - Episode: 26/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:55 - r - INFO: - Episode: 27/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:55 - r - INFO: - Episode: 28/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:56 - r - INFO: - Episode: 29/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:56 - r - INFO: - Episode: 30/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 31/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 32/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 33/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:56 - r - INFO: - Episode: 34/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 35/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 36/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 37/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:56 - r - INFO: - Episode: 38/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 39/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:56 - r - INFO: - Episode: 40/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:56 - r - INFO: - Episode: 41/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:57 - r - INFO: - Episode: 42/200, Reward: 9.000, Step: 9 +2023-03-05 11:28:57 - r - INFO: - Episode: 43/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:57 - r - INFO: - Episode: 44/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:57 - r - INFO: - Episode: 45/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:57 - r - INFO: - Episode: 46/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:57 - r - INFO: - Episode: 47/200, Reward: 11.000, Step: 11 +2023-03-05 11:28:57 - r - INFO: - Episode: 48/200, Reward: 13.000, Step: 13 +2023-03-05 11:28:57 - r - INFO: - Episode: 49/200, Reward: 10.000, Step: 10 +2023-03-05 11:28:57 - r - INFO: - Episode: 50/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:57 - r - INFO: - Episode: 51/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:58 - r - INFO: - Episode: 52/200, Reward: 24.000, Step: 24 +2023-03-05 11:28:58 - r - INFO: - Episode: 53/200, Reward: 68.000, Step: 68 +2023-03-05 11:28:58 - r - INFO: - Episode: 54/200, Reward: 12.000, Step: 12 +2023-03-05 11:28:59 - r - INFO: - Episode: 55/200, Reward: 16.000, Step: 16 +2023-03-05 11:28:59 - r - INFO: - Episode: 56/200, Reward: 83.000, Step: 83 +2023-03-05 11:29:00 - r - INFO: - Episode: 57/200, Reward: 122.000, Step: 122 +2023-03-05 11:29:00 - r - INFO: - Episode: 58/200, Reward: 28.000, Step: 28 +2023-03-05 11:29:01 - r - INFO: - Episode: 59/200, Reward: 55.000, Step: 55 +2023-03-05 11:29:01 - r - INFO: - Episode: 60/200, Reward: 32.000, Step: 32 +2023-03-05 11:29:02 - r - INFO: - Episode: 61/200, Reward: 41.000, Step: 41 +2023-03-05 11:29:02 - r - INFO: - Episode: 62/200, Reward: 33.000, Step: 33 +2023-03-05 11:29:02 - r - INFO: - Episode: 63/200, Reward: 21.000, Step: 21 +2023-03-05 11:29:02 - r - INFO: - Episode: 64/200, Reward: 19.000, Step: 19 +2023-03-05 11:29:02 - r - INFO: - Episode: 65/200, Reward: 19.000, Step: 19 +2023-03-05 11:29:03 - r - INFO: - Episode: 66/200, Reward: 27.000, Step: 27 +2023-03-05 11:29:03 - r - INFO: - Episode: 67/200, Reward: 30.000, Step: 30 +2023-03-05 11:29:03 - r - INFO: - Episode: 68/200, Reward: 25.000, Step: 25 +2023-03-05 11:29:03 - r - INFO: - Episode: 69/200, Reward: 23.000, Step: 23 +2023-03-05 11:29:04 - r - INFO: - Episode: 70/200, Reward: 27.000, Step: 27 +2023-03-05 11:29:04 - r - INFO: - Episode: 71/200, Reward: 26.000, Step: 26 +2023-03-05 11:29:04 - r - INFO: - Episode: 72/200, Reward: 26.000, Step: 26 +2023-03-05 11:29:05 - r - INFO: - Episode: 73/200, Reward: 33.000, Step: 33 +2023-03-05 11:29:05 - r - INFO: - Episode: 74/200, Reward: 43.000, Step: 43 +2023-03-05 11:29:05 - r - INFO: - Episode: 75/200, Reward: 26.000, Step: 26 +2023-03-05 11:29:05 - r - INFO: - Episode: 76/200, Reward: 25.000, Step: 25 +2023-03-05 11:29:06 - r - INFO: - Episode: 77/200, Reward: 23.000, Step: 23 +2023-03-05 11:29:06 - r - INFO: - Episode: 78/200, Reward: 41.000, Step: 41 +2023-03-05 11:29:06 - r - INFO: - Episode: 79/200, Reward: 36.000, Step: 36 +2023-03-05 11:29:07 - r - INFO: - Episode: 80/200, Reward: 36.000, Step: 36 +2023-03-05 11:29:07 - r - INFO: - Episode: 81/200, Reward: 31.000, Step: 31 +2023-03-05 11:29:07 - r - INFO: - Episode: 82/200, Reward: 27.000, Step: 27 +2023-03-05 11:29:07 - r - INFO: - Episode: 83/200, Reward: 27.000, Step: 27 +2023-03-05 11:29:08 - r - INFO: - Episode: 84/200, Reward: 28.000, Step: 28 +2023-03-05 11:29:08 - r - INFO: - Episode: 85/200, Reward: 27.000, Step: 27 +2023-03-05 11:29:08 - r - INFO: - Episode: 86/200, Reward: 29.000, Step: 29 +2023-03-05 11:29:09 - r - INFO: - Episode: 87/200, Reward: 25.000, Step: 25 +2023-03-05 11:29:09 - r - INFO: - Episode: 88/200, Reward: 59.000, Step: 59 +2023-03-05 11:29:09 - r - INFO: - Episode: 89/200, Reward: 30.000, Step: 30 +2023-03-05 11:29:10 - r - INFO: - Episode: 90/200, Reward: 91.000, Step: 91 +2023-03-05 11:29:10 - r - INFO: - Episode: 91/200, Reward: 33.000, Step: 33 +2023-03-05 11:29:11 - r - INFO: - Episode: 92/200, Reward: 77.000, Step: 77 +2023-03-05 11:29:11 - r - INFO: - Episode: 93/200, Reward: 42.000, Step: 42 +2023-03-05 11:29:12 - r - INFO: - Episode: 94/200, Reward: 57.000, Step: 57 +2023-03-05 11:29:13 - r - INFO: - Episode: 95/200, Reward: 75.000, Step: 75 +2023-03-05 11:29:13 - r - INFO: - Episode: 96/200, Reward: 55.000, Step: 55 +2023-03-05 11:29:14 - r - INFO: - Episode: 97/200, Reward: 42.000, Step: 42 +2023-03-05 11:29:14 - r - INFO: - Episode: 98/200, Reward: 43.000, Step: 43 +2023-03-05 11:29:15 - r - INFO: - Episode: 99/200, Reward: 92.000, Step: 92 +2023-03-05 11:29:15 - r - INFO: - Episode: 100/200, Reward: 41.000, Step: 41 +2023-03-05 11:29:16 - r - INFO: - Episode: 101/200, Reward: 72.000, Step: 72 +2023-03-05 11:29:18 - r - INFO: - Episode: 102/200, Reward: 200.000, Step: 200 +2023-03-05 11:29:19 - r - INFO: - Episode: 103/200, Reward: 115.000, Step: 115 +2023-03-05 11:29:20 - r - INFO: - Episode: 104/200, Reward: 90.000, Step: 90 +2023-03-05 11:29:21 - r - INFO: - Episode: 105/200, Reward: 97.000, Step: 97 +2023-03-05 11:29:21 - r - INFO: - Current episode 105 has the best eval reward: 145.500 +2023-03-05 11:29:22 - r - INFO: - Episode: 106/200, Reward: 143.000, Step: 143 +2023-03-05 11:29:23 - r - INFO: - Episode: 107/200, Reward: 109.000, Step: 109 +2023-03-05 11:29:25 - r - INFO: - Episode: 108/200, Reward: 200.000, Step: 200 +2023-03-05 11:29:26 - r - INFO: - Episode: 109/200, Reward: 168.000, Step: 168 +2023-03-05 11:29:28 - r - INFO: - Episode: 110/200, Reward: 158.000, Step: 158 +2023-03-05 11:29:28 - r - INFO: - Current episode 110 has the best eval reward: 150.100 +2023-03-05 11:29:30 - r - INFO: - Episode: 111/200, Reward: 200.000, Step: 200 +2023-03-05 11:29:31 - r - INFO: - Episode: 112/200, Reward: 133.000, Step: 133 +2023-03-05 11:29:32 - r - INFO: - Episode: 113/200, Reward: 123.000, Step: 123 +2023-03-05 11:29:33 - r - INFO: - Episode: 114/200, Reward: 135.000, Step: 135 +2023-03-05 11:29:34 - r - INFO: - Episode: 115/200, Reward: 133.000, Step: 133 +2023-03-05 11:29:35 - r - INFO: - Current episode 115 has the best eval reward: 181.400 +2023-03-05 11:29:36 - r - INFO: - Episode: 116/200, Reward: 136.000, Step: 136 +2023-03-05 11:29:37 - r - INFO: - Episode: 117/200, Reward: 141.000, Step: 141 +2023-03-05 11:29:38 - r - INFO: - Episode: 118/200, Reward: 120.000, Step: 120 +2023-03-05 11:29:39 - r - INFO: - Episode: 119/200, Reward: 156.000, Step: 156 +2023-03-05 11:29:41 - r - INFO: - Episode: 120/200, Reward: 200.000, Step: 200 +2023-03-05 11:29:41 - r - INFO: - Current episode 120 has the best eval reward: 192.200 +2023-03-05 11:29:43 - r - INFO: - Episode: 121/200, Reward: 174.000, Step: 174 +2023-03-05 11:29:44 - r - INFO: - Episode: 122/200, Reward: 158.000, Step: 158 +2023-03-05 11:29:45 - r - INFO: - Episode: 123/200, Reward: 124.000, Step: 124 +2023-03-05 11:29:47 - r - INFO: - Episode: 124/200, Reward: 200.000, Step: 200 +2023-03-05 11:29:48 - r - INFO: - Episode: 125/200, Reward: 162.000, Step: 162 +2023-03-05 11:29:51 - r - INFO: - Episode: 126/200, Reward: 200.000, Step: 200 +2023-03-05 11:29:52 - r - INFO: - Episode: 127/200, Reward: 141.000, Step: 141 +2023-03-05 11:29:53 - r - INFO: - Episode: 128/200, Reward: 192.000, Step: 192 +2023-03-05 11:29:55 - r - INFO: - Episode: 129/200, Reward: 176.000, Step: 176 +2023-03-05 11:29:56 - r - INFO: - Episode: 130/200, Reward: 127.000, Step: 127 +2023-03-05 11:29:58 - r - INFO: - Episode: 131/200, Reward: 200.000, Step: 200 +2023-03-05 11:29:59 - r - INFO: - Episode: 132/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:01 - r - INFO: - Episode: 133/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:02 - r - INFO: - Episode: 134/200, Reward: 150.000, Step: 150 +2023-03-05 11:30:03 - r - INFO: - Episode: 135/200, Reward: 139.000, Step: 139 +2023-03-05 11:30:05 - r - INFO: - Episode: 136/200, Reward: 171.000, Step: 171 +2023-03-05 11:30:07 - r - INFO: - Episode: 137/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:09 - r - INFO: - Episode: 138/200, Reward: 179.000, Step: 179 +2023-03-05 11:30:10 - r - INFO: - Episode: 139/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:12 - r - INFO: - Episode: 140/200, Reward: 187.000, Step: 187 +2023-03-05 11:30:14 - r - INFO: - Episode: 141/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:16 - r - INFO: - Episode: 142/200, Reward: 170.000, Step: 170 +2023-03-05 11:30:17 - r - INFO: - Episode: 143/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:19 - r - INFO: - Episode: 144/200, Reward: 182.000, Step: 182 +2023-03-05 11:30:20 - r - INFO: - Episode: 145/200, Reward: 117.000, Step: 117 +2023-03-05 11:30:22 - r - INFO: - Episode: 146/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:24 - r - INFO: - Episode: 147/200, Reward: 149.000, Step: 149 +2023-03-05 11:30:25 - r - INFO: - Episode: 148/200, Reward: 158.000, Step: 158 +2023-03-05 11:30:27 - r - INFO: - Episode: 149/200, Reward: 196.000, Step: 196 +2023-03-05 11:30:29 - r - INFO: - Episode: 150/200, Reward: 198.000, Step: 198 +2023-03-05 11:30:31 - r - INFO: - Episode: 151/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:33 - r - INFO: - Episode: 152/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:34 - r - INFO: - Episode: 153/200, Reward: 198.000, Step: 198 +2023-03-05 11:30:36 - r - INFO: - Episode: 154/200, Reward: 176.000, Step: 176 +2023-03-05 11:30:38 - r - INFO: - Episode: 155/200, Reward: 200.000, Step: 200 +2023-03-05 11:30:40 - r - INFO: - Episode: 156/200, Reward: 195.000, Step: 195 +2023-03-05 11:30:41 - r - INFO: - Episode: 157/200, Reward: 168.000, Step: 168 +2023-03-05 11:30:42 - r - INFO: - Episode: 158/200, Reward: 167.000, Step: 167 +2023-03-05 11:30:44 - r - INFO: - Episode: 159/200, Reward: 195.000, Step: 195 +2023-03-05 11:30:46 - r - INFO: - Episode: 160/200, Reward: 196.000, Step: 196 +2023-03-05 11:30:48 - r - INFO: - Episode: 161/200, Reward: 195.000, Step: 195 +2023-03-05 11:30:49 - r - INFO: - Episode: 162/200, Reward: 172.000, Step: 172 +2023-03-05 11:30:51 - r - INFO: - Episode: 163/200, Reward: 142.000, Step: 142 +2023-03-05 11:30:52 - r - INFO: - Episode: 164/200, Reward: 167.000, Step: 167 +2023-03-05 11:30:54 - r - INFO: - Episode: 165/200, Reward: 191.000, Step: 191 +2023-03-05 11:30:56 - r - INFO: - Episode: 166/200, Reward: 173.000, Step: 173 +2023-03-05 11:30:57 - r - INFO: - Episode: 167/200, Reward: 181.000, Step: 181 +2023-03-05 11:30:59 - r - INFO: - Episode: 168/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:01 - r - INFO: - Episode: 169/200, Reward: 180.000, Step: 180 +2023-03-05 11:31:02 - r - INFO: - Episode: 170/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:03 - r - INFO: - Current episode 170 has the best eval reward: 198.500 +2023-03-05 11:31:04 - r - INFO: - Episode: 171/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:06 - r - INFO: - Episode: 172/200, Reward: 174.000, Step: 174 +2023-03-05 11:31:07 - r - INFO: - Episode: 173/200, Reward: 189.000, Step: 189 +2023-03-05 11:31:09 - r - INFO: - Episode: 174/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:11 - r - INFO: - Episode: 175/200, Reward: 195.000, Step: 195 +2023-03-05 11:31:13 - r - INFO: - Episode: 176/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:15 - r - INFO: - Episode: 177/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:16 - r - INFO: - Episode: 178/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:18 - r - INFO: - Episode: 179/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:20 - r - INFO: - Episode: 180/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:22 - r - INFO: - Episode: 181/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:24 - r - INFO: - Episode: 182/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:25 - r - INFO: - Episode: 183/200, Reward: 184.000, Step: 184 +2023-03-05 11:31:27 - r - INFO: - Episode: 184/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:29 - r - INFO: - Episode: 185/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:29 - r - INFO: - Current episode 185 has the best eval reward: 200.000 +2023-03-05 11:31:31 - r - INFO: - Episode: 186/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:33 - r - INFO: - Episode: 187/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:34 - r - INFO: - Episode: 188/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:36 - r - INFO: - Episode: 189/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:38 - r - INFO: - Episode: 190/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:40 - r - INFO: - Episode: 191/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:42 - r - INFO: - Episode: 192/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:44 - r - INFO: - Episode: 193/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:45 - r - INFO: - Episode: 194/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:47 - r - INFO: - Episode: 195/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:47 - r - INFO: - Current episode 195 has the best eval reward: 200.000 +2023-03-05 11:31:49 - r - INFO: - Episode: 196/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:51 - r - INFO: - Episode: 197/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:53 - r - INFO: - Episode: 198/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:55 - r - INFO: - Episode: 199/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:56 - r - INFO: - Episode: 200/200, Reward: 200.000, Step: 200 +2023-03-05 11:31:57 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..c65db22c12e78156882175497b41be1c6ebff0aa --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360f671a85e43af869fcdcf35133cbea2f63d6c1c609967d3f3b457962e16f73 +size 2993695 diff --git a/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..4530541b985d5cc15c9922bfdf40d9ec045c4433 Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/results/res.csv b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..0b730e08f56c93e433cb9169a97e64c4b419d80f --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_SAC_D_20230305-112849/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,17.0,17 +1,77.0,77 +2,33.0,33 +3,25.0,25 +4,49.0,49 +5,31.0,31 +6,35.0,35 +7,15.0,15 +8,10.0,10 +9,11.0,11 +10,16.0,16 +11,10.0,10 +12,16.0,16 +13,8.0,8 +14,11.0,11 +15,14.0,14 +16,11.0,11 +17,10.0,10 +18,14.0,14 +19,14.0,14 +20,15.0,15 +21,9.0,9 +22,13.0,13 +23,12.0,12 +24,14.0,14 +25,9.0,9 +26,10.0,10 +27,10.0,10 +28,12.0,12 +29,9.0,9 +30,9.0,9 +31,9.0,9 +32,10.0,10 +33,9.0,9 +34,9.0,9 +35,9.0,9 +36,10.0,10 +37,9.0,9 +38,9.0,9 +39,12.0,12 +40,10.0,10 +41,9.0,9 +42,10.0,10 +43,10.0,10 +44,12.0,12 +45,12.0,12 +46,11.0,11 +47,13.0,13 +48,10.0,10 +49,12.0,12 +50,12.0,12 +51,24.0,24 +52,68.0,68 +53,12.0,12 +54,16.0,16 +55,83.0,83 +56,122.0,122 +57,28.0,28 +58,55.0,55 +59,32.0,32 +60,41.0,41 +61,33.0,33 +62,21.0,21 +63,19.0,19 +64,19.0,19 +65,27.0,27 +66,30.0,30 +67,25.0,25 +68,23.0,23 +69,27.0,27 +70,26.0,26 +71,26.0,26 +72,33.0,33 +73,43.0,43 +74,26.0,26 +75,25.0,25 +76,23.0,23 +77,41.0,41 +78,36.0,36 +79,36.0,36 +80,31.0,31 +81,27.0,27 +82,27.0,27 +83,28.0,28 +84,27.0,27 +85,29.0,29 +86,25.0,25 +87,59.0,59 +88,30.0,30 +89,91.0,91 +90,33.0,33 +91,77.0,77 +92,42.0,42 +93,57.0,57 +94,75.0,75 +95,55.0,55 +96,42.0,42 +97,43.0,43 +98,92.0,92 +99,41.0,41 +100,72.0,72 +101,200.0,200 +102,115.0,115 +103,90.0,90 +104,97.0,97 +105,143.0,143 +106,109.0,109 +107,200.0,200 +108,168.0,168 +109,158.0,158 +110,200.0,200 +111,133.0,133 +112,123.0,123 +113,135.0,135 +114,133.0,133 +115,136.0,136 +116,141.0,141 +117,120.0,120 +118,156.0,156 +119,200.0,200 +120,174.0,174 +121,158.0,158 +122,124.0,124 +123,200.0,200 +124,162.0,162 +125,200.0,200 +126,141.0,141 +127,192.0,192 +128,176.0,176 +129,127.0,127 +130,200.0,200 +131,200.0,200 +132,200.0,200 +133,150.0,150 +134,139.0,139 +135,171.0,171 +136,200.0,200 +137,179.0,179 +138,200.0,200 +139,187.0,187 +140,200.0,200 +141,170.0,170 +142,200.0,200 +143,182.0,182 +144,117.0,117 +145,200.0,200 +146,149.0,149 +147,158.0,158 +148,196.0,196 +149,198.0,198 +150,200.0,200 +151,200.0,200 +152,198.0,198 +153,176.0,176 +154,200.0,200 +155,195.0,195 +156,168.0,168 +157,167.0,167 +158,195.0,195 +159,196.0,196 +160,195.0,195 +161,172.0,172 +162,142.0,142 +163,167.0,167 +164,191.0,191 +165,173.0,173 +166,181.0,181 +167,200.0,200 +168,180.0,180 +169,200.0,200 +170,200.0,200 +171,174.0,174 +172,189.0,189 +173,200.0,200 +174,195.0,195 +175,200.0,200 +176,200.0,200 +177,200.0,200 +178,200.0,200 +179,200.0,200 +180,200.0,200 +181,200.0,200 +182,184.0,184 +183,200.0,200 +184,200.0,200 +185,200.0,200 +186,200.0,200 +187,200.0,200 +188,200.0,200 +189,200.0,200 +190,200.0,200 +191,200.0,200 +192,200.0,200 +193,200.0,200 +194,200.0,200 +195,200.0,200 +196,200.0,200 +197,200.0,200 +198,200.0,200 +199,200.0,200 diff --git a/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/config.yaml b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad167255a7b40391b7251467edd7d623fdbc1afc --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/config.yaml @@ -0,0 +1,44 @@ +general_cfg: + algo_name: DQN + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + n_workers: 4 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 300 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/logs/log.txt b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a6cd2150a32186c13b458fb69f5a632fd3e8d34 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/logs/log.txt @@ -0,0 +1,40 @@ +2023-03-19 21:53:40 - r - INFO: - Hyperparameters: +2023-03-19 21:53:40 - r - INFO: - ================================================================================ +2023-03-19 21:53:40 - r - INFO: - Name Value Type +2023-03-19 21:53:40 - r - INFO: - env_name CartPole-v1 +2023-03-19 21:53:40 - r - INFO: - new_step_api 1 +2023-03-19 21:53:40 - r - INFO: - wrapper None +2023-03-19 21:53:40 - r - INFO: - render 0 +2023-03-19 21:53:40 - r - INFO: - algo_name DQN +2023-03-19 21:53:40 - r - INFO: - mode train +2023-03-19 21:53:40 - r - INFO: - seed 1 +2023-03-19 21:53:40 - r - INFO: - device cpu +2023-03-19 21:53:40 - r - INFO: - train_eps 300 +2023-03-19 21:53:40 - r - INFO: - test_eps 10 +2023-03-19 21:53:40 - r - INFO: - eval_eps 10 +2023-03-19 21:53:40 - r - INFO: - eval_per_episode 5 +2023-03-19 21:53:40 - r - INFO: - max_steps 200 +2023-03-19 21:53:40 - r - INFO: - load_checkpoint 0 +2023-03-19 21:53:40 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-03-19 21:53:40 - r - INFO: - show_fig 0 +2023-03-19 21:53:40 - r - INFO: - save_fig 1 +2023-03-19 21:53:40 - r - INFO: - n_workers 4 +2023-03-19 21:53:40 - r - INFO: - epsilon_start 0.95 +2023-03-19 21:53:40 - r - INFO: - epsilon_end 0.01 +2023-03-19 21:53:40 - r - INFO: - epsilon_decay 500 +2023-03-19 21:53:40 - r - INFO: - gamma 0.95 +2023-03-19 21:53:40 - r - INFO: - lr 0.0001 +2023-03-19 21:53:40 - r - INFO: - buffer_size 100000 +2023-03-19 21:53:40 - r - INFO: - batch_size 64 +2023-03-19 21:53:40 - r - INFO: - target_update 4 +2023-03-19 21:53:40 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-03-19 21:53:40 - r - INFO: - task_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DQN_20230319-215340 +2023-03-19 21:53:40 - r - INFO: - res_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DQN_20230319-215340/results +2023-03-19 21:53:40 - r - INFO: - log_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DQN_20230319-215340/logs +2023-03-19 21:53:40 - r - INFO: - traj_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DQN_20230319-215340/traj +2023-03-19 21:53:40 - r - INFO: - tb_dir /media/disk/gsc/joyrl-offline/tasks/Train_CartPole-v1_DQN_20230319-215340/tb_logs +2023-03-19 21:53:40 - r - INFO: - ================================================================================ +2023-03-19 21:53:40 - r - INFO: - n_states: 4, n_actions: 2 +2023-03-19 21:53:40 - r - INFO: - Start training! +2023-03-19 21:53:40 - r - INFO: - Env: CartPole-v1, Algorithm: DQN, Device: cpu +2023-03-19 21:54:55 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/models/checkpoint.pt b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..267ffe076c7546d055bcebe5928f2f549f8db151 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b3cee4549ed1d3a2f559c0732f710074f472a5ebb79703c20241e2bdecac58a +size 272407 diff --git a/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..605bb0f2cd5e48ba25496b7c9f48067c3eb007ef Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/results/res.csv b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..53f039ab1e89155f531b43df4b28b2f8204b89f4 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/results/res.csv @@ -0,0 +1,302 @@ +episodes,rewards +0,18.0 +1,39.0 +2,12.0 +3,17.0 +4,16.0 +5,16.0 +6,14.0 +7,24.0 +8,18.0 +9,28.0 +10,15.0 +11,40.0 +12,37.0 +13,10.0 +14,15.0 +15,11.0 +16,17.0 +17,23.0 +18,31.0 +19,17.0 +20,30.0 +21,10.0 +22,13.0 +23,44.0 +24,31.0 +25,20.0 +26,11.0 +27,11.0 +28,14.0 +29,13.0 +30,20.0 +31,11.0 +32,21.0 +33,12.0 +34,8.0 +35,10.0 +36,12.0 +37,13.0 +38,15.0 +39,11.0 +40,24.0 +41,19.0 +42,13.0 +43,11.0 +44,13.0 +45,15.0 +46,12.0 +47,23.0 +48,31.0 +49,17.0 +50,13.0 +51,19.0 +52,20.0 +53,25.0 +54,30.0 +55,12.0 +56,15.0 +57,15.0 +58,11.0 +59,10.0 +60,35.0 +61,44.0 +62,57.0 +63,47.0 +64,29.0 +65,45.0 +66,46.0 +67,31.0 +68,42.0 +69,84.0 +70,29.0 +71,45.0 +72,27.0 +73,38.0 +74,28.0 +75,49.0 +76,44.0 +77,37.0 +78,67.0 +79,45.0 +80,46.0 +81,87.0 +82,45.0 +83,57.0 +84,51.0 +85,87.0 +86,44.0 +87,119.0 +88,133.0 +89,51.0 +90,200.0 +91,75.0 +92,141.0 +93,200.0 +94,200.0 +95,200.0 +96,200.0 +97,200.0 +98,200.0 +99,200.0 +100,169.0 +101,200.0 +102,200.0 +103,200.0 +104,189.0 +105,200.0 +106,200.0 +107,200.0 +108,200.0 +109,200.0 +110,200.0 +111,200.0 +112,200.0 +113,200.0 +114,200.0 +115,200.0 +116,200.0 +117,200.0 +118,200.0 +119,190.0 +120,200.0 +121,200.0 +122,200.0 +123,200.0 +124,200.0 +125,200.0 +126,200.0 +127,200.0 +128,200.0 +129,200.0 +130,200.0 +131,200.0 +132,200.0 +133,200.0 +134,200.0 +135,200.0 +136,200.0 +137,200.0 +138,200.0 +139,200.0 +140,200.0 +141,200.0 +142,200.0 +143,200.0 +144,200.0 +145,200.0 +146,200.0 +147,200.0 +148,200.0 +149,200.0 +150,200.0 +151,200.0 +152,200.0 +153,200.0 +154,200.0 +155,200.0 +156,200.0 +157,200.0 +158,200.0 +159,200.0 +160,200.0 +161,200.0 +162,200.0 +163,200.0 +164,200.0 +165,200.0 +166,200.0 +167,200.0 +168,200.0 +169,200.0 +170,200.0 +171,200.0 +172,200.0 +173,200.0 +174,200.0 +175,200.0 +176,200.0 +177,200.0 +178,200.0 +179,200.0 +180,200.0 +181,200.0 +182,200.0 +183,200.0 +184,200.0 +185,200.0 +186,200.0 +187,200.0 +188,200.0 +189,200.0 +190,200.0 +191,200.0 +192,200.0 +193,200.0 +194,200.0 +195,200.0 +196,200.0 +197,200.0 +198,200.0 +199,200.0 +200,200.0 +201,200.0 +202,200.0 +203,200.0 +204,200.0 +205,200.0 +206,200.0 +207,200.0 +208,200.0 +209,200.0 +210,200.0 +211,200.0 +212,200.0 +213,200.0 +214,200.0 +215,200.0 +216,200.0 +217,192.0 +218,198.0 +219,200.0 +220,196.0 +221,184.0 +222,200.0 +223,162.0 +224,170.0 +225,179.0 +226,162.0 +227,200.0 +228,175.0 +229,179.0 +230,167.0 +231,200.0 +232,170.0 +233,174.0 +234,178.0 +235,184.0 +236,200.0 +237,193.0 +238,183.0 +239,180.0 +240,169.0 +241,200.0 +242,147.0 +243,191.0 +244,192.0 +245,200.0 +246,161.0 +247,180.0 +248,200.0 +249,200.0 +250,197.0 +251,196.0 +252,200.0 +253,193.0 +254,198.0 +255,200.0 +256,192.0 +257,200.0 +258,187.0 +259,200.0 +260,200.0 +261,200.0 +262,200.0 +263,130.0 +264,200.0 +265,200.0 +266,200.0 +267,200.0 +268,200.0 +269,200.0 +270,200.0 +271,197.0 +272,200.0 +273,200.0 +274,200.0 +275,200.0 +276,200.0 +277,200.0 +278,200.0 +279,200.0 +280,200.0 +281,200.0 +282,200.0 +283,200.0 +284,200.0 +285,200.0 +286,200.0 +287,200.0 +288,200.0 +289,200.0 +290,200.0 +291,200.0 +292,200.0 +293,200.0 +294,200.0 +295,200.0 +296,200.0 +297,200.0 +298,200.0 +299,200.0 +300,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/tb_logs/events.out.tfevents.1679234020.ML3090.3787858.0 b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/tb_logs/events.out.tfevents.1679234020.ML3090.3787858.0 new file mode 100644 index 0000000000000000000000000000000000000000..448dd0d541d12a201bebc197a08df695ce1b8ff3 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_DQN_20230319-215340/tb_logs/events.out.tfevents.1679234020.ML3090.3787858.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2ae77d9e4778bd5611b745cf3f3ace7aa109c583e0ece6b3c184bc08dcdaa4 +size 40 diff --git a/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/config.yaml b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22dd6c08264765ed801605302d2b214baeb4d341 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: PPO + device: cpu + env_name: CartPole-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + n_workers: 3 + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 300 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + ppo_type: clip + sgd_batch_size: 128 + train_batch_size: 256 diff --git a/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/logs/log.txt b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..dad04efe89481d5ad679a4935eed4e6c3b15c070 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/logs/log.txt @@ -0,0 +1,43 @@ +2023-04-01 22:32:04 - r - INFO: - Hyperparameters: +2023-04-01 22:32:04 - r - INFO: - ================================================================================ +2023-04-01 22:32:04 - r - INFO: - Name Value Type +2023-04-01 22:32:04 - r - INFO: - env_name CartPole-v1 +2023-04-01 22:32:04 - r - INFO: - new_step_api 1 +2023-04-01 22:32:04 - r - INFO: - wrapper None +2023-04-01 22:32:04 - r - INFO: - render 0 +2023-04-01 22:32:04 - r - INFO: - algo_name PPO +2023-04-01 22:32:04 - r - INFO: - mode train +2023-04-01 22:32:04 - r - INFO: - seed 1 +2023-04-01 22:32:04 - r - INFO: - device cpu +2023-04-01 22:32:04 - r - INFO: - train_eps 300 +2023-04-01 22:32:04 - r - INFO: - test_eps 10 +2023-04-01 22:32:04 - r - INFO: - eval_eps 10 +2023-04-01 22:32:04 - r - INFO: - eval_per_episode 5 +2023-04-01 22:32:04 - r - INFO: - max_steps 200 +2023-04-01 22:32:04 - r - INFO: - load_checkpoint 0 +2023-04-01 22:32:04 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-01 22:32:04 - r - INFO: - show_fig 0 +2023-04-01 22:32:04 - r - INFO: - save_fig 1 +2023-04-01 22:32:04 - r - INFO: - n_workers 3 +2023-04-01 22:32:04 - r - INFO: - ppo_type clip +2023-04-01 22:32:04 - r - INFO: - continuous 0 +2023-04-01 22:32:04 - r - INFO: - gamma 0.99 +2023-04-01 22:32:04 - r - INFO: - k_epochs 4 +2023-04-01 22:32:04 - r - INFO: - actor_lr 0.0003 +2023-04-01 22:32:04 - r - INFO: - critic_lr 0.001 +2023-04-01 22:32:04 - r - INFO: - eps_clip 0.2 +2023-04-01 22:32:04 - r - INFO: - entropy_coef 0.01 +2023-04-01 22:32:04 - r - INFO: - train_batch_size 256 +2023-04-01 22:32:04 - r - INFO: - sgd_batch_size 128 +2023-04-01 22:32:04 - r - INFO: - actor_hidden_dim 256 +2023-04-01 22:32:04 - r - INFO: - critic_hidden_dim 256 +2023-04-01 22:32:04 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_mp_PPO_20230401-223204 +2023-04-01 22:32:04 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_mp_PPO_20230401-223204/results +2023-04-01 22:32:04 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_mp_PPO_20230401-223204/logs +2023-04-01 22:32:04 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_mp_PPO_20230401-223204/traj +2023-04-01 22:32:04 - r - INFO: - tb_dir /home/dingli/joyrl_offline/tasks/Train_CartPole-v1_mp_PPO_20230401-223204/tb_logs +2023-04-01 22:32:04 - r - INFO: - ================================================================================ +2023-04-01 22:32:04 - r - INFO: - n_states: 4, n_actions: 2 +2023-04-01 22:32:04 - r - INFO: - Start training! +2023-04-01 22:32:04 - r - INFO: - Env: CartPole-v1, Algorithm: PPO, Device: cpu +2023-04-01 22:33:16 - r - INFO: - Finish training! diff --git a/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/models/actor.pth b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..a22cff925f29e43f1ef84a9742bcc9a77b29ce68 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a3e48d551bcba327ff4c5d3cc464a6a94b83eda543a54d231016e021e8cbd3 +size 272151 diff --git a/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/models/critic.pth b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..10365bebec659f3c79f3521957f2814d8dfe1646 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4db7aeb3805e1deb11428a34a600a40068a0f711986f38fdf9e0f9895f8a45c +size 271127 diff --git a/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/results/learning_curve.png b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..32f2a0bb0bcdc361436a60d6b04d05927ed00adc Binary files /dev/null and b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/results/learning_curve.png differ diff --git a/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/results/res.csv b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..8c201553dc70b95f45d3774918fa217aa53d0113 --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/results/res.csv @@ -0,0 +1,302 @@ +episodes,rewards +0,39.0 +1,10.0 +2,11.0 +3,10.0 +4,30.0 +5,28.0 +6,9.0 +7,12.0 +8,13.0 +9,16.0 +10,15.0 +11,16.0 +12,38.0 +13,33.0 +14,11.0 +15,20.0 +16,14.0 +17,43.0 +18,22.0 +19,11.0 +20,34.0 +21,12.0 +22,29.0 +23,12.0 +24,39.0 +25,11.0 +26,13.0 +27,25.0 +28,26.0 +29,36.0 +30,24.0 +31,18.0 +32,40.0 +33,13.0 +34,30.0 +35,24.0 +36,22.0 +37,10.0 +38,32.0 +39,40.0 +40,15.0 +41,57.0 +42,45.0 +43,47.0 +44,38.0 +45,53.0 +46,13.0 +47,26.0 +48,19.0 +49,16.0 +50,30.0 +51,12.0 +52,18.0 +53,62.0 +54,16.0 +55,46.0 +56,40.0 +57,25.0 +58,41.0 +59,82.0 +60,18.0 +61,29.0 +62,43.0 +63,26.0 +64,28.0 +65,17.0 +66,48.0 +67,31.0 +68,35.0 +69,31.0 +70,43.0 +71,49.0 +72,25.0 +73,65.0 +74,16.0 +75,37.0 +76,44.0 +77,37.0 +78,16.0 +79,73.0 +80,23.0 +81,34.0 +82,20.0 +83,55.0 +84,18.0 +85,16.0 +86,56.0 +87,22.0 +88,40.0 +89,40.0 +90,63.0 +91,52.0 +92,43.0 +93,38.0 +94,34.0 +95,61.0 +96,34.0 +97,32.0 +98,47.0 +99,21.0 +100,24.0 +101,54.0 +102,52.0 +103,13.0 +104,26.0 +105,64.0 +106,63.0 +107,88.0 +108,65.0 +109,114.0 +110,84.0 +111,58.0 +112,148.0 +113,32.0 +114,85.0 +115,67.0 +116,45.0 +117,170.0 +118,137.0 +119,159.0 +120,60.0 +121,200.0 +122,78.0 +123,58.0 +124,60.0 +125,191.0 +126,61.0 +127,99.0 +128,26.0 +129,79.0 +130,109.0 +131,173.0 +132,61.0 +133,92.0 +134,175.0 +135,106.0 +136,32.0 +137,84.0 +138,105.0 +139,78.0 +140,78.0 +141,60.0 +142,93.0 +143,69.0 +144,85.0 +145,106.0 +146,148.0 +147,93.0 +148,119.0 +149,139.0 +150,124.0 +151,173.0 +152,200.0 +153,26.0 +154,151.0 +155,122.0 +156,182.0 +157,62.0 +158,38.0 +159,145.0 +160,75.0 +161,78.0 +162,200.0 +163,189.0 +164,94.0 +165,122.0 +166,95.0 +167,200.0 +168,154.0 +169,200.0 +170,200.0 +171,125.0 +172,200.0 +173,187.0 +174,165.0 +175,154.0 +176,200.0 +177,200.0 +178,180.0 +179,200.0 +180,191.0 +181,168.0 +182,200.0 +183,78.0 +184,124.0 +185,137.0 +186,165.0 +187,77.0 +188,144.0 +189,200.0 +190,74.0 +191,200.0 +192,99.0 +193,183.0 +194,200.0 +195,49.0 +196,127.0 +197,148.0 +198,188.0 +199,200.0 +200,113.0 +201,200.0 +202,146.0 +203,130.0 +204,128.0 +205,200.0 +206,35.0 +207,58.0 +208,130.0 +209,66.0 +210,62.0 +211,163.0 +212,147.0 +213,106.0 +214,200.0 +215,114.0 +216,147.0 +217,101.0 +218,175.0 +219,88.0 +220,144.0 +221,200.0 +222,200.0 +223,199.0 +224,200.0 +225,199.0 +226,142.0 +227,200.0 +228,200.0 +229,164.0 +230,200.0 +231,181.0 +232,200.0 +233,200.0 +234,200.0 +235,105.0 +236,200.0 +237,200.0 +238,200.0 +239,169.0 +240,200.0 +241,179.0 +242,108.0 +243,177.0 +244,132.0 +245,116.0 +246,75.0 +247,161.0 +248,192.0 +249,184.0 +250,200.0 +251,115.0 +252,200.0 +253,200.0 +254,129.0 +255,40.0 +256,80.0 +257,190.0 +258,200.0 +259,200.0 +260,151.0 +261,138.0 +262,187.0 +263,163.0 +264,101.0 +265,116.0 +266,147.0 +267,80.0 +268,200.0 +269,104.0 +270,200.0 +271,176.0 +272,157.0 +273,200.0 +274,200.0 +275,125.0 +276,77.0 +277,200.0 +278,200.0 +279,200.0 +280,200.0 +281,200.0 +282,174.0 +283,200.0 +284,200.0 +285,25.0 +286,150.0 +287,200.0 +288,200.0 +289,200.0 +290,163.0 +291,200.0 +292,200.0 +293,200.0 +294,200.0 +295,200.0 +296,200.0 +297,123.0 +298,154.0 +299,200.0 +300,200.0 diff --git a/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/tb_logs/events.out.tfevents.1680359524.dell-Precision-5820-Tower.31414.0 b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/tb_logs/events.out.tfevents.1680359524.dell-Precision-5820-Tower.31414.0 new file mode 100644 index 0000000000000000000000000000000000000000..4662940b7bbdcd853f7ce16dd77c55ba27de7d8f --- /dev/null +++ b/CartPole-v1/Train_CartPole-v1_mp_PPO_20230401-223204/tb_logs/events.out.tfevents.1680359524.dell-Precision-5820-Tower.31414.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cfeaaafbbeba1e7f3b3b9e7741cd5d212f6edfa464aa67711ba39abad513a3b +size 40 diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/config.yaml b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3f0e0accd86f07ed207f4f0648230c7befd8f0d --- /dev/null +++ b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/config.yaml @@ -0,0 +1,45 @@ +general_cfg: + algo_name: DQN + collect_traj: true + device: cpu + env_name: gym + load_checkpoint: false + load_model_step: best + load_path: Train_single_CartPole-v1_DQN_20230515-211721 + max_episode: 100 + max_step: 200 + mode: train + model_save_fre: 500 + mp_backend: ray + n_workers: 2 + online_eval: true + online_eval_episode: 10 + save_fig: true + seed: 1 + show_fig: false +algo_cfg: + batch_size: 64 + buffer_size: 100000 + buffer_type: REPLAY_QUE + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + layer_type: linear +env_cfg: + id: CartPole-v1 + ignore_params: + - wrapper + - ignore_params + render_mode: null + wrapper: null diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/logs/log.txt b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6ec72747bff623ab2fda248b2b2deaceb58ba26 --- /dev/null +++ b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/logs/log.txt @@ -0,0 +1,166 @@ +2023-05-15 22:19:16 - SimpleLog - INFO: - General Configs: +2023-05-15 22:19:16 - SimpleLog - INFO: - ================================================================================ +2023-05-15 22:19:16 - SimpleLog - INFO: - Name Value Type +2023-05-15 22:19:16 - SimpleLog - INFO: - env_name gym +2023-05-15 22:19:16 - SimpleLog - INFO: - algo_name DQN +2023-05-15 22:19:16 - SimpleLog - INFO: - mode train +2023-05-15 22:19:16 - SimpleLog - INFO: - collect_traj 1 +2023-05-15 22:19:16 - SimpleLog - INFO: - mp_backend ray +2023-05-15 22:19:16 - SimpleLog - INFO: - n_workers 2 +2023-05-15 22:19:16 - SimpleLog - INFO: - seed 1 +2023-05-15 22:19:16 - SimpleLog - INFO: - device cpu +2023-05-15 22:19:16 - SimpleLog - INFO: - max_episode 100 +2023-05-15 22:19:16 - SimpleLog - INFO: - max_step 200 +2023-05-15 22:19:16 - SimpleLog - INFO: - online_eval 1 +2023-05-15 22:19:16 - SimpleLog - INFO: - online_eval_episode 10 +2023-05-15 22:19:16 - SimpleLog - INFO: - load_checkpoint 0 +2023-05-15 22:19:16 - SimpleLog - INFO: - load_path Train_single_CartPole-v1_DQN_20230515-211721 +2023-05-15 22:19:16 - SimpleLog - INFO: - show_fig 0 +2023-05-15 22:19:16 - SimpleLog - INFO: - save_fig 1 +2023-05-15 22:19:16 - SimpleLog - INFO: - load_model_step best +2023-05-15 22:19:16 - SimpleLog - INFO: - model_save_fre 500 +2023-05-15 22:19:16 - SimpleLog - INFO: - ================================================================================ +2023-05-15 22:19:16 - SimpleLog - INFO: - Algo Configs: +2023-05-15 22:19:16 - SimpleLog - INFO: - ================================================================================ +2023-05-15 22:19:16 - SimpleLog - INFO: - Name Value Type +2023-05-15 22:19:16 - SimpleLog - INFO: - epsilon_start 0.95 +2023-05-15 22:19:16 - SimpleLog - INFO: - epsilon_end 0.01 +2023-05-15 22:19:16 - SimpleLog - INFO: - epsilon_decay 500 +2023-05-15 22:19:16 - SimpleLog - INFO: - gamma 0.95 +2023-05-15 22:19:16 - SimpleLog - INFO: - lr 0.0001 +2023-05-15 22:19:16 - SimpleLog - INFO: - buffer_size 100000 +2023-05-15 22:19:16 - SimpleLog - INFO: - batch_size 64 +2023-05-15 22:19:16 - SimpleLog - INFO: - target_update 4 +2023-05-15 22:19:16 - SimpleLog - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': [256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256], 'activation': 'relu'}] +2023-05-15 22:19:16 - SimpleLog - INFO: - buffer_type REPLAY_QUE +2023-05-15 22:19:16 - SimpleLog - INFO: - ================================================================================ +2023-05-15 22:19:16 - SimpleLog - INFO: - Env Configs: +2023-05-15 22:19:16 - SimpleLog - INFO: - ================================================================================ +2023-05-15 22:19:16 - SimpleLog - INFO: - Name Value Type +2023-05-15 22:19:16 - SimpleLog - INFO: - id CartPole-v1 +2023-05-15 22:19:16 - SimpleLog - INFO: - render_mode None +2023-05-15 22:19:16 - SimpleLog - INFO: - wrapper None +2023-05-15 22:19:16 - SimpleLog - INFO: - ignore_params ['wrapper', 'ignore_params'] +2023-05-15 22:19:16 - SimpleLog - INFO: - ================================================================================ +2023-05-15 22:19:21 - SimpleLog - INFO: - obs_space: Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32), n_actions: Discrete(2) +2023-05-15 22:19:24 - RayLog - INFO: - Worker 0 finished episode 0 with reward 12.0 in 12 steps +2023-05-15 22:19:24 - RayLog - INFO: - Worker 1 finished episode 0 with reward 22.0 in 22 steps +2023-05-15 22:19:24 - RayLog - INFO: - Worker 0 finished episode 1 with reward 21.0 in 21 steps +2023-05-15 22:19:25 - RayLog - INFO: - Worker 0 finished episode 3 with reward 18.0 in 18 steps +2023-05-15 22:19:25 - RayLog - INFO: - Worker 1 finished episode 2 with reward 32.0 in 32 steps +2023-05-15 22:19:25 - RayLog - INFO: - Worker 1 finished episode 5 with reward 13.0 in 13 steps +2023-05-15 22:19:25 - RayLog - INFO: - Worker 0 finished episode 4 with reward 23.0 in 23 steps +2023-05-15 22:19:25 - RayLog - INFO: - Worker 1 finished episode 6 with reward 9.0 in 9 steps +2023-05-15 22:19:25 - RayLog - INFO: - Worker 0 finished episode 7 with reward 12.0 in 12 steps +2023-05-15 22:19:25 - RayLog - INFO: - Worker 1 finished episode 8 with reward 11.0 in 11 steps +2023-05-15 22:19:26 - RayLog - INFO: - Worker 1 finished episode 10 with reward 17.0 in 17 steps +2023-05-15 22:19:26 - RayLog - INFO: - Worker 0 finished episode 9 with reward 19.0 in 19 steps +2023-05-15 22:19:26 - RayLog - INFO: - Worker 1 finished episode 11 with reward 9.0 in 9 steps +2023-05-15 22:19:26 - RayLog - INFO: - Worker 1 finished episode 13 with reward 14.0 in 14 steps +2023-05-15 22:19:26 - RayLog - INFO: - Worker 0 finished episode 12 with reward 25.0 in 25 steps +2023-05-15 22:19:26 - RayLog - INFO: - Worker 1 finished episode 14 with reward 13.0 in 13 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 0 finished episode 15 with reward 12.0 in 12 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 1 finished episode 16 with reward 13.0 in 13 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 0 finished episode 17 with reward 17.0 in 17 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 1 finished episode 18 with reward 9.0 in 9 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 0 finished episode 19 with reward 13.0 in 13 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 1 finished episode 20 with reward 11.0 in 11 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 0 finished episode 21 with reward 11.0 in 11 steps +2023-05-15 22:19:27 - RayLog - INFO: - Worker 1 finished episode 22 with reward 11.0 in 11 steps +2023-05-15 22:19:28 - RayLog - INFO: - Worker 1 finished episode 24 with reward 11.0 in 11 steps +2023-05-15 22:19:28 - RayLog - INFO: - Worker 1 finished episode 25 with reward 9.0 in 9 steps +2023-05-15 22:19:28 - RayLog - INFO: - Worker 0 finished episode 23 with reward 23.0 in 23 steps +2023-05-15 22:19:29 - RayLog - INFO: - Worker 1 finished episode 26 with reward 15.0 in 15 steps +2023-05-15 22:19:29 - RayLog - INFO: - Worker 1 finished episode 28 with reward 16.0 in 16 steps +2023-05-15 22:19:29 - RayLog - INFO: - Worker 0 finished episode 27 with reward 38.0 in 38 steps +2023-05-15 22:19:29 - RayLog - INFO: - Worker 1 finished episode 29 with reward 11.0 in 11 steps +2023-05-15 22:19:29 - RayLog - INFO: - Worker 0 finished episode 30 with reward 13.0 in 13 steps +2023-05-15 22:19:29 - RayLog - INFO: - Worker 1 finished episode 31 with reward 12.0 in 12 steps +2023-05-15 22:19:30 - RayLog - INFO: - Worker 1 finished episode 33 with reward 12.0 in 12 steps +2023-05-15 22:19:30 - RayLog - INFO: - Worker 0 finished episode 32 with reward 14.0 in 14 steps +2023-05-15 22:19:30 - RayLog - INFO: - Worker 0 finished episode 35 with reward 9.0 in 9 steps +2023-05-15 22:19:30 - RayLog - INFO: - Worker 1 finished episode 34 with reward 11.0 in 11 steps +2023-05-15 22:19:32 - RayLog - INFO: - update_step: 500, online_eval_reward: 9.000 +2023-05-15 22:19:32 - RayLog - INFO: - current update step obtain a better online_eval_reward: 9.000, save the best model! +2023-05-15 22:19:32 - RayLog - INFO: - Worker 0 finished episode 36 with reward 11.0 in 11 steps +2023-05-15 22:19:32 - RayLog - INFO: - Worker 1 finished episode 37 with reward 14.0 in 14 steps +2023-05-15 22:19:32 - RayLog - INFO: - Worker 0 finished episode 38 with reward 12.0 in 12 steps +2023-05-15 22:19:32 - RayLog - INFO: - Worker 1 finished episode 39 with reward 13.0 in 13 steps +2023-05-15 22:19:32 - RayLog - INFO: - Worker 0 finished episode 40 with reward 11.0 in 11 steps +2023-05-15 22:19:32 - RayLog - INFO: - Worker 1 finished episode 41 with reward 10.0 in 10 steps +2023-05-15 22:19:33 - RayLog - INFO: - Worker 0 finished episode 42 with reward 11.0 in 11 steps +2023-05-15 22:19:33 - RayLog - INFO: - Worker 1 finished episode 43 with reward 10.0 in 10 steps +2023-05-15 22:19:33 - RayLog - INFO: - Worker 0 finished episode 44 with reward 10.0 in 10 steps +2023-05-15 22:19:33 - RayLog - INFO: - Worker 1 finished episode 45 with reward 21.0 in 21 steps +2023-05-15 22:19:34 - RayLog - INFO: - Worker 0 finished episode 46 with reward 36.0 in 36 steps +2023-05-15 22:19:34 - RayLog - INFO: - Worker 1 finished episode 47 with reward 30.0 in 30 steps +2023-05-15 22:19:34 - RayLog - INFO: - Worker 1 finished episode 49 with reward 19.0 in 19 steps +2023-05-15 22:19:34 - RayLog - INFO: - Worker 0 finished episode 48 with reward 28.0 in 28 steps +2023-05-15 22:19:35 - RayLog - INFO: - Worker 1 finished episode 50 with reward 17.0 in 17 steps +2023-05-15 22:19:35 - RayLog - INFO: - Worker 0 finished episode 51 with reward 28.0 in 28 steps +2023-05-15 22:19:35 - RayLog - INFO: - Worker 1 finished episode 52 with reward 23.0 in 23 steps +2023-05-15 22:19:36 - RayLog - INFO: - Worker 0 finished episode 53 with reward 46.0 in 46 steps +2023-05-15 22:19:36 - RayLog - INFO: - Worker 1 finished episode 54 with reward 37.0 in 37 steps +2023-05-15 22:19:36 - RayLog - INFO: - Worker 1 finished episode 56 with reward 27.0 in 27 steps +2023-05-15 22:19:37 - RayLog - INFO: - Worker 0 finished episode 55 with reward 56.0 in 56 steps +2023-05-15 22:19:37 - RayLog - INFO: - update_step: 1000, online_eval_reward: 94.000 +2023-05-15 22:19:37 - RayLog - INFO: - current update step obtain a better online_eval_reward: 94.000, save the best model! +2023-05-15 22:19:37 - RayLog - INFO: - Worker 1 finished episode 57 with reward 35.0 in 35 steps +2023-05-15 22:19:38 - RayLog - INFO: - Worker 1 finished episode 59 with reward 29.0 in 29 steps +2023-05-15 22:19:38 - RayLog - INFO: - Worker 0 finished episode 58 with reward 65.0 in 65 steps +2023-05-15 22:19:39 - RayLog - INFO: - Worker 1 finished episode 60 with reward 37.0 in 37 steps +2023-05-15 22:19:39 - RayLog - INFO: - Worker 1 finished episode 62 with reward 34.0 in 34 steps +2023-05-15 22:19:40 - RayLog - INFO: - Worker 0 finished episode 61 with reward 70.0 in 70 steps +2023-05-15 22:19:40 - RayLog - INFO: - Worker 1 finished episode 63 with reward 39.0 in 39 steps +2023-05-15 22:19:41 - RayLog - INFO: - Worker 1 finished episode 65 with reward 35.0 in 35 steps +2023-05-15 22:19:41 - RayLog - INFO: - Worker 0 finished episode 64 with reward 55.0 in 55 steps +2023-05-15 22:19:42 - RayLog - INFO: - Worker 1 finished episode 66 with reward 37.0 in 37 steps +2023-05-15 22:19:42 - RayLog - INFO: - Worker 0 finished episode 67 with reward 53.0 in 53 steps +2023-05-15 22:19:42 - RayLog - INFO: - Worker 1 finished episode 68 with reward 32.0 in 32 steps +2023-05-15 22:19:42 - RayLog - INFO: - update_step: 1500, online_eval_reward: 57.000 +2023-05-15 22:19:43 - RayLog - INFO: - Worker 1 finished episode 70 with reward 51.0 in 51 steps +2023-05-15 22:19:44 - RayLog - INFO: - Worker 0 finished episode 69 with reward 67.0 in 67 steps +2023-05-15 22:19:44 - RayLog - INFO: - Worker 1 finished episode 71 with reward 40.0 in 40 steps +2023-05-15 22:19:45 - RayLog - INFO: - Worker 0 finished episode 72 with reward 68.0 in 68 steps +2023-05-15 22:19:46 - RayLog - INFO: - Worker 1 finished episode 73 with reward 79.0 in 79 steps +2023-05-15 22:19:48 - RayLog - INFO: - update_step: 2000, online_eval_reward: 138.000 +2023-05-15 22:19:48 - RayLog - INFO: - current update step obtain a better online_eval_reward: 138.000, save the best model! +2023-05-15 22:19:48 - RayLog - INFO: - Worker 0 finished episode 74 with reward 124.0 in 124 steps +2023-05-15 22:19:49 - RayLog - INFO: - Worker 1 finished episode 75 with reward 133.0 in 133 steps +2023-05-15 22:19:52 - RayLog - INFO: - Worker 0 finished episode 76 with reward 200.0 in 200 steps +2023-05-15 22:19:53 - RayLog - INFO: - update_step: 2500, online_eval_reward: 200.000 +2023-05-15 22:19:53 - RayLog - INFO: - current update step obtain a better online_eval_reward: 200.000, save the best model! +2023-05-15 22:19:53 - RayLog - INFO: - Worker 1 finished episode 77 with reward 200.0 in 200 steps +2023-05-15 22:19:56 - RayLog - INFO: - Worker 0 finished episode 78 with reward 187.0 in 187 steps +2023-05-15 22:19:57 - RayLog - INFO: - Worker 1 finished episode 79 with reward 200.0 in 200 steps +2023-05-15 22:19:58 - RayLog - INFO: - update_step: 3000, online_eval_reward: 200.000 +2023-05-15 22:20:00 - RayLog - INFO: - Worker 0 finished episode 80 with reward 200.0 in 200 steps +2023-05-15 22:20:02 - RayLog - INFO: - Worker 1 finished episode 81 with reward 200.0 in 200 steps +2023-05-15 22:20:04 - RayLog - INFO: - update_step: 3500, online_eval_reward: 165.000 +2023-05-15 22:20:04 - RayLog - INFO: - Worker 0 finished episode 82 with reward 200.0 in 200 steps +2023-05-15 22:20:06 - RayLog - INFO: - Worker 1 finished episode 83 with reward 200.0 in 200 steps +2023-05-15 22:20:08 - RayLog - INFO: - Worker 0 finished episode 84 with reward 200.0 in 200 steps +2023-05-15 22:20:09 - RayLog - INFO: - update_step: 4000, online_eval_reward: 200.000 +2023-05-15 22:20:10 - RayLog - INFO: - Worker 1 finished episode 85 with reward 200.0 in 200 steps +2023-05-15 22:20:12 - RayLog - INFO: - Worker 0 finished episode 86 with reward 200.0 in 200 steps +2023-05-15 22:20:14 - RayLog - INFO: - update_step: 4500, online_eval_reward: 200.000 +2023-05-15 22:20:14 - RayLog - INFO: - Worker 1 finished episode 87 with reward 200.0 in 200 steps +2023-05-15 22:20:16 - RayLog - INFO: - Worker 0 finished episode 88 with reward 200.0 in 200 steps +2023-05-15 22:20:18 - RayLog - INFO: - Worker 1 finished episode 89 with reward 200.0 in 200 steps +2023-05-15 22:20:19 - RayLog - INFO: - update_step: 5000, online_eval_reward: 200.000 +2023-05-15 22:20:20 - RayLog - INFO: - Worker 0 finished episode 90 with reward 200.0 in 200 steps +2023-05-15 22:20:22 - RayLog - INFO: - Worker 1 finished episode 91 with reward 200.0 in 200 steps +2023-05-15 22:20:24 - RayLog - INFO: - update_step: 5500, online_eval_reward: 200.000 +2023-05-15 22:20:24 - RayLog - INFO: - Worker 0 finished episode 92 with reward 200.0 in 200 steps +2023-05-15 22:20:26 - RayLog - INFO: - Worker 1 finished episode 93 with reward 200.0 in 200 steps +2023-05-15 22:20:28 - RayLog - INFO: - Worker 0 finished episode 94 with reward 200.0 in 200 steps +2023-05-15 22:20:29 - RayLog - INFO: - update_step: 6000, online_eval_reward: 200.000 +2023-05-15 22:20:30 - RayLog - INFO: - Worker 1 finished episode 95 with reward 200.0 in 200 steps +2023-05-15 22:20:32 - RayLog - INFO: - Worker 0 finished episode 96 with reward 200.0 in 200 steps +2023-05-15 22:20:34 - RayLog - INFO: - update_step: 6500, online_eval_reward: 200.000 +2023-05-15 22:20:34 - RayLog - INFO: - Worker 1 finished episode 97 with reward 200.0 in 200 steps +2023-05-15 22:20:37 - RayLog - INFO: - Worker 0 finished episode 98 with reward 200.0 in 200 steps +2023-05-15 22:20:38 - RayLog - INFO: - Worker 1 finished episode 99 with reward 200.0 in 200 steps +2023-05-15 22:20:40 - RayLog - INFO: - update_step: 7000, online_eval_reward: 200.000 +2023-05-15 22:20:40 - RayLog - INFO: - Worker 0 finished episode 100 with reward 200.0 in 200 steps +2023-05-15 22:20:43 - SimpleLog - INFO: - Finish training! total time consumed: 87.42s diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/1000 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/1000 new file mode 100644 index 0000000000000000000000000000000000000000..420affc154150295755e8a0feba420d5c3f9603b Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/1000 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/1500 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/1500 new file mode 100644 index 0000000000000000000000000000000000000000..8d041c47770ee04a35b3a0bf9f508b66801c7b43 Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/1500 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/2000 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/2000 new file mode 100644 index 0000000000000000000000000000000000000000..af99a2189e2272e946b2332258ce73e388df402c Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/2000 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/2500 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/2500 new file mode 100644 index 0000000000000000000000000000000000000000..9024777871ee6ddcb14c5c57466334992233c951 Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/2500 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/3000 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/3000 new file mode 100644 index 0000000000000000000000000000000000000000..2454cb0052296fba8a288a983f035f72697addfd Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/3000 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/3500 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/3500 new file mode 100644 index 0000000000000000000000000000000000000000..e6329af77934477af2a093cd2783749d1b077df7 Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/3500 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/4000 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/4000 new file mode 100644 index 0000000000000000000000000000000000000000..95485f0b5953672165cc10eaf20fd71a5fb92532 Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/4000 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/4500 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/4500 new file mode 100644 index 0000000000000000000000000000000000000000..695e655b688ede6cd96ccc8be2ebdca4bd5e0d7c Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/4500 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/500 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/500 new file mode 100644 index 0000000000000000000000000000000000000000..945009444d139d5a0050490de5d4280944c1fca4 Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/500 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/5000 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/5000 new file mode 100644 index 0000000000000000000000000000000000000000..697f3d75f0fbef8c9a2254fb7f0f5220edb9fe2a Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/5000 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/5500 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/5500 new file mode 100644 index 0000000000000000000000000000000000000000..462784933d6e0cee7746781d63464f891686efab Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/5500 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/6000 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/6000 new file mode 100644 index 0000000000000000000000000000000000000000..e034de4781176a29137e6dccded308a84e6f260f Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/6000 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/6500 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/6500 new file mode 100644 index 0000000000000000000000000000000000000000..37130f9fc68c7097b6c0af66848d867eabc774ce Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/6500 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/7000 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/7000 new file mode 100644 index 0000000000000000000000000000000000000000..43fd6d22782cb9197fe8f966fbfa603a73bf883b Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/7000 differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/best b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/best new file mode 100644 index 0000000000000000000000000000000000000000..9024777871ee6ddcb14c5c57466334992233c951 Binary files /dev/null and b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/models/best differ diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/interact/events.out.tfevents.1684160356.DESKTOP-H34HQIQ.63896.0 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/interact/events.out.tfevents.1684160356.DESKTOP-H34HQIQ.63896.0 new file mode 100644 index 0000000000000000000000000000000000000000..c1ce53e64d33577d62d915ff555249470b556213 --- /dev/null +++ b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/interact/events.out.tfevents.1684160356.DESKTOP-H34HQIQ.63896.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:696016c2758c0ccea9cb50504c6968dadacdaae4c399a656535de3d8cbd0dc05 +size 40 diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/interact/events.out.tfevents.1684160363.DESKTOP-H34HQIQ.19952.0 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/interact/events.out.tfevents.1684160363.DESKTOP-H34HQIQ.19952.0 new file mode 100644 index 0000000000000000000000000000000000000000..2a88fbf7d25c597103e74228db51d47a06bdb4a9 --- /dev/null +++ b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/interact/events.out.tfevents.1684160363.DESKTOP-H34HQIQ.19952.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c39d72aaeb990179b01ad4a2bbaba1d46da5d2f48bdc6e892248d0641c70f310 +size 10028 diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/model/events.out.tfevents.1684160356.DESKTOP-H34HQIQ.63896.1 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/model/events.out.tfevents.1684160356.DESKTOP-H34HQIQ.63896.1 new file mode 100644 index 0000000000000000000000000000000000000000..1b0d971ed2287aa916ae13e39e26c5ae1bab6211 --- /dev/null +++ b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/model/events.out.tfevents.1684160356.DESKTOP-H34HQIQ.63896.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe9bc5429757d5aaaa9a50bfc3728575b26888176dd0cab2a5566229a9165e50 +size 40 diff --git a/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/model/events.out.tfevents.1684160363.DESKTOP-H34HQIQ.19952.1 b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/model/events.out.tfevents.1684160363.DESKTOP-H34HQIQ.19952.1 new file mode 100644 index 0000000000000000000000000000000000000000..444406ca55374a8f3a2d824cafe3f61577e7a65a --- /dev/null +++ b/CartPole-v1/Train_ray_CartPole-v1_DQN_20230515-221916/tb_logs/model/events.out.tfevents.1684160363.DESKTOP-H34HQIQ.19952.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0a82a1d116fc74439b132d0e405286bf1aa52a0883adcfe3b91391fabc89e5a +size 344824 diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/config.yaml b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75110d577c14ac7a1b2fd656506e4e380280b325 --- /dev/null +++ b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/config.yaml @@ -0,0 +1,45 @@ +general_cfg: + algo_name: DQN + collect_traj: false + device: cpu + env_name: gym + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_episode: 100 + max_step: 200 + mode: train + model_save_fre: 500 + mp_backend: single + n_workers: 1 + online_eval: true + online_eval_episode: 10 + render_mode: null + save_fig: true + seed: 1 + show_fig: false +algo_cfg: + batch_size: 64 + buffer_size: 100000 + buffer_type: REPLAY_QUE + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.95 + lr: 0.0001 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + layer_type: linear +env_cfg: + id: CartPole-v1 + ignore_params: + - wrapper + - ignore_params + render_mode: null + wrapper: null diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/logs/log.txt b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..c7d2c227f466b49b663bebf2274cd8434afaf2f6 --- /dev/null +++ b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/logs/log.txt @@ -0,0 +1,168 @@ +2023-05-15 21:17:21 - SimpleLog - INFO: - General Configs: +2023-05-15 21:17:21 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:17:21 - SimpleLog - INFO: - Name Value Type +2023-05-15 21:17:21 - SimpleLog - INFO: - env_name gym +2023-05-15 21:17:21 - SimpleLog - INFO: - render_mode None +2023-05-15 21:17:21 - SimpleLog - INFO: - algo_name DQN +2023-05-15 21:17:21 - SimpleLog - INFO: - mode train +2023-05-15 21:17:21 - SimpleLog - INFO: - collect_traj 0 +2023-05-15 21:17:21 - SimpleLog - INFO: - mp_backend single +2023-05-15 21:17:21 - SimpleLog - INFO: - n_workers 1 +2023-05-15 21:17:21 - SimpleLog - INFO: - seed 1 +2023-05-15 21:17:21 - SimpleLog - INFO: - device cpu +2023-05-15 21:17:21 - SimpleLog - INFO: - max_episode 100 +2023-05-15 21:17:21 - SimpleLog - INFO: - max_step 200 +2023-05-15 21:17:21 - SimpleLog - INFO: - online_eval 1 +2023-05-15 21:17:21 - SimpleLog - INFO: - online_eval_episode 10 +2023-05-15 21:17:21 - SimpleLog - INFO: - load_checkpoint 0 +2023-05-15 21:17:21 - SimpleLog - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-05-15 21:17:21 - SimpleLog - INFO: - show_fig 0 +2023-05-15 21:17:21 - SimpleLog - INFO: - save_fig 1 +2023-05-15 21:17:21 - SimpleLog - INFO: - model_save_fre 500 +2023-05-15 21:17:21 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:17:21 - SimpleLog - INFO: - Algo Configs: +2023-05-15 21:17:21 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:17:21 - SimpleLog - INFO: - Name Value Type +2023-05-15 21:17:21 - SimpleLog - INFO: - epsilon_start 0.95 +2023-05-15 21:17:21 - SimpleLog - INFO: - epsilon_end 0.01 +2023-05-15 21:17:21 - SimpleLog - INFO: - epsilon_decay 500 +2023-05-15 21:17:21 - SimpleLog - INFO: - gamma 0.95 +2023-05-15 21:17:21 - SimpleLog - INFO: - lr 0.0001 +2023-05-15 21:17:21 - SimpleLog - INFO: - buffer_size 100000 +2023-05-15 21:17:21 - SimpleLog - INFO: - batch_size 64 +2023-05-15 21:17:21 - SimpleLog - INFO: - target_update 4 +2023-05-15 21:17:21 - SimpleLog - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': [256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256], 'activation': 'relu'}] +2023-05-15 21:17:21 - SimpleLog - INFO: - buffer_type REPLAY_QUE +2023-05-15 21:17:21 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:17:21 - SimpleLog - INFO: - Env Configs: +2023-05-15 21:17:21 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:17:21 - SimpleLog - INFO: - Name Value Type +2023-05-15 21:17:21 - SimpleLog - INFO: - id CartPole-v1 +2023-05-15 21:17:21 - SimpleLog - INFO: - render_mode None +2023-05-15 21:17:21 - SimpleLog - INFO: - wrapper None +2023-05-15 21:17:21 - SimpleLog - INFO: - ignore_params ['wrapper', 'ignore_params'] +2023-05-15 21:17:21 - SimpleLog - INFO: - ================================================================================ +2023-05-15 21:17:21 - SimpleLog - INFO: - obs_space: Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32), n_actions: Discrete(2) +2023-05-15 21:17:21 - SimpleLog - INFO: - Start training! +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 0, ep_reward: 25.0, ep_step: 25 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 1, ep_reward: 10.0, ep_step: 10 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 2, ep_reward: 13.0, ep_step: 13 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 3, ep_reward: 11.0, ep_step: 11 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 4, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 5, ep_reward: 20.0, ep_step: 20 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 6, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 7, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 8, ep_reward: 11.0, ep_step: 11 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 9, ep_reward: 14.0, ep_step: 14 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 10, ep_reward: 22.0, ep_step: 22 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 11, ep_reward: 14.0, ep_step: 14 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 12, ep_reward: 18.0, ep_step: 18 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 13, ep_reward: 10.0, ep_step: 10 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 14, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 15, ep_reward: 13.0, ep_step: 13 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 16, ep_reward: 13.0, ep_step: 13 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 17, ep_reward: 19.0, ep_step: 19 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 18, ep_reward: 10.0, ep_step: 10 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 19, ep_reward: 11.0, ep_step: 11 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 20, ep_reward: 9.0, ep_step: 9 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 21, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 22, ep_reward: 10.0, ep_step: 10 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 23, ep_reward: 13.0, ep_step: 13 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 24, ep_reward: 14.0, ep_step: 14 +2023-05-15 21:17:21 - SimpleLog - INFO: - episode: 25, ep_reward: 20.0, ep_step: 20 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 26, ep_reward: 37.0, ep_step: 37 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 27, ep_reward: 15.0, ep_step: 15 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 28, ep_reward: 10.0, ep_step: 10 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 29, ep_reward: 11.0, ep_step: 11 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 30, ep_reward: 11.0, ep_step: 11 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 31, ep_reward: 11.0, ep_step: 11 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 32, ep_reward: 11.0, ep_step: 11 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 33, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 34, ep_reward: 9.0, ep_step: 9 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 35, ep_reward: 9.0, ep_step: 9 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 36, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 37, ep_reward: 9.0, ep_step: 9 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 38, ep_reward: 14.0, ep_step: 14 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 39, ep_reward: 13.0, ep_step: 13 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 40, ep_reward: 10.0, ep_step: 10 +2023-05-15 21:17:22 - SimpleLog - INFO: - update_step: 500, online_eval_reward: 17.900 +2023-05-15 21:17:22 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 17.900, save the best model! +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 41, ep_reward: 12.0, ep_step: 12 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 42, ep_reward: 19.0, ep_step: 19 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 43, ep_reward: 16.0, ep_step: 16 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 44, ep_reward: 14.0, ep_step: 14 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 45, ep_reward: 18.0, ep_step: 18 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 46, ep_reward: 24.0, ep_step: 24 +2023-05-15 21:17:22 - SimpleLog - INFO: - episode: 47, ep_reward: 27.0, ep_step: 27 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 48, ep_reward: 27.0, ep_step: 27 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 49, ep_reward: 19.0, ep_step: 19 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 50, ep_reward: 20.0, ep_step: 20 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 51, ep_reward: 37.0, ep_step: 37 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 52, ep_reward: 26.0, ep_step: 26 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 53, ep_reward: 31.0, ep_step: 31 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 54, ep_reward: 23.0, ep_step: 23 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 55, ep_reward: 27.0, ep_step: 27 +2023-05-15 21:17:23 - SimpleLog - INFO: - episode: 56, ep_reward: 35.0, ep_step: 35 +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 57, ep_reward: 33.0, ep_step: 33 +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 58, ep_reward: 24.0, ep_step: 24 +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 59, ep_reward: 33.0, ep_step: 33 +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 60, ep_reward: 36.0, ep_step: 36 +2023-05-15 21:17:24 - SimpleLog - INFO: - update_step: 1000, online_eval_reward: 29.100 +2023-05-15 21:17:24 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 29.100, save the best model! +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 61, ep_reward: 33.0, ep_step: 33 +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 62, ep_reward: 34.0, ep_step: 34 +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 63, ep_reward: 51.0, ep_step: 51 +2023-05-15 21:17:24 - SimpleLog - INFO: - episode: 64, ep_reward: 41.0, ep_step: 41 +2023-05-15 21:17:25 - SimpleLog - INFO: - episode: 65, ep_reward: 54.0, ep_step: 54 +2023-05-15 21:17:25 - SimpleLog - INFO: - episode: 66, ep_reward: 54.0, ep_step: 54 +2023-05-15 21:17:25 - SimpleLog - INFO: - episode: 67, ep_reward: 74.0, ep_step: 74 +2023-05-15 21:17:25 - SimpleLog - INFO: - episode: 68, ep_reward: 44.0, ep_step: 44 +2023-05-15 21:17:25 - SimpleLog - INFO: - episode: 69, ep_reward: 38.0, ep_step: 38 +2023-05-15 21:17:25 - SimpleLog - INFO: - episode: 70, ep_reward: 49.0, ep_step: 49 +2023-05-15 21:17:26 - SimpleLog - INFO: - update_step: 1500, online_eval_reward: 57.300 +2023-05-15 21:17:26 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 57.300, save the best model! +2023-05-15 21:17:26 - SimpleLog - INFO: - episode: 71, ep_reward: 64.0, ep_step: 64 +2023-05-15 21:17:26 - SimpleLog - INFO: - episode: 72, ep_reward: 89.0, ep_step: 89 +2023-05-15 21:17:26 - SimpleLog - INFO: - episode: 73, ep_reward: 90.0, ep_step: 90 +2023-05-15 21:17:26 - SimpleLog - INFO: - episode: 74, ep_reward: 66.0, ep_step: 66 +2023-05-15 21:17:27 - SimpleLog - INFO: - episode: 75, ep_reward: 60.0, ep_step: 60 +2023-05-15 21:17:27 - SimpleLog - INFO: - episode: 76, ep_reward: 66.0, ep_step: 66 +2023-05-15 21:17:27 - SimpleLog - INFO: - episode: 77, ep_reward: 67.0, ep_step: 67 +2023-05-15 21:17:27 - SimpleLog - INFO: - update_step: 2000, online_eval_reward: 91.800 +2023-05-15 21:17:27 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 91.800, save the best model! +2023-05-15 21:17:27 - SimpleLog - INFO: - episode: 78, ep_reward: 141.0, ep_step: 141 +2023-05-15 21:17:28 - SimpleLog - INFO: - episode: 79, ep_reward: 138.0, ep_step: 138 +2023-05-15 21:17:28 - SimpleLog - INFO: - episode: 80, ep_reward: 141.0, ep_step: 141 +2023-05-15 21:17:29 - SimpleLog - INFO: - update_step: 2500, online_eval_reward: 178.900 +2023-05-15 21:17:29 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 178.900, save the best model! +2023-05-15 21:17:29 - SimpleLog - INFO: - episode: 81, ep_reward: 192.0, ep_step: 192 +2023-05-15 21:17:30 - SimpleLog - INFO: - episode: 82, ep_reward: 163.0, ep_step: 163 +2023-05-15 21:17:31 - SimpleLog - INFO: - episode: 83, ep_reward: 183.0, ep_step: 183 +2023-05-15 21:17:31 - SimpleLog - INFO: - update_step: 3000, online_eval_reward: 165.900 +2023-05-15 21:17:31 - SimpleLog - INFO: - episode: 84, ep_reward: 199.0, ep_step: 199 +2023-05-15 21:17:32 - SimpleLog - INFO: - episode: 85, ep_reward: 175.0, ep_step: 175 +2023-05-15 21:17:32 - SimpleLog - INFO: - episode: 86, ep_reward: 191.0, ep_step: 191 +2023-05-15 21:17:33 - SimpleLog - INFO: - update_step: 3500, online_eval_reward: 186.300 +2023-05-15 21:17:33 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 186.300, save the best model! +2023-05-15 21:17:33 - SimpleLog - INFO: - episode: 87, ep_reward: 180.0, ep_step: 180 +2023-05-15 21:17:34 - SimpleLog - INFO: - episode: 88, ep_reward: 185.0, ep_step: 185 +2023-05-15 21:17:35 - SimpleLog - INFO: - update_step: 4000, online_eval_reward: 187.300 +2023-05-15 21:17:35 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 187.300, save the best model! +2023-05-15 21:17:35 - SimpleLog - INFO: - episode: 89, ep_reward: 194.0, ep_step: 194 +2023-05-15 21:17:35 - SimpleLog - INFO: - episode: 90, ep_reward: 181.0, ep_step: 181 +2023-05-15 21:17:36 - SimpleLog - INFO: - episode: 91, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:36 - SimpleLog - INFO: - update_step: 4500, online_eval_reward: 197.500 +2023-05-15 21:17:36 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 197.500, save the best model! +2023-05-15 21:17:37 - SimpleLog - INFO: - episode: 92, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:37 - SimpleLog - INFO: - episode: 93, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:38 - SimpleLog - INFO: - update_step: 5000, online_eval_reward: 196.200 +2023-05-15 21:17:38 - SimpleLog - INFO: - episode: 94, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:39 - SimpleLog - INFO: - episode: 95, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:39 - SimpleLog - INFO: - episode: 96, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:40 - SimpleLog - INFO: - update_step: 5500, online_eval_reward: 199.500 +2023-05-15 21:17:40 - SimpleLog - INFO: - current update step obtain a better online_eval_reward: 199.500, save the best model! +2023-05-15 21:17:40 - SimpleLog - INFO: - episode: 97, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:41 - SimpleLog - INFO: - episode: 98, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:42 - SimpleLog - INFO: - update_step: 6000, online_eval_reward: 196.200 +2023-05-15 21:17:42 - SimpleLog - INFO: - episode: 99, ep_reward: 200.0, ep_step: 200 +2023-05-15 21:17:42 - SimpleLog - INFO: - Finish training! total time consumed: 21.14s diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/1000 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/1000 new file mode 100644 index 0000000000000000000000000000000000000000..dbf3fa9fda9fb47fdcf28b3d2e1b8cb1d6b50996 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/1000 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/1500 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/1500 new file mode 100644 index 0000000000000000000000000000000000000000..5ee7115ec7afb544054aee808779e08ed2699b9f Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/1500 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/2000 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/2000 new file mode 100644 index 0000000000000000000000000000000000000000..258d4873833124a5fcd1163fe9756e977a852d57 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/2000 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/2500 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/2500 new file mode 100644 index 0000000000000000000000000000000000000000..30eb4a77f2882a1141aaf0566d96ca4890ad5061 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/2500 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/3000 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/3000 new file mode 100644 index 0000000000000000000000000000000000000000..1b7adc295072abcbc5d58100bde674b9ee166d3f Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/3000 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/3500 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/3500 new file mode 100644 index 0000000000000000000000000000000000000000..a4f70eb9c16b6482fd4bdf5b88ba9efedea01a91 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/3500 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/4000 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/4000 new file mode 100644 index 0000000000000000000000000000000000000000..e5802f1d33b2e5bfcd042c183d6baeb9684fd010 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/4000 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/4500 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/4500 new file mode 100644 index 0000000000000000000000000000000000000000..16e4ba299793db1f8533e35289e7f2532557c16b Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/4500 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/500 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/500 new file mode 100644 index 0000000000000000000000000000000000000000..eca897a0bbd68625355e7b428f650d27effb00c3 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/500 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/5000 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/5000 new file mode 100644 index 0000000000000000000000000000000000000000..dd0c84d8cd4c59bbc8ea7c2a238da855afedbad1 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/5000 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/5500 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/5500 new file mode 100644 index 0000000000000000000000000000000000000000..554bffc8fafc5efcfab15798da33e11c9fbeebb7 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/5500 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/6000 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/6000 new file mode 100644 index 0000000000000000000000000000000000000000..0f432672e133a8be11116a3d574e92f37a205004 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/6000 differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/best b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/best new file mode 100644 index 0000000000000000000000000000000000000000..554bffc8fafc5efcfab15798da33e11c9fbeebb7 Binary files /dev/null and b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/models/best differ diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/tb_logs/interact/events.out.tfevents.1684156641.DESKTOP-H34HQIQ.65936.0 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/tb_logs/interact/events.out.tfevents.1684156641.DESKTOP-H34HQIQ.65936.0 new file mode 100644 index 0000000000000000000000000000000000000000..7d0a6c3605bfd40626cc871d584af84282c89bc1 --- /dev/null +++ b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/tb_logs/interact/events.out.tfevents.1684156641.DESKTOP-H34HQIQ.65936.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88c8b584bf2023d80b8da777a9aa1d6799df06ab44841a1350e3779314f0dff5 +size 10436 diff --git a/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/tb_logs/model/events.out.tfevents.1684156641.DESKTOP-H34HQIQ.65936.1 b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/tb_logs/model/events.out.tfevents.1684156641.DESKTOP-H34HQIQ.65936.1 new file mode 100644 index 0000000000000000000000000000000000000000..77772ec0a8df42b45d1c7a26ce2a784e43b1bd63 --- /dev/null +++ b/CartPole-v1/Train_single_CartPole-v1_DQN_20230515-211721/tb_logs/model/events.out.tfevents.1684156641.DESKTOP-H34HQIQ.65936.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34aa745aea2a3b867783177a4a48344c1e8a6218893f768de6a7cc79eb74d6d8 +size 295334 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/config.yaml b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..888f3870de1ac1a5d13ea580654cb9474a0736b3 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/config.yaml @@ -0,0 +1,26 @@ +general_cfg: + algo_name: DynaQ + device: cpu + env_name: CliffWalking-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CliffWalking-v0_DynaQ_20221210-095808 + max_steps: 100 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: envs.wrappers.CliffWalkingWapper +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + exploration_type: e-greedy + gamma: 0.99 + lr: 0.1 + n_planning: 10 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/logs/log.txt b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..aabda5e18769c0050babfa8461f3c7f092e1be10 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/logs/log.txt @@ -0,0 +1,42 @@ +2022-12-10 09:59:44 - r - INFO: - Hyperparameters: +2022-12-10 09:59:44 - r - INFO: - ================================================================================ +2022-12-10 09:59:44 - r - INFO: - Name Value Type +2022-12-10 09:59:44 - r - INFO: - env_name CliffWalking-v0 +2022-12-10 09:59:44 - r - INFO: - new_step_api 1 +2022-12-10 09:59:44 - r - INFO: - wrapper envs.wrappers.CliffWalkingWapper +2022-12-10 09:59:44 - r - INFO: - render 0 +2022-12-10 09:59:44 - r - INFO: - algo_name DynaQ +2022-12-10 09:59:44 - r - INFO: - mode test +2022-12-10 09:59:44 - r - INFO: - seed 1 +2022-12-10 09:59:44 - r - INFO: - device cpu +2022-12-10 09:59:44 - r - INFO: - train_eps 100 +2022-12-10 09:59:44 - r - INFO: - test_eps 10 +2022-12-10 09:59:44 - r - INFO: - eval_eps 10 +2022-12-10 09:59:44 - r - INFO: - eval_per_episode 5 +2022-12-10 09:59:44 - r - INFO: - max_steps 100 +2022-12-10 09:59:44 - r - INFO: - load_checkpoint 1 +2022-12-10 09:59:44 - r - INFO: - load_path Train_CliffWalking-v0_DynaQ_20221210-095808 +2022-12-10 09:59:44 - r - INFO: - show_fig 0 +2022-12-10 09:59:44 - r - INFO: - save_fig 1 +2022-12-10 09:59:44 - r - INFO: - epsilon_start 0.95 +2022-12-10 09:59:44 - r - INFO: - epsilon_end 0.01 +2022-12-10 09:59:44 - r - INFO: - epsilon_decay 300 +2022-12-10 09:59:44 - r - INFO: - gamma 0.99 +2022-12-10 09:59:44 - r - INFO: - lr 0.1 +2022-12-10 09:59:44 - r - INFO: - n_planning 10 +2022-12-10 09:59:44 - r - INFO: - exploration_type e-greedy +2022-12-10 09:59:44 - r - INFO: - ================================================================================ +2022-12-10 09:59:44 - r - INFO: - n_states: 48, n_actions: 4 +2022-12-10 09:59:44 - r - INFO: - Start testing! +2022-12-10 09:59:44 - r - INFO: - Env: CliffWalking-v0, Algorithm: DynaQ, Device: cpu +2022-12-10 09:59:44 - r - INFO: - Episode: 1/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 2/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 3/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 4/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 5/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 6/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 7/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 8/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 9/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Episode: 10/10, Reward: -13.000, Step: 13 +2022-12-10 09:59:44 - r - INFO: - Finish testing! diff --git a/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/modelsDynaQ_model.pkl b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/modelsDynaQ_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..5bf73409cd4c1e4fa4c52b791f0f4fd08745031c --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/modelsDynaQ_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f30300751586760b2a448b16a5ba8051c11f00882a4cdd9560d4a48ee781d0 +size 14127 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/results/learning_curve.png b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..c33f6c925c7c445a48063a6e39b5d33b72475a14 Binary files /dev/null and b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/results/learning_curve.png differ diff --git a/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/results/res.csv b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d69fb8e7ddf3a69ca3e85cbc39fd580d2160ba5 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_DynaQ_20221210-095944/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,-13,13 +1,-13,13 +2,-13,13 +3,-13,13 +4,-13,13 +5,-13,13 +6,-13,13 +7,-13,13 +8,-13,13 +9,-13,13 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/config.yaml b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0579a96367861a1ecbc06ec72861a9ea60a66edd --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/config.yaml @@ -0,0 +1,24 @@ +general_cfg: + algo_name: QLearning + device: cpu + env_name: CliffWalking-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CliffWalking-v0_QLearning_20221128-144520 + max_steps: 100 + mode: test + new_step_api: true + render: true + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 300 + wrapper: envs.wrappers.CliffWalkingWapper +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.1 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/logs/log.txt b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a7968b9e5d088b7998840f204da58d2afe612c7 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/logs/log.txt @@ -0,0 +1,14 @@ +2022-11-28 15:05:39 - r - INFO: - n_states: 48, n_actions: 4 +2022-11-28 15:05:39 - r - INFO: - Start testing! +2022-11-28 15:05:39 - r - INFO: - Env: CliffWalking-v0, Algorithm: QLearning, Device: cpu +2022-11-28 15:05:44 - r - INFO: - Episode: 1/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:45 - r - INFO: - Episode: 2/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:45 - r - INFO: - Episode: 3/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:46 - r - INFO: - Episode: 4/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:46 - r - INFO: - Episode: 5/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:46 - r - INFO: - Episode: 6/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:47 - r - INFO: - Episode: 7/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:47 - r - INFO: - Episode: 8/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:48 - r - INFO: - Episode: 9/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:48 - r - INFO: - Episode: 10/10, Reward: -13.000, Step: 13 +2022-11-28 15:05:48 - r - INFO: - Finish testing! diff --git a/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/models/Qleaning_model.pkl b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/models/Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..11b3246418fe8e689ff25f81da33205b94378416 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/models/Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a5088e21ba086ef3631d0381b7b57ec20f56261d403a159a6c8d0cae3d6ec63 +size 5999 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/results/learning_curve.png b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..c565c2a416cfb9116b0b3200fb6d0a5617472d76 Binary files /dev/null and b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/results/learning_curve.png differ diff --git a/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/results/res.csv b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d69fb8e7ddf3a69ca3e85cbc39fd580d2160ba5 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_QLearning_20221128-150537/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,-13,13 +1,-13,13 +2,-13,13 +3,-13,13 +4,-13,13 +5,-13,13 +6,-13,13 +7,-13,13 +8,-13,13 +9,-13,13 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/config.yaml b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70b113e3ce74eb89033815e63b9d477ce198aa8b --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/config.yaml @@ -0,0 +1,25 @@ +general_cfg: + algo_name: Sarsa + device: cpu + env_name: CliffWalking-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_CliffWalking-v0_Sarsa_20230311-001504 + max_steps: 100 + mode: test + new_step_api: true + render: true + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 300 + wrapper: envs.wrappers.CliffWalkingWapper +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.1 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/logs/log.txt b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3961d6c76485db66fd4a519566e62f5f7717ec54 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/logs/log.txt @@ -0,0 +1,46 @@ +2023-03-11 00:15:30 - r - INFO: - Hyperparameters: +2023-03-11 00:15:30 - r - INFO: - ================================================================================ +2023-03-11 00:15:30 - r - INFO: - Name Value Type +2023-03-11 00:15:30 - r - INFO: - env_name CliffWalking-v0 +2023-03-11 00:15:30 - r - INFO: - new_step_api 1 +2023-03-11 00:15:30 - r - INFO: - wrapper envs.wrappers.CliffWalkingWapper +2023-03-11 00:15:30 - r - INFO: - render 1 +2023-03-11 00:15:30 - r - INFO: - algo_name Sarsa +2023-03-11 00:15:30 - r - INFO: - mode test +2023-03-11 00:15:30 - r - INFO: - seed 1 +2023-03-11 00:15:30 - r - INFO: - device cpu +2023-03-11 00:15:30 - r - INFO: - train_eps 300 +2023-03-11 00:15:30 - r - INFO: - test_eps 10 +2023-03-11 00:15:30 - r - INFO: - eval_eps 10 +2023-03-11 00:15:30 - r - INFO: - eval_per_episode 5 +2023-03-11 00:15:30 - r - INFO: - max_steps 100 +2023-03-11 00:15:30 - r - INFO: - load_checkpoint 1 +2023-03-11 00:15:30 - r - INFO: - load_path Train_CliffWalking-v0_Sarsa_20230311-001504 +2023-03-11 00:15:30 - r - INFO: - show_fig 0 +2023-03-11 00:15:30 - r - INFO: - save_fig 1 +2023-03-11 00:15:30 - r - INFO: - render_mode human +2023-03-11 00:15:30 - r - INFO: - epsilon_start 0.95 +2023-03-11 00:15:30 - r - INFO: - epsilon_end 0.01 +2023-03-11 00:15:30 - r - INFO: - epsilon_decay 300 +2023-03-11 00:15:30 - r - INFO: - gamma 0.99 +2023-03-11 00:15:30 - r - INFO: - lr 0.1 +2023-03-11 00:15:30 - r - INFO: - task_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CliffWalking-v0_Sarsa_20230311-001530 +2023-03-11 00:15:30 - r - INFO: - res_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CliffWalking-v0_Sarsa_20230311-001530/results +2023-03-11 00:15:30 - r - INFO: - log_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CliffWalking-v0_Sarsa_20230311-001530/logs +2023-03-11 00:15:30 - r - INFO: - traj_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CliffWalking-v0_Sarsa_20230311-001530/traj +2023-03-11 00:15:30 - r - INFO: - tb_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Test_CliffWalking-v0_Sarsa_20230311-001530/tb_logs +2023-03-11 00:15:30 - r - INFO: - ================================================================================ +2023-03-11 00:15:31 - r - INFO: - n_states: 48, n_actions: 4 +2023-03-11 00:15:31 - r - INFO: - Start testing! +2023-03-11 00:15:31 - r - INFO: - Env: CliffWalking-v0, Algorithm: Sarsa, Device: cpu +2023-03-11 00:15:38 - r - INFO: - Episode: 1/10, Reward: -13.000, Step: 13 +2023-03-11 00:15:42 - r - INFO: - Episode: 2/10, Reward: -13.000, Step: 13 +2023-03-11 00:15:45 - r - INFO: - Episode: 3/10, Reward: -13.000, Step: 13 +2023-03-11 00:15:49 - r - INFO: - Episode: 4/10, Reward: -13.000, Step: 13 +2023-03-11 00:15:52 - r - INFO: - Episode: 5/10, Reward: -13.000, Step: 13 +2023-03-11 00:15:56 - r - INFO: - Episode: 6/10, Reward: -13.000, Step: 13 +2023-03-11 00:15:59 - r - INFO: - Episode: 7/10, Reward: -13.000, Step: 13 +2023-03-11 00:16:03 - r - INFO: - Episode: 8/10, Reward: -13.000, Step: 13 +2023-03-11 00:16:06 - r - INFO: - Episode: 9/10, Reward: -13.000, Step: 13 +2023-03-11 00:16:10 - r - INFO: - Episode: 10/10, Reward: -13.000, Step: 13 +2023-03-11 00:16:10 - r - INFO: - Finish testing! diff --git a/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/models/checkpoint.pkl b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/models/checkpoint.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ccccfb56162441b23fef7debf21e39446752490d --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/models/checkpoint.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b49e03abcf8f10412bffd219b9d2a522e02d39fce0365e81456762229100643d +size 6127 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/results/learning_curve.png b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..3ca0e8af938b60b78f60774120bd40ebe65fcd59 Binary files /dev/null and b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/results/learning_curve.png differ diff --git a/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/results/res.csv b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d69fb8e7ddf3a69ca3e85cbc39fd580d2160ba5 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,-13,13 +1,-13,13 +2,-13,13 +3,-13,13 +4,-13,13 +5,-13,13 +6,-13,13 +7,-13,13 +8,-13,13 +9,-13,13 diff --git a/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/tb_logs/events.out.tfevents.1678464930.DESKTOP-H34HQIQ.51560.0 b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/tb_logs/events.out.tfevents.1678464930.DESKTOP-H34HQIQ.51560.0 new file mode 100644 index 0000000000000000000000000000000000000000..994fe3d0d8fb17787e203f494706870fee02c349 --- /dev/null +++ b/CliffWalking-v0/Test_CliffWalking-v0_Sarsa_20230311-001530/tb_logs/events.out.tfevents.1678464930.DESKTOP-H34HQIQ.51560.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e83cf950f12dc53dd89a40339f0b381110e98b72b10b044c3d17d454d4301a +size 40 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/config.yaml b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c036ff9c7c1c0338e3cef87517fc16ca7d5b277 --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/config.yaml @@ -0,0 +1,26 @@ +general_cfg: + algo_name: DynaQ + device: cpu + env_name: CliffWalking-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CliffWalking-v0_DynaQ_20221210-095208 + max_steps: 100 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: envs.wrappers.CliffWalkingWapper +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + exploration_type: e-greedy + gamma: 0.99 + lr: 0.1 + n_planning: 10 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/logs/log.txt b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..1df556b3287f1f47305df89a4f0a0069f4806bc1 --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/logs/log.txt @@ -0,0 +1,141 @@ +2022-12-10 09:58:08 - r - INFO: - Hyperparameters: +2022-12-10 09:58:08 - r - INFO: - ================================================================================ +2022-12-10 09:58:08 - r - INFO: - Name Value Type +2022-12-10 09:58:08 - r - INFO: - env_name CliffWalking-v0 +2022-12-10 09:58:08 - r - INFO: - new_step_api 1 +2022-12-10 09:58:08 - r - INFO: - wrapper envs.wrappers.CliffWalkingWapper +2022-12-10 09:58:08 - r - INFO: - render 0 +2022-12-10 09:58:08 - r - INFO: - algo_name DynaQ +2022-12-10 09:58:08 - r - INFO: - mode train +2022-12-10 09:58:08 - r - INFO: - seed 1 +2022-12-10 09:58:08 - r - INFO: - device cpu +2022-12-10 09:58:08 - r - INFO: - train_eps 100 +2022-12-10 09:58:08 - r - INFO: - test_eps 10 +2022-12-10 09:58:08 - r - INFO: - eval_eps 10 +2022-12-10 09:58:08 - r - INFO: - eval_per_episode 5 +2022-12-10 09:58:08 - r - INFO: - max_steps 100 +2022-12-10 09:58:08 - r - INFO: - load_checkpoint 0 +2022-12-10 09:58:08 - r - INFO: - load_path Train_CliffWalking-v0_DynaQ_20221210-095208 +2022-12-10 09:58:08 - r - INFO: - show_fig 0 +2022-12-10 09:58:08 - r - INFO: - save_fig 1 +2022-12-10 09:58:08 - r - INFO: - epsilon_start 0.95 +2022-12-10 09:58:08 - r - INFO: - epsilon_end 0.01 +2022-12-10 09:58:08 - r - INFO: - epsilon_decay 300 +2022-12-10 09:58:08 - r - INFO: - gamma 0.99 +2022-12-10 09:58:08 - r - INFO: - lr 0.1 +2022-12-10 09:58:08 - r - INFO: - n_planning 10 +2022-12-10 09:58:08 - r - INFO: - exploration_type e-greedy +2022-12-10 09:58:08 - r - INFO: - ================================================================================ +2022-12-10 09:58:09 - r - INFO: - n_states: 48, n_actions: 4 +2022-12-10 09:58:09 - r - INFO: - Start training! +2022-12-10 09:58:09 - r - INFO: - Env: CliffWalking-v0, Algorithm: DynaQ, Device: cpu +2022-12-10 09:58:09 - r - INFO: - Episode: 1/100, Reward: -397.000, Step: 100 +2022-12-10 09:58:09 - r - INFO: - Episode: 2/100, Reward: -78.000, Step: 78 +2022-12-10 09:58:09 - r - INFO: - Episode: 3/100, Reward: -496.000, Step: 100 +2022-12-10 09:58:09 - r - INFO: - Episode: 4/100, Reward: -496.000, Step: 100 +2022-12-10 09:58:09 - r - INFO: - Episode: 5/100, Reward: -694.000, Step: 100 +2022-12-10 09:58:09 - r - INFO: - Current episode 5 has the best eval reward: -694.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 6/100, Reward: -595.000, Step: 100 +2022-12-10 09:58:09 - r - INFO: - Episode: 7/100, Reward: -298.000, Step: 100 +2022-12-10 09:58:09 - r - INFO: - Episode: 8/100, Reward: -96.000, Step: 96 +2022-12-10 09:58:09 - r - INFO: - Episode: 9/100, Reward: -27.000, Step: 27 +2022-12-10 09:58:09 - r - INFO: - Episode: 10/100, Reward: -134.000, Step: 35 +2022-12-10 09:58:09 - r - INFO: - Current episode 10 has the best eval reward: -100.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 11/100, Reward: -31.000, Step: 31 +2022-12-10 09:58:09 - r - INFO: - Episode: 12/100, Reward: -46.000, Step: 46 +2022-12-10 09:58:09 - r - INFO: - Episode: 13/100, Reward: -19.000, Step: 19 +2022-12-10 09:58:09 - r - INFO: - Episode: 14/100, Reward: -19.000, Step: 19 +2022-12-10 09:58:09 - r - INFO: - Episode: 15/100, Reward: -29.000, Step: 29 +2022-12-10 09:58:09 - r - INFO: - Current episode 15 has the best eval reward: -100.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 16/100, Reward: -19.000, Step: 19 +2022-12-10 09:58:09 - r - INFO: - Episode: 17/100, Reward: -29.000, Step: 29 +2022-12-10 09:58:09 - r - INFO: - Episode: 18/100, Reward: -31.000, Step: 31 +2022-12-10 09:58:09 - r - INFO: - Episode: 19/100, Reward: -25.000, Step: 25 +2022-12-10 09:58:09 - r - INFO: - Episode: 20/100, Reward: -32.000, Step: 32 +2022-12-10 09:58:09 - r - INFO: - Current episode 20 has the best eval reward: -100.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 21/100, Reward: -22.000, Step: 22 +2022-12-10 09:58:09 - r - INFO: - Episode: 22/100, Reward: -29.000, Step: 29 +2022-12-10 09:58:09 - r - INFO: - Episode: 23/100, Reward: -32.000, Step: 32 +2022-12-10 09:58:09 - r - INFO: - Episode: 24/100, Reward: -134.000, Step: 35 +2022-12-10 09:58:09 - r - INFO: - Episode: 25/100, Reward: -130.000, Step: 31 +2022-12-10 09:58:09 - r - INFO: - Current episode 25 has the best eval reward: -15.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 26/100, Reward: -30.000, Step: 30 +2022-12-10 09:58:09 - r - INFO: - Episode: 27/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 28/100, Reward: -21.000, Step: 21 +2022-12-10 09:58:09 - r - INFO: - Episode: 29/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 30/100, Reward: -30.000, Step: 30 +2022-12-10 09:58:09 - r - INFO: - Episode: 31/100, Reward: -48.000, Step: 48 +2022-12-10 09:58:09 - r - INFO: - Episode: 32/100, Reward: -25.000, Step: 25 +2022-12-10 09:58:09 - r - INFO: - Episode: 33/100, Reward: -27.000, Step: 27 +2022-12-10 09:58:09 - r - INFO: - Episode: 34/100, Reward: -27.000, Step: 27 +2022-12-10 09:58:09 - r - INFO: - Episode: 35/100, Reward: -22.000, Step: 22 +2022-12-10 09:58:09 - r - INFO: - Episode: 36/100, Reward: -33.000, Step: 33 +2022-12-10 09:58:09 - r - INFO: - Episode: 37/100, Reward: -30.000, Step: 30 +2022-12-10 09:58:09 - r - INFO: - Episode: 38/100, Reward: -32.000, Step: 32 +2022-12-10 09:58:09 - r - INFO: - Episode: 39/100, Reward: -22.000, Step: 22 +2022-12-10 09:58:09 - r - INFO: - Episode: 40/100, Reward: -29.000, Step: 29 +2022-12-10 09:58:09 - r - INFO: - Episode: 41/100, Reward: -35.000, Step: 35 +2022-12-10 09:58:09 - r - INFO: - Episode: 42/100, Reward: -39.000, Step: 39 +2022-12-10 09:58:09 - r - INFO: - Episode: 43/100, Reward: -55.000, Step: 55 +2022-12-10 09:58:09 - r - INFO: - Episode: 44/100, Reward: -30.000, Step: 30 +2022-12-10 09:58:09 - r - INFO: - Episode: 45/100, Reward: -56.000, Step: 56 +2022-12-10 09:58:09 - r - INFO: - Episode: 46/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 47/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 48/100, Reward: -23.000, Step: 23 +2022-12-10 09:58:09 - r - INFO: - Episode: 49/100, Reward: -30.000, Step: 30 +2022-12-10 09:58:09 - r - INFO: - Episode: 50/100, Reward: -31.000, Step: 31 +2022-12-10 09:58:09 - r - INFO: - Episode: 51/100, Reward: -52.000, Step: 52 +2022-12-10 09:58:09 - r - INFO: - Episode: 52/100, Reward: -14.000, Step: 14 +2022-12-10 09:58:09 - r - INFO: - Episode: 53/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 54/100, Reward: -31.000, Step: 31 +2022-12-10 09:58:09 - r - INFO: - Episode: 55/100, Reward: -25.000, Step: 25 +2022-12-10 09:58:09 - r - INFO: - Episode: 56/100, Reward: -19.000, Step: 19 +2022-12-10 09:58:09 - r - INFO: - Episode: 57/100, Reward: -27.000, Step: 27 +2022-12-10 09:58:09 - r - INFO: - Episode: 58/100, Reward: -30.000, Step: 30 +2022-12-10 09:58:09 - r - INFO: - Episode: 59/100, Reward: -30.000, Step: 30 +2022-12-10 09:58:09 - r - INFO: - Episode: 60/100, Reward: -27.000, Step: 27 +2022-12-10 09:58:09 - r - INFO: - Episode: 61/100, Reward: -29.000, Step: 29 +2022-12-10 09:58:09 - r - INFO: - Episode: 62/100, Reward: -19.000, Step: 19 +2022-12-10 09:58:09 - r - INFO: - Episode: 63/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 64/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 65/100, Reward: -14.000, Step: 14 +2022-12-10 09:58:09 - r - INFO: - Current episode 65 has the best eval reward: -13.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 66/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 67/100, Reward: -21.000, Step: 21 +2022-12-10 09:58:09 - r - INFO: - Episode: 68/100, Reward: -25.000, Step: 25 +2022-12-10 09:58:09 - r - INFO: - Episode: 69/100, Reward: -20.000, Step: 20 +2022-12-10 09:58:09 - r - INFO: - Episode: 70/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 71/100, Reward: -15.000, Step: 15 +2022-12-10 09:58:09 - r - INFO: - Episode: 72/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 73/100, Reward: -27.000, Step: 27 +2022-12-10 09:58:09 - r - INFO: - Episode: 74/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 75/100, Reward: -14.000, Step: 14 +2022-12-10 09:58:09 - r - INFO: - Current episode 75 has the best eval reward: -13.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 76/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 77/100, Reward: -16.000, Step: 16 +2022-12-10 09:58:09 - r - INFO: - Episode: 78/100, Reward: -28.000, Step: 28 +2022-12-10 09:58:09 - r - INFO: - Episode: 79/100, Reward: -15.000, Step: 15 +2022-12-10 09:58:09 - r - INFO: - Episode: 80/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Current episode 80 has the best eval reward: -13.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 81/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 82/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 83/100, Reward: -117.000, Step: 18 +2022-12-10 09:58:09 - r - INFO: - Episode: 84/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 85/100, Reward: -19.000, Step: 19 +2022-12-10 09:58:09 - r - INFO: - Episode: 86/100, Reward: -15.000, Step: 15 +2022-12-10 09:58:09 - r - INFO: - Episode: 87/100, Reward: -21.000, Step: 21 +2022-12-10 09:58:09 - r - INFO: - Episode: 88/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 89/100, Reward: -15.000, Step: 15 +2022-12-10 09:58:09 - r - INFO: - Episode: 90/100, Reward: -15.000, Step: 15 +2022-12-10 09:58:09 - r - INFO: - Current episode 90 has the best eval reward: -13.000 +2022-12-10 09:58:09 - r - INFO: - Episode: 91/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 92/100, Reward: -19.000, Step: 19 +2022-12-10 09:58:09 - r - INFO: - Episode: 93/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 94/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 95/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 96/100, Reward: -23.000, Step: 23 +2022-12-10 09:58:09 - r - INFO: - Episode: 97/100, Reward: -13.000, Step: 13 +2022-12-10 09:58:09 - r - INFO: - Episode: 98/100, Reward: -17.000, Step: 17 +2022-12-10 09:58:09 - r - INFO: - Episode: 99/100, Reward: -21.000, Step: 21 +2022-12-10 09:58:09 - r - INFO: - Episode: 100/100, Reward: -15.000, Step: 15 +2022-12-10 09:58:09 - r - INFO: - Finish training! diff --git a/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/modelsDynaQ_model.pkl b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/modelsDynaQ_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..5bf73409cd4c1e4fa4c52b791f0f4fd08745031c --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/modelsDynaQ_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f30300751586760b2a448b16a5ba8051c11f00882a4cdd9560d4a48ee781d0 +size 14127 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/results/learning_curve.png b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..c550aca02d28b40cdaf63c32c64317015158bebc Binary files /dev/null and b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/results/learning_curve.png differ diff --git a/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/results/res.csv b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..444c022ac1304a288ee2b24725591638d48f4122 --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_DynaQ_20221210-095808/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,-397,100 +1,-78,78 +2,-496,100 +3,-496,100 +4,-694,100 +5,-595,100 +6,-298,100 +7,-96,96 +8,-27,27 +9,-134,35 +10,-31,31 +11,-46,46 +12,-19,19 +13,-19,19 +14,-29,29 +15,-19,19 +16,-29,29 +17,-31,31 +18,-25,25 +19,-32,32 +20,-22,22 +21,-29,29 +22,-32,32 +23,-134,35 +24,-130,31 +25,-30,30 +26,-17,17 +27,-21,21 +28,-17,17 +29,-30,30 +30,-48,48 +31,-25,25 +32,-27,27 +33,-27,27 +34,-22,22 +35,-33,33 +36,-30,30 +37,-32,32 +38,-22,22 +39,-29,29 +40,-35,35 +41,-39,39 +42,-55,55 +43,-30,30 +44,-56,56 +45,-17,17 +46,-17,17 +47,-23,23 +48,-30,30 +49,-31,31 +50,-52,52 +51,-14,14 +52,-17,17 +53,-31,31 +54,-25,25 +55,-19,19 +56,-27,27 +57,-30,30 +58,-30,30 +59,-27,27 +60,-29,29 +61,-19,19 +62,-13,13 +63,-13,13 +64,-14,14 +65,-13,13 +66,-21,21 +67,-25,25 +68,-20,20 +69,-17,17 +70,-15,15 +71,-17,17 +72,-27,27 +73,-13,13 +74,-14,14 +75,-13,13 +76,-16,16 +77,-28,28 +78,-15,15 +79,-13,13 +80,-13,13 +81,-13,13 +82,-117,18 +83,-13,13 +84,-19,19 +85,-15,15 +86,-21,21 +87,-13,13 +88,-15,15 +89,-15,15 +90,-13,13 +91,-19,19 +92,-13,13 +93,-13,13 +94,-13,13 +95,-23,23 +96,-13,13 +97,-17,17 +98,-21,21 +99,-15,15 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/config.yaml b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aea7ad063e6aedffd91853a1dcc659229e6ec15e --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/config.yaml @@ -0,0 +1,23 @@ +general_cfg: + algo_name: QLearning + device: cpu + env_name: CliffWalking-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 100 + mode: train + new_step_api: true + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 300 + wrapper: envs.wrappers.CliffWalkingWapper +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.1 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/logs/log.txt b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..b7991b770c307671d75d96851aba9c27779f1dfa --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/logs/log.txt @@ -0,0 +1,342 @@ +2022-11-28 14:45:22 - r - INFO: - n_states: 48, n_actions: 4 +2022-11-28 14:45:22 - r - INFO: - Start training! +2022-11-28 14:45:22 - r - INFO: - Env: CliffWalking-v0, Algorithm: QLearning, Device: cpu +2022-11-28 14:45:22 - r - INFO: - Episode: 1/300, Reward: -397.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 2/300, Reward: -1090.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 3/300, Reward: -694.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 4/300, Reward: -397.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 5/300, Reward: -496.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Current episode 5 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 6/300, Reward: -298.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 7/300, Reward: -199.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 8/300, Reward: -298.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 9/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 10/300, Reward: -199.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Current episode 10 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 11/300, Reward: -93.000, Step: 93 +2022-11-28 14:45:22 - r - INFO: - Episode: 12/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 13/300, Reward: -199.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 14/300, Reward: -199.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 15/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Current episode 15 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 16/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 17/300, Reward: -33.000, Step: 33 +2022-11-28 14:45:22 - r - INFO: - Episode: 18/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 19/300, Reward: -66.000, Step: 66 +2022-11-28 14:45:22 - r - INFO: - Episode: 20/300, Reward: -99.000, Step: 99 +2022-11-28 14:45:22 - r - INFO: - Current episode 20 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 21/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 22/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 23/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 24/300, Reward: -54.000, Step: 54 +2022-11-28 14:45:22 - r - INFO: - Episode: 25/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Current episode 25 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 26/300, Reward: -73.000, Step: 73 +2022-11-28 14:45:22 - r - INFO: - Episode: 27/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 28/300, Reward: -90.000, Step: 90 +2022-11-28 14:45:22 - r - INFO: - Episode: 29/300, Reward: -87.000, Step: 87 +2022-11-28 14:45:22 - r - INFO: - Episode: 30/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Current episode 30 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 31/300, Reward: -46.000, Step: 46 +2022-11-28 14:45:22 - r - INFO: - Episode: 32/300, Reward: -82.000, Step: 82 +2022-11-28 14:45:22 - r - INFO: - Episode: 33/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 34/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 35/300, Reward: -67.000, Step: 67 +2022-11-28 14:45:22 - r - INFO: - Current episode 35 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 36/300, Reward: -75.000, Step: 75 +2022-11-28 14:45:22 - r - INFO: - Episode: 37/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 38/300, Reward: -45.000, Step: 45 +2022-11-28 14:45:22 - r - INFO: - Episode: 39/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 40/300, Reward: -37.000, Step: 37 +2022-11-28 14:45:22 - r - INFO: - Current episode 40 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 41/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 42/300, Reward: -43.000, Step: 43 +2022-11-28 14:45:22 - r - INFO: - Episode: 43/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 44/300, Reward: -56.000, Step: 56 +2022-11-28 14:45:22 - r - INFO: - Episode: 45/300, Reward: -70.000, Step: 70 +2022-11-28 14:45:22 - r - INFO: - Current episode 45 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 46/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 47/300, Reward: -47.000, Step: 47 +2022-11-28 14:45:22 - r - INFO: - Episode: 48/300, Reward: -85.000, Step: 85 +2022-11-28 14:45:22 - r - INFO: - Episode: 49/300, Reward: -82.000, Step: 82 +2022-11-28 14:45:22 - r - INFO: - Episode: 50/300, Reward: -51.000, Step: 51 +2022-11-28 14:45:22 - r - INFO: - Current episode 50 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 51/300, Reward: -50.000, Step: 50 +2022-11-28 14:45:22 - r - INFO: - Episode: 52/300, Reward: -86.000, Step: 86 +2022-11-28 14:45:22 - r - INFO: - Episode: 53/300, Reward: -100.000, Step: 100 +2022-11-28 14:45:22 - r - INFO: - Episode: 54/300, Reward: -40.000, Step: 40 +2022-11-28 14:45:22 - r - INFO: - Episode: 55/300, Reward: -87.000, Step: 87 +2022-11-28 14:45:22 - r - INFO: - Current episode 55 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 56/300, Reward: -59.000, Step: 59 +2022-11-28 14:45:22 - r - INFO: - Episode: 57/300, Reward: -51.000, Step: 51 +2022-11-28 14:45:22 - r - INFO: - Episode: 58/300, Reward: -98.000, Step: 98 +2022-11-28 14:45:22 - r - INFO: - Episode: 59/300, Reward: -67.000, Step: 67 +2022-11-28 14:45:22 - r - INFO: - Episode: 60/300, Reward: -60.000, Step: 60 +2022-11-28 14:45:22 - r - INFO: - Current episode 60 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 61/300, Reward: -38.000, Step: 38 +2022-11-28 14:45:22 - r - INFO: - Episode: 62/300, Reward: -76.000, Step: 76 +2022-11-28 14:45:22 - r - INFO: - Episode: 63/300, Reward: -70.000, Step: 70 +2022-11-28 14:45:22 - r - INFO: - Episode: 64/300, Reward: -60.000, Step: 60 +2022-11-28 14:45:22 - r - INFO: - Episode: 65/300, Reward: -68.000, Step: 68 +2022-11-28 14:45:22 - r - INFO: - Current episode 65 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 66/300, Reward: -25.000, Step: 25 +2022-11-28 14:45:22 - r - INFO: - Episode: 67/300, Reward: -69.000, Step: 69 +2022-11-28 14:45:22 - r - INFO: - Episode: 68/300, Reward: -183.000, Step: 84 +2022-11-28 14:45:22 - r - INFO: - Episode: 69/300, Reward: -62.000, Step: 62 +2022-11-28 14:45:22 - r - INFO: - Episode: 70/300, Reward: -44.000, Step: 44 +2022-11-28 14:45:22 - r - INFO: - Current episode 70 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 71/300, Reward: -71.000, Step: 71 +2022-11-28 14:45:22 - r - INFO: - Episode: 72/300, Reward: -55.000, Step: 55 +2022-11-28 14:45:22 - r - INFO: - Episode: 73/300, Reward: -65.000, Step: 65 +2022-11-28 14:45:22 - r - INFO: - Episode: 74/300, Reward: -61.000, Step: 61 +2022-11-28 14:45:22 - r - INFO: - Episode: 75/300, Reward: -36.000, Step: 36 +2022-11-28 14:45:22 - r - INFO: - Current episode 75 has the best eval reward: -100.000 +2022-11-28 14:45:22 - r - INFO: - Episode: 76/300, Reward: -47.000, Step: 47 +2022-11-28 14:45:22 - r - INFO: - Episode: 77/300, Reward: -63.000, Step: 63 +2022-11-28 14:45:22 - r - INFO: - Episode: 78/300, Reward: -45.000, Step: 45 +2022-11-28 14:45:22 - r - INFO: - Episode: 79/300, Reward: -59.000, Step: 59 +2022-11-28 14:45:22 - r - INFO: - Episode: 80/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:23 - r - INFO: - Current episode 80 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 81/300, Reward: -62.000, Step: 62 +2022-11-28 14:45:23 - r - INFO: - Episode: 82/300, Reward: -38.000, Step: 38 +2022-11-28 14:45:23 - r - INFO: - Episode: 83/300, Reward: -64.000, Step: 64 +2022-11-28 14:45:23 - r - INFO: - Episode: 84/300, Reward: -40.000, Step: 40 +2022-11-28 14:45:23 - r - INFO: - Episode: 85/300, Reward: -59.000, Step: 59 +2022-11-28 14:45:23 - r - INFO: - Current episode 85 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 86/300, Reward: -48.000, Step: 48 +2022-11-28 14:45:23 - r - INFO: - Episode: 87/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:23 - r - INFO: - Episode: 88/300, Reward: -70.000, Step: 70 +2022-11-28 14:45:23 - r - INFO: - Episode: 89/300, Reward: -34.000, Step: 34 +2022-11-28 14:45:23 - r - INFO: - Episode: 90/300, Reward: -40.000, Step: 40 +2022-11-28 14:45:23 - r - INFO: - Current episode 90 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 91/300, Reward: -27.000, Step: 27 +2022-11-28 14:45:23 - r - INFO: - Episode: 92/300, Reward: -65.000, Step: 65 +2022-11-28 14:45:23 - r - INFO: - Episode: 93/300, Reward: -79.000, Step: 79 +2022-11-28 14:45:23 - r - INFO: - Episode: 94/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Episode: 95/300, Reward: -38.000, Step: 38 +2022-11-28 14:45:23 - r - INFO: - Current episode 95 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 96/300, Reward: -23.000, Step: 23 +2022-11-28 14:45:23 - r - INFO: - Episode: 97/300, Reward: -60.000, Step: 60 +2022-11-28 14:45:23 - r - INFO: - Episode: 98/300, Reward: -41.000, Step: 41 +2022-11-28 14:45:23 - r - INFO: - Episode: 99/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:23 - r - INFO: - Episode: 100/300, Reward: -42.000, Step: 42 +2022-11-28 14:45:23 - r - INFO: - Current episode 100 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 101/300, Reward: -41.000, Step: 41 +2022-11-28 14:45:23 - r - INFO: - Episode: 102/300, Reward: -59.000, Step: 59 +2022-11-28 14:45:23 - r - INFO: - Episode: 103/300, Reward: -31.000, Step: 31 +2022-11-28 14:45:23 - r - INFO: - Episode: 104/300, Reward: -43.000, Step: 43 +2022-11-28 14:45:23 - r - INFO: - Episode: 105/300, Reward: -70.000, Step: 70 +2022-11-28 14:45:23 - r - INFO: - Current episode 105 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 106/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 107/300, Reward: -161.000, Step: 62 +2022-11-28 14:45:23 - r - INFO: - Episode: 108/300, Reward: -38.000, Step: 38 +2022-11-28 14:45:23 - r - INFO: - Episode: 109/300, Reward: -45.000, Step: 45 +2022-11-28 14:45:23 - r - INFO: - Episode: 110/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Current episode 110 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 111/300, Reward: -47.000, Step: 47 +2022-11-28 14:45:23 - r - INFO: - Episode: 112/300, Reward: -23.000, Step: 23 +2022-11-28 14:45:23 - r - INFO: - Episode: 113/300, Reward: -62.000, Step: 62 +2022-11-28 14:45:23 - r - INFO: - Episode: 114/300, Reward: -34.000, Step: 34 +2022-11-28 14:45:23 - r - INFO: - Episode: 115/300, Reward: -46.000, Step: 46 +2022-11-28 14:45:23 - r - INFO: - Current episode 115 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 116/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 117/300, Reward: -46.000, Step: 46 +2022-11-28 14:45:23 - r - INFO: - Episode: 118/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Episode: 119/300, Reward: -24.000, Step: 24 +2022-11-28 14:45:23 - r - INFO: - Episode: 120/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Current episode 120 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 121/300, Reward: -42.000, Step: 42 +2022-11-28 14:45:23 - r - INFO: - Episode: 122/300, Reward: -62.000, Step: 62 +2022-11-28 14:45:23 - r - INFO: - Episode: 123/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Episode: 124/300, Reward: -31.000, Step: 31 +2022-11-28 14:45:23 - r - INFO: - Episode: 125/300, Reward: -37.000, Step: 37 +2022-11-28 14:45:23 - r - INFO: - Current episode 125 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 126/300, Reward: -51.000, Step: 51 +2022-11-28 14:45:23 - r - INFO: - Episode: 127/300, Reward: -32.000, Step: 32 +2022-11-28 14:45:23 - r - INFO: - Episode: 128/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:23 - r - INFO: - Episode: 129/300, Reward: -34.000, Step: 34 +2022-11-28 14:45:23 - r - INFO: - Episode: 130/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:23 - r - INFO: - Current episode 130 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 131/300, Reward: -53.000, Step: 53 +2022-11-28 14:45:23 - r - INFO: - Episode: 132/300, Reward: -28.000, Step: 28 +2022-11-28 14:45:23 - r - INFO: - Episode: 133/300, Reward: -33.000, Step: 33 +2022-11-28 14:45:23 - r - INFO: - Episode: 134/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:23 - r - INFO: - Episode: 135/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:23 - r - INFO: - Current episode 135 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 136/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:23 - r - INFO: - Episode: 137/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Episode: 138/300, Reward: -42.000, Step: 42 +2022-11-28 14:45:23 - r - INFO: - Episode: 139/300, Reward: -154.000, Step: 55 +2022-11-28 14:45:23 - r - INFO: - Episode: 140/300, Reward: -34.000, Step: 34 +2022-11-28 14:45:23 - r - INFO: - Current episode 140 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 141/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 142/300, Reward: -21.000, Step: 21 +2022-11-28 14:45:23 - r - INFO: - Episode: 143/300, Reward: -38.000, Step: 38 +2022-11-28 14:45:23 - r - INFO: - Episode: 144/300, Reward: -21.000, Step: 21 +2022-11-28 14:45:23 - r - INFO: - Episode: 145/300, Reward: -46.000, Step: 46 +2022-11-28 14:45:23 - r - INFO: - Current episode 145 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 146/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:23 - r - INFO: - Episode: 147/300, Reward: -37.000, Step: 37 +2022-11-28 14:45:23 - r - INFO: - Episode: 148/300, Reward: -27.000, Step: 27 +2022-11-28 14:45:23 - r - INFO: - Episode: 149/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:23 - r - INFO: - Episode: 150/300, Reward: -18.000, Step: 18 +2022-11-28 14:45:23 - r - INFO: - Current episode 150 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 151/300, Reward: -34.000, Step: 34 +2022-11-28 14:45:23 - r - INFO: - Episode: 152/300, Reward: -25.000, Step: 25 +2022-11-28 14:45:23 - r - INFO: - Episode: 153/300, Reward: -32.000, Step: 32 +2022-11-28 14:45:23 - r - INFO: - Episode: 154/300, Reward: -40.000, Step: 40 +2022-11-28 14:45:23 - r - INFO: - Episode: 155/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Current episode 155 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 156/300, Reward: -36.000, Step: 36 +2022-11-28 14:45:23 - r - INFO: - Episode: 157/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 158/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Episode: 159/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 160/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:23 - r - INFO: - Current episode 160 has the best eval reward: -100.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 161/300, Reward: -18.000, Step: 18 +2022-11-28 14:45:23 - r - INFO: - Episode: 162/300, Reward: -46.000, Step: 46 +2022-11-28 14:45:23 - r - INFO: - Episode: 163/300, Reward: -27.000, Step: 27 +2022-11-28 14:45:23 - r - INFO: - Episode: 164/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:23 - r - INFO: - Episode: 165/300, Reward: -34.000, Step: 34 +2022-11-28 14:45:23 - r - INFO: - Current episode 165 has the best eval reward: -15.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 166/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:23 - r - INFO: - Episode: 167/300, Reward: -24.000, Step: 24 +2022-11-28 14:45:23 - r - INFO: - Episode: 168/300, Reward: -45.000, Step: 45 +2022-11-28 14:45:23 - r - INFO: - Episode: 169/300, Reward: -36.000, Step: 36 +2022-11-28 14:45:23 - r - INFO: - Episode: 170/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:23 - r - INFO: - Episode: 171/300, Reward: -26.000, Step: 26 +2022-11-28 14:45:23 - r - INFO: - Episode: 172/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 173/300, Reward: -32.000, Step: 32 +2022-11-28 14:45:23 - r - INFO: - Episode: 174/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:23 - r - INFO: - Episode: 175/300, Reward: -31.000, Step: 31 +2022-11-28 14:45:23 - r - INFO: - Current episode 175 has the best eval reward: -15.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 176/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:23 - r - INFO: - Episode: 177/300, Reward: -31.000, Step: 31 +2022-11-28 14:45:23 - r - INFO: - Episode: 178/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 179/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 180/300, Reward: -21.000, Step: 21 +2022-11-28 14:45:23 - r - INFO: - Episode: 181/300, Reward: -31.000, Step: 31 +2022-11-28 14:45:23 - r - INFO: - Episode: 182/300, Reward: -21.000, Step: 21 +2022-11-28 14:45:23 - r - INFO: - Episode: 183/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:23 - r - INFO: - Episode: 184/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Episode: 185/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:23 - r - INFO: - Episode: 186/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:23 - r - INFO: - Episode: 187/300, Reward: -27.000, Step: 27 +2022-11-28 14:45:23 - r - INFO: - Episode: 188/300, Reward: -18.000, Step: 18 +2022-11-28 14:45:23 - r - INFO: - Episode: 189/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:23 - r - INFO: - Episode: 190/300, Reward: -24.000, Step: 24 +2022-11-28 14:45:23 - r - INFO: - Episode: 191/300, Reward: -28.000, Step: 28 +2022-11-28 14:45:23 - r - INFO: - Episode: 192/300, Reward: -36.000, Step: 36 +2022-11-28 14:45:23 - r - INFO: - Episode: 193/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:23 - r - INFO: - Episode: 194/300, Reward: -40.000, Step: 40 +2022-11-28 14:45:23 - r - INFO: - Episode: 195/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:23 - r - INFO: - Episode: 196/300, Reward: -23.000, Step: 23 +2022-11-28 14:45:23 - r - INFO: - Episode: 197/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:23 - r - INFO: - Episode: 198/300, Reward: -28.000, Step: 28 +2022-11-28 14:45:23 - r - INFO: - Episode: 199/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:23 - r - INFO: - Episode: 200/300, Reward: -27.000, Step: 27 +2022-11-28 14:45:23 - r - INFO: - Episode: 201/300, Reward: -14.000, Step: 14 +2022-11-28 14:45:23 - r - INFO: - Episode: 202/300, Reward: -31.000, Step: 31 +2022-11-28 14:45:23 - r - INFO: - Episode: 203/300, Reward: -25.000, Step: 25 +2022-11-28 14:45:23 - r - INFO: - Episode: 204/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:23 - r - INFO: - Episode: 205/300, Reward: -36.000, Step: 36 +2022-11-28 14:45:23 - r - INFO: - Episode: 206/300, Reward: -29.000, Step: 29 +2022-11-28 14:45:23 - r - INFO: - Episode: 207/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:23 - r - INFO: - Episode: 208/300, Reward: -23.000, Step: 23 +2022-11-28 14:45:23 - r - INFO: - Episode: 209/300, Reward: -14.000, Step: 14 +2022-11-28 14:45:23 - r - INFO: - Episode: 210/300, Reward: -28.000, Step: 28 +2022-11-28 14:45:23 - r - INFO: - Episode: 211/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:23 - r - INFO: - Episode: 212/300, Reward: -26.000, Step: 26 +2022-11-28 14:45:23 - r - INFO: - Episode: 213/300, Reward: -18.000, Step: 18 +2022-11-28 14:45:23 - r - INFO: - Episode: 214/300, Reward: -26.000, Step: 26 +2022-11-28 14:45:23 - r - INFO: - Episode: 215/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:23 - r - INFO: - Episode: 216/300, Reward: -23.000, Step: 23 +2022-11-28 14:45:23 - r - INFO: - Episode: 217/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:23 - r - INFO: - Episode: 218/300, Reward: -48.000, Step: 48 +2022-11-28 14:45:23 - r - INFO: - Episode: 219/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:23 - r - INFO: - Episode: 220/300, Reward: -133.000, Step: 34 +2022-11-28 14:45:23 - r - INFO: - Episode: 221/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:23 - r - INFO: - Episode: 222/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:23 - r - INFO: - Episode: 223/300, Reward: -18.000, Step: 18 +2022-11-28 14:45:23 - r - INFO: - Episode: 224/300, Reward: -35.000, Step: 35 +2022-11-28 14:45:23 - r - INFO: - Episode: 225/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:23 - r - INFO: - Current episode 225 has the best eval reward: -13.000 +2022-11-28 14:45:23 - r - INFO: - Episode: 226/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:23 - r - INFO: - Episode: 227/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:23 - r - INFO: - Episode: 228/300, Reward: -25.000, Step: 25 +2022-11-28 14:45:23 - r - INFO: - Episode: 229/300, Reward: -25.000, Step: 25 +2022-11-28 14:45:23 - r - INFO: - Episode: 230/300, Reward: -26.000, Step: 26 +2022-11-28 14:45:24 - r - INFO: - Episode: 231/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:24 - r - INFO: - Episode: 232/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:24 - r - INFO: - Episode: 233/300, Reward: -27.000, Step: 27 +2022-11-28 14:45:24 - r - INFO: - Episode: 234/300, Reward: -30.000, Step: 30 +2022-11-28 14:45:24 - r - INFO: - Episode: 235/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:24 - r - INFO: - Episode: 236/300, Reward: -18.000, Step: 18 +2022-11-28 14:45:24 - r - INFO: - Episode: 237/300, Reward: -122.000, Step: 23 +2022-11-28 14:45:24 - r - INFO: - Episode: 238/300, Reward: -25.000, Step: 25 +2022-11-28 14:45:24 - r - INFO: - Episode: 239/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:24 - r - INFO: - Episode: 240/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 241/300, Reward: -39.000, Step: 39 +2022-11-28 14:45:24 - r - INFO: - Episode: 242/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:24 - r - INFO: - Episode: 243/300, Reward: -14.000, Step: 14 +2022-11-28 14:45:24 - r - INFO: - Episode: 244/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 245/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:24 - r - INFO: - Episode: 246/300, Reward: -31.000, Step: 31 +2022-11-28 14:45:24 - r - INFO: - Episode: 247/300, Reward: -20.000, Step: 20 +2022-11-28 14:45:24 - r - INFO: - Episode: 248/300, Reward: -28.000, Step: 28 +2022-11-28 14:45:24 - r - INFO: - Episode: 249/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 250/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:24 - r - INFO: - Episode: 251/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 252/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 253/300, Reward: -21.000, Step: 21 +2022-11-28 14:45:24 - r - INFO: - Episode: 254/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:24 - r - INFO: - Episode: 255/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:24 - r - INFO: - Episode: 256/300, Reward: -18.000, Step: 18 +2022-11-28 14:45:24 - r - INFO: - Episode: 257/300, Reward: -23.000, Step: 23 +2022-11-28 14:45:24 - r - INFO: - Episode: 258/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:24 - r - INFO: - Episode: 259/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:24 - r - INFO: - Episode: 260/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Current episode 260 has the best eval reward: -13.000 +2022-11-28 14:45:24 - r - INFO: - Episode: 261/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 262/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:24 - r - INFO: - Episode: 263/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:24 - r - INFO: - Episode: 264/300, Reward: -32.000, Step: 32 +2022-11-28 14:45:24 - r - INFO: - Episode: 265/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:24 - r - INFO: - Episode: 266/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 267/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:24 - r - INFO: - Episode: 268/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 269/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 270/300, Reward: -28.000, Step: 28 +2022-11-28 14:45:24 - r - INFO: - Episode: 271/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 272/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 273/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:24 - r - INFO: - Episode: 274/300, Reward: -128.000, Step: 29 +2022-11-28 14:45:24 - r - INFO: - Episode: 275/300, Reward: -32.000, Step: 32 +2022-11-28 14:45:24 - r - INFO: - Episode: 276/300, Reward: -24.000, Step: 24 +2022-11-28 14:45:24 - r - INFO: - Episode: 277/300, Reward: -26.000, Step: 26 +2022-11-28 14:45:24 - r - INFO: - Episode: 278/300, Reward: -14.000, Step: 14 +2022-11-28 14:45:24 - r - INFO: - Episode: 279/300, Reward: -25.000, Step: 25 +2022-11-28 14:45:24 - r - INFO: - Episode: 280/300, Reward: -22.000, Step: 22 +2022-11-28 14:45:24 - r - INFO: - Current episode 280 has the best eval reward: -13.000 +2022-11-28 14:45:24 - r - INFO: - Episode: 281/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 282/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 283/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 284/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 285/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 286/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:24 - r - INFO: - Episode: 287/300, Reward: -21.000, Step: 21 +2022-11-28 14:45:24 - r - INFO: - Episode: 288/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:24 - r - INFO: - Episode: 289/300, Reward: -26.000, Step: 26 +2022-11-28 14:45:24 - r - INFO: - Episode: 290/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:24 - r - INFO: - Current episode 290 has the best eval reward: -13.000 +2022-11-28 14:45:24 - r - INFO: - Episode: 291/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 292/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 293/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 294/300, Reward: -15.000, Step: 15 +2022-11-28 14:45:24 - r - INFO: - Episode: 295/300, Reward: -13.000, Step: 13 +2022-11-28 14:45:24 - r - INFO: - Episode: 296/300, Reward: -24.000, Step: 24 +2022-11-28 14:45:24 - r - INFO: - Episode: 297/300, Reward: -19.000, Step: 19 +2022-11-28 14:45:24 - r - INFO: - Episode: 298/300, Reward: -14.000, Step: 14 +2022-11-28 14:45:24 - r - INFO: - Episode: 299/300, Reward: -16.000, Step: 16 +2022-11-28 14:45:24 - r - INFO: - Episode: 300/300, Reward: -17.000, Step: 17 +2022-11-28 14:45:24 - r - INFO: - Finish training! diff --git a/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/models/Qleaning_model.pkl b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/models/Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..11b3246418fe8e689ff25f81da33205b94378416 --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/models/Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a5088e21ba086ef3631d0381b7b57ec20f56261d403a159a6c8d0cae3d6ec63 +size 5999 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/results/learning_curve.png b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..9a7d7b53cd632bf99f7c9b1335f1816c33df8bea Binary files /dev/null and b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/results/learning_curve.png differ diff --git a/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/results/res.csv b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e5550faa60c3fd0b9aed2945e73ed281a843f5e --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_QLearning_20221128-144520/results/res.csv @@ -0,0 +1,301 @@ +episodes,rewards,steps +0,-397,100 +1,-1090,100 +2,-694,100 +3,-397,100 +4,-496,100 +5,-298,100 +6,-199,100 +7,-298,100 +8,-100,100 +9,-199,100 +10,-93,93 +11,-100,100 +12,-199,100 +13,-199,100 +14,-100,100 +15,-100,100 +16,-33,33 +17,-100,100 +18,-66,66 +19,-99,99 +20,-100,100 +21,-100,100 +22,-100,100 +23,-54,54 +24,-100,100 +25,-73,73 +26,-100,100 +27,-90,90 +28,-87,87 +29,-100,100 +30,-46,46 +31,-82,82 +32,-100,100 +33,-100,100 +34,-67,67 +35,-75,75 +36,-100,100 +37,-45,45 +38,-100,100 +39,-37,37 +40,-100,100 +41,-43,43 +42,-100,100 +43,-56,56 +44,-70,70 +45,-100,100 +46,-47,47 +47,-85,85 +48,-82,82 +49,-51,51 +50,-50,50 +51,-86,86 +52,-100,100 +53,-40,40 +54,-87,87 +55,-59,59 +56,-51,51 +57,-98,98 +58,-67,67 +59,-60,60 +60,-38,38 +61,-76,76 +62,-70,70 +63,-60,60 +64,-68,68 +65,-25,25 +66,-69,69 +67,-183,84 +68,-62,62 +69,-44,44 +70,-71,71 +71,-55,55 +72,-65,65 +73,-61,61 +74,-36,36 +75,-47,47 +76,-63,63 +77,-45,45 +78,-59,59 +79,-39,39 +80,-62,62 +81,-38,38 +82,-64,64 +83,-40,40 +84,-59,59 +85,-48,48 +86,-39,39 +87,-70,70 +88,-34,34 +89,-40,40 +90,-27,27 +91,-65,65 +92,-79,79 +93,-30,30 +94,-38,38 +95,-23,23 +96,-60,60 +97,-41,41 +98,-39,39 +99,-42,42 +100,-41,41 +101,-59,59 +102,-31,31 +103,-43,43 +104,-70,70 +105,-29,29 +106,-161,62 +107,-38,38 +108,-45,45 +109,-29,29 +110,-47,47 +111,-23,23 +112,-62,62 +113,-34,34 +114,-46,46 +115,-29,29 +116,-46,46 +117,-30,30 +118,-24,24 +119,-30,30 +120,-42,42 +121,-62,62 +122,-30,30 +123,-31,31 +124,-37,37 +125,-51,51 +126,-32,32 +127,-22,22 +128,-34,34 +129,-22,22 +130,-53,53 +131,-28,28 +132,-33,33 +133,-39,39 +134,-39,39 +135,-20,20 +136,-30,30 +137,-42,42 +138,-154,55 +139,-34,34 +140,-29,29 +141,-21,21 +142,-38,38 +143,-21,21 +144,-46,46 +145,-19,19 +146,-37,37 +147,-27,27 +148,-39,39 +149,-18,18 +150,-34,34 +151,-25,25 +152,-32,32 +153,-40,40 +154,-30,30 +155,-36,36 +156,-29,29 +157,-30,30 +158,-29,29 +159,-22,22 +160,-18,18 +161,-46,46 +162,-27,27 +163,-22,22 +164,-34,34 +165,-15,15 +166,-24,24 +167,-45,45 +168,-36,36 +169,-20,20 +170,-26,26 +171,-29,29 +172,-32,32 +173,-17,17 +174,-31,31 +175,-15,15 +176,-31,31 +177,-29,29 +178,-29,29 +179,-21,21 +180,-31,31 +181,-21,21 +182,-39,39 +183,-30,30 +184,-13,13 +185,-30,30 +186,-27,27 +187,-18,18 +188,-17,17 +189,-24,24 +190,-28,28 +191,-36,36 +192,-15,15 +193,-40,40 +194,-20,20 +195,-23,23 +196,-17,17 +197,-28,28 +198,-15,15 +199,-27,27 +200,-14,14 +201,-31,31 +202,-25,25 +203,-19,19 +204,-36,36 +205,-29,29 +206,-20,20 +207,-23,23 +208,-14,14 +209,-28,28 +210,-19,19 +211,-26,26 +212,-18,18 +213,-26,26 +214,-19,19 +215,-23,23 +216,-20,20 +217,-48,48 +218,-13,13 +219,-133,34 +220,-17,17 +221,-20,20 +222,-18,18 +223,-35,35 +224,-17,17 +225,-13,13 +226,-16,16 +227,-25,25 +228,-25,25 +229,-26,26 +230,-19,19 +231,-17,17 +232,-27,27 +233,-30,30 +234,-17,17 +235,-18,18 +236,-122,23 +237,-25,25 +238,-20,20 +239,-13,13 +240,-39,39 +241,-17,17 +242,-14,14 +243,-15,15 +244,-22,22 +245,-31,31 +246,-20,20 +247,-28,28 +248,-15,15 +249,-19,19 +250,-15,15 +251,-13,13 +252,-21,21 +253,-16,16 +254,-22,22 +255,-18,18 +256,-23,23 +257,-16,16 +258,-22,22 +259,-13,13 +260,-13,13 +261,-19,19 +262,-17,17 +263,-32,32 +264,-16,16 +265,-15,15 +266,-17,17 +267,-13,13 +268,-15,15 +269,-28,28 +270,-15,15 +271,-13,13 +272,-16,16 +273,-128,29 +274,-32,32 +275,-24,24 +276,-26,26 +277,-14,14 +278,-25,25 +279,-22,22 +280,-13,13 +281,-15,15 +282,-15,15 +283,-13,13 +284,-13,13 +285,-16,16 +286,-21,21 +287,-19,19 +288,-26,26 +289,-16,16 +290,-13,13 +291,-13,13 +292,-13,13 +293,-15,15 +294,-13,13 +295,-24,24 +296,-19,19 +297,-14,14 +298,-16,16 +299,-17,17 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/config.yaml b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7df6ea4f615fa0de19ff9043f5b567ff2cfdcce --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/config.yaml @@ -0,0 +1,25 @@ +general_cfg: + algo_name: Sarsa + device: cpu + env_name: CliffWalking-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 100 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: envs.wrappers.CliffWalkingWapper +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + exploration_type: e-greedy + gamma: 0.99 + lr: 0.1 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/logs/log.txt b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6bc447c26eafe85b9215db161eabfc8ebe19c576 --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/logs/log.txt @@ -0,0 +1,475 @@ +2023-03-11 00:15:04 - r - INFO: - Hyperparameters: +2023-03-11 00:15:04 - r - INFO: - ================================================================================ +2023-03-11 00:15:04 - r - INFO: - Name Value Type +2023-03-11 00:15:04 - r - INFO: - env_name CliffWalking-v0 +2023-03-11 00:15:04 - r - INFO: - new_step_api 1 +2023-03-11 00:15:04 - r - INFO: - wrapper envs.wrappers.CliffWalkingWapper +2023-03-11 00:15:04 - r - INFO: - render 0 +2023-03-11 00:15:04 - r - INFO: - algo_name Sarsa +2023-03-11 00:15:04 - r - INFO: - mode train +2023-03-11 00:15:04 - r - INFO: - seed 1 +2023-03-11 00:15:04 - r - INFO: - device cpu +2023-03-11 00:15:04 - r - INFO: - train_eps 400 +2023-03-11 00:15:04 - r - INFO: - test_eps 10 +2023-03-11 00:15:04 - r - INFO: - eval_eps 10 +2023-03-11 00:15:04 - r - INFO: - eval_per_episode 5 +2023-03-11 00:15:04 - r - INFO: - max_steps 100 +2023-03-11 00:15:04 - r - INFO: - load_checkpoint 0 +2023-03-11 00:15:04 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-03-11 00:15:04 - r - INFO: - show_fig 0 +2023-03-11 00:15:04 - r - INFO: - save_fig 1 +2023-03-11 00:15:04 - r - INFO: - epsilon_start 0.95 +2023-03-11 00:15:04 - r - INFO: - epsilon_end 0.01 +2023-03-11 00:15:04 - r - INFO: - epsilon_decay 300 +2023-03-11 00:15:04 - r - INFO: - gamma 0.99 +2023-03-11 00:15:04 - r - INFO: - lr 0.1 +2023-03-11 00:15:04 - r - INFO: - exploration_type e-greedy +2023-03-11 00:15:04 - r - INFO: - task_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CliffWalking-v0_Sarsa_20230311-001504 +2023-03-11 00:15:04 - r - INFO: - res_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CliffWalking-v0_Sarsa_20230311-001504/results +2023-03-11 00:15:04 - r - INFO: - log_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CliffWalking-v0_Sarsa_20230311-001504/logs +2023-03-11 00:15:04 - r - INFO: - traj_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CliffWalking-v0_Sarsa_20230311-001504/traj +2023-03-11 00:15:04 - r - INFO: - tb_dir C:\Users\24438\Desktop\joyrl-offline/tasks/Train_CliffWalking-v0_Sarsa_20230311-001504/tb_logs +2023-03-11 00:15:04 - r - INFO: - ================================================================================ +2023-03-11 00:15:04 - r - INFO: - n_states: 48, n_actions: 4 +2023-03-11 00:15:04 - r - INFO: - Start training! +2023-03-11 00:15:04 - r - INFO: - Env: CliffWalking-v0, Algorithm: Sarsa, Device: cpu +2023-03-11 00:15:04 - r - INFO: - Episode: 1/400, Reward: -595.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 2/400, Reward: -397.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 3/400, Reward: -397.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 4/400, Reward: -199.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 5/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Current episode 5 has the best eval reward: -100.000 +2023-03-11 00:15:04 - r - INFO: - Episode: 6/400, Reward: -36.000, Step: 36 +2023-03-11 00:15:04 - r - INFO: - Episode: 7/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 8/400, Reward: -199.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 9/400, Reward: -199.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 10/400, Reward: -199.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Current episode 10 has the best eval reward: -100.000 +2023-03-11 00:15:04 - r - INFO: - Episode: 11/400, Reward: -199.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 12/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 13/400, Reward: -48.000, Step: 48 +2023-03-11 00:15:04 - r - INFO: - Episode: 14/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 15/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Current episode 15 has the best eval reward: -100.000 +2023-03-11 00:15:04 - r - INFO: - Episode: 16/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 17/400, Reward: -46.000, Step: 46 +2023-03-11 00:15:04 - r - INFO: - Episode: 18/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 19/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 20/400, Reward: -64.000, Step: 64 +2023-03-11 00:15:04 - r - INFO: - Current episode 20 has the best eval reward: -100.000 +2023-03-11 00:15:04 - r - INFO: - Episode: 21/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 22/400, Reward: -74.000, Step: 74 +2023-03-11 00:15:04 - r - INFO: - Episode: 23/400, Reward: -89.000, Step: 89 +2023-03-11 00:15:04 - r - INFO: - Episode: 24/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 25/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Current episode 25 has the best eval reward: -100.000 +2023-03-11 00:15:04 - r - INFO: - Episode: 26/400, Reward: -68.000, Step: 68 +2023-03-11 00:15:04 - r - INFO: - Episode: 27/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:04 - r - INFO: - Episode: 28/400, Reward: -52.000, Step: 52 +2023-03-11 00:15:04 - r - INFO: - Episode: 29/400, Reward: -86.000, Step: 86 +2023-03-11 00:15:04 - r - INFO: - Episode: 30/400, Reward: -66.000, Step: 66 +2023-03-11 00:15:04 - r - INFO: - Current episode 30 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 31/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 32/400, Reward: -45.000, Step: 45 +2023-03-11 00:15:05 - r - INFO: - Episode: 33/400, Reward: -77.000, Step: 77 +2023-03-11 00:15:05 - r - INFO: - Episode: 34/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 35/400, Reward: -56.000, Step: 56 +2023-03-11 00:15:05 - r - INFO: - Current episode 35 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 36/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 37/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Episode: 38/400, Reward: -81.000, Step: 81 +2023-03-11 00:15:05 - r - INFO: - Episode: 39/400, Reward: -79.000, Step: 79 +2023-03-11 00:15:05 - r - INFO: - Episode: 40/400, Reward: -73.000, Step: 73 +2023-03-11 00:15:05 - r - INFO: - Current episode 40 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 41/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 42/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 43/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 44/400, Reward: -82.000, Step: 82 +2023-03-11 00:15:05 - r - INFO: - Episode: 45/400, Reward: -44.000, Step: 44 +2023-03-11 00:15:05 - r - INFO: - Current episode 45 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 46/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 47/400, Reward: -42.000, Step: 42 +2023-03-11 00:15:05 - r - INFO: - Episode: 48/400, Reward: -61.000, Step: 61 +2023-03-11 00:15:05 - r - INFO: - Episode: 49/400, Reward: -71.000, Step: 71 +2023-03-11 00:15:05 - r - INFO: - Episode: 50/400, Reward: -96.000, Step: 96 +2023-03-11 00:15:05 - r - INFO: - Current episode 50 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 51/400, Reward: -76.000, Step: 76 +2023-03-11 00:15:05 - r - INFO: - Episode: 52/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 53/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Episode: 54/400, Reward: -99.000, Step: 99 +2023-03-11 00:15:05 - r - INFO: - Episode: 55/400, Reward: -41.000, Step: 41 +2023-03-11 00:15:05 - r - INFO: - Current episode 55 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 56/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 57/400, Reward: -34.000, Step: 34 +2023-03-11 00:15:05 - r - INFO: - Episode: 58/400, Reward: -61.000, Step: 61 +2023-03-11 00:15:05 - r - INFO: - Episode: 59/400, Reward: -55.000, Step: 55 +2023-03-11 00:15:05 - r - INFO: - Episode: 60/400, Reward: -168.000, Step: 69 +2023-03-11 00:15:05 - r - INFO: - Current episode 60 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 61/400, Reward: -100.000, Step: 100 +2023-03-11 00:15:05 - r - INFO: - Episode: 62/400, Reward: -76.000, Step: 76 +2023-03-11 00:15:05 - r - INFO: - Episode: 63/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 64/400, Reward: -54.000, Step: 54 +2023-03-11 00:15:05 - r - INFO: - Episode: 65/400, Reward: -54.000, Step: 54 +2023-03-11 00:15:05 - r - INFO: - Current episode 65 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 66/400, Reward: -65.000, Step: 65 +2023-03-11 00:15:05 - r - INFO: - Episode: 67/400, Reward: -42.000, Step: 42 +2023-03-11 00:15:05 - r - INFO: - Episode: 68/400, Reward: -67.000, Step: 67 +2023-03-11 00:15:05 - r - INFO: - Episode: 69/400, Reward: -61.000, Step: 61 +2023-03-11 00:15:05 - r - INFO: - Episode: 70/400, Reward: -54.000, Step: 54 +2023-03-11 00:15:05 - r - INFO: - Current episode 70 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 71/400, Reward: -42.000, Step: 42 +2023-03-11 00:15:05 - r - INFO: - Episode: 72/400, Reward: -64.000, Step: 64 +2023-03-11 00:15:05 - r - INFO: - Episode: 73/400, Reward: -52.000, Step: 52 +2023-03-11 00:15:05 - r - INFO: - Episode: 74/400, Reward: -45.000, Step: 45 +2023-03-11 00:15:05 - r - INFO: - Episode: 75/400, Reward: -57.000, Step: 57 +2023-03-11 00:15:05 - r - INFO: - Current episode 75 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 76/400, Reward: -59.000, Step: 59 +2023-03-11 00:15:05 - r - INFO: - Episode: 77/400, Reward: -67.000, Step: 67 +2023-03-11 00:15:05 - r - INFO: - Episode: 78/400, Reward: -46.000, Step: 46 +2023-03-11 00:15:05 - r - INFO: - Episode: 79/400, Reward: -48.000, Step: 48 +2023-03-11 00:15:05 - r - INFO: - Episode: 80/400, Reward: -63.000, Step: 63 +2023-03-11 00:15:05 - r - INFO: - Current episode 80 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 81/400, Reward: -60.000, Step: 60 +2023-03-11 00:15:05 - r - INFO: - Episode: 82/400, Reward: -47.000, Step: 47 +2023-03-11 00:15:05 - r - INFO: - Episode: 83/400, Reward: -34.000, Step: 34 +2023-03-11 00:15:05 - r - INFO: - Episode: 84/400, Reward: -188.000, Step: 89 +2023-03-11 00:15:05 - r - INFO: - Episode: 85/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:05 - r - INFO: - Current episode 85 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 86/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 87/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Episode: 88/400, Reward: -61.000, Step: 61 +2023-03-11 00:15:05 - r - INFO: - Episode: 89/400, Reward: -39.000, Step: 39 +2023-03-11 00:15:05 - r - INFO: - Episode: 90/400, Reward: -57.000, Step: 57 +2023-03-11 00:15:05 - r - INFO: - Current episode 90 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 91/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Episode: 92/400, Reward: -56.000, Step: 56 +2023-03-11 00:15:05 - r - INFO: - Episode: 93/400, Reward: -60.000, Step: 60 +2023-03-11 00:15:05 - r - INFO: - Episode: 94/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:05 - r - INFO: - Episode: 95/400, Reward: -61.000, Step: 61 +2023-03-11 00:15:05 - r - INFO: - Current episode 95 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 96/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 97/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:05 - r - INFO: - Episode: 98/400, Reward: -73.000, Step: 73 +2023-03-11 00:15:05 - r - INFO: - Episode: 99/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:05 - r - INFO: - Episode: 100/400, Reward: -44.000, Step: 44 +2023-03-11 00:15:05 - r - INFO: - Current episode 100 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 101/400, Reward: -55.000, Step: 55 +2023-03-11 00:15:05 - r - INFO: - Episode: 102/400, Reward: -56.000, Step: 56 +2023-03-11 00:15:05 - r - INFO: - Episode: 103/400, Reward: -42.000, Step: 42 +2023-03-11 00:15:05 - r - INFO: - Episode: 104/400, Reward: -48.000, Step: 48 +2023-03-11 00:15:05 - r - INFO: - Episode: 105/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:05 - r - INFO: - Current episode 105 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 106/400, Reward: -49.000, Step: 49 +2023-03-11 00:15:05 - r - INFO: - Episode: 107/400, Reward: -42.000, Step: 42 +2023-03-11 00:15:05 - r - INFO: - Episode: 108/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 109/400, Reward: -49.000, Step: 49 +2023-03-11 00:15:05 - r - INFO: - Episode: 110/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Current episode 110 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 111/400, Reward: -37.000, Step: 37 +2023-03-11 00:15:05 - r - INFO: - Episode: 112/400, Reward: -34.000, Step: 34 +2023-03-11 00:15:05 - r - INFO: - Episode: 113/400, Reward: -64.000, Step: 64 +2023-03-11 00:15:05 - r - INFO: - Episode: 114/400, Reward: -32.000, Step: 32 +2023-03-11 00:15:05 - r - INFO: - Episode: 115/400, Reward: -40.000, Step: 40 +2023-03-11 00:15:05 - r - INFO: - Current episode 115 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 116/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:05 - r - INFO: - Episode: 117/400, Reward: -41.000, Step: 41 +2023-03-11 00:15:05 - r - INFO: - Episode: 118/400, Reward: -44.000, Step: 44 +2023-03-11 00:15:05 - r - INFO: - Episode: 119/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:05 - r - INFO: - Episode: 120/400, Reward: -39.000, Step: 39 +2023-03-11 00:15:05 - r - INFO: - Current episode 120 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 121/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 122/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Episode: 123/400, Reward: -32.000, Step: 32 +2023-03-11 00:15:05 - r - INFO: - Episode: 124/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:05 - r - INFO: - Episode: 125/400, Reward: -45.000, Step: 45 +2023-03-11 00:15:05 - r - INFO: - Current episode 125 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 126/400, Reward: -44.000, Step: 44 +2023-03-11 00:15:05 - r - INFO: - Episode: 127/400, Reward: -50.000, Step: 50 +2023-03-11 00:15:05 - r - INFO: - Episode: 128/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 129/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 130/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Current episode 130 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 131/400, Reward: -48.000, Step: 48 +2023-03-11 00:15:05 - r - INFO: - Episode: 132/400, Reward: -55.000, Step: 55 +2023-03-11 00:15:05 - r - INFO: - Episode: 133/400, Reward: -30.000, Step: 30 +2023-03-11 00:15:05 - r - INFO: - Episode: 134/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 135/400, Reward: -49.000, Step: 49 +2023-03-11 00:15:05 - r - INFO: - Current episode 135 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 136/400, Reward: -34.000, Step: 34 +2023-03-11 00:15:05 - r - INFO: - Episode: 137/400, Reward: -41.000, Step: 41 +2023-03-11 00:15:05 - r - INFO: - Episode: 138/400, Reward: -41.000, Step: 41 +2023-03-11 00:15:05 - r - INFO: - Episode: 139/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:05 - r - INFO: - Episode: 140/400, Reward: -48.000, Step: 48 +2023-03-11 00:15:05 - r - INFO: - Current episode 140 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 141/400, Reward: -174.000, Step: 75 +2023-03-11 00:15:05 - r - INFO: - Episode: 142/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:05 - r - INFO: - Episode: 143/400, Reward: -39.000, Step: 39 +2023-03-11 00:15:05 - r - INFO: - Episode: 144/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 145/400, Reward: -47.000, Step: 47 +2023-03-11 00:15:05 - r - INFO: - Current episode 145 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 146/400, Reward: -47.000, Step: 47 +2023-03-11 00:15:05 - r - INFO: - Episode: 147/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 148/400, Reward: -32.000, Step: 32 +2023-03-11 00:15:05 - r - INFO: - Episode: 149/400, Reward: -49.000, Step: 49 +2023-03-11 00:15:05 - r - INFO: - Episode: 150/400, Reward: -38.000, Step: 38 +2023-03-11 00:15:05 - r - INFO: - Current episode 150 has the best eval reward: -100.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 151/400, Reward: -38.000, Step: 38 +2023-03-11 00:15:05 - r - INFO: - Episode: 152/400, Reward: -37.000, Step: 37 +2023-03-11 00:15:05 - r - INFO: - Episode: 153/400, Reward: -57.000, Step: 57 +2023-03-11 00:15:05 - r - INFO: - Episode: 154/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:05 - r - INFO: - Episode: 155/400, Reward: -38.000, Step: 38 +2023-03-11 00:15:05 - r - INFO: - Current episode 155 has the best eval reward: -15.000 +2023-03-11 00:15:05 - r - INFO: - Episode: 156/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:05 - r - INFO: - Episode: 157/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Episode: 158/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 159/400, Reward: -45.000, Step: 45 +2023-03-11 00:15:05 - r - INFO: - Episode: 160/400, Reward: -40.000, Step: 40 +2023-03-11 00:15:05 - r - INFO: - Episode: 161/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 162/400, Reward: -28.000, Step: 28 +2023-03-11 00:15:05 - r - INFO: - Episode: 163/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 164/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:05 - r - INFO: - Episode: 165/400, Reward: -30.000, Step: 30 +2023-03-11 00:15:05 - r - INFO: - Episode: 166/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:05 - r - INFO: - Episode: 167/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 168/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:05 - r - INFO: - Episode: 169/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:05 - r - INFO: - Episode: 170/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:05 - r - INFO: - Episode: 171/400, Reward: -38.000, Step: 38 +2023-03-11 00:15:05 - r - INFO: - Episode: 172/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:05 - r - INFO: - Episode: 173/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 174/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:05 - r - INFO: - Episode: 175/400, Reward: -38.000, Step: 38 +2023-03-11 00:15:05 - r - INFO: - Episode: 176/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:05 - r - INFO: - Episode: 177/400, Reward: -44.000, Step: 44 +2023-03-11 00:15:05 - r - INFO: - Episode: 178/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 179/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:05 - r - INFO: - Episode: 180/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 181/400, Reward: -31.000, Step: 31 +2023-03-11 00:15:05 - r - INFO: - Episode: 182/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:05 - r - INFO: - Episode: 183/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 184/400, Reward: -37.000, Step: 37 +2023-03-11 00:15:05 - r - INFO: - Episode: 185/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 186/400, Reward: -33.000, Step: 33 +2023-03-11 00:15:05 - r - INFO: - Episode: 187/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:05 - r - INFO: - Episode: 188/400, Reward: -31.000, Step: 31 +2023-03-11 00:15:05 - r - INFO: - Episode: 189/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:05 - r - INFO: - Episode: 190/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 191/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:05 - r - INFO: - Episode: 192/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:05 - r - INFO: - Episode: 193/400, Reward: -43.000, Step: 43 +2023-03-11 00:15:05 - r - INFO: - Episode: 194/400, Reward: -30.000, Step: 30 +2023-03-11 00:15:05 - r - INFO: - Episode: 195/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 196/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 197/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:05 - r - INFO: - Episode: 198/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:05 - r - INFO: - Episode: 199/400, Reward: -31.000, Step: 31 +2023-03-11 00:15:05 - r - INFO: - Episode: 200/400, Reward: -33.000, Step: 33 +2023-03-11 00:15:05 - r - INFO: - Episode: 201/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 202/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:05 - r - INFO: - Episode: 203/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 204/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 205/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 206/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:05 - r - INFO: - Episode: 207/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:05 - r - INFO: - Episode: 208/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:05 - r - INFO: - Episode: 209/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:05 - r - INFO: - Episode: 210/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 211/400, Reward: -31.000, Step: 31 +2023-03-11 00:15:05 - r - INFO: - Episode: 212/400, Reward: -42.000, Step: 42 +2023-03-11 00:15:05 - r - INFO: - Episode: 213/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 214/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:05 - r - INFO: - Episode: 215/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 216/400, Reward: -30.000, Step: 30 +2023-03-11 00:15:05 - r - INFO: - Episode: 217/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:05 - r - INFO: - Episode: 218/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:05 - r - INFO: - Episode: 219/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:05 - r - INFO: - Episode: 220/400, Reward: -28.000, Step: 28 +2023-03-11 00:15:05 - r - INFO: - Episode: 221/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:05 - r - INFO: - Episode: 222/400, Reward: -38.000, Step: 38 +2023-03-11 00:15:05 - r - INFO: - Episode: 223/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 224/400, Reward: -31.000, Step: 31 +2023-03-11 00:15:05 - r - INFO: - Episode: 225/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:05 - r - INFO: - Episode: 226/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:05 - r - INFO: - Episode: 227/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:05 - r - INFO: - Episode: 228/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 229/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 230/400, Reward: -30.000, Step: 30 +2023-03-11 00:15:05 - r - INFO: - Episode: 231/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 232/400, Reward: -33.000, Step: 33 +2023-03-11 00:15:05 - r - INFO: - Episode: 233/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 234/400, Reward: -35.000, Step: 35 +2023-03-11 00:15:05 - r - INFO: - Episode: 235/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 236/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 237/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 238/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:05 - r - INFO: - Episode: 239/400, Reward: -33.000, Step: 33 +2023-03-11 00:15:05 - r - INFO: - Episode: 240/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 241/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:05 - r - INFO: - Episode: 242/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:05 - r - INFO: - Episode: 243/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:05 - r - INFO: - Episode: 244/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:05 - r - INFO: - Episode: 245/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:05 - r - INFO: - Episode: 246/400, Reward: -126.000, Step: 27 +2023-03-11 00:15:05 - r - INFO: - Episode: 247/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:05 - r - INFO: - Episode: 248/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 249/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:05 - r - INFO: - Episode: 250/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:05 - r - INFO: - Episode: 251/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:05 - r - INFO: - Episode: 252/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:05 - r - INFO: - Episode: 253/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:05 - r - INFO: - Episode: 254/400, Reward: -28.000, Step: 28 +2023-03-11 00:15:05 - r - INFO: - Episode: 255/400, Reward: -31.000, Step: 31 +2023-03-11 00:15:06 - r - INFO: - Episode: 256/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:06 - r - INFO: - Episode: 257/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:06 - r - INFO: - Episode: 258/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 259/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 260/400, Reward: -31.000, Step: 31 +2023-03-11 00:15:06 - r - INFO: - Current episode 260 has the best eval reward: -15.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 261/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 262/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:06 - r - INFO: - Episode: 263/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 264/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 265/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 266/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 267/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 268/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 269/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 270/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 271/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:06 - r - INFO: - Episode: 272/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 273/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 274/400, Reward: -30.000, Step: 30 +2023-03-11 00:15:06 - r - INFO: - Episode: 275/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 276/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 277/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 278/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 279/400, Reward: -27.000, Step: 27 +2023-03-11 00:15:06 - r - INFO: - Episode: 280/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 281/400, Reward: -28.000, Step: 28 +2023-03-11 00:15:06 - r - INFO: - Episode: 282/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 283/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:06 - r - INFO: - Episode: 284/400, Reward: -28.000, Step: 28 +2023-03-11 00:15:06 - r - INFO: - Episode: 285/400, Reward: -37.000, Step: 37 +2023-03-11 00:15:06 - r - INFO: - Episode: 286/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 287/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 288/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 289/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 290/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 291/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 292/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 293/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 294/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 295/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 296/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:06 - r - INFO: - Episode: 297/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:06 - r - INFO: - Episode: 298/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 299/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 300/400, Reward: -30.000, Step: 30 +2023-03-11 00:15:06 - r - INFO: - Episode: 301/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:06 - r - INFO: - Episode: 302/400, Reward: -29.000, Step: 29 +2023-03-11 00:15:06 - r - INFO: - Episode: 303/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 304/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:06 - r - INFO: - Episode: 305/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:06 - r - INFO: - Current episode 305 has the best eval reward: -15.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 306/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 307/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 308/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:06 - r - INFO: - Episode: 309/400, Reward: -32.000, Step: 32 +2023-03-11 00:15:06 - r - INFO: - Episode: 310/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 311/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:06 - r - INFO: - Episode: 312/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 313/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 314/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 315/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Current episode 315 has the best eval reward: -15.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 316/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 317/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 318/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 319/400, Reward: -14.000, Step: 14 +2023-03-11 00:15:06 - r - INFO: - Episode: 320/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 321/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 322/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 323/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:06 - r - INFO: - Episode: 324/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:06 - r - INFO: - Episode: 325/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:06 - r - INFO: - Episode: 326/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:06 - r - INFO: - Episode: 327/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 328/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 329/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:06 - r - INFO: - Episode: 330/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 331/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 332/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 333/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 334/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 335/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 336/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:06 - r - INFO: - Episode: 337/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 338/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 339/400, Reward: -25.000, Step: 25 +2023-03-11 00:15:06 - r - INFO: - Episode: 340/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 341/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 342/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:06 - r - INFO: - Episode: 343/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 344/400, Reward: -14.000, Step: 14 +2023-03-11 00:15:06 - r - INFO: - Episode: 345/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 346/400, Reward: -20.000, Step: 20 +2023-03-11 00:15:06 - r - INFO: - Episode: 347/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 348/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:06 - r - INFO: - Episode: 349/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 350/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 351/400, Reward: -24.000, Step: 24 +2023-03-11 00:15:06 - r - INFO: - Episode: 352/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 353/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 354/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:06 - r - INFO: - Episode: 355/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:06 - r - INFO: - Current episode 355 has the best eval reward: -15.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 356/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 357/400, Reward: -26.000, Step: 26 +2023-03-11 00:15:06 - r - INFO: - Episode: 358/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 359/400, Reward: -23.000, Step: 23 +2023-03-11 00:15:06 - r - INFO: - Episode: 360/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 361/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 362/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:06 - r - INFO: - Episode: 363/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 364/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 365/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:06 - r - INFO: - Current episode 365 has the best eval reward: -15.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 366/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 367/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 368/400, Reward: -19.000, Step: 19 +2023-03-11 00:15:06 - r - INFO: - Episode: 369/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 370/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 371/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Episode: 372/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 373/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 374/400, Reward: -22.000, Step: 22 +2023-03-11 00:15:06 - r - INFO: - Episode: 375/400, Reward: -14.000, Step: 14 +2023-03-11 00:15:06 - r - INFO: - Current episode 375 has the best eval reward: -13.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 376/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 377/400, Reward: -16.000, Step: 16 +2023-03-11 00:15:06 - r - INFO: - Episode: 378/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 379/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 380/400, Reward: -17.000, Step: 17 +2023-03-11 00:15:06 - r - INFO: - Current episode 380 has the best eval reward: -13.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 381/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 382/400, Reward: -14.000, Step: 14 +2023-03-11 00:15:06 - r - INFO: - Episode: 383/400, Reward: -21.000, Step: 21 +2023-03-11 00:15:06 - r - INFO: - Episode: 384/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 385/400, Reward: -15.000, Step: 15 +2023-03-11 00:15:06 - r - INFO: - Episode: 386/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:06 - r - INFO: - Episode: 387/400, Reward: -14.000, Step: 14 +2023-03-11 00:15:06 - r - INFO: - Episode: 388/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 389/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 390/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 391/400, Reward: -18.000, Step: 18 +2023-03-11 00:15:06 - r - INFO: - Episode: 392/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 393/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 394/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 395/400, Reward: -14.000, Step: 14 +2023-03-11 00:15:06 - r - INFO: - Current episode 395 has the best eval reward: -13.000 +2023-03-11 00:15:06 - r - INFO: - Episode: 396/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 397/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 398/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 399/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Episode: 400/400, Reward: -13.000, Step: 13 +2023-03-11 00:15:06 - r - INFO: - Finish training! diff --git a/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/models/checkpoint.pkl b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/models/checkpoint.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ccccfb56162441b23fef7debf21e39446752490d --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/models/checkpoint.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b49e03abcf8f10412bffd219b9d2a522e02d39fce0365e81456762229100643d +size 6127 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/results/learning_curve.png b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..34ee9f739b8e642211ad95d5fce6c5d411815ba1 Binary files /dev/null and b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/results/learning_curve.png differ diff --git a/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/results/res.csv b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..a7a633a245ff8e328db140bef84c2c32c1ddb845 --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/results/res.csv @@ -0,0 +1,401 @@ +episodes,rewards,steps +0,-595,100 +1,-397,100 +2,-397,100 +3,-199,100 +4,-100,100 +5,-36,36 +6,-100,100 +7,-199,100 +8,-199,100 +9,-199,100 +10,-199,100 +11,-100,100 +12,-48,48 +13,-100,100 +14,-100,100 +15,-100,100 +16,-46,46 +17,-100,100 +18,-100,100 +19,-64,64 +20,-100,100 +21,-74,74 +22,-89,89 +23,-100,100 +24,-100,100 +25,-68,68 +26,-100,100 +27,-52,52 +28,-86,86 +29,-66,66 +30,-100,100 +31,-45,45 +32,-77,77 +33,-100,100 +34,-56,56 +35,-100,100 +36,-43,43 +37,-81,81 +38,-79,79 +39,-73,73 +40,-100,100 +41,-35,35 +42,-100,100 +43,-82,82 +44,-44,44 +45,-100,100 +46,-42,42 +47,-61,61 +48,-71,71 +49,-96,96 +50,-76,76 +51,-100,100 +52,-43,43 +53,-99,99 +54,-41,41 +55,-100,100 +56,-34,34 +57,-61,61 +58,-55,55 +59,-168,69 +60,-100,100 +61,-76,76 +62,-22,22 +63,-54,54 +64,-54,54 +65,-65,65 +66,-42,42 +67,-67,67 +68,-61,61 +69,-54,54 +70,-42,42 +71,-64,64 +72,-52,52 +73,-45,45 +74,-57,57 +75,-59,59 +76,-67,67 +77,-46,46 +78,-48,48 +79,-63,63 +80,-60,60 +81,-47,47 +82,-34,34 +83,-188,89 +84,-23,23 +85,-35,35 +86,-43,43 +87,-61,61 +88,-39,39 +89,-57,57 +90,-43,43 +91,-56,56 +92,-60,60 +93,-24,24 +94,-61,61 +95,-22,22 +96,-27,27 +97,-73,73 +98,-19,19 +99,-44,44 +100,-55,55 +101,-56,56 +102,-42,42 +103,-48,48 +104,-24,24 +105,-49,49 +106,-42,42 +107,-21,21 +108,-49,49 +109,-25,25 +110,-37,37 +111,-34,34 +112,-64,64 +113,-32,32 +114,-40,40 +115,-29,29 +116,-41,41 +117,-44,44 +118,-29,29 +119,-39,39 +120,-25,25 +121,-43,43 +122,-32,32 +123,-20,20 +124,-45,45 +125,-44,44 +126,-50,50 +127,-21,21 +128,-25,25 +129,-43,43 +130,-48,48 +131,-55,55 +132,-30,30 +133,-25,25 +134,-49,49 +135,-34,34 +136,-41,41 +137,-41,41 +138,-24,24 +139,-48,48 +140,-174,75 +141,-20,20 +142,-39,39 +143,-25,25 +144,-47,47 +145,-47,47 +146,-22,22 +147,-32,32 +148,-49,49 +149,-38,38 +150,-38,38 +151,-37,37 +152,-57,57 +153,-27,27 +154,-38,38 +155,-15,15 +156,-43,43 +157,-35,35 +158,-45,45 +159,-40,40 +160,-35,35 +161,-28,28 +162,-22,22 +163,-26,26 +164,-30,30 +165,-29,29 +166,-35,35 +167,-19,19 +168,-20,20 +169,-20,20 +170,-38,38 +171,-23,23 +172,-35,35 +173,-24,24 +174,-38,38 +175,-24,24 +176,-44,44 +177,-22,22 +178,-27,27 +179,-21,21 +180,-31,31 +181,-26,26 +182,-22,22 +183,-37,37 +184,-25,25 +185,-33,33 +186,-15,15 +187,-31,31 +188,-29,29 +189,-25,25 +190,-18,18 +191,-19,19 +192,-43,43 +193,-30,30 +194,-21,21 +195,-21,21 +196,-27,27 +197,-23,23 +198,-31,31 +199,-33,33 +200,-35,35 +201,-24,24 +202,-22,22 +203,-25,25 +204,-35,35 +205,-20,20 +206,-23,23 +207,-29,29 +208,-26,26 +209,-25,25 +210,-31,31 +211,-42,42 +212,-22,22 +213,-29,29 +214,-25,25 +215,-30,30 +216,-16,16 +217,-26,26 +218,-23,23 +219,-28,28 +220,-17,17 +221,-38,38 +222,-21,21 +223,-31,31 +224,-23,23 +225,-27,27 +226,-19,19 +227,-21,21 +228,-25,25 +229,-30,30 +230,-22,22 +231,-33,33 +232,-21,21 +233,-35,35 +234,-25,25 +235,-22,22 +236,-25,25 +237,-15,15 +238,-33,33 +239,-25,25 +240,-18,18 +241,-23,23 +242,-21,21 +243,-22,22 +244,-26,26 +245,-126,27 +246,-27,27 +247,-25,25 +248,-26,26 +249,-16,16 +250,-26,26 +251,-19,19 +252,-25,25 +253,-28,28 +254,-31,31 +255,-25,25 +256,-25,25 +257,-15,15 +258,-19,19 +259,-31,31 +260,-15,15 +261,-27,27 +262,-20,20 +263,-23,23 +264,-23,23 +265,-24,24 +266,-20,20 +267,-24,24 +268,-24,24 +269,-17,17 +270,-29,29 +271,-15,15 +272,-23,23 +273,-30,30 +274,-17,17 +275,-19,19 +276,-23,23 +277,-19,19 +278,-27,27 +279,-20,20 +280,-28,28 +281,-19,19 +282,-22,22 +283,-28,28 +284,-37,37 +285,-19,19 +286,-15,15 +287,-19,19 +288,-19,19 +289,-19,19 +290,-20,20 +291,-15,15 +292,-20,20 +293,-23,23 +294,-20,20 +295,-21,21 +296,-22,22 +297,-17,17 +298,-19,19 +299,-30,30 +300,-16,16 +301,-29,29 +302,-15,15 +303,-18,18 +304,-21,21 +305,-15,15 +306,-20,20 +307,-22,22 +308,-32,32 +309,-24,24 +310,-21,21 +311,-17,17 +312,-17,17 +313,-20,20 +314,-19,19 +315,-15,15 +316,-23,23 +317,-17,17 +318,-14,14 +319,-19,19 +320,-23,23 +321,-24,24 +322,-22,22 +323,-21,21 +324,-18,18 +325,-26,26 +326,-17,17 +327,-13,13 +328,-18,18 +329,-15,15 +330,-19,19 +331,-20,20 +332,-13,13 +333,-20,20 +334,-17,17 +335,-18,18 +336,-17,17 +337,-24,24 +338,-25,25 +339,-13,13 +340,-24,24 +341,-16,16 +342,-17,17 +343,-14,14 +344,-15,15 +345,-20,20 +346,-13,13 +347,-16,16 +348,-15,15 +349,-13,13 +350,-24,24 +351,-19,19 +352,-13,13 +353,-16,16 +354,-16,16 +355,-15,15 +356,-26,26 +357,-15,15 +358,-23,23 +359,-13,13 +360,-17,17 +361,-22,22 +362,-17,17 +363,-17,17 +364,-16,16 +365,-15,15 +366,-13,13 +367,-19,19 +368,-15,15 +369,-13,13 +370,-17,17 +371,-15,15 +372,-13,13 +373,-22,22 +374,-14,14 +375,-13,13 +376,-16,16 +377,-13,13 +378,-13,13 +379,-17,17 +380,-13,13 +381,-14,14 +382,-21,21 +383,-15,15 +384,-15,15 +385,-18,18 +386,-14,14 +387,-13,13 +388,-13,13 +389,-13,13 +390,-18,18 +391,-13,13 +392,-13,13 +393,-13,13 +394,-14,14 +395,-13,13 +396,-13,13 +397,-13,13 +398,-13,13 +399,-13,13 diff --git a/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/tb_logs/events.out.tfevents.1678464904.DESKTOP-H34HQIQ.27128.0 b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/tb_logs/events.out.tfevents.1678464904.DESKTOP-H34HQIQ.27128.0 new file mode 100644 index 0000000000000000000000000000000000000000..75f8aba19bb697dfffee4c1e4a26b06203881d60 --- /dev/null +++ b/CliffWalking-v0/Train_CliffWalking-v0_Sarsa_20230311-001504/tb_logs/events.out.tfevents.1678464904.DESKTOP-H34HQIQ.27128.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406bc6b455783f40d4d9b3be306d06875ce10f5931c052b9b696a29baff18a63 +size 40 diff --git a/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/config.yaml b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68f2af895e0ce7e1fdb3dd9fb24bbd09fa38c2ae --- /dev/null +++ b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: MO-QLearning + device: cpu + env_name: deep-sea-treasure-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: tasks + max_steps: 200 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 0 + show_fig: false + test_eps: 20 + train_eps: 100 + wrapper: null +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.9 + lr: 0.1 + weights: + - 0.5 + - 0.5 diff --git a/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/logs/log.txt b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a24688dc36c80104931f6d3cc74da54f8dabc0af --- /dev/null +++ b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/logs/log.txt @@ -0,0 +1,56 @@ +2023-03-29 23:48:02 - r - INFO: - Hyperparameters: +2023-03-29 23:48:02 - r - INFO: - ================================================================================ +2023-03-29 23:48:02 - r - INFO: - Name Value Type +2023-03-29 23:48:02 - r - INFO: - env_name deep-sea-treasure-v0 +2023-03-29 23:48:02 - r - INFO: - new_step_api 1 +2023-03-29 23:48:02 - r - INFO: - wrapper None +2023-03-29 23:48:02 - r - INFO: - render 0 +2023-03-29 23:48:02 - r - INFO: - algo_name MO-QLearning +2023-03-29 23:48:02 - r - INFO: - mode test +2023-03-29 23:48:02 - r - INFO: - seed 0 +2023-03-29 23:48:02 - r - INFO: - device cpu +2023-03-29 23:48:02 - r - INFO: - train_eps 100 +2023-03-29 23:48:02 - r - INFO: - test_eps 20 +2023-03-29 23:48:02 - r - INFO: - eval_eps 10 +2023-03-29 23:48:02 - r - INFO: - eval_per_episode 5 +2023-03-29 23:48:02 - r - INFO: - max_steps 200 +2023-03-29 23:48:02 - r - INFO: - load_checkpoint 0 +2023-03-29 23:48:02 - r - INFO: - load_path tasks +2023-03-29 23:48:02 - r - INFO: - show_fig 0 +2023-03-29 23:48:02 - r - INFO: - save_fig 1 +2023-03-29 23:48:02 - r - INFO: - epsilon_start 0.95 +2023-03-29 23:48:02 - r - INFO: - epsilon_end 0.01 +2023-03-29 23:48:02 - r - INFO: - epsilon_decay 300 +2023-03-29 23:48:02 - r - INFO: - gamma 0.9 +2023-03-29 23:48:02 - r - INFO: - lr 0.1 +2023-03-29 23:48:02 - r - INFO: - weights [0.5, 0.5] +2023-03-29 23:48:02 - r - INFO: - task_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Test_deep-sea-treasure-v0_MO-QLearning_20230329-234802 +2023-03-29 23:48:02 - r - INFO: - res_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Test_deep-sea-treasure-v0_MO-QLearning_20230329-234802/results +2023-03-29 23:48:02 - r - INFO: - log_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Test_deep-sea-treasure-v0_MO-QLearning_20230329-234802/logs +2023-03-29 23:48:02 - r - INFO: - traj_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Test_deep-sea-treasure-v0_MO-QLearning_20230329-234802/traj +2023-03-29 23:48:02 - r - INFO: - tb_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Test_deep-sea-treasure-v0_MO-QLearning_20230329-234802/tb_logs +2023-03-29 23:48:02 - r - INFO: - ================================================================================ +2023-03-29 23:48:02 - r - INFO: - n_states: 2, n_actions: 4 +2023-03-29 23:48:02 - r - INFO: - Start testing! +2023-03-29 23:48:02 - r - INFO: - Env: deep-sea-treasure-v0, Algorithm: MO-QLearning, Device: cpu +2023-03-29 23:48:02 - r - INFO: - Episode: 1/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 2/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 3/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 4/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 5/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 6/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 7/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 8/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 9/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 10/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 11/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 12/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 13/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 14/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 15/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 16/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 17/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 18/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 19/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Episode: 20/20, Reward: 0.000,-200.000, Step: 200 +2023-03-29 23:48:02 - r - INFO: - Finish testing! diff --git a/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/modelsMO-Qleaning_model.pkl b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/modelsMO-Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..583e9dcb05db9599efb8c1ca002f8f644dae8920 --- /dev/null +++ b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/modelsMO-Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30484015fa82af0af854080dd953112dc7cdfe0e240fcb7d99a431f42db75458 +size 1647 diff --git a/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/results/learning_curve.png b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..2b9e26ddc94cfcaff891e7ced934342d6cc60953 Binary files /dev/null and b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/results/learning_curve.png differ diff --git a/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/results/res.csv b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..0b0e6aba6eeb9f04b7ee20cae06c539b27390f22 --- /dev/null +++ b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,[ 0. -200.],200 +1,[ 0. -200.],200 +2,[ 0. -200.],200 +3,[ 0. -200.],200 +4,[ 0. -200.],200 +5,[ 0. -200.],200 +6,[ 0. -200.],200 +7,[ 0. -200.],200 +8,[ 0. -200.],200 +9,[ 0. -200.],200 +10,[ 0. -200.],200 +11,[ 0. -200.],200 +12,[ 0. -200.],200 +13,[ 0. -200.],200 +14,[ 0. -200.],200 +15,[ 0. -200.],200 +16,[ 0. -200.],200 +17,[ 0. -200.],200 +18,[ 0. -200.],200 +19,[ 0. -200.],200 diff --git a/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/tb_logs/events.out.tfevents.1680104882.LAPTOP-NVQGGDIH.21028.0 b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/tb_logs/events.out.tfevents.1680104882.LAPTOP-NVQGGDIH.21028.0 new file mode 100644 index 0000000000000000000000000000000000000000..0c42f0b6c69e78eb629f2ce7d57bfe81931a545d --- /dev/null +++ b/Deep-Sea-Treasure-v0/Test_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234802/tb_logs/events.out.tfevents.1680104882.LAPTOP-NVQGGDIH.21028.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60c778d757eac82153e0359a27f54539e4545cfff63ec1a31a7cc9523d6ea1c1 +size 88 diff --git a/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/config.yaml b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5945b4c42b0d3e31b1ee06f03102d303039352ab --- /dev/null +++ b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: MO-QLearning + device: cpu + env_name: deep-sea-treasure-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: tasks + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 0 + show_fig: false + test_eps: 20 + train_eps: 100 + wrapper: null +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.9 + lr: 0.1 + weights: + - 0.5 + - 0.5 diff --git a/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/logs/log.txt b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..7509473a837c019e7e55195abb67f58b2a7665ab --- /dev/null +++ b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/logs/log.txt @@ -0,0 +1,156 @@ +2023-03-29 23:43:19 - r - INFO: - Hyperparameters: +2023-03-29 23:43:19 - r - INFO: - ================================================================================ +2023-03-29 23:43:19 - r - INFO: - Name Value Type +2023-03-29 23:43:19 - r - INFO: - env_name deep-sea-treasure-v0 +2023-03-29 23:43:19 - r - INFO: - new_step_api 1 +2023-03-29 23:43:19 - r - INFO: - wrapper None +2023-03-29 23:43:19 - r - INFO: - render 0 +2023-03-29 23:43:19 - r - INFO: - algo_name MO-QLearning +2023-03-29 23:43:19 - r - INFO: - mode train +2023-03-29 23:43:19 - r - INFO: - seed 0 +2023-03-29 23:43:19 - r - INFO: - device cpu +2023-03-29 23:43:19 - r - INFO: - train_eps 100 +2023-03-29 23:43:19 - r - INFO: - test_eps 20 +2023-03-29 23:43:19 - r - INFO: - eval_eps 10 +2023-03-29 23:43:19 - r - INFO: - eval_per_episode 5 +2023-03-29 23:43:19 - r - INFO: - max_steps 200 +2023-03-29 23:43:19 - r - INFO: - load_checkpoint 0 +2023-03-29 23:43:19 - r - INFO: - load_path tasks +2023-03-29 23:43:19 - r - INFO: - show_fig 0 +2023-03-29 23:43:19 - r - INFO: - save_fig 1 +2023-03-29 23:43:19 - r - INFO: - epsilon_start 0.95 +2023-03-29 23:43:19 - r - INFO: - epsilon_end 0.01 +2023-03-29 23:43:19 - r - INFO: - epsilon_decay 300 +2023-03-29 23:43:19 - r - INFO: - gamma 0.9 +2023-03-29 23:43:19 - r - INFO: - lr 0.1 +2023-03-29 23:43:19 - r - INFO: - weights [0.5, 0.5] +2023-03-29 23:43:19 - r - INFO: - task_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Train_deep-sea-treasure-v0_MO-QLearning_20230329-234319 +2023-03-29 23:43:19 - r - INFO: - res_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Train_deep-sea-treasure-v0_MO-QLearning_20230329-234319/results +2023-03-29 23:43:19 - r - INFO: - log_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Train_deep-sea-treasure-v0_MO-QLearning_20230329-234319/logs +2023-03-29 23:43:19 - r - INFO: - traj_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Train_deep-sea-treasure-v0_MO-QLearning_20230329-234319/traj +2023-03-29 23:43:19 - r - INFO: - tb_dir E:\LAMDA\ongoing work\joyrl\joyrl-offline/tasks/Train_deep-sea-treasure-v0_MO-QLearning_20230329-234319/tb_logs +2023-03-29 23:43:19 - r - INFO: - ================================================================================ +2023-03-29 23:43:19 - r - INFO: - n_states: 2, n_actions: 4 +2023-03-29 23:43:19 - r - INFO: - Start training! +2023-03-29 23:43:19 - r - INFO: - Env: deep-sea-treasure-v0, Algorithm: MO-QLearning, Device: cpu +2023-03-29 23:43:19 - r - INFO: - Episode: 1/100, Reward: 16.100, -131.000, Step: 131 +2023-03-29 23:43:19 - r - INFO: - Episode: 2/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 3/100, Reward: 0.700, -4.000, Step: 4 +2023-03-29 23:43:19 - r - INFO: - Episode: 4/100, Reward: 0.700, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 5/100, Reward: 8.200, -15.000, Step: 15 +2023-03-29 23:43:19 - r - INFO: - Current episode 5 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 6/100, Reward: 0.700, -4.000, Step: 4 +2023-03-29 23:43:19 - r - INFO: - Episode: 7/100, Reward: 0.700, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 8/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 9/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 10/100, Reward: 8.200, -5.000, Step: 5 +2023-03-29 23:43:19 - r - INFO: - Current episode 10 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 11/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 12/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 13/100, Reward: 8.200, -5.000, Step: 5 +2023-03-29 23:43:19 - r - INFO: - Episode: 14/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 15/100, Reward: 0.700, -5.000, Step: 5 +2023-03-29 23:43:19 - r - INFO: - Current episode 15 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 16/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 17/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 18/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 19/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 20/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 20 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 21/100, Reward: 8.200, -6.000, Step: 6 +2023-03-29 23:43:19 - r - INFO: - Episode: 22/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 23/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 24/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 25/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 25 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 26/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 27/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 28/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 29/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 30/100, Reward: 8.200, -4.000, Step: 4 +2023-03-29 23:43:19 - r - INFO: - Current episode 30 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 31/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 32/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 33/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 34/100, Reward: 8.200, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 35/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Current episode 35 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 36/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 37/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 38/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 39/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 40/100, Reward: 8.200, -4.000, Step: 4 +2023-03-29 23:43:19 - r - INFO: - Current episode 40 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 41/100, Reward: 0.700, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 42/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 43/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 44/100, Reward: 0.700, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 45/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 45 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 46/100, Reward: 8.200, -7.000, Step: 7 +2023-03-29 23:43:19 - r - INFO: - Episode: 47/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 48/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 49/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 50/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Current episode 50 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 51/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 52/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 53/100, Reward: 8.200, -25.000, Step: 25 +2023-03-29 23:43:19 - r - INFO: - Episode: 54/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 55/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Current episode 55 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 56/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 57/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 58/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 59/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 60/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 60 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 61/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 62/100, Reward: 8.200, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 63/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 64/100, Reward: 8.200, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 65/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 65 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 66/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 67/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 68/100, Reward: 0.700, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Episode: 69/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 70/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 70 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 71/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 72/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 73/100, Reward: 8.200, -6.000, Step: 6 +2023-03-29 23:43:19 - r - INFO: - Episode: 74/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 75/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 75 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 76/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 77/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 78/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 79/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 80/100, Reward: 8.200, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Current episode 80 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 81/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 82/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 83/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 84/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 85/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 85 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 86/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Episode: 87/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 88/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 89/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 90/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Current episode 90 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 91/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 92/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 93/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 94/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 95/100, Reward: 0.700, -3.000, Step: 3 +2023-03-29 23:43:19 - r - INFO: - Current episode 95 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Episode: 96/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 97/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 98/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 99/100, Reward: 0.700, -1.000, Step: 1 +2023-03-29 23:43:19 - r - INFO: - Episode: 100/100, Reward: 0.700, -2.000, Step: 2 +2023-03-29 23:43:19 - r - INFO: - Current episode 100 has the best eval reward: 0.700 +2023-03-29 23:43:19 - r - INFO: - Finish training! diff --git a/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/models/MO-Qleaning_model.pkl b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/models/MO-Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a690008490d4ec471352a445c2cc7703b42bf007 --- /dev/null +++ b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/models/MO-Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ba6c97edafaf585186cbf8af5eb445a7e95f211016807acffa2d1d287489069 +size 7599 diff --git a/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/results/learning_curve.png b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..82144f8208fac779486e0739772174f49a26f14a Binary files /dev/null and b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/results/learning_curve.png differ diff --git a/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/results/res.csv b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..4fd422ed57b25ed7158518fad359c7881d0686f2 --- /dev/null +++ b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,[ 16.1 -131. ],131 +1,[ 0.7 -2. ],2 +2,[ 0.7 -4. ],4 +3,[ 0.7 -3. ],3 +4,[ 8.2 -15. ],15 +5,[ 0.7 -4. ],4 +6,[ 0.7 -3. ],3 +7,[ 0.7 -2. ],2 +8,[ 0.7 -1. ],1 +9,[ 8.2 -5. ],5 +10,[ 0.7 -1. ],1 +11,[ 0.7 -2. ],2 +12,[ 8.2 -5. ],5 +13,[ 0.7 -1. ],1 +14,[ 0.7 -5. ],5 +15,[ 0.7 -1. ],1 +16,[ 0.7 -1. ],1 +17,[ 0.7 -1. ],1 +18,[ 0.7 -2. ],2 +19,[ 0.7 -1. ],1 +20,[ 8.2 -6. ],6 +21,[ 0.7 -1. ],1 +22,[ 0.7 -1. ],1 +23,[ 0.7 -1. ],1 +24,[ 0.7 -1. ],1 +25,[ 0.7 -1. ],1 +26,[ 0.7 -1. ],1 +27,[ 0.7 -1. ],1 +28,[ 0.7 -1. ],1 +29,[ 8.2 -4. ],4 +30,[ 0.7 -1. ],1 +31,[ 0.7 -1. ],1 +32,[ 0.7 -1. ],1 +33,[ 8.2 -3. ],3 +34,[ 0.7 -2. ],2 +35,[ 0.7 -2. ],2 +36,[ 0.7 -2. ],2 +37,[ 0.7 -2. ],2 +38,[ 0.7 -2. ],2 +39,[ 8.2 -4. ],4 +40,[ 0.7 -3. ],3 +41,[ 0.7 -1. ],1 +42,[ 0.7 -1. ],1 +43,[ 0.7 -3. ],3 +44,[ 0.7 -1. ],1 +45,[ 8.2 -7. ],7 +46,[ 0.7 -1. ],1 +47,[ 0.7 -2. ],2 +48,[ 0.7 -2. ],2 +49,[ 0.7 -2. ],2 +50,[ 0.7 -1. ],1 +51,[ 0.7 -1. ],1 +52,[ 8.2 -25. ],25 +53,[ 0.7 -1. ],1 +54,[ 0.7 -2. ],2 +55,[ 0.7 -1. ],1 +56,[ 0.7 -2. ],2 +57,[ 0.7 -1. ],1 +58,[ 0.7 -2. ],2 +59,[ 0.7 -1. ],1 +60,[ 0.7 -1. ],1 +61,[ 8.2 -3. ],3 +62,[ 0.7 -1. ],1 +63,[ 8.2 -3. ],3 +64,[ 0.7 -1. ],1 +65,[ 0.7 -1. ],1 +66,[ 0.7 -2. ],2 +67,[ 0.7 -3. ],3 +68,[ 0.7 -1. ],1 +69,[ 0.7 -1. ],1 +70,[ 0.7 -1. ],1 +71,[ 0.7 -1. ],1 +72,[ 8.2 -6. ],6 +73,[ 0.7 -1. ],1 +74,[ 0.7 -1. ],1 +75,[ 0.7 -1. ],1 +76,[ 0.7 -1. ],1 +77,[ 0.7 -1. ],1 +78,[ 0.7 -1. ],1 +79,[ 8.2 -3. ],3 +80,[ 0.7 -1. ],1 +81,[ 0.7 -1. ],1 +82,[ 0.7 -1. ],1 +83,[ 0.7 -1. ],1 +84,[ 0.7 -1. ],1 +85,[ 0.7 -2. ],2 +86,[ 0.7 -1. ],1 +87,[ 0.7 -1. ],1 +88,[ 0.7 -1. ],1 +89,[ 0.7 -1. ],1 +90,[ 0.7 -1. ],1 +91,[ 0.7 -1. ],1 +92,[ 0.7 -1. ],1 +93,[ 0.7 -1. ],1 +94,[ 0.7 -3. ],3 +95,[ 0.7 -1. ],1 +96,[ 0.7 -1. ],1 +97,[ 0.7 -1. ],1 +98,[ 0.7 -1. ],1 +99,[ 0.7 -2. ],2 diff --git a/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/tb_logs/events.out.tfevents.1680104599.LAPTOP-NVQGGDIH.32188.0 b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/tb_logs/events.out.tfevents.1680104599.LAPTOP-NVQGGDIH.32188.0 new file mode 100644 index 0000000000000000000000000000000000000000..a455c8eb665d252b0b77a1dec4785fb3d4647d68 --- /dev/null +++ b/Deep-Sea-Treasure-v0/Train_Deep-Sea-Treasure-v0_MO-QLearning_20230329-234319/tb_logs/events.out.tfevents.1680104599.LAPTOP-NVQGGDIH.32188.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e639ae8e66b64df246bbb4edaf7065ce31c515270d62a702801b343e5f8e779b +size 88 diff --git a/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/config.yaml b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d760074016206c2f2ea51c5d8481ea546f649ac9 --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/config.yaml @@ -0,0 +1,25 @@ +general_cfg: + algo_name: QLearning + device: cpu + env_name: FrozenLakeNoSlippery-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949 + max_steps: 200 + mode: test + new_step_api: true + render: true + render_mode: human + save_fig: true + seed: 10 + show_fig: false + test_eps: 10 + train_eps: 800 + wrapper: null +algo_cfg: + epsilon_decay: 2000 + epsilon_end: 0.1 + epsilon_start: 0.7 + gamma: 0.95 + lr: 0.9 diff --git a/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/logs/log.txt b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..24da427671432318f1f54231d88386ce16db8054 --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/logs/log.txt @@ -0,0 +1,41 @@ +2022-11-28 15:46:29 - r - INFO: - Hyperparameters: +2022-11-28 15:46:29 - r - INFO: - ================================================================================ +2022-11-28 15:46:29 - r - INFO: - Name Value Type +2022-11-28 15:46:29 - r - INFO: - env_name FrozenLakeNoSlippery-v1 +2022-11-28 15:46:29 - r - INFO: - new_step_api 1 +2022-11-28 15:46:29 - r - INFO: - wrapper None +2022-11-28 15:46:29 - r - INFO: - render 1 +2022-11-28 15:46:29 - r - INFO: - algo_name QLearning +2022-11-28 15:46:29 - r - INFO: - mode test +2022-11-28 15:46:29 - r - INFO: - seed 10 +2022-11-28 15:46:29 - r - INFO: - device cpu +2022-11-28 15:46:29 - r - INFO: - train_eps 800 +2022-11-28 15:46:29 - r - INFO: - test_eps 10 +2022-11-28 15:46:29 - r - INFO: - eval_eps 10 +2022-11-28 15:46:29 - r - INFO: - eval_per_episode 5 +2022-11-28 15:46:29 - r - INFO: - max_steps 200 +2022-11-28 15:46:29 - r - INFO: - load_checkpoint 1 +2022-11-28 15:46:29 - r - INFO: - load_path Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949 +2022-11-28 15:46:29 - r - INFO: - show_fig 0 +2022-11-28 15:46:29 - r - INFO: - save_fig 1 +2022-11-28 15:46:29 - r - INFO: - render_mode human +2022-11-28 15:46:29 - r - INFO: - epsilon_start 0.7 +2022-11-28 15:46:29 - r - INFO: - epsilon_end 0.1 +2022-11-28 15:46:29 - r - INFO: - epsilon_decay 2000 +2022-11-28 15:46:29 - r - INFO: - gamma 0.95 +2022-11-28 15:46:29 - r - INFO: - lr 0.9 +2022-11-28 15:46:29 - r - INFO: - ================================================================================ +2022-11-28 15:46:30 - r - INFO: - n_states: 16, n_actions: 4 +2022-11-28 15:46:30 - r - INFO: - Start testing! +2022-11-28 15:46:30 - r - INFO: - Env: FrozenLakeNoSlippery-v1, Algorithm: QLearning, Device: cpu +2022-11-28 15:46:33 - r - INFO: - Episode: 1/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:35 - r - INFO: - Episode: 2/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:37 - r - INFO: - Episode: 3/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:39 - r - INFO: - Episode: 4/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:40 - r - INFO: - Episode: 5/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:42 - r - INFO: - Episode: 6/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:44 - r - INFO: - Episode: 7/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:46 - r - INFO: - Episode: 8/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:47 - r - INFO: - Episode: 9/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:49 - r - INFO: - Episode: 10/10, Reward: 1.000, Step: 6 +2022-11-28 15:46:49 - r - INFO: - Finish testing! diff --git a/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/models/Qleaning_model.pkl b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/models/Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c0b10db9f2404d272ed7739f1190ffaa3867dd92 --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/models/Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd5782c99cad77fe3383f02d3f430ac7bd22fd8aed9973645d3c5ed600eafbb +size 2479 diff --git a/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/results/learning_curve.png b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..8bb97b7dd58f78acaf6ee22eca30f5cfe2663158 Binary files /dev/null and b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/results/learning_curve.png differ diff --git a/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/results/res.csv b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..1562cd757dccf9cb8d35ff9ab51f2e16bafd4896 --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Test_FrozenLakeNoSlippery-v1_QLearning_20221128-154629/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,1.0,6 +1,1.0,6 +2,1.0,6 +3,1.0,6 +4,1.0,6 +5,1.0,6 +6,1.0,6 +7,1.0,6 +8,1.0,6 +9,1.0,6 diff --git a/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/config.yaml b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..025d85a0c9273574251a4d969cfc772d80f4838e --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/config.yaml @@ -0,0 +1,24 @@ +general_cfg: + algo_name: QLearning + device: cpu + env_name: FrozenLakeNoSlippery-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 10 + show_fig: false + test_eps: 20 + train_eps: 800 + wrapper: null +algo_cfg: + epsilon_decay: 2000 + epsilon_end: 0.1 + epsilon_start: 0.7 + gamma: 0.95 + lr: 0.9 diff --git a/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/logs/log.txt b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..5afc4f4c3ae6251638f8dda9414f65ec827e1b59 --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/logs/log.txt @@ -0,0 +1,990 @@ +2022-11-28 15:29:49 - r - INFO: - Hyperparameters: +2022-11-28 15:29:49 - r - INFO: - ================================================================================ +2022-11-28 15:29:49 - r - INFO: - Name Value Type +2022-11-28 15:29:49 - r - INFO: - env_name FrozenLakeNoSlippery-v1 +2022-11-28 15:29:49 - r - INFO: - new_step_api 1 +2022-11-28 15:29:49 - r - INFO: - wrapper None +2022-11-28 15:29:49 - r - INFO: - render 0 +2022-11-28 15:29:49 - r - INFO: - algo_name QLearning +2022-11-28 15:29:49 - r - INFO: - mode train +2022-11-28 15:29:49 - r - INFO: - seed 10 +2022-11-28 15:29:49 - r - INFO: - device cpu +2022-11-28 15:29:49 - r - INFO: - train_eps 800 +2022-11-28 15:29:49 - r - INFO: - test_eps 20 +2022-11-28 15:29:49 - r - INFO: - eval_eps 10 +2022-11-28 15:29:49 - r - INFO: - eval_per_episode 5 +2022-11-28 15:29:49 - r - INFO: - max_steps 200 +2022-11-28 15:29:49 - r - INFO: - load_checkpoint 0 +2022-11-28 15:29:49 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2022-11-28 15:29:49 - r - INFO: - show_fig 0 +2022-11-28 15:29:49 - r - INFO: - save_fig 1 +2022-11-28 15:29:49 - r - INFO: - epsilon_start 0.7 +2022-11-28 15:29:49 - r - INFO: - epsilon_end 0.1 +2022-11-28 15:29:49 - r - INFO: - epsilon_decay 2000 +2022-11-28 15:29:49 - r - INFO: - gamma 0.95 +2022-11-28 15:29:49 - r - INFO: - lr 0.9 +2022-11-28 15:29:49 - r - INFO: - ================================================================================ +2022-11-28 15:29:50 - r - INFO: - n_states: 16, n_actions: 4 +2022-11-28 15:29:50 - r - INFO: - Start training! +2022-11-28 15:29:50 - r - INFO: - Env: FrozenLakeNoSlippery-v1, Algorithm: QLearning, Device: cpu +2022-11-28 15:29:50 - r - INFO: - Episode: 1/800, Reward: 0.000, Step: 20 +2022-11-28 15:29:50 - r - INFO: - Episode: 2/800, Reward: 0.000, Step: 14 +2022-11-28 15:29:50 - r - INFO: - Episode: 3/800, Reward: 0.000, Step: 13 +2022-11-28 15:29:50 - r - INFO: - Episode: 4/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:50 - r - INFO: - Episode: 5/800, Reward: 0.000, Step: 10 +2022-11-28 15:29:50 - r - INFO: - Current episode 5 has the best eval reward: 0.000 +2022-11-28 15:29:50 - r - INFO: - Episode: 6/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:50 - r - INFO: - Episode: 7/800, Reward: 0.000, Step: 11 +2022-11-28 15:29:50 - r - INFO: - Episode: 8/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:50 - r - INFO: - Episode: 9/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:50 - r - INFO: - Episode: 10/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:50 - r - INFO: - Current episode 10 has the best eval reward: 0.000 +2022-11-28 15:29:50 - r - INFO: - Episode: 11/800, Reward: 0.000, Step: 11 +2022-11-28 15:29:50 - r - INFO: - Episode: 12/800, Reward: 0.000, Step: 22 +2022-11-28 15:29:50 - r - INFO: - Episode: 13/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:50 - r - INFO: - Episode: 14/800, Reward: 0.000, Step: 16 +2022-11-28 15:29:50 - r - INFO: - Episode: 15/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:50 - r - INFO: - Current episode 15 has the best eval reward: 0.000 +2022-11-28 15:29:50 - r - INFO: - Episode: 16/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:50 - r - INFO: - Episode: 17/800, Reward: 0.000, Step: 18 +2022-11-28 15:29:50 - r - INFO: - Episode: 18/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:50 - r - INFO: - Episode: 19/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:50 - r - INFO: - Episode: 20/800, Reward: 0.000, Step: 8 +2022-11-28 15:29:50 - r - INFO: - Current episode 20 has the best eval reward: 0.000 +2022-11-28 15:29:50 - r - INFO: - Episode: 21/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:50 - r - INFO: - Episode: 22/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:50 - r - INFO: - Episode: 23/800, Reward: 0.000, Step: 22 +2022-11-28 15:29:50 - r - INFO: - Episode: 24/800, Reward: 0.000, Step: 15 +2022-11-28 15:29:50 - r - INFO: - Episode: 25/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:50 - r - INFO: - Current episode 25 has the best eval reward: 0.000 +2022-11-28 15:29:50 - r - INFO: - Episode: 26/800, Reward: 0.000, Step: 16 +2022-11-28 15:29:50 - r - INFO: - Episode: 27/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:50 - r - INFO: - Episode: 28/800, Reward: 0.000, Step: 19 +2022-11-28 15:29:50 - r - INFO: - Episode: 29/800, Reward: 0.000, Step: 22 +2022-11-28 15:29:50 - r - INFO: - Episode: 30/800, Reward: 0.000, Step: 16 +2022-11-28 15:29:50 - r - INFO: - Current episode 30 has the best eval reward: 0.000 +2022-11-28 15:29:50 - r - INFO: - Episode: 31/800, Reward: 0.000, Step: 11 +2022-11-28 15:29:50 - r - INFO: - Episode: 32/800, Reward: 0.000, Step: 22 +2022-11-28 15:29:50 - r - INFO: - Episode: 33/800, Reward: 0.000, Step: 28 +2022-11-28 15:29:50 - r - INFO: - Episode: 34/800, Reward: 0.000, Step: 23 +2022-11-28 15:29:50 - r - INFO: - Episode: 35/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:51 - r - INFO: - Current episode 35 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 36/800, Reward: 0.000, Step: 11 +2022-11-28 15:29:51 - r - INFO: - Episode: 37/800, Reward: 0.000, Step: 8 +2022-11-28 15:29:51 - r - INFO: - Episode: 38/800, Reward: 0.000, Step: 15 +2022-11-28 15:29:51 - r - INFO: - Episode: 39/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:51 - r - INFO: - Episode: 40/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:51 - r - INFO: - Current episode 40 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 41/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:51 - r - INFO: - Episode: 42/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:51 - r - INFO: - Episode: 43/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:51 - r - INFO: - Episode: 44/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:51 - r - INFO: - Episode: 45/800, Reward: 0.000, Step: 41 +2022-11-28 15:29:51 - r - INFO: - Current episode 45 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 46/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:51 - r - INFO: - Episode: 47/800, Reward: 0.000, Step: 23 +2022-11-28 15:29:51 - r - INFO: - Episode: 48/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:51 - r - INFO: - Episode: 49/800, Reward: 1.000, Step: 38 +2022-11-28 15:29:51 - r - INFO: - Episode: 50/800, Reward: 0.000, Step: 29 +2022-11-28 15:29:51 - r - INFO: - Current episode 50 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 51/800, Reward: 0.000, Step: 17 +2022-11-28 15:29:51 - r - INFO: - Episode: 52/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:51 - r - INFO: - Episode: 53/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:51 - r - INFO: - Episode: 54/800, Reward: 0.000, Step: 25 +2022-11-28 15:29:51 - r - INFO: - Episode: 55/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:51 - r - INFO: - Current episode 55 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 56/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:51 - r - INFO: - Episode: 57/800, Reward: 0.000, Step: 30 +2022-11-28 15:29:51 - r - INFO: - Episode: 58/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:51 - r - INFO: - Episode: 59/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:51 - r - INFO: - Episode: 60/800, Reward: 0.000, Step: 11 +2022-11-28 15:29:51 - r - INFO: - Current episode 60 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 61/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:51 - r - INFO: - Episode: 62/800, Reward: 0.000, Step: 8 +2022-11-28 15:29:51 - r - INFO: - Episode: 63/800, Reward: 0.000, Step: 23 +2022-11-28 15:29:51 - r - INFO: - Episode: 64/800, Reward: 0.000, Step: 10 +2022-11-28 15:29:51 - r - INFO: - Episode: 65/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:51 - r - INFO: - Current episode 65 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 66/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:51 - r - INFO: - Episode: 67/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:51 - r - INFO: - Episode: 68/800, Reward: 0.000, Step: 18 +2022-11-28 15:29:51 - r - INFO: - Episode: 69/800, Reward: 0.000, Step: 8 +2022-11-28 15:29:51 - r - INFO: - Episode: 70/800, Reward: 0.000, Step: 26 +2022-11-28 15:29:51 - r - INFO: - Current episode 70 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 71/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:51 - r - INFO: - Episode: 72/800, Reward: 0.000, Step: 14 +2022-11-28 15:29:51 - r - INFO: - Episode: 73/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:51 - r - INFO: - Episode: 74/800, Reward: 0.000, Step: 25 +2022-11-28 15:29:51 - r - INFO: - Episode: 75/800, Reward: 0.000, Step: 21 +2022-11-28 15:29:51 - r - INFO: - Current episode 75 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 76/800, Reward: 0.000, Step: 13 +2022-11-28 15:29:51 - r - INFO: - Episode: 77/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:51 - r - INFO: - Episode: 78/800, Reward: 0.000, Step: 29 +2022-11-28 15:29:51 - r - INFO: - Episode: 79/800, Reward: 0.000, Step: 21 +2022-11-28 15:29:51 - r - INFO: - Episode: 80/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:51 - r - INFO: - Current episode 80 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 81/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:51 - r - INFO: - Episode: 82/800, Reward: 0.000, Step: 11 +2022-11-28 15:29:51 - r - INFO: - Episode: 83/800, Reward: 0.000, Step: 21 +2022-11-28 15:29:51 - r - INFO: - Episode: 84/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:51 - r - INFO: - Episode: 85/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:51 - r - INFO: - Current episode 85 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 86/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:51 - r - INFO: - Episode: 87/800, Reward: 0.000, Step: 48 +2022-11-28 15:29:51 - r - INFO: - Episode: 88/800, Reward: 0.000, Step: 23 +2022-11-28 15:29:51 - r - INFO: - Episode: 89/800, Reward: 0.000, Step: 160 +2022-11-28 15:29:51 - r - INFO: - Episode: 90/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:51 - r - INFO: - Current episode 90 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 91/800, Reward: 0.000, Step: 10 +2022-11-28 15:29:51 - r - INFO: - Episode: 92/800, Reward: 0.000, Step: 24 +2022-11-28 15:29:51 - r - INFO: - Episode: 93/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:51 - r - INFO: - Episode: 94/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:51 - r - INFO: - Episode: 95/800, Reward: 0.000, Step: 17 +2022-11-28 15:29:51 - r - INFO: - Current episode 95 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 96/800, Reward: 0.000, Step: 87 +2022-11-28 15:29:51 - r - INFO: - Episode: 97/800, Reward: 0.000, Step: 28 +2022-11-28 15:29:51 - r - INFO: - Episode: 98/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:51 - r - INFO: - Episode: 99/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:51 - r - INFO: - Episode: 100/800, Reward: 0.000, Step: 12 +2022-11-28 15:29:51 - r - INFO: - Current episode 100 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 101/800, Reward: 0.000, Step: 14 +2022-11-28 15:29:51 - r - INFO: - Episode: 102/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:51 - r - INFO: - Episode: 103/800, Reward: 0.000, Step: 13 +2022-11-28 15:29:51 - r - INFO: - Episode: 104/800, Reward: 0.000, Step: 93 +2022-11-28 15:29:51 - r - INFO: - Episode: 105/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:51 - r - INFO: - Current episode 105 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 106/800, Reward: 0.000, Step: 50 +2022-11-28 15:29:51 - r - INFO: - Episode: 107/800, Reward: 0.000, Step: 8 +2022-11-28 15:29:51 - r - INFO: - Episode: 108/800, Reward: 0.000, Step: 12 +2022-11-28 15:29:51 - r - INFO: - Episode: 109/800, Reward: 0.000, Step: 43 +2022-11-28 15:29:51 - r - INFO: - Episode: 110/800, Reward: 0.000, Step: 30 +2022-11-28 15:29:51 - r - INFO: - Current episode 110 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 111/800, Reward: 0.000, Step: 15 +2022-11-28 15:29:51 - r - INFO: - Episode: 112/800, Reward: 0.000, Step: 19 +2022-11-28 15:29:51 - r - INFO: - Episode: 113/800, Reward: 0.000, Step: 182 +2022-11-28 15:29:51 - r - INFO: - Episode: 114/800, Reward: 0.000, Step: 40 +2022-11-28 15:29:51 - r - INFO: - Episode: 115/800, Reward: 0.000, Step: 88 +2022-11-28 15:29:51 - r - INFO: - Current episode 115 has the best eval reward: 0.000 +2022-11-28 15:29:51 - r - INFO: - Episode: 116/800, Reward: 0.000, Step: 19 +2022-11-28 15:29:51 - r - INFO: - Episode: 117/800, Reward: 0.000, Step: 30 +2022-11-28 15:29:51 - r - INFO: - Episode: 118/800, Reward: 0.000, Step: 27 +2022-11-28 15:29:51 - r - INFO: - Episode: 119/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:51 - r - INFO: - Episode: 120/800, Reward: 0.000, Step: 87 +2022-11-28 15:29:52 - r - INFO: - Current episode 120 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 121/800, Reward: 0.000, Step: 9 +2022-11-28 15:29:52 - r - INFO: - Episode: 122/800, Reward: 0.000, Step: 64 +2022-11-28 15:29:52 - r - INFO: - Episode: 123/800, Reward: 0.000, Step: 27 +2022-11-28 15:29:52 - r - INFO: - Episode: 124/800, Reward: 0.000, Step: 68 +2022-11-28 15:29:52 - r - INFO: - Episode: 125/800, Reward: 0.000, Step: 81 +2022-11-28 15:29:52 - r - INFO: - Current episode 125 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 126/800, Reward: 0.000, Step: 86 +2022-11-28 15:29:52 - r - INFO: - Episode: 127/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:52 - r - INFO: - Episode: 128/800, Reward: 0.000, Step: 27 +2022-11-28 15:29:52 - r - INFO: - Episode: 129/800, Reward: 0.000, Step: 41 +2022-11-28 15:29:52 - r - INFO: - Episode: 130/800, Reward: 0.000, Step: 70 +2022-11-28 15:29:52 - r - INFO: - Current episode 130 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 131/800, Reward: 0.000, Step: 27 +2022-11-28 15:29:52 - r - INFO: - Episode: 132/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:52 - r - INFO: - Episode: 133/800, Reward: 0.000, Step: 18 +2022-11-28 15:29:52 - r - INFO: - Episode: 134/800, Reward: 0.000, Step: 38 +2022-11-28 15:29:52 - r - INFO: - Episode: 135/800, Reward: 0.000, Step: 26 +2022-11-28 15:29:52 - r - INFO: - Current episode 135 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 136/800, Reward: 0.000, Step: 36 +2022-11-28 15:29:52 - r - INFO: - Episode: 137/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:52 - r - INFO: - Episode: 138/800, Reward: 0.000, Step: 61 +2022-11-28 15:29:52 - r - INFO: - Episode: 139/800, Reward: 0.000, Step: 105 +2022-11-28 15:29:52 - r - INFO: - Episode: 140/800, Reward: 0.000, Step: 38 +2022-11-28 15:29:52 - r - INFO: - Current episode 140 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 141/800, Reward: 0.000, Step: 18 +2022-11-28 15:29:52 - r - INFO: - Episode: 142/800, Reward: 0.000, Step: 33 +2022-11-28 15:29:52 - r - INFO: - Episode: 143/800, Reward: 0.000, Step: 29 +2022-11-28 15:29:52 - r - INFO: - Episode: 144/800, Reward: 0.000, Step: 49 +2022-11-28 15:29:52 - r - INFO: - Episode: 145/800, Reward: 0.000, Step: 88 +2022-11-28 15:29:52 - r - INFO: - Current episode 145 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 146/800, Reward: 0.000, Step: 22 +2022-11-28 15:29:52 - r - INFO: - Episode: 147/800, Reward: 0.000, Step: 65 +2022-11-28 15:29:52 - r - INFO: - Episode: 148/800, Reward: 0.000, Step: 36 +2022-11-28 15:29:52 - r - INFO: - Episode: 149/800, Reward: 0.000, Step: 30 +2022-11-28 15:29:52 - r - INFO: - Episode: 150/800, Reward: 0.000, Step: 58 +2022-11-28 15:29:52 - r - INFO: - Current episode 150 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 151/800, Reward: 0.000, Step: 43 +2022-11-28 15:29:52 - r - INFO: - Episode: 152/800, Reward: 0.000, Step: 53 +2022-11-28 15:29:52 - r - INFO: - Episode: 153/800, Reward: 0.000, Step: 43 +2022-11-28 15:29:52 - r - INFO: - Episode: 154/800, Reward: 0.000, Step: 13 +2022-11-28 15:29:52 - r - INFO: - Episode: 155/800, Reward: 0.000, Step: 8 +2022-11-28 15:29:52 - r - INFO: - Current episode 155 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 156/800, Reward: 0.000, Step: 39 +2022-11-28 15:29:52 - r - INFO: - Episode: 157/800, Reward: 0.000, Step: 29 +2022-11-28 15:29:52 - r - INFO: - Episode: 158/800, Reward: 0.000, Step: 26 +2022-11-28 15:29:52 - r - INFO: - Episode: 159/800, Reward: 0.000, Step: 60 +2022-11-28 15:29:52 - r - INFO: - Episode: 160/800, Reward: 0.000, Step: 153 +2022-11-28 15:29:52 - r - INFO: - Current episode 160 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 161/800, Reward: 0.000, Step: 116 +2022-11-28 15:29:52 - r - INFO: - Episode: 162/800, Reward: 0.000, Step: 53 +2022-11-28 15:29:52 - r - INFO: - Episode: 163/800, Reward: 0.000, Step: 54 +2022-11-28 15:29:52 - r - INFO: - Episode: 164/800, Reward: 0.000, Step: 8 +2022-11-28 15:29:52 - r - INFO: - Episode: 165/800, Reward: 0.000, Step: 58 +2022-11-28 15:29:52 - r - INFO: - Current episode 165 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 166/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:52 - r - INFO: - Episode: 167/800, Reward: 0.000, Step: 47 +2022-11-28 15:29:52 - r - INFO: - Episode: 168/800, Reward: 0.000, Step: 16 +2022-11-28 15:29:52 - r - INFO: - Episode: 169/800, Reward: 0.000, Step: 21 +2022-11-28 15:29:52 - r - INFO: - Episode: 170/800, Reward: 0.000, Step: 44 +2022-11-28 15:29:52 - r - INFO: - Current episode 170 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 171/800, Reward: 0.000, Step: 29 +2022-11-28 15:29:52 - r - INFO: - Episode: 172/800, Reward: 0.000, Step: 104 +2022-11-28 15:29:52 - r - INFO: - Episode: 173/800, Reward: 0.000, Step: 158 +2022-11-28 15:29:52 - r - INFO: - Episode: 174/800, Reward: 0.000, Step: 83 +2022-11-28 15:29:52 - r - INFO: - Episode: 175/800, Reward: 0.000, Step: 26 +2022-11-28 15:29:52 - r - INFO: - Current episode 175 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 176/800, Reward: 0.000, Step: 24 +2022-11-28 15:29:52 - r - INFO: - Episode: 177/800, Reward: 0.000, Step: 10 +2022-11-28 15:29:52 - r - INFO: - Episode: 178/800, Reward: 0.000, Step: 12 +2022-11-28 15:29:52 - r - INFO: - Episode: 179/800, Reward: 0.000, Step: 40 +2022-11-28 15:29:52 - r - INFO: - Episode: 180/800, Reward: 0.000, Step: 25 +2022-11-28 15:29:52 - r - INFO: - Current episode 180 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 181/800, Reward: 0.000, Step: 18 +2022-11-28 15:29:52 - r - INFO: - Episode: 182/800, Reward: 0.000, Step: 60 +2022-11-28 15:29:52 - r - INFO: - Episode: 183/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:52 - r - INFO: - Episode: 184/800, Reward: 0.000, Step: 24 +2022-11-28 15:29:52 - r - INFO: - Episode: 185/800, Reward: 0.000, Step: 56 +2022-11-28 15:29:52 - r - INFO: - Current episode 185 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 186/800, Reward: 0.000, Step: 71 +2022-11-28 15:29:52 - r - INFO: - Episode: 187/800, Reward: 0.000, Step: 19 +2022-11-28 15:29:52 - r - INFO: - Episode: 188/800, Reward: 0.000, Step: 118 +2022-11-28 15:29:52 - r - INFO: - Episode: 189/800, Reward: 0.000, Step: 26 +2022-11-28 15:29:52 - r - INFO: - Episode: 190/800, Reward: 0.000, Step: 41 +2022-11-28 15:29:52 - r - INFO: - Current episode 190 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 191/800, Reward: 0.000, Step: 41 +2022-11-28 15:29:52 - r - INFO: - Episode: 192/800, Reward: 0.000, Step: 60 +2022-11-28 15:29:52 - r - INFO: - Episode: 193/800, Reward: 0.000, Step: 31 +2022-11-28 15:29:52 - r - INFO: - Episode: 194/800, Reward: 0.000, Step: 34 +2022-11-28 15:29:52 - r - INFO: - Episode: 195/800, Reward: 0.000, Step: 35 +2022-11-28 15:29:52 - r - INFO: - Current episode 195 has the best eval reward: 0.000 +2022-11-28 15:29:52 - r - INFO: - Episode: 196/800, Reward: 0.000, Step: 59 +2022-11-28 15:29:52 - r - INFO: - Episode: 197/800, Reward: 0.000, Step: 51 +2022-11-28 15:29:52 - r - INFO: - Episode: 198/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:52 - r - INFO: - Episode: 199/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:52 - r - INFO: - Episode: 200/800, Reward: 0.000, Step: 37 +2022-11-28 15:29:53 - r - INFO: - Current episode 200 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 201/800, Reward: 0.000, Step: 68 +2022-11-28 15:29:53 - r - INFO: - Episode: 202/800, Reward: 0.000, Step: 40 +2022-11-28 15:29:53 - r - INFO: - Episode: 203/800, Reward: 0.000, Step: 17 +2022-11-28 15:29:53 - r - INFO: - Episode: 204/800, Reward: 0.000, Step: 79 +2022-11-28 15:29:53 - r - INFO: - Episode: 205/800, Reward: 0.000, Step: 126 +2022-11-28 15:29:53 - r - INFO: - Current episode 205 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 206/800, Reward: 0.000, Step: 61 +2022-11-28 15:29:53 - r - INFO: - Episode: 207/800, Reward: 0.000, Step: 25 +2022-11-28 15:29:53 - r - INFO: - Episode: 208/800, Reward: 0.000, Step: 18 +2022-11-28 15:29:53 - r - INFO: - Episode: 209/800, Reward: 0.000, Step: 27 +2022-11-28 15:29:53 - r - INFO: - Episode: 210/800, Reward: 0.000, Step: 13 +2022-11-28 15:29:53 - r - INFO: - Current episode 210 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 211/800, Reward: 0.000, Step: 187 +2022-11-28 15:29:53 - r - INFO: - Episode: 212/800, Reward: 0.000, Step: 160 +2022-11-28 15:29:53 - r - INFO: - Episode: 213/800, Reward: 0.000, Step: 32 +2022-11-28 15:29:53 - r - INFO: - Episode: 214/800, Reward: 0.000, Step: 108 +2022-11-28 15:29:53 - r - INFO: - Episode: 215/800, Reward: 0.000, Step: 164 +2022-11-28 15:29:53 - r - INFO: - Current episode 215 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 216/800, Reward: 0.000, Step: 17 +2022-11-28 15:29:53 - r - INFO: - Episode: 217/800, Reward: 0.000, Step: 82 +2022-11-28 15:29:53 - r - INFO: - Episode: 218/800, Reward: 0.000, Step: 194 +2022-11-28 15:29:53 - r - INFO: - Episode: 219/800, Reward: 0.000, Step: 7 +2022-11-28 15:29:53 - r - INFO: - Episode: 220/800, Reward: 0.000, Step: 36 +2022-11-28 15:29:53 - r - INFO: - Current episode 220 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 221/800, Reward: 0.000, Step: 156 +2022-11-28 15:29:53 - r - INFO: - Episode: 222/800, Reward: 0.000, Step: 17 +2022-11-28 15:29:53 - r - INFO: - Episode: 223/800, Reward: 0.000, Step: 183 +2022-11-28 15:29:53 - r - INFO: - Episode: 224/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:53 - r - INFO: - Episode: 225/800, Reward: 0.000, Step: 43 +2022-11-28 15:29:53 - r - INFO: - Current episode 225 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 226/800, Reward: 0.000, Step: 87 +2022-11-28 15:29:53 - r - INFO: - Episode: 227/800, Reward: 0.000, Step: 42 +2022-11-28 15:29:53 - r - INFO: - Episode: 228/800, Reward: 0.000, Step: 80 +2022-11-28 15:29:53 - r - INFO: - Episode: 229/800, Reward: 0.000, Step: 54 +2022-11-28 15:29:53 - r - INFO: - Episode: 230/800, Reward: 0.000, Step: 82 +2022-11-28 15:29:53 - r - INFO: - Current episode 230 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 231/800, Reward: 0.000, Step: 97 +2022-11-28 15:29:53 - r - INFO: - Episode: 232/800, Reward: 0.000, Step: 65 +2022-11-28 15:29:53 - r - INFO: - Episode: 233/800, Reward: 0.000, Step: 83 +2022-11-28 15:29:53 - r - INFO: - Episode: 234/800, Reward: 0.000, Step: 159 +2022-11-28 15:29:53 - r - INFO: - Episode: 235/800, Reward: 0.000, Step: 178 +2022-11-28 15:29:53 - r - INFO: - Current episode 235 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 236/800, Reward: 0.000, Step: 104 +2022-11-28 15:29:53 - r - INFO: - Episode: 237/800, Reward: 0.000, Step: 21 +2022-11-28 15:29:53 - r - INFO: - Episode: 238/800, Reward: 0.000, Step: 118 +2022-11-28 15:29:53 - r - INFO: - Episode: 239/800, Reward: 0.000, Step: 80 +2022-11-28 15:29:53 - r - INFO: - Episode: 240/800, Reward: 0.000, Step: 170 +2022-11-28 15:29:53 - r - INFO: - Current episode 240 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 241/800, Reward: 0.000, Step: 94 +2022-11-28 15:29:53 - r - INFO: - Episode: 242/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:53 - r - INFO: - Episode: 243/800, Reward: 0.000, Step: 37 +2022-11-28 15:29:53 - r - INFO: - Episode: 244/800, Reward: 0.000, Step: 11 +2022-11-28 15:29:53 - r - INFO: - Episode: 245/800, Reward: 0.000, Step: 31 +2022-11-28 15:29:53 - r - INFO: - Current episode 245 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 246/800, Reward: 0.000, Step: 134 +2022-11-28 15:29:53 - r - INFO: - Episode: 247/800, Reward: 0.000, Step: 32 +2022-11-28 15:29:53 - r - INFO: - Episode: 248/800, Reward: 0.000, Step: 58 +2022-11-28 15:29:53 - r - INFO: - Episode: 249/800, Reward: 0.000, Step: 38 +2022-11-28 15:29:53 - r - INFO: - Episode: 250/800, Reward: 0.000, Step: 28 +2022-11-28 15:29:53 - r - INFO: - Current episode 250 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 251/800, Reward: 0.000, Step: 159 +2022-11-28 15:29:53 - r - INFO: - Episode: 252/800, Reward: 0.000, Step: 182 +2022-11-28 15:29:53 - r - INFO: - Episode: 253/800, Reward: 0.000, Step: 51 +2022-11-28 15:29:53 - r - INFO: - Episode: 254/800, Reward: 0.000, Step: 25 +2022-11-28 15:29:53 - r - INFO: - Episode: 255/800, Reward: 0.000, Step: 73 +2022-11-28 15:29:53 - r - INFO: - Current episode 255 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 256/800, Reward: 0.000, Step: 56 +2022-11-28 15:29:53 - r - INFO: - Episode: 257/800, Reward: 0.000, Step: 55 +2022-11-28 15:29:53 - r - INFO: - Episode: 258/800, Reward: 0.000, Step: 38 +2022-11-28 15:29:53 - r - INFO: - Episode: 259/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:53 - r - INFO: - Episode: 260/800, Reward: 0.000, Step: 92 +2022-11-28 15:29:53 - r - INFO: - Current episode 260 has the best eval reward: 0.000 +2022-11-28 15:29:53 - r - INFO: - Episode: 261/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:53 - r - INFO: - Episode: 262/800, Reward: 0.000, Step: 119 +2022-11-28 15:29:53 - r - INFO: - Episode: 263/800, Reward: 0.000, Step: 100 +2022-11-28 15:29:53 - r - INFO: - Episode: 264/800, Reward: 0.000, Step: 84 +2022-11-28 15:29:53 - r - INFO: - Episode: 265/800, Reward: 0.000, Step: 24 +2022-11-28 15:29:54 - r - INFO: - Current episode 265 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 266/800, Reward: 0.000, Step: 17 +2022-11-28 15:29:54 - r - INFO: - Episode: 267/800, Reward: 0.000, Step: 159 +2022-11-28 15:29:54 - r - INFO: - Episode: 268/800, Reward: 0.000, Step: 25 +2022-11-28 15:29:54 - r - INFO: - Episode: 269/800, Reward: 0.000, Step: 73 +2022-11-28 15:29:54 - r - INFO: - Episode: 270/800, Reward: 0.000, Step: 130 +2022-11-28 15:29:54 - r - INFO: - Current episode 270 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 271/800, Reward: 0.000, Step: 111 +2022-11-28 15:29:54 - r - INFO: - Episode: 272/800, Reward: 0.000, Step: 65 +2022-11-28 15:29:54 - r - INFO: - Episode: 273/800, Reward: 1.000, Step: 58 +2022-11-28 15:29:54 - r - INFO: - Episode: 274/800, Reward: 0.000, Step: 47 +2022-11-28 15:29:54 - r - INFO: - Episode: 275/800, Reward: 0.000, Step: 48 +2022-11-28 15:29:54 - r - INFO: - Current episode 275 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 276/800, Reward: 0.000, Step: 13 +2022-11-28 15:29:54 - r - INFO: - Episode: 277/800, Reward: 0.000, Step: 100 +2022-11-28 15:29:54 - r - INFO: - Episode: 278/800, Reward: 0.000, Step: 38 +2022-11-28 15:29:54 - r - INFO: - Episode: 279/800, Reward: 0.000, Step: 111 +2022-11-28 15:29:54 - r - INFO: - Episode: 280/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:54 - r - INFO: - Current episode 280 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 281/800, Reward: 0.000, Step: 26 +2022-11-28 15:29:54 - r - INFO: - Episode: 282/800, Reward: 0.000, Step: 38 +2022-11-28 15:29:54 - r - INFO: - Episode: 283/800, Reward: 0.000, Step: 83 +2022-11-28 15:29:54 - r - INFO: - Episode: 284/800, Reward: 0.000, Step: 42 +2022-11-28 15:29:54 - r - INFO: - Episode: 285/800, Reward: 0.000, Step: 199 +2022-11-28 15:29:54 - r - INFO: - Current episode 285 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 286/800, Reward: 0.000, Step: 83 +2022-11-28 15:29:54 - r - INFO: - Episode: 287/800, Reward: 0.000, Step: 28 +2022-11-28 15:29:54 - r - INFO: - Episode: 288/800, Reward: 0.000, Step: 46 +2022-11-28 15:29:54 - r - INFO: - Episode: 289/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:54 - r - INFO: - Episode: 290/800, Reward: 0.000, Step: 62 +2022-11-28 15:29:54 - r - INFO: - Current episode 290 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 291/800, Reward: 0.000, Step: 123 +2022-11-28 15:29:54 - r - INFO: - Episode: 292/800, Reward: 0.000, Step: 91 +2022-11-28 15:29:54 - r - INFO: - Episode: 293/800, Reward: 0.000, Step: 53 +2022-11-28 15:29:54 - r - INFO: - Episode: 294/800, Reward: 0.000, Step: 19 +2022-11-28 15:29:54 - r - INFO: - Episode: 295/800, Reward: 0.000, Step: 26 +2022-11-28 15:29:54 - r - INFO: - Current episode 295 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 296/800, Reward: 0.000, Step: 93 +2022-11-28 15:29:54 - r - INFO: - Episode: 297/800, Reward: 0.000, Step: 38 +2022-11-28 15:29:54 - r - INFO: - Episode: 298/800, Reward: 0.000, Step: 22 +2022-11-28 15:29:54 - r - INFO: - Episode: 299/800, Reward: 0.000, Step: 43 +2022-11-28 15:29:54 - r - INFO: - Episode: 300/800, Reward: 0.000, Step: 163 +2022-11-28 15:29:54 - r - INFO: - Current episode 300 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 301/800, Reward: 0.000, Step: 25 +2022-11-28 15:29:54 - r - INFO: - Episode: 302/800, Reward: 0.000, Step: 59 +2022-11-28 15:29:54 - r - INFO: - Episode: 303/800, Reward: 0.000, Step: 71 +2022-11-28 15:29:54 - r - INFO: - Episode: 304/800, Reward: 0.000, Step: 20 +2022-11-28 15:29:54 - r - INFO: - Episode: 305/800, Reward: 0.000, Step: 115 +2022-11-28 15:29:54 - r - INFO: - Current episode 305 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 306/800, Reward: 0.000, Step: 200 +2022-11-28 15:29:54 - r - INFO: - Episode: 307/800, Reward: 0.000, Step: 48 +2022-11-28 15:29:54 - r - INFO: - Episode: 308/800, Reward: 0.000, Step: 66 +2022-11-28 15:29:54 - r - INFO: - Episode: 309/800, Reward: 0.000, Step: 58 +2022-11-28 15:29:54 - r - INFO: - Episode: 310/800, Reward: 0.000, Step: 129 +2022-11-28 15:29:54 - r - INFO: - Current episode 310 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 311/800, Reward: 0.000, Step: 122 +2022-11-28 15:29:54 - r - INFO: - Episode: 312/800, Reward: 0.000, Step: 47 +2022-11-28 15:29:54 - r - INFO: - Episode: 313/800, Reward: 0.000, Step: 60 +2022-11-28 15:29:54 - r - INFO: - Episode: 314/800, Reward: 0.000, Step: 79 +2022-11-28 15:29:54 - r - INFO: - Episode: 315/800, Reward: 1.000, Step: 137 +2022-11-28 15:29:54 - r - INFO: - Current episode 315 has the best eval reward: 0.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 316/800, Reward: 0.000, Step: 27 +2022-11-28 15:29:54 - r - INFO: - Episode: 317/800, Reward: 1.000, Step: 93 +2022-11-28 15:29:54 - r - INFO: - Episode: 318/800, Reward: 0.000, Step: 46 +2022-11-28 15:29:54 - r - INFO: - Episode: 319/800, Reward: 1.000, Step: 83 +2022-11-28 15:29:54 - r - INFO: - Episode: 320/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Current episode 320 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 321/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 322/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 323/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:54 - r - INFO: - Episode: 324/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 325/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:54 - r - INFO: - Current episode 325 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 326/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 327/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 328/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 329/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 330/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Current episode 330 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 331/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:54 - r - INFO: - Episode: 332/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 333/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 334/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:54 - r - INFO: - Episode: 335/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 335 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 336/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 337/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Episode: 338/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 339/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 340/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 340 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 341/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 342/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 343/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 344/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:54 - r - INFO: - Episode: 345/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Current episode 345 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 346/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:54 - r - INFO: - Episode: 347/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 348/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 349/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 350/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 350 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 351/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 352/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 353/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 354/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 355/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 355 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 356/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 357/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 358/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 359/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 360/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 360 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 361/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 362/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 363/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:54 - r - INFO: - Episode: 364/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Episode: 365/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Current episode 365 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 366/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 367/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 368/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Episode: 369/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 370/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 370 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 371/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 372/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 373/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 374/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Episode: 375/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Current episode 375 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 376/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 377/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 378/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:54 - r - INFO: - Episode: 379/800, Reward: 1.000, Step: 11 +2022-11-28 15:29:54 - r - INFO: - Episode: 380/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 380 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 381/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Episode: 382/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:54 - r - INFO: - Episode: 383/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 384/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 385/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 385 has the best eval reward: 1.000 +2022-11-28 15:29:54 - r - INFO: - Episode: 386/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 387/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:54 - r - INFO: - Episode: 388/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Episode: 389/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:54 - r - INFO: - Episode: 390/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:54 - r - INFO: - Current episode 390 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 391/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 392/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 393/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 394/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 395/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 395 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 396/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 397/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 398/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 399/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:55 - r - INFO: - Episode: 400/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Current episode 400 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 401/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 402/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 403/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 404/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 405/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 405 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 406/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 407/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:55 - r - INFO: - Episode: 408/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 409/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 410/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 410 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 411/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:55 - r - INFO: - Episode: 412/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 413/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 414/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 415/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 415 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 416/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 417/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 418/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 419/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 420/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 420 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 421/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 422/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 423/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 424/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 425/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Current episode 425 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 426/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 427/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 428/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 429/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 430/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 430 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 431/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 432/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 433/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 434/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 435/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 435 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 436/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 437/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 438/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:55 - r - INFO: - Episode: 439/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 440/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Current episode 440 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 441/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 442/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 443/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 444/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:55 - r - INFO: - Episode: 445/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 445 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 446/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 447/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 448/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 449/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:55 - r - INFO: - Episode: 450/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 450 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 451/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 452/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 453/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 454/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 455/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:55 - r - INFO: - Current episode 455 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 456/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 457/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 458/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 459/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:55 - r - INFO: - Episode: 460/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 460 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 461/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 462/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:55 - r - INFO: - Episode: 463/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 464/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 465/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 465 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 466/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:55 - r - INFO: - Episode: 467/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 468/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:55 - r - INFO: - Episode: 469/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 470/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 470 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 471/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 472/800, Reward: 1.000, Step: 15 +2022-11-28 15:29:55 - r - INFO: - Episode: 473/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 474/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 475/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 475 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 476/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 477/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 478/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 479/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 480/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 480 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 481/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 482/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 483/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 484/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 485/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 485 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 486/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 487/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 488/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 489/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 490/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 490 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 491/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 492/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:55 - r - INFO: - Episode: 493/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 494/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 495/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 495 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 496/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 497/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 498/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 499/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 500/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 500 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 501/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 502/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 503/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 504/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 505/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 505 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 506/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 507/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 508/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 509/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 510/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 510 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 511/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 512/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 513/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 514/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 515/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 515 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 516/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 517/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 518/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 519/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:55 - r - INFO: - Episode: 520/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Current episode 520 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 521/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 522/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 523/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 524/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 525/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 525 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 526/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 527/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 528/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 529/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 530/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 530 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 531/800, Reward: 1.000, Step: 9 +2022-11-28 15:29:55 - r - INFO: - Episode: 532/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 533/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 534/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 535/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 535 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 536/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 537/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 538/800, Reward: 1.000, Step: 9 +2022-11-28 15:29:55 - r - INFO: - Episode: 539/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 540/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:55 - r - INFO: - Current episode 540 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 541/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 542/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 543/800, Reward: 1.000, Step: 11 +2022-11-28 15:29:55 - r - INFO: - Episode: 544/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 545/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 545 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 546/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 547/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 548/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 549/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 550/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 550 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 551/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 552/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 553/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 554/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 555/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 555 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 556/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 557/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:55 - r - INFO: - Episode: 558/800, Reward: 1.000, Step: 9 +2022-11-28 15:29:55 - r - INFO: - Episode: 559/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 560/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 560 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 561/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 562/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 563/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 564/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 565/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:55 - r - INFO: - Current episode 565 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 566/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:55 - r - INFO: - Episode: 567/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 568/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 569/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 570/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 570 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 571/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 572/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 573/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 574/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 575/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 575 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 576/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 577/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 578/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 579/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 580/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 580 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 581/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 582/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 583/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 584/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 585/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:55 - r - INFO: - Current episode 585 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 586/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 587/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 588/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 589/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 590/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 590 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 591/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 592/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:55 - r - INFO: - Episode: 593/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 594/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 595/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 595 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 596/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 597/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 598/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 599/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:55 - r - INFO: - Episode: 600/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 600 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 601/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 602/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 603/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 604/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 605/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:55 - r - INFO: - Current episode 605 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 606/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 607/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 608/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 609/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 610/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 610 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 611/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 612/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 613/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 614/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 615/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 615 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 616/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Episode: 617/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 618/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 619/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:55 - r - INFO: - Episode: 620/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:55 - r - INFO: - Current episode 620 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 621/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:55 - r - INFO: - Episode: 622/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 623/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:55 - r - INFO: - Episode: 624/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 625/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 625 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 626/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 627/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 628/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:55 - r - INFO: - Episode: 629/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 630/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 630 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 631/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 632/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 633/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 634/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 635/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:55 - r - INFO: - Current episode 635 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 636/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 637/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 638/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 639/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 640/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Current episode 640 has the best eval reward: 1.000 +2022-11-28 15:29:55 - r - INFO: - Episode: 641/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 642/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 643/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Episode: 644/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:55 - r - INFO: - Episode: 645/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:55 - r - INFO: - Current episode 645 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 646/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 647/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 648/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 649/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 650/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:56 - r - INFO: - Current episode 650 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 651/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:56 - r - INFO: - Episode: 652/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:56 - r - INFO: - Episode: 653/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 654/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 655/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 655 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 656/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 657/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 658/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 659/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:56 - r - INFO: - Episode: 660/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 660 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 661/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 662/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 663/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 664/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 665/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 665 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 666/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 667/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 668/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 669/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 670/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Current episode 670 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 671/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 672/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 673/800, Reward: 1.000, Step: 9 +2022-11-28 15:29:56 - r - INFO: - Episode: 674/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 675/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 675 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 676/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 677/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 678/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:56 - r - INFO: - Episode: 679/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 680/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 680 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 681/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 682/800, Reward: 1.000, Step: 11 +2022-11-28 15:29:56 - r - INFO: - Episode: 683/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:56 - r - INFO: - Episode: 684/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 685/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 685 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 686/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 687/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 688/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:56 - r - INFO: - Episode: 689/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 690/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:56 - r - INFO: - Current episode 690 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 691/800, Reward: 1.000, Step: 9 +2022-11-28 15:29:56 - r - INFO: - Episode: 692/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 693/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 694/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:56 - r - INFO: - Episode: 695/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 695 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 696/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 697/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:56 - r - INFO: - Episode: 698/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:56 - r - INFO: - Episode: 699/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:56 - r - INFO: - Episode: 700/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:56 - r - INFO: - Current episode 700 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 701/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 702/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 703/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 704/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 705/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 705 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 706/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 707/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:56 - r - INFO: - Episode: 708/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 709/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 710/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Current episode 710 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 711/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 712/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 713/800, Reward: 1.000, Step: 9 +2022-11-28 15:29:56 - r - INFO: - Episode: 714/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 715/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Current episode 715 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 716/800, Reward: 1.000, Step: 11 +2022-11-28 15:29:56 - r - INFO: - Episode: 717/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 718/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 719/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 720/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 720 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 721/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 722/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 723/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 724/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 725/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:56 - r - INFO: - Current episode 725 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 726/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 727/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 728/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 729/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 730/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 730 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 731/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:56 - r - INFO: - Episode: 732/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 733/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 734/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 735/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 735 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 736/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:56 - r - INFO: - Episode: 737/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 738/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 739/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 740/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 740 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 741/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 742/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:56 - r - INFO: - Episode: 743/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 744/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 745/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:56 - r - INFO: - Current episode 745 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 746/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 747/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 748/800, Reward: 1.000, Step: 14 +2022-11-28 15:29:56 - r - INFO: - Episode: 749/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 750/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 750 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 751/800, Reward: 1.000, Step: 12 +2022-11-28 15:29:56 - r - INFO: - Episode: 752/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 753/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 754/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 755/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 755 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 756/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 757/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 758/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:56 - r - INFO: - Episode: 759/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 760/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 760 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 761/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 762/800, Reward: 1.000, Step: 7 +2022-11-28 15:29:56 - r - INFO: - Episode: 763/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 764/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 765/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 765 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 766/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 767/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:56 - r - INFO: - Episode: 768/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 769/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 770/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 770 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 771/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 772/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 773/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 774/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 775/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 775 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 776/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 777/800, Reward: 0.000, Step: 4 +2022-11-28 15:29:56 - r - INFO: - Episode: 778/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 779/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 780/800, Reward: 0.000, Step: 2 +2022-11-28 15:29:56 - r - INFO: - Current episode 780 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 781/800, Reward: 1.000, Step: 10 +2022-11-28 15:29:56 - r - INFO: - Episode: 782/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 783/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 784/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 785/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 785 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 786/800, Reward: 0.000, Step: 3 +2022-11-28 15:29:56 - r - INFO: - Episode: 787/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 788/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 789/800, Reward: 0.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 790/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Current episode 790 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 791/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 792/800, Reward: 1.000, Step: 9 +2022-11-28 15:29:56 - r - INFO: - Episode: 793/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 794/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 795/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Current episode 795 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Episode: 796/800, Reward: 1.000, Step: 8 +2022-11-28 15:29:56 - r - INFO: - Episode: 797/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 798/800, Reward: 0.000, Step: 5 +2022-11-28 15:29:56 - r - INFO: - Episode: 799/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Episode: 800/800, Reward: 1.000, Step: 6 +2022-11-28 15:29:56 - r - INFO: - Current episode 800 has the best eval reward: 1.000 +2022-11-28 15:29:56 - r - INFO: - Finish training! diff --git a/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/models/Qleaning_model.pkl b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/models/Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c0b10db9f2404d272ed7739f1190ffaa3867dd92 --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/models/Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd5782c99cad77fe3383f02d3f430ac7bd22fd8aed9973645d3c5ed600eafbb +size 2479 diff --git a/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/results/learning_curve.png b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..59a00477e26e04f2e1b8e21b930243f77103fa20 Binary files /dev/null and b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/results/learning_curve.png differ diff --git a/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/results/res.csv b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..335c1d84e615b85651608b2d6268d94022665132 --- /dev/null +++ b/FrozenLakeNoSlippery-v1/Train_FrozenLakeNoSlippery-v1_QLearning_20221128-152949/results/res.csv @@ -0,0 +1,801 @@ +episodes,rewards,steps +0,0.0,20 +1,0.0,14 +2,0.0,13 +3,0.0,9 +4,0.0,10 +5,0.0,6 +6,0.0,11 +7,0.0,6 +8,0.0,3 +9,0.0,9 +10,0.0,11 +11,0.0,22 +12,0.0,5 +13,0.0,16 +14,0.0,4 +15,0.0,9 +16,0.0,18 +17,0.0,2 +18,0.0,4 +19,0.0,8 +20,0.0,7 +21,0.0,4 +22,0.0,22 +23,0.0,15 +24,0.0,5 +25,0.0,16 +26,0.0,7 +27,0.0,19 +28,0.0,22 +29,0.0,16 +30,0.0,11 +31,0.0,22 +32,0.0,28 +33,0.0,23 +34,0.0,4 +35,0.0,11 +36,0.0,8 +37,0.0,15 +38,0.0,5 +39,0.0,7 +40,0.0,9 +41,0.0,4 +42,0.0,3 +43,0.0,6 +44,0.0,41 +45,0.0,9 +46,0.0,23 +47,0.0,3 +48,1.0,38 +49,0.0,29 +50,0.0,17 +51,0.0,4 +52,0.0,2 +53,0.0,25 +54,0.0,6 +55,0.0,2 +56,0.0,30 +57,0.0,6 +58,0.0,7 +59,0.0,11 +60,0.0,9 +61,0.0,8 +62,0.0,23 +63,0.0,10 +64,0.0,3 +65,0.0,5 +66,0.0,7 +67,0.0,18 +68,0.0,8 +69,0.0,26 +70,0.0,6 +71,0.0,14 +72,0.0,4 +73,0.0,25 +74,0.0,21 +75,0.0,13 +76,0.0,4 +77,0.0,29 +78,0.0,21 +79,0.0,6 +80,0.0,6 +81,0.0,11 +82,0.0,21 +83,0.0,9 +84,0.0,9 +85,0.0,7 +86,0.0,48 +87,0.0,23 +88,0.0,160 +89,0.0,7 +90,0.0,10 +91,0.0,24 +92,0.0,4 +93,0.0,7 +94,0.0,17 +95,0.0,87 +96,0.0,28 +97,0.0,7 +98,0.0,5 +99,0.0,12 +100,0.0,14 +101,0.0,6 +102,0.0,13 +103,0.0,93 +104,0.0,4 +105,0.0,50 +106,0.0,8 +107,0.0,12 +108,0.0,43 +109,0.0,30 +110,0.0,15 +111,0.0,19 +112,0.0,182 +113,0.0,40 +114,0.0,88 +115,0.0,19 +116,0.0,30 +117,0.0,27 +118,0.0,5 +119,0.0,87 +120,0.0,9 +121,0.0,64 +122,0.0,27 +123,0.0,68 +124,0.0,81 +125,0.0,86 +126,0.0,200 +127,0.0,27 +128,0.0,41 +129,0.0,70 +130,0.0,27 +131,0.0,6 +132,0.0,18 +133,0.0,38 +134,0.0,26 +135,0.0,36 +136,0.0,3 +137,0.0,61 +138,0.0,105 +139,0.0,38 +140,0.0,18 +141,0.0,33 +142,0.0,29 +143,0.0,49 +144,0.0,88 +145,0.0,22 +146,0.0,65 +147,0.0,36 +148,0.0,30 +149,0.0,58 +150,0.0,43 +151,0.0,53 +152,0.0,43 +153,0.0,13 +154,0.0,8 +155,0.0,39 +156,0.0,29 +157,0.0,26 +158,0.0,60 +159,0.0,153 +160,0.0,116 +161,0.0,53 +162,0.0,54 +163,0.0,8 +164,0.0,58 +165,0.0,3 +166,0.0,47 +167,0.0,16 +168,0.0,21 +169,0.0,44 +170,0.0,29 +171,0.0,104 +172,0.0,158 +173,0.0,83 +174,0.0,26 +175,0.0,24 +176,0.0,10 +177,0.0,12 +178,0.0,40 +179,0.0,25 +180,0.0,18 +181,0.0,60 +182,0.0,200 +183,0.0,24 +184,0.0,56 +185,0.0,71 +186,0.0,19 +187,0.0,118 +188,0.0,26 +189,0.0,41 +190,0.0,41 +191,0.0,60 +192,0.0,31 +193,0.0,34 +194,0.0,35 +195,0.0,59 +196,0.0,51 +197,0.0,200 +198,0.0,200 +199,0.0,37 +200,0.0,68 +201,0.0,40 +202,0.0,17 +203,0.0,79 +204,0.0,126 +205,0.0,61 +206,0.0,25 +207,0.0,18 +208,0.0,27 +209,0.0,13 +210,0.0,187 +211,0.0,160 +212,0.0,32 +213,0.0,108 +214,0.0,164 +215,0.0,17 +216,0.0,82 +217,0.0,194 +218,0.0,7 +219,0.0,36 +220,0.0,156 +221,0.0,17 +222,0.0,183 +223,0.0,200 +224,0.0,43 +225,0.0,87 +226,0.0,42 +227,0.0,80 +228,0.0,54 +229,0.0,82 +230,0.0,97 +231,0.0,65 +232,0.0,83 +233,0.0,159 +234,0.0,178 +235,0.0,104 +236,0.0,21 +237,0.0,118 +238,0.0,80 +239,0.0,170 +240,0.0,94 +241,0.0,200 +242,0.0,37 +243,0.0,11 +244,0.0,31 +245,0.0,134 +246,0.0,32 +247,0.0,58 +248,0.0,38 +249,0.0,28 +250,0.0,159 +251,0.0,182 +252,0.0,51 +253,0.0,25 +254,0.0,73 +255,0.0,56 +256,0.0,55 +257,0.0,38 +258,0.0,200 +259,0.0,92 +260,0.0,200 +261,0.0,119 +262,0.0,100 +263,0.0,84 +264,0.0,24 +265,0.0,17 +266,0.0,159 +267,0.0,25 +268,0.0,73 +269,0.0,130 +270,0.0,111 +271,0.0,65 +272,1.0,58 +273,0.0,47 +274,0.0,48 +275,0.0,13 +276,0.0,100 +277,0.0,38 +278,0.0,111 +279,0.0,200 +280,0.0,26 +281,0.0,38 +282,0.0,83 +283,0.0,42 +284,0.0,199 +285,0.0,83 +286,0.0,28 +287,0.0,46 +288,0.0,200 +289,0.0,62 +290,0.0,123 +291,0.0,91 +292,0.0,53 +293,0.0,19 +294,0.0,26 +295,0.0,93 +296,0.0,38 +297,0.0,22 +298,0.0,43 +299,0.0,163 +300,0.0,25 +301,0.0,59 +302,0.0,71 +303,0.0,20 +304,0.0,115 +305,0.0,200 +306,0.0,48 +307,0.0,66 +308,0.0,58 +309,0.0,129 +310,0.0,122 +311,0.0,47 +312,0.0,60 +313,0.0,79 +314,1.0,137 +315,0.0,27 +316,1.0,93 +317,0.0,46 +318,1.0,83 +319,1.0,8 +320,1.0,6 +321,1.0,6 +322,0.0,4 +323,1.0,6 +324,0.0,2 +325,1.0,6 +326,1.0,6 +327,1.0,6 +328,1.0,6 +329,1.0,8 +330,0.0,5 +331,1.0,6 +332,1.0,7 +333,0.0,5 +334,1.0,6 +335,1.0,6 +336,1.0,8 +337,1.0,6 +338,1.0,6 +339,1.0,6 +340,1.0,7 +341,1.0,6 +342,1.0,6 +343,0.0,3 +344,1.0,7 +345,0.0,4 +346,1.0,6 +347,1.0,6 +348,1.0,7 +349,1.0,6 +350,1.0,6 +351,1.0,7 +352,1.0,7 +353,1.0,7 +354,1.0,6 +355,1.0,6 +356,1.0,6 +357,1.0,6 +358,1.0,6 +359,1.0,6 +360,1.0,6 +361,1.0,7 +362,0.0,4 +363,1.0,8 +364,1.0,8 +365,1.0,7 +366,1.0,6 +367,1.0,8 +368,1.0,6 +369,1.0,6 +370,1.0,7 +371,1.0,6 +372,1.0,6 +373,1.0,8 +374,1.0,7 +375,1.0,6 +376,1.0,6 +377,0.0,3 +378,1.0,11 +379,1.0,6 +380,1.0,8 +381,0.0,2 +382,1.0,6 +383,1.0,6 +384,1.0,6 +385,1.0,6 +386,1.0,8 +387,1.0,6 +388,1.0,7 +389,1.0,6 +390,1.0,7 +391,1.0,6 +392,1.0,8 +393,0.0,2 +394,1.0,6 +395,1.0,7 +396,1.0,6 +397,1.0,6 +398,1.0,10 +399,1.0,7 +400,1.0,6 +401,1.0,6 +402,1.0,6 +403,1.0,6 +404,1.0,6 +405,1.0,7 +406,0.0,4 +407,1.0,7 +408,1.0,6 +409,1.0,8 +410,0.0,3 +411,1.0,6 +412,1.0,6 +413,1.0,6 +414,1.0,6 +415,0.0,2 +416,1.0,6 +417,1.0,6 +418,1.0,6 +419,1.0,6 +420,1.0,6 +421,1.0,7 +422,1.0,6 +423,1.0,6 +424,1.0,7 +425,1.0,6 +426,1.0,6 +427,1.0,6 +428,1.0,6 +429,1.0,6 +430,1.0,6 +431,1.0,6 +432,1.0,8 +433,1.0,6 +434,1.0,8 +435,1.0,7 +436,1.0,6 +437,0.0,3 +438,1.0,6 +439,1.0,7 +440,1.0,6 +441,1.0,6 +442,1.0,6 +443,1.0,10 +444,1.0,6 +445,1.0,6 +446,1.0,6 +447,1.0,6 +448,1.0,10 +449,1.0,6 +450,1.0,8 +451,1.0,8 +452,1.0,7 +453,1.0,6 +454,0.0,5 +455,0.0,2 +456,1.0,8 +457,1.0,6 +458,1.0,10 +459,1.0,6 +460,1.0,8 +461,1.0,10 +462,1.0,6 +463,1.0,6 +464,1.0,6 +465,1.0,10 +466,1.0,6 +467,0.0,4 +468,1.0,6 +469,1.0,6 +470,1.0,6 +471,1.0,15 +472,1.0,6 +473,1.0,6 +474,1.0,6 +475,1.0,6 +476,1.0,6 +477,1.0,6 +478,1.0,8 +479,1.0,6 +480,1.0,7 +481,1.0,6 +482,1.0,6 +483,1.0,8 +484,1.0,6 +485,1.0,6 +486,1.0,8 +487,1.0,8 +488,1.0,6 +489,1.0,6 +490,1.0,6 +491,1.0,10 +492,1.0,6 +493,1.0,6 +494,1.0,6 +495,1.0,6 +496,1.0,6 +497,1.0,6 +498,1.0,6 +499,1.0,8 +500,1.0,8 +501,1.0,6 +502,1.0,6 +503,0.0,2 +504,1.0,6 +505,1.0,6 +506,1.0,6 +507,1.0,8 +508,1.0,6 +509,1.0,6 +510,1.0,6 +511,1.0,6 +512,1.0,6 +513,1.0,6 +514,1.0,6 +515,1.0,6 +516,1.0,6 +517,1.0,7 +518,0.0,3 +519,1.0,7 +520,1.0,6 +521,1.0,6 +522,1.0,6 +523,0.0,2 +524,1.0,6 +525,1.0,8 +526,1.0,6 +527,1.0,6 +528,1.0,6 +529,1.0,6 +530,1.0,9 +531,1.0,6 +532,1.0,6 +533,1.0,6 +534,1.0,6 +535,1.0,6 +536,1.0,6 +537,1.0,9 +538,1.0,7 +539,0.0,4 +540,1.0,6 +541,1.0,8 +542,1.0,11 +543,1.0,6 +544,1.0,6 +545,1.0,6 +546,1.0,6 +547,1.0,6 +548,1.0,8 +549,1.0,6 +550,1.0,6 +551,1.0,8 +552,1.0,7 +553,1.0,6 +554,1.0,8 +555,1.0,6 +556,0.0,5 +557,1.0,9 +558,1.0,8 +559,1.0,8 +560,1.0,6 +561,1.0,8 +562,1.0,8 +563,1.0,6 +564,0.0,5 +565,0.0,3 +566,0.0,2 +567,1.0,8 +568,1.0,6 +569,1.0,6 +570,1.0,6 +571,1.0,6 +572,1.0,6 +573,1.0,6 +574,1.0,6 +575,1.0,6 +576,1.0,6 +577,1.0,6 +578,1.0,6 +579,1.0,6 +580,1.0,6 +581,1.0,6 +582,0.0,2 +583,1.0,6 +584,0.0,4 +585,1.0,6 +586,1.0,6 +587,1.0,6 +588,1.0,6 +589,1.0,6 +590,1.0,8 +591,0.0,5 +592,1.0,6 +593,1.0,6 +594,1.0,6 +595,1.0,6 +596,1.0,6 +597,1.0,6 +598,0.0,3 +599,1.0,6 +600,1.0,6 +601,1.0,6 +602,0.0,2 +603,1.0,6 +604,0.0,4 +605,1.0,6 +606,1.0,6 +607,1.0,6 +608,1.0,6 +609,1.0,8 +610,1.0,6 +611,1.0,7 +612,1.0,6 +613,1.0,7 +614,1.0,6 +615,0.0,2 +616,1.0,6 +617,1.0,6 +618,0.0,5 +619,0.0,3 +620,0.0,3 +621,1.0,6 +622,0.0,5 +623,1.0,8 +624,1.0,8 +625,1.0,6 +626,1.0,6 +627,1.0,7 +628,1.0,6 +629,1.0,6 +630,1.0,6 +631,1.0,6 +632,1.0,6 +633,1.0,8 +634,0.0,2 +635,1.0,6 +636,1.0,6 +637,1.0,6 +638,1.0,6 +639,1.0,6 +640,1.0,6 +641,1.0,6 +642,1.0,8 +643,1.0,6 +644,1.0,8 +645,1.0,6 +646,1.0,6 +647,1.0,8 +648,1.0,8 +649,0.0,5 +650,0.0,4 +651,0.0,4 +652,1.0,6 +653,1.0,6 +654,1.0,6 +655,1.0,6 +656,1.0,8 +657,1.0,6 +658,0.0,4 +659,1.0,6 +660,1.0,8 +661,1.0,6 +662,1.0,6 +663,1.0,6 +664,1.0,6 +665,1.0,6 +666,1.0,6 +667,1.0,6 +668,1.0,8 +669,1.0,8 +670,1.0,6 +671,1.0,8 +672,1.0,9 +673,1.0,6 +674,1.0,6 +675,1.0,6 +676,1.0,6 +677,1.0,10 +678,1.0,6 +679,1.0,6 +680,1.0,6 +681,1.0,11 +682,1.0,10 +683,1.0,8 +684,1.0,6 +685,1.0,6 +686,1.0,6 +687,0.0,5 +688,1.0,6 +689,0.0,2 +690,1.0,9 +691,1.0,6 +692,1.0,8 +693,1.0,7 +694,1.0,6 +695,1.0,6 +696,1.0,7 +697,0.0,3 +698,1.0,7 +699,0.0,2 +700,1.0,6 +701,1.0,6 +702,1.0,8 +703,1.0,8 +704,1.0,6 +705,1.0,6 +706,0.0,2 +707,1.0,8 +708,1.0,6 +709,1.0,8 +710,1.0,6 +711,1.0,6 +712,1.0,9 +713,1.0,6 +714,1.0,8 +715,1.0,11 +716,1.0,6 +717,1.0,6 +718,1.0,6 +719,1.0,6 +720,1.0,8 +721,1.0,6 +722,1.0,6 +723,1.0,6 +724,0.0,5 +725,1.0,6 +726,1.0,6 +727,1.0,6 +728,1.0,6 +729,1.0,6 +730,1.0,7 +731,1.0,6 +732,1.0,6 +733,1.0,6 +734,1.0,6 +735,1.0,10 +736,1.0,6 +737,1.0,6 +738,1.0,6 +739,1.0,6 +740,1.0,6 +741,1.0,7 +742,1.0,6 +743,1.0,8 +744,1.0,7 +745,1.0,6 +746,1.0,6 +747,1.0,14 +748,1.0,6 +749,1.0,6 +750,1.0,12 +751,1.0,6 +752,1.0,6 +753,1.0,6 +754,1.0,6 +755,1.0,6 +756,1.0,6 +757,0.0,3 +758,1.0,6 +759,1.0,6 +760,1.0,6 +761,1.0,7 +762,1.0,6 +763,1.0,6 +764,1.0,6 +765,1.0,8 +766,0.0,2 +767,1.0,6 +768,1.0,6 +769,1.0,6 +770,1.0,6 +771,1.0,6 +772,1.0,6 +773,1.0,6 +774,1.0,6 +775,1.0,6 +776,0.0,4 +777,1.0,8 +778,1.0,6 +779,0.0,2 +780,1.0,10 +781,1.0,8 +782,1.0,6 +783,1.0,6 +784,1.0,6 +785,0.0,3 +786,1.0,6 +787,1.0,6 +788,0.0,6 +789,1.0,8 +790,1.0,6 +791,1.0,9 +792,1.0,6 +793,1.0,6 +794,1.0,8 +795,1.0,8 +796,1.0,6 +797,0.0,5 +798,1.0,6 +799,1.0,6 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/config.yaml b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69d9b79f3e0cfd2d7715c8a9e245d476d3033254 --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/config.yaml @@ -0,0 +1,50 @@ +general_cfg: + algo_name: BCQ + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_gym_BCQ_20230417-141811 + max_steps: 200 + mode: test + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 0 + show_fig: false + test_eps: 200 + train_eps: 1000 + wrapper: null +algo_cfg: + actor_hidden_dims: + - 400 + - 300 + actor_lr: 0.001 + batch_size: 128 + behavior_agent_name: DDPG + behavior_agent_parameters_path: /behave_param.yaml + behavior_policy_path: /behaviour_models + buffer_size: 100000 + collect_eps: 500 + collect_explore_data: true + critic_hidden_dims: + - 400 + - 300 + critic_lr: 0.001 + gamma: 0.99 + iters_per_ep: 10 + lmbda: 0.75 + phi: 0.05 + start_learn_buffer_size: 1000.0 + tau: 0.005 + vae_hidden_dims: + - 750 + - 750 + vae_lr: 0.001 +env_cfg: + id: HalfCheetah-v3 + new_step_api: true + render_mode: null diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/logs/log.txt b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..cc91638ba01769d121b9c8c4ee4178db20a12010 --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/logs/log.txt @@ -0,0 +1,253 @@ +2023-04-17 14:43:16 - r - INFO: - Hyperparameters: +2023-04-17 14:43:16 - r - INFO: - ================================================================================ +2023-04-17 14:43:16 - r - INFO: - Name Value Type +2023-04-17 14:43:16 - r - INFO: - env_name gym +2023-04-17 14:43:16 - r - INFO: - new_step_api 1 +2023-04-17 14:43:16 - r - INFO: - wrapper None +2023-04-17 14:43:16 - r - INFO: - render 0 +2023-04-17 14:43:16 - r - INFO: - render_mode None +2023-04-17 14:43:16 - r - INFO: - algo_name BCQ +2023-04-17 14:43:16 - r - INFO: - mode test +2023-04-17 14:43:16 - r - INFO: - mp_backend mp +2023-04-17 14:43:16 - r - INFO: - seed 0 +2023-04-17 14:43:16 - r - INFO: - device cpu +2023-04-17 14:43:16 - r - INFO: - train_eps 1000 +2023-04-17 14:43:16 - r - INFO: - test_eps 200 +2023-04-17 14:43:16 - r - INFO: - eval_eps 10 +2023-04-17 14:43:16 - r - INFO: - eval_per_episode 5 +2023-04-17 14:43:16 - r - INFO: - max_steps 200 +2023-04-17 14:43:16 - r - INFO: - load_checkpoint 1 +2023-04-17 14:43:16 - r - INFO: - load_path Train_gym_BCQ_20230417-141811 +2023-04-17 14:43:16 - r - INFO: - show_fig 0 +2023-04-17 14:43:16 - r - INFO: - save_fig 1 +2023-04-17 14:43:16 - r - INFO: - critic_hidden_dims [400, 300] +2023-04-17 14:43:16 - r - INFO: - actor_hidden_dims [400, 300] +2023-04-17 14:43:16 - r - INFO: - vae_hidden_dims [750, 750] +2023-04-17 14:43:16 - r - INFO: - critic_lr 0.001 +2023-04-17 14:43:16 - r - INFO: - actor_lr 0.001 +2023-04-17 14:43:16 - r - INFO: - vae_lr 0.001 +2023-04-17 14:43:16 - r - INFO: - batch_size 128 +2023-04-17 14:43:16 - r - INFO: - gamma 0.99 +2023-04-17 14:43:16 - r - INFO: - tau 0.005 +2023-04-17 14:43:16 - r - INFO: - lmbda 0.75 +2023-04-17 14:43:16 - r - INFO: - phi 0.05 +2023-04-17 14:43:16 - r - INFO: - iters_per_ep 10 +2023-04-17 14:43:16 - r - INFO: - buffer_size 100000 +2023-04-17 14:43:16 - r - INFO: - start_learn_buffer_size 1000.0 +2023-04-17 14:43:16 - r - INFO: - collect_explore_data 1 +2023-04-17 14:43:16 - r - INFO: - behavior_agent_name DDPG +2023-04-17 14:43:16 - r - INFO: - behavior_agent_parameters_path /behave_param.yaml +2023-04-17 14:43:16 - r - INFO: - behavior_policy_path /behaviour_models +2023-04-17 14:43:16 - r - INFO: - collect_eps 500 +2023-04-17 14:43:16 - r - INFO: - id HalfCheetah-v3 +2023-04-17 14:43:16 - r - INFO: - task_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Test_gym_BCQ_20230417-144316 +2023-04-17 14:43:16 - r - INFO: - res_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Test_gym_BCQ_20230417-144316/results +2023-04-17 14:43:16 - r - INFO: - log_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Test_gym_BCQ_20230417-144316/logs +2023-04-17 14:43:16 - r - INFO: - traj_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Test_gym_BCQ_20230417-144316/traj +2023-04-17 14:43:16 - r - INFO: - video_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Test_gym_BCQ_20230417-144316/videos +2023-04-17 14:43:16 - r - INFO: - ================================================================================ +2023-04-17 14:43:17 - r - INFO: - action_bound: 1.0 +2023-04-17 14:43:17 - r - INFO: - n_states: 17, n_actions: 6 +2023-04-17 14:43:17 - r - INFO: - Start testing! +2023-04-17 14:43:17 - r - INFO: - Env: gym, Algorithm: BCQ, Device: cpu +2023-04-17 14:43:17 - r - INFO: - Episode: 1/200, Reward: 1308.262, Step: 200 +2023-04-17 14:43:17 - r - INFO: - Episode: 2/200, Reward: 1270.460, Step: 200 +2023-04-17 14:43:17 - r - INFO: - Episode: 3/200, Reward: 1232.738, Step: 200 +2023-04-17 14:43:18 - r - INFO: - Episode: 4/200, Reward: 1301.504, Step: 200 +2023-04-17 14:43:18 - r - INFO: - Episode: 5/200, Reward: 1377.433, Step: 200 +2023-04-17 14:43:18 - r - INFO: - Episode: 6/200, Reward: 1360.484, Step: 200 +2023-04-17 14:43:18 - r - INFO: - Episode: 7/200, Reward: 1209.330, Step: 200 +2023-04-17 14:43:19 - r - INFO: - Episode: 8/200, Reward: 1366.542, Step: 200 +2023-04-17 14:43:19 - r - INFO: - Episode: 9/200, Reward: 1332.486, Step: 200 +2023-04-17 14:43:19 - r - INFO: - Episode: 10/200, Reward: 1190.748, Step: 200 +2023-04-17 14:43:19 - r - INFO: - Episode: 11/200, Reward: 1329.531, Step: 200 +2023-04-17 14:43:20 - r - INFO: - Episode: 12/200, Reward: 554.371, Step: 200 +2023-04-17 14:43:20 - r - INFO: - Episode: 13/200, Reward: 1328.543, Step: 200 +2023-04-17 14:43:20 - r - INFO: - Episode: 14/200, Reward: 1327.361, Step: 200 +2023-04-17 14:43:21 - r - INFO: - Episode: 15/200, Reward: 1232.017, Step: 200 +2023-04-17 14:43:21 - r - INFO: - Episode: 16/200, Reward: 1392.045, Step: 200 +2023-04-17 14:43:21 - r - INFO: - Episode: 17/200, Reward: 1354.818, Step: 200 +2023-04-17 14:43:22 - r - INFO: - Episode: 18/200, Reward: 1305.356, Step: 200 +2023-04-17 14:43:22 - r - INFO: - Episode: 19/200, Reward: 1308.354, Step: 200 +2023-04-17 14:43:22 - r - INFO: - Episode: 20/200, Reward: 1328.289, Step: 200 +2023-04-17 14:43:22 - r - INFO: - Episode: 21/200, Reward: 1213.166, Step: 200 +2023-04-17 14:43:23 - r - INFO: - Episode: 22/200, Reward: 1335.024, Step: 200 +2023-04-17 14:43:23 - r - INFO: - Episode: 23/200, Reward: 1157.548, Step: 200 +2023-04-17 14:43:23 - r - INFO: - Episode: 24/200, Reward: 1258.234, Step: 200 +2023-04-17 14:43:23 - r - INFO: - Episode: 25/200, Reward: 1277.436, Step: 200 +2023-04-17 14:43:24 - r - INFO: - Episode: 26/200, Reward: 1267.203, Step: 200 +2023-04-17 14:43:24 - r - INFO: - Episode: 27/200, Reward: 1255.470, Step: 200 +2023-04-17 14:43:24 - r - INFO: - Episode: 28/200, Reward: 1226.970, Step: 200 +2023-04-17 14:43:25 - r - INFO: - Episode: 29/200, Reward: 1248.535, Step: 200 +2023-04-17 14:43:25 - r - INFO: - Episode: 30/200, Reward: 1320.410, Step: 200 +2023-04-17 14:43:25 - r - INFO: - Episode: 31/200, Reward: 1170.146, Step: 200 +2023-04-17 14:43:25 - r - INFO: - Episode: 32/200, Reward: 1228.375, Step: 200 +2023-04-17 14:43:26 - r - INFO: - Episode: 33/200, Reward: 1299.463, Step: 200 +2023-04-17 14:43:26 - r - INFO: - Episode: 34/200, Reward: 1309.056, Step: 200 +2023-04-17 14:43:26 - r - INFO: - Episode: 35/200, Reward: 1278.642, Step: 200 +2023-04-17 14:43:27 - r - INFO: - Episode: 36/200, Reward: 1203.290, Step: 200 +2023-04-17 14:43:27 - r - INFO: - Episode: 37/200, Reward: 1228.405, Step: 200 +2023-04-17 14:43:27 - r - INFO: - Episode: 38/200, Reward: 1319.337, Step: 200 +2023-04-17 14:43:27 - r - INFO: - Episode: 39/200, Reward: 1204.985, Step: 200 +2023-04-17 14:43:28 - r - INFO: - Episode: 40/200, Reward: 1317.443, Step: 200 +2023-04-17 14:43:28 - r - INFO: - Episode: 41/200, Reward: 1370.977, Step: 200 +2023-04-17 14:43:28 - r - INFO: - Episode: 42/200, Reward: 1275.394, Step: 200 +2023-04-17 14:43:29 - r - INFO: - Episode: 43/200, Reward: 1159.293, Step: 200 +2023-04-17 14:43:29 - r - INFO: - Episode: 44/200, Reward: 1295.355, Step: 200 +2023-04-17 14:43:29 - r - INFO: - Episode: 45/200, Reward: 1321.710, Step: 200 +2023-04-17 14:43:30 - r - INFO: - Episode: 46/200, Reward: 1315.284, Step: 200 +2023-04-17 14:43:30 - r - INFO: - Episode: 47/200, Reward: 1255.528, Step: 200 +2023-04-17 14:43:30 - r - INFO: - Episode: 48/200, Reward: 1347.399, Step: 200 +2023-04-17 14:43:30 - r - INFO: - Episode: 49/200, Reward: 1372.131, Step: 200 +2023-04-17 14:43:31 - r - INFO: - Episode: 50/200, Reward: 1258.952, Step: 200 +2023-04-17 14:43:31 - r - INFO: - Episode: 51/200, Reward: 1268.580, Step: 200 +2023-04-17 14:43:31 - r - INFO: - Episode: 52/200, Reward: 1248.082, Step: 200 +2023-04-17 14:43:31 - r - INFO: - Episode: 53/200, Reward: 1146.066, Step: 200 +2023-04-17 14:43:32 - r - INFO: - Episode: 54/200, Reward: 1319.450, Step: 200 +2023-04-17 14:43:32 - r - INFO: - Episode: 55/200, Reward: 1267.772, Step: 200 +2023-04-17 14:43:32 - r - INFO: - Episode: 56/200, Reward: 1323.475, Step: 200 +2023-04-17 14:43:32 - r - INFO: - Episode: 57/200, Reward: 1299.357, Step: 200 +2023-04-17 14:43:33 - r - INFO: - Episode: 58/200, Reward: 1327.757, Step: 200 +2023-04-17 14:43:33 - r - INFO: - Episode: 59/200, Reward: 1336.869, Step: 200 +2023-04-17 14:43:33 - r - INFO: - Episode: 60/200, Reward: 1402.798, Step: 200 +2023-04-17 14:43:33 - r - INFO: - Episode: 61/200, Reward: 1383.733, Step: 200 +2023-04-17 14:43:34 - r - INFO: - Episode: 62/200, Reward: 1339.146, Step: 200 +2023-04-17 14:43:34 - r - INFO: - Episode: 63/200, Reward: 1333.416, Step: 200 +2023-04-17 14:43:34 - r - INFO: - Episode: 64/200, Reward: 1263.389, Step: 200 +2023-04-17 14:43:34 - r - INFO: - Episode: 65/200, Reward: 1166.727, Step: 200 +2023-04-17 14:43:35 - r - INFO: - Episode: 66/200, Reward: 1349.627, Step: 200 +2023-04-17 14:43:35 - r - INFO: - Episode: 67/200, Reward: 1347.438, Step: 200 +2023-04-17 14:43:35 - r - INFO: - Episode: 68/200, Reward: 1326.951, Step: 200 +2023-04-17 14:43:36 - r - INFO: - Episode: 69/200, Reward: 1255.685, Step: 200 +2023-04-17 14:43:36 - r - INFO: - Episode: 70/200, Reward: 1353.316, Step: 200 +2023-04-17 14:43:36 - r - INFO: - Episode: 71/200, Reward: 1312.619, Step: 200 +2023-04-17 14:43:36 - r - INFO: - Episode: 72/200, Reward: 1235.518, Step: 200 +2023-04-17 14:43:37 - r - INFO: - Episode: 73/200, Reward: 1289.647, Step: 200 +2023-04-17 14:43:37 - r - INFO: - Episode: 74/200, Reward: 1113.164, Step: 200 +2023-04-17 14:43:37 - r - INFO: - Episode: 75/200, Reward: 1275.012, Step: 200 +2023-04-17 14:43:38 - r - INFO: - Episode: 76/200, Reward: 1281.044, Step: 200 +2023-04-17 14:43:38 - r - INFO: - Episode: 77/200, Reward: 1294.306, Step: 200 +2023-04-17 14:43:38 - r - INFO: - Episode: 78/200, Reward: 1248.278, Step: 200 +2023-04-17 14:43:38 - r - INFO: - Episode: 79/200, Reward: 1294.375, Step: 200 +2023-04-17 14:43:39 - r - INFO: - Episode: 80/200, Reward: 1383.276, Step: 200 +2023-04-17 14:43:39 - r - INFO: - Episode: 81/200, Reward: 1331.390, Step: 200 +2023-04-17 14:43:39 - r - INFO: - Episode: 82/200, Reward: 1417.109, Step: 200 +2023-04-17 14:43:40 - r - INFO: - Episode: 83/200, Reward: 1411.741, Step: 200 +2023-04-17 14:43:40 - r - INFO: - Episode: 84/200, Reward: 1349.740, Step: 200 +2023-04-17 14:43:40 - r - INFO: - Episode: 85/200, Reward: 1216.375, Step: 200 +2023-04-17 14:43:41 - r - INFO: - Episode: 86/200, Reward: 1332.206, Step: 200 +2023-04-17 14:43:41 - r - INFO: - Episode: 87/200, Reward: 1268.855, Step: 200 +2023-04-17 14:43:41 - r - INFO: - Episode: 88/200, Reward: 1289.923, Step: 200 +2023-04-17 14:43:41 - r - INFO: - Episode: 89/200, Reward: 1305.528, Step: 200 +2023-04-17 14:43:42 - r - INFO: - Episode: 90/200, Reward: 1364.474, Step: 200 +2023-04-17 14:43:42 - r - INFO: - Episode: 91/200, Reward: 1337.275, Step: 200 +2023-04-17 14:43:42 - r - INFO: - Episode: 92/200, Reward: 1325.012, Step: 200 +2023-04-17 14:43:43 - r - INFO: - Episode: 93/200, Reward: 1276.419, Step: 200 +2023-04-17 14:43:43 - r - INFO: - Episode: 94/200, Reward: 1223.767, Step: 200 +2023-04-17 14:43:43 - r - INFO: - Episode: 95/200, Reward: 1219.303, Step: 200 +2023-04-17 14:43:43 - r - INFO: - Episode: 96/200, Reward: 1313.126, Step: 200 +2023-04-17 14:43:44 - r - INFO: - Episode: 97/200, Reward: 1266.540, Step: 200 +2023-04-17 14:43:44 - r - INFO: - Episode: 98/200, Reward: 1313.752, Step: 200 +2023-04-17 14:43:44 - r - INFO: - Episode: 99/200, Reward: 1322.603, Step: 200 +2023-04-17 14:43:45 - r - INFO: - Episode: 100/200, Reward: 1327.168, Step: 200 +2023-04-17 14:43:45 - r - INFO: - Episode: 101/200, Reward: 1306.251, Step: 200 +2023-04-17 14:43:45 - r - INFO: - Episode: 102/200, Reward: 1311.810, Step: 200 +2023-04-17 14:43:46 - r - INFO: - Episode: 103/200, Reward: 1343.756, Step: 200 +2023-04-17 14:43:46 - r - INFO: - Episode: 104/200, Reward: 1318.610, Step: 200 +2023-04-17 14:43:46 - r - INFO: - Episode: 105/200, Reward: 1343.098, Step: 200 +2023-04-17 14:43:46 - r - INFO: - Episode: 106/200, Reward: 1314.142, Step: 200 +2023-04-17 14:43:47 - r - INFO: - Episode: 107/200, Reward: 1300.201, Step: 200 +2023-04-17 14:43:47 - r - INFO: - Episode: 108/200, Reward: 1368.704, Step: 200 +2023-04-17 14:43:47 - r - INFO: - Episode: 109/200, Reward: 1338.619, Step: 200 +2023-04-17 14:43:48 - r - INFO: - Episode: 110/200, Reward: 1347.307, Step: 200 +2023-04-17 14:43:48 - r - INFO: - Episode: 111/200, Reward: 1301.271, Step: 200 +2023-04-17 14:43:48 - r - INFO: - Episode: 112/200, Reward: 1343.502, Step: 200 +2023-04-17 14:43:49 - r - INFO: - Episode: 113/200, Reward: 1296.342, Step: 200 +2023-04-17 14:43:49 - r - INFO: - Episode: 114/200, Reward: 1145.894, Step: 200 +2023-04-17 14:43:49 - r - INFO: - Episode: 115/200, Reward: 1326.472, Step: 200 +2023-04-17 14:43:50 - r - INFO: - Episode: 116/200, Reward: 1310.095, Step: 200 +2023-04-17 14:43:50 - r - INFO: - Episode: 117/200, Reward: 1306.200, Step: 200 +2023-04-17 14:43:50 - r - INFO: - Episode: 118/200, Reward: 1269.625, Step: 200 +2023-04-17 14:43:51 - r - INFO: - Episode: 119/200, Reward: 1330.256, Step: 200 +2023-04-17 14:43:51 - r - INFO: - Episode: 120/200, Reward: 1176.782, Step: 200 +2023-04-17 14:43:51 - r - INFO: - Episode: 121/200, Reward: 1228.143, Step: 200 +2023-04-17 14:43:52 - r - INFO: - Episode: 122/200, Reward: 1292.745, Step: 200 +2023-04-17 14:43:52 - r - INFO: - Episode: 123/200, Reward: 1261.806, Step: 200 +2023-04-17 14:43:52 - r - INFO: - Episode: 124/200, Reward: 1148.581, Step: 200 +2023-04-17 14:43:52 - r - INFO: - Episode: 125/200, Reward: 1376.888, Step: 200 +2023-04-17 14:43:53 - r - INFO: - Episode: 126/200, Reward: 1270.227, Step: 200 +2023-04-17 14:43:53 - r - INFO: - Episode: 127/200, Reward: 1297.554, Step: 200 +2023-04-17 14:43:53 - r - INFO: - Episode: 128/200, Reward: 119.449, Step: 200 +2023-04-17 14:43:54 - r - INFO: - Episode: 129/200, Reward: 1283.901, Step: 200 +2023-04-17 14:43:54 - r - INFO: - Episode: 130/200, Reward: 1320.992, Step: 200 +2023-04-17 14:43:54 - r - INFO: - Episode: 131/200, Reward: 1242.699, Step: 200 +2023-04-17 14:43:54 - r - INFO: - Episode: 132/200, Reward: 1263.889, Step: 200 +2023-04-17 14:43:55 - r - INFO: - Episode: 133/200, Reward: 1265.926, Step: 200 +2023-04-17 14:43:55 - r - INFO: - Episode: 134/200, Reward: 1312.005, Step: 200 +2023-04-17 14:43:55 - r - INFO: - Episode: 135/200, Reward: 1283.035, Step: 200 +2023-04-17 14:43:56 - r - INFO: - Episode: 136/200, Reward: 1298.766, Step: 200 +2023-04-17 14:43:56 - r - INFO: - Episode: 137/200, Reward: 1229.121, Step: 200 +2023-04-17 14:43:56 - r - INFO: - Episode: 138/200, Reward: 1333.515, Step: 200 +2023-04-17 14:43:57 - r - INFO: - Episode: 139/200, Reward: 1250.741, Step: 200 +2023-04-17 14:43:57 - r - INFO: - Episode: 140/200, Reward: 1344.817, Step: 200 +2023-04-17 14:43:57 - r - INFO: - Episode: 141/200, Reward: 1194.106, Step: 200 +2023-04-17 14:43:58 - r - INFO: - Episode: 142/200, Reward: 1306.963, Step: 200 +2023-04-17 14:43:58 - r - INFO: - Episode: 143/200, Reward: 1254.498, Step: 200 +2023-04-17 14:43:58 - r - INFO: - Episode: 144/200, Reward: 1348.872, Step: 200 +2023-04-17 14:43:58 - r - INFO: - Episode: 145/200, Reward: 1267.701, Step: 200 +2023-04-17 14:43:59 - r - INFO: - Episode: 146/200, Reward: 1314.007, Step: 200 +2023-04-17 14:43:59 - r - INFO: - Episode: 147/200, Reward: 1282.027, Step: 200 +2023-04-17 14:43:59 - r - INFO: - Episode: 148/200, Reward: 1351.543, Step: 200 +2023-04-17 14:43:59 - r - INFO: - Episode: 149/200, Reward: 1338.997, Step: 200 +2023-04-17 14:44:00 - r - INFO: - Episode: 150/200, Reward: 1279.964, Step: 200 +2023-04-17 14:44:00 - r - INFO: - Episode: 151/200, Reward: 1196.204, Step: 200 +2023-04-17 14:44:00 - r - INFO: - Episode: 152/200, Reward: 1237.979, Step: 200 +2023-04-17 14:44:00 - r - INFO: - Episode: 153/200, Reward: 1299.837, Step: 200 +2023-04-17 14:44:01 - r - INFO: - Episode: 154/200, Reward: 1302.113, Step: 200 +2023-04-17 14:44:01 - r - INFO: - Episode: 155/200, Reward: 1371.231, Step: 200 +2023-04-17 14:44:01 - r - INFO: - Episode: 156/200, Reward: 1352.787, Step: 200 +2023-04-17 14:44:02 - r - INFO: - Episode: 157/200, Reward: 1328.460, Step: 200 +2023-04-17 14:44:02 - r - INFO: - Episode: 158/200, Reward: 1351.489, Step: 200 +2023-04-17 14:44:02 - r - INFO: - Episode: 159/200, Reward: 1352.125, Step: 200 +2023-04-17 14:44:03 - r - INFO: - Episode: 160/200, Reward: 1254.551, Step: 200 +2023-04-17 14:44:03 - r - INFO: - Episode: 161/200, Reward: 1350.503, Step: 200 +2023-04-17 14:44:03 - r - INFO: - Episode: 162/200, Reward: 1319.012, Step: 200 +2023-04-17 14:44:03 - r - INFO: - Episode: 163/200, Reward: 1208.436, Step: 200 +2023-04-17 14:44:04 - r - INFO: - Episode: 164/200, Reward: 1251.329, Step: 200 +2023-04-17 14:44:04 - r - INFO: - Episode: 165/200, Reward: 1239.774, Step: 200 +2023-04-17 14:44:04 - r - INFO: - Episode: 166/200, Reward: 1316.650, Step: 200 +2023-04-17 14:44:04 - r - INFO: - Episode: 167/200, Reward: 1361.254, Step: 200 +2023-04-17 14:44:05 - r - INFO: - Episode: 168/200, Reward: 1234.934, Step: 200 +2023-04-17 14:44:05 - r - INFO: - Episode: 169/200, Reward: 925.280, Step: 200 +2023-04-17 14:44:05 - r - INFO: - Episode: 170/200, Reward: 1315.849, Step: 200 +2023-04-17 14:44:05 - r - INFO: - Episode: 171/200, Reward: 1234.899, Step: 200 +2023-04-17 14:44:06 - r - INFO: - Episode: 172/200, Reward: 1222.966, Step: 200 +2023-04-17 14:44:06 - r - INFO: - Episode: 173/200, Reward: 1226.339, Step: 200 +2023-04-17 14:44:06 - r - INFO: - Episode: 174/200, Reward: 1263.257, Step: 200 +2023-04-17 14:44:06 - r - INFO: - Episode: 175/200, Reward: 1326.118, Step: 200 +2023-04-17 14:44:07 - r - INFO: - Episode: 176/200, Reward: 1337.059, Step: 200 +2023-04-17 14:44:07 - r - INFO: - Episode: 177/200, Reward: 1317.011, Step: 200 +2023-04-17 14:44:07 - r - INFO: - Episode: 178/200, Reward: 1280.764, Step: 200 +2023-04-17 14:44:08 - r - INFO: - Episode: 179/200, Reward: 1301.730, Step: 200 +2023-04-17 14:44:08 - r - INFO: - Episode: 180/200, Reward: 1261.635, Step: 200 +2023-04-17 14:44:08 - r - INFO: - Episode: 181/200, Reward: 1366.866, Step: 200 +2023-04-17 14:44:08 - r - INFO: - Episode: 182/200, Reward: 1307.725, Step: 200 +2023-04-17 14:44:09 - r - INFO: - Episode: 183/200, Reward: 1406.161, Step: 200 +2023-04-17 14:44:09 - r - INFO: - Episode: 184/200, Reward: 1276.682, Step: 200 +2023-04-17 14:44:09 - r - INFO: - Episode: 185/200, Reward: 1256.515, Step: 200 +2023-04-17 14:44:09 - r - INFO: - Episode: 186/200, Reward: 1326.052, Step: 200 +2023-04-17 14:44:10 - r - INFO: - Episode: 187/200, Reward: 985.071, Step: 200 +2023-04-17 14:44:10 - r - INFO: - Episode: 188/200, Reward: 1304.430, Step: 200 +2023-04-17 14:44:10 - r - INFO: - Episode: 189/200, Reward: 1359.663, Step: 200 +2023-04-17 14:44:11 - r - INFO: - Episode: 190/200, Reward: 1312.544, Step: 200 +2023-04-17 14:44:11 - r - INFO: - Episode: 191/200, Reward: 1142.080, Step: 200 +2023-04-17 14:44:11 - r - INFO: - Episode: 192/200, Reward: 1345.347, Step: 200 +2023-04-17 14:44:11 - r - INFO: - Episode: 193/200, Reward: 1204.062, Step: 200 +2023-04-17 14:44:12 - r - INFO: - Episode: 194/200, Reward: 1301.517, Step: 200 +2023-04-17 14:44:12 - r - INFO: - Episode: 195/200, Reward: 1236.015, Step: 200 +2023-04-17 14:44:12 - r - INFO: - Episode: 196/200, Reward: 1308.347, Step: 200 +2023-04-17 14:44:12 - r - INFO: - Episode: 197/200, Reward: 1379.318, Step: 200 +2023-04-17 14:44:13 - r - INFO: - Episode: 198/200, Reward: 1313.630, Step: 200 +2023-04-17 14:44:13 - r - INFO: - Episode: 199/200, Reward: 1373.463, Step: 200 +2023-04-17 14:44:13 - r - INFO: - Episode: 200/200, Reward: 1253.490, Step: 200 +2023-04-17 14:44:13 - r - INFO: - Finish testing! diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/action_bias.npy b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/action_bias.npy new file mode 100644 index 0000000000000000000000000000000000000000..6c6fe1c76066c93672fb5706a997a503b0673162 --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/action_bias.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c515444fcd73a5aa350c5bf219d2dc5eeaaff903072d1d8589fb51c62673d171 +size 152 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/action_scale.npy b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/action_scale.npy new file mode 100644 index 0000000000000000000000000000000000000000..451bf3e54e2dabc6c3b3b4a7997636eab067c68d --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/action_scale.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d5962f5d28bcd9bd926f0ec6edd6b85da90b0c02d79cec3c35d4591b3587fc +size 152 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/actor_checkpoint.pt b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeee1181236eb2269b6168f8f66d8e127f2dd58b --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc44577ce8999b631c1d268ad806aba2b8ebd79814e3936660308e1cde36f3d5 +size 529119 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/actor_target_checkpoint.pt b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/actor_target_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4989a0f64b2b65d68470466e95fb9c39b730f029 --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/actor_target_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:192189a0990ca082589e42218f4f390d4934ba7762f9f8c1b9e264c12b71d34d +size 529239 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/critic_checkpoint.pt b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/critic_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..40d26067ac8487bf7a41dcee31c26e15a07ec1ba --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/critic_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35f32a5624dc18fe39c4052a622f04f6b1aefdeae09d2d7013e5ff8142616234 +size 1045645 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/critic_target_checkpoint.pt b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/critic_target_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ab6372531f1c5cab7a20f995968a2f2ad32c4b5 --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/critic_target_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:445561576af6b0e059b5f45dd52ee0b93f64b19e13c2d8c4e797c00659c57cb8 +size 1046127 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/vae_checkpoint.pt b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/vae_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..32c790ac6a4cbbe2dccc3f273ff3074f5ecaf508 --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/models/vae_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f53b8347886813683817d2983722a1b9a59279ac4030bd177a023db5640c5c +size 4762475 diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/results/learning_curve.png b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..ea4de12025ce8cda14540b41a4320251a0152d50 Binary files /dev/null and b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/results/learning_curve.png differ diff --git a/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/results/res.csv b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..99720cbb5bf4864836bdfedf74f363580a8b5860 --- /dev/null +++ b/HalfCheetah-v3/Test_gym_BCQ_20230417-144316/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,1308.2619674528387,200 +1,1270.4598858580123,200 +2,1232.738042208589,200 +3,1301.50408327569,200 +4,1377.4333899080552,200 +5,1360.4837180757902,200 +6,1209.3300964298783,200 +7,1366.5422485188215,200 +8,1332.4864487779203,200 +9,1190.7475300653034,200 +10,1329.530946446529,200 +11,554.3709626880774,200 +12,1328.5427748847312,200 +13,1327.360897337637,200 +14,1232.017457879202,200 +15,1392.0448956735493,200 +16,1354.8176537556556,200 +17,1305.3555134461901,200 +18,1308.3537042352089,200 +19,1328.2892394751216,200 +20,1213.1658654312446,200 +21,1335.0242873394182,200 +22,1157.548158321714,200 +23,1258.2344423377285,200 +24,1277.4359570924576,200 +25,1267.2026759534451,200 +26,1255.4700396397604,200 +27,1226.969606794796,200 +28,1248.5354676742747,200 +29,1320.4095665708503,200 +30,1170.145678287014,200 +31,1228.3745421397875,200 +32,1299.4627399222688,200 +33,1309.0558543891716,200 +34,1278.6422334963186,200 +35,1203.2896830411469,200 +36,1228.4053800157217,200 +37,1319.3370814739403,200 +38,1204.984693214043,200 +39,1317.4431288121411,200 +40,1370.976865302254,200 +41,1275.3938286832004,200 +42,1159.2932779536045,200 +43,1295.3552564764502,200 +44,1321.7101350730525,200 +45,1315.2840089194603,200 +46,1255.527849688872,200 +47,1347.399036593072,200 +48,1372.1313190592762,200 +49,1258.951594258188,200 +50,1268.5803131654252,200 +51,1248.0824245739786,200 +52,1146.0657832771146,200 +53,1319.4496190985674,200 +54,1267.7720088557194,200 +55,1323.475391792521,200 +56,1299.3574031840724,200 +57,1327.757494482645,200 +58,1336.8685776034336,200 +59,1402.79795155143,200 +60,1383.7333822340768,200 +61,1339.145554225169,200 +62,1333.416067631717,200 +63,1263.3890608866425,200 +64,1166.7270354330508,200 +65,1349.6271820800835,200 +66,1347.4376674478895,200 +67,1326.9510723269393,200 +68,1255.6850259289645,200 +69,1353.3160034627613,200 +70,1312.6188522201915,200 +71,1235.5179486884265,200 +72,1289.6469051043755,200 +73,1113.1637504361497,200 +74,1275.0116311941774,200 +75,1281.0435890778997,200 +76,1294.3063179234116,200 +77,1248.2779313763344,200 +78,1294.3750099996098,200 +79,1383.2760738828285,200 +80,1331.389835464806,200 +81,1417.1091267316813,200 +82,1411.7413782577878,200 +83,1349.7404973970954,200 +84,1216.3745389946757,200 +85,1332.206322983513,200 +86,1268.8548862579837,200 +87,1289.9229804617362,200 +88,1305.5283448346543,200 +89,1364.4742853212904,200 +90,1337.2747306049341,200 +91,1325.0122978856398,200 +92,1276.418891743612,200 +93,1223.7673301772454,200 +94,1219.302742269,200 +95,1313.1256445274155,200 +96,1266.540110512444,200 +97,1313.7520142289025,200 +98,1322.6033034346785,200 +99,1327.167921519437,200 +100,1306.2512433642737,200 +101,1311.8095326863313,200 +102,1343.7559494426505,200 +103,1318.610243404628,200 +104,1343.0977555701204,200 +105,1314.142169793057,200 +106,1300.2008582781848,200 +107,1368.7042376191973,200 +108,1338.618601238244,200 +109,1347.3071314275398,200 +110,1301.2713959558073,200 +111,1343.5023092652673,200 +112,1296.3421288530942,200 +113,1145.8937922299538,200 +114,1326.4723617155694,200 +115,1310.0948536777855,200 +116,1306.2000225649788,200 +117,1269.6246995545744,200 +118,1330.2561884962045,200 +119,1176.781867673625,200 +120,1228.1431094237435,200 +121,1292.7445004451072,200 +122,1261.8063101799141,200 +123,1148.5808072210982,200 +124,1376.8876002702034,200 +125,1270.2266242905089,200 +126,1297.5540842144867,200 +127,119.44900226478956,200 +128,1283.9011032433573,200 +129,1320.9916578321024,200 +130,1242.6994638492492,200 +131,1263.8888546860508,200 +132,1265.9264769020285,200 +133,1312.0048001052564,200 +134,1283.0348640989625,200 +135,1298.7660769426818,200 +136,1229.1205146540442,200 +137,1333.5154637009468,200 +138,1250.7407107194388,200 +139,1344.8174269307906,200 +140,1194.1062039052754,200 +141,1306.962852255905,200 +142,1254.4975867384592,200 +143,1348.8721936433271,200 +144,1267.7010330346998,200 +145,1314.0071127290557,200 +146,1282.0266876520295,200 +147,1351.542992558106,200 +148,1338.996997673138,200 +149,1279.9644693952291,200 +150,1196.2042887620498,200 +151,1237.9786444050783,200 +152,1299.836694135456,200 +153,1302.113472310689,200 +154,1371.2306570632488,200 +155,1352.7868939641226,200 +156,1328.4600105676209,200 +157,1351.489276978419,200 +158,1352.1248356969763,200 +159,1254.550578366483,200 +160,1350.5034684327127,200 +161,1319.0123269647354,200 +162,1208.4355318474331,200 +163,1251.3286981545907,200 +164,1239.773644713926,200 +165,1316.6499460155617,200 +166,1361.2544272937716,200 +167,1234.9336000636656,200 +168,925.2797056655064,200 +169,1315.848719767156,200 +170,1234.8994940389596,200 +171,1222.9658731554498,200 +172,1226.3387131909212,200 +173,1263.2569318547637,200 +174,1326.1178530320587,200 +175,1337.059436199117,200 +176,1317.0108736790994,200 +177,1280.7636838172125,200 +178,1301.730392552132,200 +179,1261.63501183927,200 +180,1366.8655490159574,200 +181,1307.725051167091,200 +182,1406.1605827088163,200 +183,1276.6815882472104,200 +184,1256.5149771958486,200 +185,1326.052220576704,200 +186,985.0705447892553,200 +187,1304.4303700287994,200 +188,1359.6632694642876,200 +189,1312.5436149796483,200 +190,1142.0804208507298,200 +191,1345.3472434109558,200 +192,1204.0618446762921,200 +193,1301.5165719191598,200 +194,1236.0145461272527,200 +195,1308.347394931438,200 +196,1379.3180273907155,200 +197,1313.6295956965005,200 +198,1373.4626366844288,200 +199,1253.490113978808,200 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/config.yaml b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6055aa85a55e79b2987fc21f5b9fd5ed7659f99d --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/config.yaml @@ -0,0 +1,50 @@ +general_cfg: + algo_name: BCQ + device: cpu + env_name: gym + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: tasks + max_steps: 200 + mode: train + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 0 + show_fig: false + test_eps: 20 + train_eps: 1000 + wrapper: null +algo_cfg: + actor_hidden_dims: + - 400 + - 300 + actor_lr: 0.001 + batch_size: 128 + behavior_agent_name: DDPG + behavior_agent_parameters_path: /behave_param.yaml + behavior_policy_path: /behaviour_models + buffer_size: 100000 + collect_eps: 500 + collect_explore_data: true + critic_hidden_dims: + - 400 + - 300 + critic_lr: 0.001 + gamma: 0.99 + iters_per_ep: 10 + lmbda: 0.75 + phi: 0.05 + start_learn_buffer_size: 1000.0 + tau: 0.005 + vae_hidden_dims: + - 750 + - 750 + vae_lr: 0.001 +env_cfg: + id: HalfCheetah-v3 + new_step_api: true + render_mode: null diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/logs/log.txt b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6a236e8f0080156aa9160020c47f91ff1efd88eb --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/logs/log.txt @@ -0,0 +1,1082 @@ +2023-04-17 14:18:11 - r - INFO: - Hyperparameters: +2023-04-17 14:18:11 - r - INFO: - ================================================================================ +2023-04-17 14:18:11 - r - INFO: - Name Value Type +2023-04-17 14:18:11 - r - INFO: - env_name gym +2023-04-17 14:18:11 - r - INFO: - new_step_api 1 +2023-04-17 14:18:11 - r - INFO: - wrapper None +2023-04-17 14:18:11 - r - INFO: - render 0 +2023-04-17 14:18:11 - r - INFO: - render_mode None +2023-04-17 14:18:11 - r - INFO: - algo_name BCQ +2023-04-17 14:18:11 - r - INFO: - mode train +2023-04-17 14:18:11 - r - INFO: - mp_backend mp +2023-04-17 14:18:11 - r - INFO: - seed 0 +2023-04-17 14:18:11 - r - INFO: - device cpu +2023-04-17 14:18:11 - r - INFO: - train_eps 1000 +2023-04-17 14:18:11 - r - INFO: - test_eps 20 +2023-04-17 14:18:11 - r - INFO: - eval_eps 10 +2023-04-17 14:18:11 - r - INFO: - eval_per_episode 5 +2023-04-17 14:18:11 - r - INFO: - max_steps 200 +2023-04-17 14:18:11 - r - INFO: - load_checkpoint 0 +2023-04-17 14:18:11 - r - INFO: - load_path tasks +2023-04-17 14:18:11 - r - INFO: - show_fig 0 +2023-04-17 14:18:11 - r - INFO: - save_fig 1 +2023-04-17 14:18:11 - r - INFO: - critic_hidden_dims [400, 300] +2023-04-17 14:18:11 - r - INFO: - actor_hidden_dims [400, 300] +2023-04-17 14:18:11 - r - INFO: - vae_hidden_dims [750, 750] +2023-04-17 14:18:11 - r - INFO: - critic_lr 0.001 +2023-04-17 14:18:11 - r - INFO: - actor_lr 0.001 +2023-04-17 14:18:11 - r - INFO: - vae_lr 0.001 +2023-04-17 14:18:11 - r - INFO: - batch_size 128 +2023-04-17 14:18:11 - r - INFO: - gamma 0.99 +2023-04-17 14:18:11 - r - INFO: - tau 0.005 +2023-04-17 14:18:11 - r - INFO: - lmbda 0.75 +2023-04-17 14:18:11 - r - INFO: - phi 0.05 +2023-04-17 14:18:11 - r - INFO: - iters_per_ep 10 +2023-04-17 14:18:11 - r - INFO: - buffer_size 100000 +2023-04-17 14:18:11 - r - INFO: - start_learn_buffer_size 1000.0 +2023-04-17 14:18:11 - r - INFO: - collect_explore_data 1 +2023-04-17 14:18:11 - r - INFO: - behavior_agent_name DDPG +2023-04-17 14:18:11 - r - INFO: - behavior_agent_parameters_path /behave_param.yaml +2023-04-17 14:18:11 - r - INFO: - behavior_policy_path /behaviour_models +2023-04-17 14:18:11 - r - INFO: - collect_eps 500 +2023-04-17 14:18:11 - r - INFO: - id HalfCheetah-v3 +2023-04-17 14:18:11 - r - INFO: - task_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Train_gym_BCQ_20230417-141811 +2023-04-17 14:18:11 - r - INFO: - res_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Train_gym_BCQ_20230417-141811/results +2023-04-17 14:18:11 - r - INFO: - log_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Train_gym_BCQ_20230417-141811/logs +2023-04-17 14:18:11 - r - INFO: - traj_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Train_gym_BCQ_20230417-141811/traj +2023-04-17 14:18:11 - r - INFO: - video_dir /home/chaoqun/桌面/gitcode/joyrl-offline/tasks/Train_gym_BCQ_20230417-141811/videos +2023-04-17 14:18:11 - r - INFO: - ================================================================================ +2023-04-17 14:18:11 - r - INFO: - action_bound: 1.0 +2023-04-17 14:18:11 - r - INFO: - n_states: 17, n_actions: 6 +2023-04-17 14:18:11 - r - INFO: - Start training! +2023-04-17 14:18:11 - r - INFO: - Env: gym, Algorithm: BCQ, Device: cpu +2023-04-17 14:18:13 - r - INFO: - Episode: 1/1000, Reward: 36.767, Step: 200 +2023-04-17 14:18:13 - r - INFO: - Episode: 2/1000, Reward: 21.783, Step: 200 +2023-04-17 14:18:13 - r - INFO: - Episode: 3/1000, Reward: -60.802, Step: 200 +2023-04-17 14:18:14 - r - INFO: - Episode: 4/1000, Reward: -30.031, Step: 200 +2023-04-17 14:18:14 - r - INFO: - Episode: 5/1000, Reward: 68.186, Step: 200 +2023-04-17 14:18:16 - r - INFO: - Current episode 5 has the best eval reward: 42.989 +2023-04-17 14:18:17 - r - INFO: - Episode: 6/1000, Reward: 434.885, Step: 200 +2023-04-17 14:18:17 - r - INFO: - Episode: 7/1000, Reward: -39.494, Step: 200 +2023-04-17 14:18:18 - r - INFO: - Episode: 8/1000, Reward: -8.455, Step: 200 +2023-04-17 14:18:18 - r - INFO: - Episode: 9/1000, Reward: 47.446, Step: 200 +2023-04-17 14:18:19 - r - INFO: - Episode: 10/1000, Reward: -39.634, Step: 200 +2023-04-17 14:18:22 - r - INFO: - Episode: 11/1000, Reward: 7.466, Step: 200 +2023-04-17 14:18:22 - r - INFO: - Episode: 12/1000, Reward: -26.828, Step: 200 +2023-04-17 14:18:23 - r - INFO: - Episode: 13/1000, Reward: 137.514, Step: 200 +2023-04-17 14:18:23 - r - INFO: - Episode: 14/1000, Reward: -23.069, Step: 200 +2023-04-17 14:18:24 - r - INFO: - Episode: 15/1000, Reward: 76.067, Step: 200 +2023-04-17 14:18:26 - r - INFO: - Current episode 15 has the best eval reward: 70.009 +2023-04-17 14:18:26 - r - INFO: - Episode: 16/1000, Reward: -3.251, Step: 200 +2023-04-17 14:18:26 - r - INFO: - Episode: 17/1000, Reward: 65.348, Step: 200 +2023-04-17 14:18:27 - r - INFO: - Episode: 18/1000, Reward: -11.215, Step: 200 +2023-04-17 14:18:27 - r - INFO: - Episode: 19/1000, Reward: 11.328, Step: 200 +2023-04-17 14:18:28 - r - INFO: - Episode: 20/1000, Reward: 199.265, Step: 200 +2023-04-17 14:18:30 - r - INFO: - Current episode 20 has the best eval reward: 151.140 +2023-04-17 14:18:30 - r - INFO: - Episode: 21/1000, Reward: 408.710, Step: 200 +2023-04-17 14:18:30 - r - INFO: - Episode: 22/1000, Reward: 338.869, Step: 200 +2023-04-17 14:18:31 - r - INFO: - Episode: 23/1000, Reward: -32.201, Step: 200 +2023-04-17 14:18:31 - r - INFO: - Episode: 24/1000, Reward: 367.254, Step: 200 +2023-04-17 14:18:32 - r - INFO: - Episode: 25/1000, Reward: 134.399, Step: 200 +2023-04-17 14:18:34 - r - INFO: - Episode: 26/1000, Reward: 28.196, Step: 200 +2023-04-17 14:18:34 - r - INFO: - Episode: 27/1000, Reward: 76.629, Step: 200 +2023-04-17 14:18:35 - r - INFO: - Episode: 28/1000, Reward: 193.268, Step: 200 +2023-04-17 14:18:35 - r - INFO: - Episode: 29/1000, Reward: 227.018, Step: 200 +2023-04-17 14:18:36 - r - INFO: - Episode: 30/1000, Reward: 276.609, Step: 200 +2023-04-17 14:18:38 - r - INFO: - Current episode 30 has the best eval reward: 169.620 +2023-04-17 14:18:38 - r - INFO: - Episode: 31/1000, Reward: 22.316, Step: 200 +2023-04-17 14:18:38 - r - INFO: - Episode: 32/1000, Reward: 94.406, Step: 200 +2023-04-17 14:18:39 - r - INFO: - Episode: 33/1000, Reward: -31.564, Step: 200 +2023-04-17 14:18:39 - r - INFO: - Episode: 34/1000, Reward: 31.355, Step: 200 +2023-04-17 14:18:40 - r - INFO: - Episode: 35/1000, Reward: 403.364, Step: 200 +2023-04-17 14:18:42 - r - INFO: - Current episode 35 has the best eval reward: 187.440 +2023-04-17 14:18:42 - r - INFO: - Episode: 36/1000, Reward: 574.459, Step: 200 +2023-04-17 14:18:43 - r - INFO: - Episode: 37/1000, Reward: 280.111, Step: 200 +2023-04-17 14:18:43 - r - INFO: - Episode: 38/1000, Reward: -94.247, Step: 200 +2023-04-17 14:18:43 - r - INFO: - Episode: 39/1000, Reward: 125.289, Step: 200 +2023-04-17 14:18:44 - r - INFO: - Episode: 40/1000, Reward: 775.172, Step: 200 +2023-04-17 14:18:46 - r - INFO: - Current episode 40 has the best eval reward: 192.895 +2023-04-17 14:18:46 - r - INFO: - Episode: 41/1000, Reward: 13.564, Step: 200 +2023-04-17 14:18:47 - r - INFO: - Episode: 42/1000, Reward: 23.385, Step: 200 +2023-04-17 14:18:47 - r - INFO: - Episode: 43/1000, Reward: 348.051, Step: 200 +2023-04-17 14:18:48 - r - INFO: - Episode: 44/1000, Reward: 165.466, Step: 200 +2023-04-17 14:18:48 - r - INFO: - Episode: 45/1000, Reward: 113.787, Step: 200 +2023-04-17 14:18:51 - r - INFO: - Episode: 46/1000, Reward: 164.293, Step: 200 +2023-04-17 14:18:52 - r - INFO: - Episode: 47/1000, Reward: 192.775, Step: 200 +2023-04-17 14:18:52 - r - INFO: - Episode: 48/1000, Reward: 109.055, Step: 200 +2023-04-17 14:18:52 - r - INFO: - Episode: 49/1000, Reward: -26.554, Step: 200 +2023-04-17 14:18:53 - r - INFO: - Episode: 50/1000, Reward: 321.206, Step: 200 +2023-04-17 14:18:56 - r - INFO: - Current episode 50 has the best eval reward: 238.171 +2023-04-17 14:18:56 - r - INFO: - Episode: 51/1000, Reward: 252.379, Step: 200 +2023-04-17 14:18:57 - r - INFO: - Episode: 52/1000, Reward: 53.372, Step: 200 +2023-04-17 14:18:57 - r - INFO: - Episode: 53/1000, Reward: 227.691, Step: 200 +2023-04-17 14:18:58 - r - INFO: - Episode: 54/1000, Reward: 99.686, Step: 200 +2023-04-17 14:18:58 - r - INFO: - Episode: 55/1000, Reward: 125.093, Step: 200 +2023-04-17 14:19:02 - r - INFO: - Episode: 56/1000, Reward: 637.904, Step: 200 +2023-04-17 14:19:04 - r - INFO: - Episode: 57/1000, Reward: 75.350, Step: 200 +2023-04-17 14:19:05 - r - INFO: - Episode: 58/1000, Reward: 157.205, Step: 200 +2023-04-17 14:19:06 - r - INFO: - Episode: 59/1000, Reward: -80.936, Step: 200 +2023-04-17 14:19:06 - r - INFO: - Episode: 60/1000, Reward: 305.868, Step: 200 +2023-04-17 14:19:10 - r - INFO: - Episode: 61/1000, Reward: 358.851, Step: 200 +2023-04-17 14:19:11 - r - INFO: - Episode: 62/1000, Reward: -73.471, Step: 200 +2023-04-17 14:19:12 - r - INFO: - Episode: 63/1000, Reward: 319.353, Step: 200 +2023-04-17 14:19:12 - r - INFO: - Episode: 64/1000, Reward: 82.729, Step: 200 +2023-04-17 14:19:13 - r - INFO: - Episode: 65/1000, Reward: 92.988, Step: 200 +2023-04-17 14:19:16 - r - INFO: - Episode: 66/1000, Reward: 145.374, Step: 200 +2023-04-17 14:19:16 - r - INFO: - Episode: 67/1000, Reward: -12.395, Step: 200 +2023-04-17 14:19:17 - r - INFO: - Episode: 68/1000, Reward: -52.084, Step: 200 +2023-04-17 14:19:17 - r - INFO: - Episode: 69/1000, Reward: 230.191, Step: 200 +2023-04-17 14:19:18 - r - INFO: - Episode: 70/1000, Reward: 157.504, Step: 200 +2023-04-17 14:19:21 - r - INFO: - Episode: 71/1000, Reward: 26.734, Step: 200 +2023-04-17 14:19:21 - r - INFO: - Episode: 72/1000, Reward: 1077.251, Step: 200 +2023-04-17 14:19:22 - r - INFO: - Episode: 73/1000, Reward: 316.201, Step: 200 +2023-04-17 14:19:22 - r - INFO: - Episode: 74/1000, Reward: 186.210, Step: 200 +2023-04-17 14:19:23 - r - INFO: - Episode: 75/1000, Reward: 330.830, Step: 200 +2023-04-17 14:19:25 - r - INFO: - Episode: 76/1000, Reward: 62.513, Step: 200 +2023-04-17 14:19:26 - r - INFO: - Episode: 77/1000, Reward: 94.116, Step: 200 +2023-04-17 14:19:26 - r - INFO: - Episode: 78/1000, Reward: 105.742, Step: 200 +2023-04-17 14:19:27 - r - INFO: - Episode: 79/1000, Reward: 318.653, Step: 200 +2023-04-17 14:19:28 - r - INFO: - Episode: 80/1000, Reward: 221.205, Step: 200 +2023-04-17 14:19:30 - r - INFO: - Episode: 81/1000, Reward: 394.372, Step: 200 +2023-04-17 14:19:31 - r - INFO: - Episode: 82/1000, Reward: 130.508, Step: 200 +2023-04-17 14:19:31 - r - INFO: - Episode: 83/1000, Reward: 352.314, Step: 200 +2023-04-17 14:19:31 - r - INFO: - Episode: 84/1000, Reward: 119.763, Step: 200 +2023-04-17 14:19:32 - r - INFO: - Episode: 85/1000, Reward: 197.989, Step: 200 +2023-04-17 14:19:34 - r - INFO: - Current episode 85 has the best eval reward: 263.313 +2023-04-17 14:19:34 - r - INFO: - Episode: 86/1000, Reward: 85.460, Step: 200 +2023-04-17 14:19:35 - r - INFO: - Episode: 87/1000, Reward: 363.022, Step: 200 +2023-04-17 14:19:35 - r - INFO: - Episode: 88/1000, Reward: 16.903, Step: 200 +2023-04-17 14:19:36 - r - INFO: - Episode: 89/1000, Reward: 391.962, Step: 200 +2023-04-17 14:19:37 - r - INFO: - Episode: 90/1000, Reward: 164.657, Step: 200 +2023-04-17 14:19:40 - r - INFO: - Current episode 90 has the best eval reward: 371.576 +2023-04-17 14:19:41 - r - INFO: - Episode: 91/1000, Reward: 551.290, Step: 200 +2023-04-17 14:19:42 - r - INFO: - Episode: 92/1000, Reward: 358.540, Step: 200 +2023-04-17 14:19:42 - r - INFO: - Episode: 93/1000, Reward: 233.183, Step: 200 +2023-04-17 14:19:43 - r - INFO: - Episode: 94/1000, Reward: 155.593, Step: 200 +2023-04-17 14:19:44 - r - INFO: - Episode: 95/1000, Reward: 220.060, Step: 200 +2023-04-17 14:19:48 - r - INFO: - Episode: 96/1000, Reward: 267.437, Step: 200 +2023-04-17 14:19:49 - r - INFO: - Episode: 97/1000, Reward: 99.229, Step: 200 +2023-04-17 14:19:49 - r - INFO: - Episode: 98/1000, Reward: 161.359, Step: 200 +2023-04-17 14:19:50 - r - INFO: - Episode: 99/1000, Reward: -1.241, Step: 200 +2023-04-17 14:19:50 - r - INFO: - Episode: 100/1000, Reward: 116.920, Step: 200 +2023-04-17 14:19:54 - r - INFO: - Episode: 101/1000, Reward: 188.344, Step: 200 +2023-04-17 14:19:54 - r - INFO: - Episode: 102/1000, Reward: 9.056, Step: 200 +2023-04-17 14:19:55 - r - INFO: - Episode: 103/1000, Reward: 928.376, Step: 200 +2023-04-17 14:19:55 - r - INFO: - Episode: 104/1000, Reward: 297.736, Step: 200 +2023-04-17 14:19:56 - r - INFO: - Episode: 105/1000, Reward: 122.111, Step: 200 +2023-04-17 14:19:59 - r - INFO: - Episode: 106/1000, Reward: 442.425, Step: 200 +2023-04-17 14:19:59 - r - INFO: - Episode: 107/1000, Reward: 67.100, Step: 200 +2023-04-17 14:20:00 - r - INFO: - Episode: 108/1000, Reward: -47.063, Step: 200 +2023-04-17 14:20:01 - r - INFO: - Episode: 109/1000, Reward: 369.440, Step: 200 +2023-04-17 14:20:01 - r - INFO: - Episode: 110/1000, Reward: 66.838, Step: 200 +2023-04-17 14:20:05 - r - INFO: - Episode: 111/1000, Reward: 118.007, Step: 200 +2023-04-17 14:20:06 - r - INFO: - Episode: 112/1000, Reward: 289.216, Step: 200 +2023-04-17 14:20:07 - r - INFO: - Episode: 113/1000, Reward: 89.995, Step: 200 +2023-04-17 14:20:07 - r - INFO: - Episode: 114/1000, Reward: 99.790, Step: 200 +2023-04-17 14:20:08 - r - INFO: - Episode: 115/1000, Reward: 26.232, Step: 200 +2023-04-17 14:20:11 - r - INFO: - Episode: 116/1000, Reward: 375.304, Step: 200 +2023-04-17 14:20:11 - r - INFO: - Episode: 117/1000, Reward: 113.852, Step: 200 +2023-04-17 14:20:12 - r - INFO: - Episode: 118/1000, Reward: 236.113, Step: 200 +2023-04-17 14:20:12 - r - INFO: - Episode: 119/1000, Reward: 206.119, Step: 200 +2023-04-17 14:20:13 - r - INFO: - Episode: 120/1000, Reward: 78.333, Step: 200 +2023-04-17 14:20:16 - r - INFO: - Episode: 121/1000, Reward: 605.236, Step: 200 +2023-04-17 14:20:17 - r - INFO: - Episode: 122/1000, Reward: 16.320, Step: 200 +2023-04-17 14:20:17 - r - INFO: - Episode: 123/1000, Reward: 312.691, Step: 200 +2023-04-17 14:20:18 - r - INFO: - Episode: 124/1000, Reward: 409.327, Step: 200 +2023-04-17 14:20:18 - r - INFO: - Episode: 125/1000, Reward: 301.539, Step: 200 +2023-04-17 14:20:22 - r - INFO: - Episode: 126/1000, Reward: 474.213, Step: 200 +2023-04-17 14:20:23 - r - INFO: - Episode: 127/1000, Reward: 63.957, Step: 200 +2023-04-17 14:20:23 - r - INFO: - Episode: 128/1000, Reward: 134.605, Step: 200 +2023-04-17 14:20:24 - r - INFO: - Episode: 129/1000, Reward: 180.447, Step: 200 +2023-04-17 14:20:24 - r - INFO: - Episode: 130/1000, Reward: 388.013, Step: 200 +2023-04-17 14:20:27 - r - INFO: - Current episode 130 has the best eval reward: 523.746 +2023-04-17 14:20:28 - r - INFO: - Episode: 131/1000, Reward: 125.378, Step: 200 +2023-04-17 14:20:28 - r - INFO: - Episode: 132/1000, Reward: 332.917, Step: 200 +2023-04-17 14:20:29 - r - INFO: - Episode: 133/1000, Reward: 251.745, Step: 200 +2023-04-17 14:20:29 - r - INFO: - Episode: 134/1000, Reward: 787.607, Step: 200 +2023-04-17 14:20:30 - r - INFO: - Episode: 135/1000, Reward: 139.888, Step: 200 +2023-04-17 14:20:33 - r - INFO: - Episode: 136/1000, Reward: 224.870, Step: 200 +2023-04-17 14:20:34 - r - INFO: - Episode: 137/1000, Reward: 410.017, Step: 200 +2023-04-17 14:20:34 - r - INFO: - Episode: 138/1000, Reward: 224.096, Step: 200 +2023-04-17 14:20:35 - r - INFO: - Episode: 139/1000, Reward: 701.013, Step: 200 +2023-04-17 14:20:36 - r - INFO: - Episode: 140/1000, Reward: 960.260, Step: 200 +2023-04-17 14:20:39 - r - INFO: - Episode: 141/1000, Reward: -34.193, Step: 200 +2023-04-17 14:20:39 - r - INFO: - Episode: 142/1000, Reward: 63.261, Step: 200 +2023-04-17 14:20:40 - r - INFO: - Episode: 143/1000, Reward: 166.859, Step: 200 +2023-04-17 14:20:40 - r - INFO: - Episode: 144/1000, Reward: -62.683, Step: 200 +2023-04-17 14:20:41 - r - INFO: - Episode: 145/1000, Reward: 213.826, Step: 200 +2023-04-17 14:20:44 - r - INFO: - Episode: 146/1000, Reward: 94.539, Step: 200 +2023-04-17 14:20:44 - r - INFO: - Episode: 147/1000, Reward: 117.668, Step: 200 +2023-04-17 14:20:45 - r - INFO: - Episode: 148/1000, Reward: 99.363, Step: 200 +2023-04-17 14:20:46 - r - INFO: - Episode: 149/1000, Reward: 480.802, Step: 200 +2023-04-17 14:20:46 - r - INFO: - Episode: 150/1000, Reward: 182.572, Step: 200 +2023-04-17 14:20:49 - r - INFO: - Episode: 151/1000, Reward: 3.159, Step: 200 +2023-04-17 14:20:50 - r - INFO: - Episode: 152/1000, Reward: 146.942, Step: 200 +2023-04-17 14:20:50 - r - INFO: - Episode: 153/1000, Reward: 43.636, Step: 200 +2023-04-17 14:20:51 - r - INFO: - Episode: 154/1000, Reward: 33.393, Step: 200 +2023-04-17 14:20:51 - r - INFO: - Episode: 155/1000, Reward: 591.482, Step: 200 +2023-04-17 14:20:54 - r - INFO: - Episode: 156/1000, Reward: 169.522, Step: 200 +2023-04-17 14:20:55 - r - INFO: - Episode: 157/1000, Reward: 486.412, Step: 200 +2023-04-17 14:20:55 - r - INFO: - Episode: 158/1000, Reward: 429.272, Step: 200 +2023-04-17 14:20:56 - r - INFO: - Episode: 159/1000, Reward: 626.850, Step: 200 +2023-04-17 14:20:57 - r - INFO: - Episode: 160/1000, Reward: 193.040, Step: 200 +2023-04-17 14:21:00 - r - INFO: - Episode: 161/1000, Reward: 305.280, Step: 200 +2023-04-17 14:21:00 - r - INFO: - Episode: 162/1000, Reward: 537.074, Step: 200 +2023-04-17 14:21:01 - r - INFO: - Episode: 163/1000, Reward: 614.044, Step: 200 +2023-04-17 14:21:01 - r - INFO: - Episode: 164/1000, Reward: 16.916, Step: 200 +2023-04-17 14:21:02 - r - INFO: - Episode: 165/1000, Reward: 162.581, Step: 200 +2023-04-17 14:21:05 - r - INFO: - Episode: 166/1000, Reward: 187.667, Step: 200 +2023-04-17 14:21:06 - r - INFO: - Episode: 167/1000, Reward: 87.235, Step: 200 +2023-04-17 14:21:07 - r - INFO: - Episode: 168/1000, Reward: -62.937, Step: 200 +2023-04-17 14:21:07 - r - INFO: - Episode: 169/1000, Reward: 23.235, Step: 200 +2023-04-17 14:21:08 - r - INFO: - Episode: 170/1000, Reward: 863.124, Step: 200 +2023-04-17 14:21:12 - r - INFO: - Episode: 171/1000, Reward: -27.099, Step: 200 +2023-04-17 14:21:12 - r - INFO: - Episode: 172/1000, Reward: 323.274, Step: 200 +2023-04-17 14:21:13 - r - INFO: - Episode: 173/1000, Reward: 245.846, Step: 200 +2023-04-17 14:21:14 - r - INFO: - Episode: 174/1000, Reward: 27.394, Step: 200 +2023-04-17 14:21:14 - r - INFO: - Episode: 175/1000, Reward: 292.128, Step: 200 +2023-04-17 14:21:18 - r - INFO: - Episode: 176/1000, Reward: 225.704, Step: 200 +2023-04-17 14:21:18 - r - INFO: - Episode: 177/1000, Reward: 568.517, Step: 200 +2023-04-17 14:21:19 - r - INFO: - Episode: 178/1000, Reward: 277.103, Step: 200 +2023-04-17 14:21:19 - r - INFO: - Episode: 179/1000, Reward: 438.246, Step: 200 +2023-04-17 14:21:20 - r - INFO: - Episode: 180/1000, Reward: 130.911, Step: 200 +2023-04-17 14:21:24 - r - INFO: - Episode: 181/1000, Reward: -31.328, Step: 200 +2023-04-17 14:21:24 - r - INFO: - Episode: 182/1000, Reward: 156.361, Step: 200 +2023-04-17 14:21:25 - r - INFO: - Episode: 183/1000, Reward: -87.087, Step: 200 +2023-04-17 14:21:25 - r - INFO: - Episode: 184/1000, Reward: 26.352, Step: 200 +2023-04-17 14:21:26 - r - INFO: - Episode: 185/1000, Reward: 283.009, Step: 200 +2023-04-17 14:21:29 - r - INFO: - Episode: 186/1000, Reward: 131.234, Step: 200 +2023-04-17 14:21:30 - r - INFO: - Episode: 187/1000, Reward: 352.672, Step: 200 +2023-04-17 14:21:30 - r - INFO: - Episode: 188/1000, Reward: -49.442, Step: 200 +2023-04-17 14:21:31 - r - INFO: - Episode: 189/1000, Reward: 261.836, Step: 200 +2023-04-17 14:21:31 - r - INFO: - Episode: 190/1000, Reward: 121.849, Step: 200 +2023-04-17 14:21:35 - r - INFO: - Episode: 191/1000, Reward: 259.229, Step: 200 +2023-04-17 14:21:35 - r - INFO: - Episode: 192/1000, Reward: 2.797, Step: 200 +2023-04-17 14:21:36 - r - INFO: - Episode: 193/1000, Reward: 88.652, Step: 200 +2023-04-17 14:21:36 - r - INFO: - Episode: 194/1000, Reward: -112.533, Step: 200 +2023-04-17 14:21:37 - r - INFO: - Episode: 195/1000, Reward: 590.822, Step: 200 +2023-04-17 14:21:40 - r - INFO: - Episode: 196/1000, Reward: 246.962, Step: 200 +2023-04-17 14:21:41 - r - INFO: - Episode: 197/1000, Reward: 152.954, Step: 200 +2023-04-17 14:21:42 - r - INFO: - Episode: 198/1000, Reward: 120.679, Step: 200 +2023-04-17 14:21:42 - r - INFO: - Episode: 199/1000, Reward: 673.171, Step: 200 +2023-04-17 14:21:43 - r - INFO: - Episode: 200/1000, Reward: 522.125, Step: 200 +2023-04-17 14:21:46 - r - INFO: - Episode: 201/1000, Reward: 340.316, Step: 200 +2023-04-17 14:21:47 - r - INFO: - Episode: 202/1000, Reward: 67.554, Step: 200 +2023-04-17 14:21:47 - r - INFO: - Episode: 203/1000, Reward: 134.180, Step: 200 +2023-04-17 14:21:48 - r - INFO: - Episode: 204/1000, Reward: 262.443, Step: 200 +2023-04-17 14:21:48 - r - INFO: - Episode: 205/1000, Reward: 233.219, Step: 200 +2023-04-17 14:21:52 - r - INFO: - Episode: 206/1000, Reward: -81.757, Step: 200 +2023-04-17 14:21:52 - r - INFO: - Episode: 207/1000, Reward: 102.885, Step: 200 +2023-04-17 14:21:53 - r - INFO: - Episode: 208/1000, Reward: 347.454, Step: 200 +2023-04-17 14:21:53 - r - INFO: - Episode: 209/1000, Reward: -32.425, Step: 200 +2023-04-17 14:21:54 - r - INFO: - Episode: 210/1000, Reward: 704.177, Step: 200 +2023-04-17 14:21:58 - r - INFO: - Episode: 211/1000, Reward: 227.647, Step: 200 +2023-04-17 14:21:58 - r - INFO: - Episode: 212/1000, Reward: 451.534, Step: 200 +2023-04-17 14:21:59 - r - INFO: - Episode: 213/1000, Reward: 676.279, Step: 200 +2023-04-17 14:22:00 - r - INFO: - Episode: 214/1000, Reward: 401.667, Step: 200 +2023-04-17 14:22:00 - r - INFO: - Episode: 215/1000, Reward: 377.560, Step: 200 +2023-04-17 14:22:03 - r - INFO: - Current episode 215 has the best eval reward: 537.317 +2023-04-17 14:22:04 - r - INFO: - Episode: 216/1000, Reward: 374.650, Step: 200 +2023-04-17 14:22:04 - r - INFO: - Episode: 217/1000, Reward: 6.814, Step: 200 +2023-04-17 14:22:05 - r - INFO: - Episode: 218/1000, Reward: 843.697, Step: 200 +2023-04-17 14:22:05 - r - INFO: - Episode: 219/1000, Reward: 930.712, Step: 200 +2023-04-17 14:22:06 - r - INFO: - Episode: 220/1000, Reward: 74.585, Step: 200 +2023-04-17 14:22:09 - r - INFO: - Episode: 221/1000, Reward: -62.471, Step: 200 +2023-04-17 14:22:10 - r - INFO: - Episode: 222/1000, Reward: 91.411, Step: 200 +2023-04-17 14:22:11 - r - INFO: - Episode: 223/1000, Reward: 439.858, Step: 200 +2023-04-17 14:22:11 - r - INFO: - Episode: 224/1000, Reward: 828.926, Step: 200 +2023-04-17 14:22:12 - r - INFO: - Episode: 225/1000, Reward: 49.018, Step: 200 +2023-04-17 14:22:15 - r - INFO: - Episode: 226/1000, Reward: 199.457, Step: 200 +2023-04-17 14:22:16 - r - INFO: - Episode: 227/1000, Reward: 108.963, Step: 200 +2023-04-17 14:22:17 - r - INFO: - Episode: 228/1000, Reward: -73.162, Step: 200 +2023-04-17 14:22:17 - r - INFO: - Episode: 229/1000, Reward: 292.108, Step: 200 +2023-04-17 14:22:18 - r - INFO: - Episode: 230/1000, Reward: 74.100, Step: 200 +2023-04-17 14:22:21 - r - INFO: - Episode: 231/1000, Reward: 563.871, Step: 200 +2023-04-17 14:22:22 - r - INFO: - Episode: 232/1000, Reward: -0.594, Step: 200 +2023-04-17 14:22:22 - r - INFO: - Episode: 233/1000, Reward: 1001.469, Step: 200 +2023-04-17 14:22:23 - r - INFO: - Episode: 234/1000, Reward: 101.065, Step: 200 +2023-04-17 14:22:23 - r - INFO: - Episode: 235/1000, Reward: 334.962, Step: 200 +2023-04-17 14:22:26 - r - INFO: - Current episode 235 has the best eval reward: 632.590 +2023-04-17 14:22:27 - r - INFO: - Episode: 236/1000, Reward: 616.074, Step: 200 +2023-04-17 14:22:28 - r - INFO: - Episode: 237/1000, Reward: 534.571, Step: 200 +2023-04-17 14:22:29 - r - INFO: - Episode: 238/1000, Reward: 159.166, Step: 200 +2023-04-17 14:22:30 - r - INFO: - Episode: 239/1000, Reward: 1088.036, Step: 200 +2023-04-17 14:22:31 - r - INFO: - Episode: 240/1000, Reward: 818.456, Step: 200 +2023-04-17 14:22:35 - r - INFO: - Episode: 241/1000, Reward: 297.688, Step: 200 +2023-04-17 14:22:35 - r - INFO: - Episode: 242/1000, Reward: 678.415, Step: 200 +2023-04-17 14:22:36 - r - INFO: - Episode: 243/1000, Reward: 77.522, Step: 200 +2023-04-17 14:22:37 - r - INFO: - Episode: 244/1000, Reward: 10.843, Step: 200 +2023-04-17 14:22:37 - r - INFO: - Episode: 245/1000, Reward: 14.063, Step: 200 +2023-04-17 14:22:40 - r - INFO: - Episode: 246/1000, Reward: 1165.727, Step: 200 +2023-04-17 14:22:41 - r - INFO: - Episode: 247/1000, Reward: 473.052, Step: 200 +2023-04-17 14:22:42 - r - INFO: - Episode: 248/1000, Reward: 114.722, Step: 200 +2023-04-17 14:22:42 - r - INFO: - Episode: 249/1000, Reward: 98.678, Step: 200 +2023-04-17 14:22:43 - r - INFO: - Episode: 250/1000, Reward: 541.536, Step: 200 +2023-04-17 14:22:48 - r - INFO: - Episode: 251/1000, Reward: 170.182, Step: 200 +2023-04-17 14:22:48 - r - INFO: - Episode: 252/1000, Reward: 532.782, Step: 200 +2023-04-17 14:22:49 - r - INFO: - Episode: 253/1000, Reward: 19.611, Step: 200 +2023-04-17 14:22:50 - r - INFO: - Episode: 254/1000, Reward: 1142.534, Step: 200 +2023-04-17 14:22:50 - r - INFO: - Episode: 255/1000, Reward: 1006.831, Step: 200 +2023-04-17 14:22:56 - r - INFO: - Episode: 256/1000, Reward: 1038.001, Step: 200 +2023-04-17 14:22:57 - r - INFO: - Episode: 257/1000, Reward: 291.740, Step: 200 +2023-04-17 14:22:57 - r - INFO: - Episode: 258/1000, Reward: 91.170, Step: 200 +2023-04-17 14:22:58 - r - INFO: - Episode: 259/1000, Reward: 325.686, Step: 200 +2023-04-17 14:22:59 - r - INFO: - Episode: 260/1000, Reward: 93.240, Step: 200 +2023-04-17 14:23:03 - r - INFO: - Episode: 261/1000, Reward: 28.747, Step: 200 +2023-04-17 14:23:03 - r - INFO: - Episode: 262/1000, Reward: 447.162, Step: 200 +2023-04-17 14:23:04 - r - INFO: - Episode: 263/1000, Reward: 292.820, Step: 200 +2023-04-17 14:23:05 - r - INFO: - Episode: 264/1000, Reward: 51.004, Step: 200 +2023-04-17 14:23:05 - r - INFO: - Episode: 265/1000, Reward: 1314.805, Step: 200 +2023-04-17 14:23:09 - r - INFO: - Episode: 266/1000, Reward: 274.492, Step: 200 +2023-04-17 14:23:09 - r - INFO: - Episode: 267/1000, Reward: 242.027, Step: 200 +2023-04-17 14:23:10 - r - INFO: - Episode: 268/1000, Reward: 754.570, Step: 200 +2023-04-17 14:23:11 - r - INFO: - Episode: 269/1000, Reward: 76.262, Step: 200 +2023-04-17 14:23:11 - r - INFO: - Episode: 270/1000, Reward: 382.932, Step: 200 +2023-04-17 14:23:16 - r - INFO: - Episode: 271/1000, Reward: 183.575, Step: 200 +2023-04-17 14:23:16 - r - INFO: - Episode: 272/1000, Reward: 906.285, Step: 200 +2023-04-17 14:23:17 - r - INFO: - Episode: 273/1000, Reward: 663.803, Step: 200 +2023-04-17 14:23:18 - r - INFO: - Episode: 274/1000, Reward: 112.702, Step: 200 +2023-04-17 14:23:18 - r - INFO: - Episode: 275/1000, Reward: 163.363, Step: 200 +2023-04-17 14:23:22 - r - INFO: - Episode: 276/1000, Reward: 728.186, Step: 200 +2023-04-17 14:23:22 - r - INFO: - Episode: 277/1000, Reward: 654.437, Step: 200 +2023-04-17 14:23:23 - r - INFO: - Episode: 278/1000, Reward: 455.033, Step: 200 +2023-04-17 14:23:23 - r - INFO: - Episode: 279/1000, Reward: 789.555, Step: 200 +2023-04-17 14:23:24 - r - INFO: - Episode: 280/1000, Reward: -71.630, Step: 200 +2023-04-17 14:23:28 - r - INFO: - Current episode 280 has the best eval reward: 645.866 +2023-04-17 14:23:29 - r - INFO: - Episode: 281/1000, Reward: 86.629, Step: 200 +2023-04-17 14:23:30 - r - INFO: - Episode: 282/1000, Reward: 502.362, Step: 200 +2023-04-17 14:23:30 - r - INFO: - Episode: 283/1000, Reward: 229.235, Step: 200 +2023-04-17 14:23:31 - r - INFO: - Episode: 284/1000, Reward: 39.444, Step: 200 +2023-04-17 14:23:31 - r - INFO: - Episode: 285/1000, Reward: 969.390, Step: 200 +2023-04-17 14:23:34 - r - INFO: - Episode: 286/1000, Reward: -49.349, Step: 200 +2023-04-17 14:23:35 - r - INFO: - Episode: 287/1000, Reward: 557.668, Step: 200 +2023-04-17 14:23:35 - r - INFO: - Episode: 288/1000, Reward: 180.889, Step: 200 +2023-04-17 14:23:36 - r - INFO: - Episode: 289/1000, Reward: 1270.018, Step: 200 +2023-04-17 14:23:36 - r - INFO: - Episode: 290/1000, Reward: -52.145, Step: 200 +2023-04-17 14:23:40 - r - INFO: - Episode: 291/1000, Reward: 338.981, Step: 200 +2023-04-17 14:23:40 - r - INFO: - Episode: 292/1000, Reward: -10.908, Step: 200 +2023-04-17 14:23:41 - r - INFO: - Episode: 293/1000, Reward: 845.803, Step: 200 +2023-04-17 14:23:41 - r - INFO: - Episode: 294/1000, Reward: 896.835, Step: 200 +2023-04-17 14:23:42 - r - INFO: - Episode: 295/1000, Reward: 403.896, Step: 200 +2023-04-17 14:23:45 - r - INFO: - Episode: 296/1000, Reward: 66.396, Step: 200 +2023-04-17 14:23:46 - r - INFO: - Episode: 297/1000, Reward: 207.754, Step: 200 +2023-04-17 14:23:46 - r - INFO: - Episode: 298/1000, Reward: 632.779, Step: 200 +2023-04-17 14:23:47 - r - INFO: - Episode: 299/1000, Reward: 361.880, Step: 200 +2023-04-17 14:23:47 - r - INFO: - Episode: 300/1000, Reward: 1098.759, Step: 200 +2023-04-17 14:23:51 - r - INFO: - Episode: 301/1000, Reward: 125.698, Step: 200 +2023-04-17 14:23:51 - r - INFO: - Episode: 302/1000, Reward: 105.351, Step: 200 +2023-04-17 14:23:52 - r - INFO: - Episode: 303/1000, Reward: -51.808, Step: 200 +2023-04-17 14:23:52 - r - INFO: - Episode: 304/1000, Reward: 122.586, Step: 200 +2023-04-17 14:23:53 - r - INFO: - Episode: 305/1000, Reward: 131.504, Step: 200 +2023-04-17 14:23:57 - r - INFO: - Episode: 306/1000, Reward: -54.181, Step: 200 +2023-04-17 14:23:58 - r - INFO: - Episode: 307/1000, Reward: 1209.181, Step: 200 +2023-04-17 14:23:58 - r - INFO: - Episode: 308/1000, Reward: -49.109, Step: 200 +2023-04-17 14:23:59 - r - INFO: - Episode: 309/1000, Reward: 229.582, Step: 200 +2023-04-17 14:24:00 - r - INFO: - Episode: 310/1000, Reward: 1087.966, Step: 200 +2023-04-17 14:24:04 - r - INFO: - Episode: 311/1000, Reward: 649.488, Step: 200 +2023-04-17 14:24:04 - r - INFO: - Episode: 312/1000, Reward: -19.740, Step: 200 +2023-04-17 14:24:05 - r - INFO: - Episode: 313/1000, Reward: 588.880, Step: 200 +2023-04-17 14:24:05 - r - INFO: - Episode: 314/1000, Reward: 515.671, Step: 200 +2023-04-17 14:24:06 - r - INFO: - Episode: 315/1000, Reward: 47.394, Step: 200 +2023-04-17 14:24:09 - r - INFO: - Episode: 316/1000, Reward: -27.627, Step: 200 +2023-04-17 14:24:10 - r - INFO: - Episode: 317/1000, Reward: 1056.537, Step: 200 +2023-04-17 14:24:11 - r - INFO: - Episode: 318/1000, Reward: 454.216, Step: 200 +2023-04-17 14:24:11 - r - INFO: - Episode: 319/1000, Reward: 963.842, Step: 200 +2023-04-17 14:24:12 - r - INFO: - Episode: 320/1000, Reward: -12.392, Step: 200 +2023-04-17 14:24:15 - r - INFO: - Current episode 320 has the best eval reward: 655.069 +2023-04-17 14:24:15 - r - INFO: - Episode: 321/1000, Reward: -21.301, Step: 200 +2023-04-17 14:24:16 - r - INFO: - Episode: 322/1000, Reward: 905.088, Step: 200 +2023-04-17 14:24:17 - r - INFO: - Episode: 323/1000, Reward: 43.727, Step: 200 +2023-04-17 14:24:17 - r - INFO: - Episode: 324/1000, Reward: 1173.265, Step: 200 +2023-04-17 14:24:18 - r - INFO: - Episode: 325/1000, Reward: 148.092, Step: 200 +2023-04-17 14:24:21 - r - INFO: - Episode: 326/1000, Reward: 1195.893, Step: 200 +2023-04-17 14:24:22 - r - INFO: - Episode: 327/1000, Reward: 47.574, Step: 200 +2023-04-17 14:24:23 - r - INFO: - Episode: 328/1000, Reward: -79.854, Step: 200 +2023-04-17 14:24:23 - r - INFO: - Episode: 329/1000, Reward: 28.465, Step: 200 +2023-04-17 14:24:24 - r - INFO: - Episode: 330/1000, Reward: 671.698, Step: 200 +2023-04-17 14:24:27 - r - INFO: - Current episode 330 has the best eval reward: 689.900 +2023-04-17 14:24:27 - r - INFO: - Episode: 331/1000, Reward: -23.250, Step: 200 +2023-04-17 14:24:28 - r - INFO: - Episode: 332/1000, Reward: 451.186, Step: 200 +2023-04-17 14:24:28 - r - INFO: - Episode: 333/1000, Reward: 592.159, Step: 200 +2023-04-17 14:24:29 - r - INFO: - Episode: 334/1000, Reward: 1009.069, Step: 200 +2023-04-17 14:24:30 - r - INFO: - Episode: 335/1000, Reward: 481.500, Step: 200 +2023-04-17 14:24:33 - r - INFO: - Episode: 336/1000, Reward: 992.449, Step: 200 +2023-04-17 14:24:34 - r - INFO: - Episode: 337/1000, Reward: 716.702, Step: 200 +2023-04-17 14:24:34 - r - INFO: - Episode: 338/1000, Reward: 697.781, Step: 200 +2023-04-17 14:24:35 - r - INFO: - Episode: 339/1000, Reward: -34.708, Step: 200 +2023-04-17 14:24:36 - r - INFO: - Episode: 340/1000, Reward: 231.961, Step: 200 +2023-04-17 14:24:39 - r - INFO: - Episode: 341/1000, Reward: 798.434, Step: 200 +2023-04-17 14:24:40 - r - INFO: - Episode: 342/1000, Reward: 146.498, Step: 200 +2023-04-17 14:24:40 - r - INFO: - Episode: 343/1000, Reward: 1223.739, Step: 200 +2023-04-17 14:24:41 - r - INFO: - Episode: 344/1000, Reward: 326.800, Step: 200 +2023-04-17 14:24:41 - r - INFO: - Episode: 345/1000, Reward: 619.586, Step: 200 +2023-04-17 14:24:45 - r - INFO: - Episode: 346/1000, Reward: 681.383, Step: 200 +2023-04-17 14:24:46 - r - INFO: - Episode: 347/1000, Reward: 118.500, Step: 200 +2023-04-17 14:24:46 - r - INFO: - Episode: 348/1000, Reward: 281.490, Step: 200 +2023-04-17 14:24:47 - r - INFO: - Episode: 349/1000, Reward: 101.647, Step: 200 +2023-04-17 14:24:47 - r - INFO: - Episode: 350/1000, Reward: 1102.037, Step: 200 +2023-04-17 14:24:50 - r - INFO: - Current episode 350 has the best eval reward: 933.649 +2023-04-17 14:24:51 - r - INFO: - Episode: 351/1000, Reward: 470.470, Step: 200 +2023-04-17 14:24:51 - r - INFO: - Episode: 352/1000, Reward: 1139.708, Step: 200 +2023-04-17 14:24:52 - r - INFO: - Episode: 353/1000, Reward: 662.397, Step: 200 +2023-04-17 14:24:53 - r - INFO: - Episode: 354/1000, Reward: 704.077, Step: 200 +2023-04-17 14:24:53 - r - INFO: - Episode: 355/1000, Reward: 1056.759, Step: 200 +2023-04-17 14:24:57 - r - INFO: - Episode: 356/1000, Reward: 98.834, Step: 200 +2023-04-17 14:24:58 - r - INFO: - Episode: 357/1000, Reward: 465.316, Step: 200 +2023-04-17 14:24:58 - r - INFO: - Episode: 358/1000, Reward: 252.658, Step: 200 +2023-04-17 14:24:59 - r - INFO: - Episode: 359/1000, Reward: 907.869, Step: 200 +2023-04-17 14:25:00 - r - INFO: - Episode: 360/1000, Reward: 783.853, Step: 200 +2023-04-17 14:25:03 - r - INFO: - Episode: 361/1000, Reward: 1008.177, Step: 200 +2023-04-17 14:25:04 - r - INFO: - Episode: 362/1000, Reward: 550.195, Step: 200 +2023-04-17 14:25:05 - r - INFO: - Episode: 363/1000, Reward: 467.910, Step: 200 +2023-04-17 14:25:05 - r - INFO: - Episode: 364/1000, Reward: 80.382, Step: 200 +2023-04-17 14:25:06 - r - INFO: - Episode: 365/1000, Reward: 817.459, Step: 200 +2023-04-17 14:25:09 - r - INFO: - Episode: 366/1000, Reward: 760.718, Step: 200 +2023-04-17 14:25:10 - r - INFO: - Episode: 367/1000, Reward: 804.009, Step: 200 +2023-04-17 14:25:10 - r - INFO: - Episode: 368/1000, Reward: 1197.992, Step: 200 +2023-04-17 14:25:11 - r - INFO: - Episode: 369/1000, Reward: 730.026, Step: 200 +2023-04-17 14:25:12 - r - INFO: - Episode: 370/1000, Reward: 1089.508, Step: 200 +2023-04-17 14:25:15 - r - INFO: - Current episode 370 has the best eval reward: 953.187 +2023-04-17 14:25:15 - r - INFO: - Episode: 371/1000, Reward: 448.971, Step: 200 +2023-04-17 14:25:16 - r - INFO: - Episode: 372/1000, Reward: 788.668, Step: 200 +2023-04-17 14:25:16 - r - INFO: - Episode: 373/1000, Reward: 880.668, Step: 200 +2023-04-17 14:25:17 - r - INFO: - Episode: 374/1000, Reward: 1206.582, Step: 200 +2023-04-17 14:25:17 - r - INFO: - Episode: 375/1000, Reward: 460.745, Step: 200 +2023-04-17 14:25:21 - r - INFO: - Episode: 376/1000, Reward: 331.947, Step: 200 +2023-04-17 14:25:22 - r - INFO: - Episode: 377/1000, Reward: 683.261, Step: 200 +2023-04-17 14:25:22 - r - INFO: - Episode: 378/1000, Reward: 612.485, Step: 200 +2023-04-17 14:25:23 - r - INFO: - Episode: 379/1000, Reward: 1333.897, Step: 200 +2023-04-17 14:25:23 - r - INFO: - Episode: 380/1000, Reward: 1174.568, Step: 200 +2023-04-17 14:25:27 - r - INFO: - Episode: 381/1000, Reward: 216.481, Step: 200 +2023-04-17 14:25:28 - r - INFO: - Episode: 382/1000, Reward: 1095.802, Step: 200 +2023-04-17 14:25:28 - r - INFO: - Episode: 383/1000, Reward: 1235.750, Step: 200 +2023-04-17 14:25:29 - r - INFO: - Episode: 384/1000, Reward: 334.865, Step: 200 +2023-04-17 14:25:29 - r - INFO: - Episode: 385/1000, Reward: 467.626, Step: 200 +2023-04-17 14:25:33 - r - INFO: - Episode: 386/1000, Reward: 315.888, Step: 200 +2023-04-17 14:25:34 - r - INFO: - Episode: 387/1000, Reward: 116.421, Step: 200 +2023-04-17 14:25:35 - r - INFO: - Episode: 388/1000, Reward: 18.114, Step: 200 +2023-04-17 14:25:35 - r - INFO: - Episode: 389/1000, Reward: 1196.012, Step: 200 +2023-04-17 14:25:36 - r - INFO: - Episode: 390/1000, Reward: 1231.013, Step: 200 +2023-04-17 14:25:39 - r - INFO: - Episode: 391/1000, Reward: 770.481, Step: 200 +2023-04-17 14:25:40 - r - INFO: - Episode: 392/1000, Reward: 703.682, Step: 200 +2023-04-17 14:25:41 - r - INFO: - Episode: 393/1000, Reward: 1135.027, Step: 200 +2023-04-17 14:25:41 - r - INFO: - Episode: 394/1000, Reward: 158.624, Step: 200 +2023-04-17 14:25:42 - r - INFO: - Episode: 395/1000, Reward: 921.914, Step: 200 +2023-04-17 14:25:45 - r - INFO: - Current episode 395 has the best eval reward: 1012.412 +2023-04-17 14:25:46 - r - INFO: - Episode: 396/1000, Reward: 1255.950, Step: 200 +2023-04-17 14:25:46 - r - INFO: - Episode: 397/1000, Reward: 1134.413, Step: 200 +2023-04-17 14:25:47 - r - INFO: - Episode: 398/1000, Reward: 1040.113, Step: 200 +2023-04-17 14:25:47 - r - INFO: - Episode: 399/1000, Reward: 176.982, Step: 200 +2023-04-17 14:25:48 - r - INFO: - Episode: 400/1000, Reward: 152.716, Step: 200 +2023-04-17 14:25:52 - r - INFO: - Episode: 401/1000, Reward: 1274.312, Step: 200 +2023-04-17 14:25:53 - r - INFO: - Episode: 402/1000, Reward: 278.023, Step: 200 +2023-04-17 14:25:53 - r - INFO: - Episode: 403/1000, Reward: 1282.718, Step: 200 +2023-04-17 14:25:54 - r - INFO: - Episode: 404/1000, Reward: 1035.864, Step: 200 +2023-04-17 14:25:55 - r - INFO: - Episode: 405/1000, Reward: 1018.727, Step: 200 +2023-04-17 14:25:58 - r - INFO: - Episode: 406/1000, Reward: 691.938, Step: 200 +2023-04-17 14:25:59 - r - INFO: - Episode: 407/1000, Reward: 279.530, Step: 200 +2023-04-17 14:26:00 - r - INFO: - Episode: 408/1000, Reward: 749.076, Step: 200 +2023-04-17 14:26:00 - r - INFO: - Episode: 409/1000, Reward: 1310.240, Step: 200 +2023-04-17 14:26:01 - r - INFO: - Episode: 410/1000, Reward: 1023.312, Step: 200 +2023-04-17 14:26:05 - r - INFO: - Episode: 411/1000, Reward: 328.899, Step: 200 +2023-04-17 14:26:05 - r - INFO: - Episode: 412/1000, Reward: 1060.744, Step: 200 +2023-04-17 14:26:06 - r - INFO: - Episode: 413/1000, Reward: 536.737, Step: 200 +2023-04-17 14:26:06 - r - INFO: - Episode: 414/1000, Reward: 959.221, Step: 200 +2023-04-17 14:26:07 - r - INFO: - Episode: 415/1000, Reward: 1066.303, Step: 200 +2023-04-17 14:26:10 - r - INFO: - Episode: 416/1000, Reward: 1148.971, Step: 200 +2023-04-17 14:26:11 - r - INFO: - Episode: 417/1000, Reward: 1274.566, Step: 200 +2023-04-17 14:26:12 - r - INFO: - Episode: 418/1000, Reward: 1257.960, Step: 200 +2023-04-17 14:26:12 - r - INFO: - Episode: 419/1000, Reward: 984.619, Step: 200 +2023-04-17 14:26:13 - r - INFO: - Episode: 420/1000, Reward: 1159.036, Step: 200 +2023-04-17 14:26:16 - r - INFO: - Episode: 421/1000, Reward: 74.465, Step: 200 +2023-04-17 14:26:17 - r - INFO: - Episode: 422/1000, Reward: 10.670, Step: 200 +2023-04-17 14:26:18 - r - INFO: - Episode: 423/1000, Reward: 1274.714, Step: 200 +2023-04-17 14:26:18 - r - INFO: - Episode: 424/1000, Reward: 892.079, Step: 200 +2023-04-17 14:26:19 - r - INFO: - Episode: 425/1000, Reward: 698.716, Step: 200 +2023-04-17 14:26:22 - r - INFO: - Current episode 425 has the best eval reward: 1028.427 +2023-04-17 14:26:22 - r - INFO: - Episode: 426/1000, Reward: 1225.864, Step: 200 +2023-04-17 14:26:23 - r - INFO: - Episode: 427/1000, Reward: 72.346, Step: 200 +2023-04-17 14:26:24 - r - INFO: - Episode: 428/1000, Reward: 962.215, Step: 200 +2023-04-17 14:26:24 - r - INFO: - Episode: 429/1000, Reward: 14.879, Step: 200 +2023-04-17 14:26:25 - r - INFO: - Episode: 430/1000, Reward: 970.588, Step: 200 +2023-04-17 14:26:28 - r - INFO: - Episode: 431/1000, Reward: 1158.391, Step: 200 +2023-04-17 14:26:29 - r - INFO: - Episode: 432/1000, Reward: 568.867, Step: 200 +2023-04-17 14:26:30 - r - INFO: - Episode: 433/1000, Reward: 1334.252, Step: 200 +2023-04-17 14:26:30 - r - INFO: - Episode: 434/1000, Reward: 1251.842, Step: 200 +2023-04-17 14:26:31 - r - INFO: - Episode: 435/1000, Reward: 827.310, Step: 200 +2023-04-17 14:26:35 - r - INFO: - Episode: 436/1000, Reward: 657.988, Step: 200 +2023-04-17 14:26:35 - r - INFO: - Episode: 437/1000, Reward: 34.566, Step: 200 +2023-04-17 14:26:36 - r - INFO: - Episode: 438/1000, Reward: 1077.560, Step: 200 +2023-04-17 14:26:37 - r - INFO: - Episode: 439/1000, Reward: 35.438, Step: 200 +2023-04-17 14:26:37 - r - INFO: - Episode: 440/1000, Reward: 1220.925, Step: 200 +2023-04-17 14:26:41 - r - INFO: - Episode: 441/1000, Reward: 880.140, Step: 200 +2023-04-17 14:26:41 - r - INFO: - Episode: 442/1000, Reward: 1262.243, Step: 200 +2023-04-17 14:26:42 - r - INFO: - Episode: 443/1000, Reward: 375.213, Step: 200 +2023-04-17 14:26:43 - r - INFO: - Episode: 444/1000, Reward: 1335.946, Step: 200 +2023-04-17 14:26:43 - r - INFO: - Episode: 445/1000, Reward: -71.372, Step: 200 +2023-04-17 14:26:47 - r - INFO: - Episode: 446/1000, Reward: 1068.740, Step: 200 +2023-04-17 14:26:47 - r - INFO: - Episode: 447/1000, Reward: 225.259, Step: 200 +2023-04-17 14:26:48 - r - INFO: - Episode: 448/1000, Reward: 1033.825, Step: 200 +2023-04-17 14:26:49 - r - INFO: - Episode: 449/1000, Reward: 1154.308, Step: 200 +2023-04-17 14:26:49 - r - INFO: - Episode: 450/1000, Reward: 535.141, Step: 200 +2023-04-17 14:26:53 - r - INFO: - Current episode 450 has the best eval reward: 1102.113 +2023-04-17 14:26:53 - r - INFO: - Episode: 451/1000, Reward: 883.874, Step: 200 +2023-04-17 14:26:54 - r - INFO: - Episode: 452/1000, Reward: 1253.952, Step: 200 +2023-04-17 14:26:54 - r - INFO: - Episode: 453/1000, Reward: 949.978, Step: 200 +2023-04-17 14:26:55 - r - INFO: - Episode: 454/1000, Reward: 565.115, Step: 200 +2023-04-17 14:26:56 - r - INFO: - Episode: 455/1000, Reward: 1343.220, Step: 200 +2023-04-17 14:26:59 - r - INFO: - Episode: 456/1000, Reward: 1164.005, Step: 200 +2023-04-17 14:27:00 - r - INFO: - Episode: 457/1000, Reward: 1324.921, Step: 200 +2023-04-17 14:27:01 - r - INFO: - Episode: 458/1000, Reward: 539.195, Step: 200 +2023-04-17 14:27:01 - r - INFO: - Episode: 459/1000, Reward: 972.374, Step: 200 +2023-04-17 14:27:02 - r - INFO: - Episode: 460/1000, Reward: 1019.584, Step: 200 +2023-04-17 14:27:05 - r - INFO: - Episode: 461/1000, Reward: 1355.197, Step: 200 +2023-04-17 14:27:06 - r - INFO: - Episode: 462/1000, Reward: 68.088, Step: 200 +2023-04-17 14:27:06 - r - INFO: - Episode: 463/1000, Reward: 604.801, Step: 200 +2023-04-17 14:27:07 - r - INFO: - Episode: 464/1000, Reward: 1186.173, Step: 200 +2023-04-17 14:27:08 - r - INFO: - Episode: 465/1000, Reward: 945.381, Step: 200 +2023-04-17 14:27:11 - r - INFO: - Episode: 466/1000, Reward: 789.604, Step: 200 +2023-04-17 14:27:12 - r - INFO: - Episode: 467/1000, Reward: 1279.780, Step: 200 +2023-04-17 14:27:12 - r - INFO: - Episode: 468/1000, Reward: 293.803, Step: 200 +2023-04-17 14:27:13 - r - INFO: - Episode: 469/1000, Reward: 805.219, Step: 200 +2023-04-17 14:27:14 - r - INFO: - Episode: 470/1000, Reward: 909.764, Step: 200 +2023-04-17 14:27:17 - r - INFO: - Episode: 471/1000, Reward: 331.214, Step: 200 +2023-04-17 14:27:18 - r - INFO: - Episode: 472/1000, Reward: 1313.807, Step: 200 +2023-04-17 14:27:18 - r - INFO: - Episode: 473/1000, Reward: 1155.083, Step: 200 +2023-04-17 14:27:19 - r - INFO: - Episode: 474/1000, Reward: 700.048, Step: 200 +2023-04-17 14:27:20 - r - INFO: - Episode: 475/1000, Reward: 1053.117, Step: 200 +2023-04-17 14:27:23 - r - INFO: - Episode: 476/1000, Reward: 1165.505, Step: 200 +2023-04-17 14:27:24 - r - INFO: - Episode: 477/1000, Reward: 954.894, Step: 200 +2023-04-17 14:27:24 - r - INFO: - Episode: 478/1000, Reward: 1241.416, Step: 200 +2023-04-17 14:27:25 - r - INFO: - Episode: 479/1000, Reward: 1226.657, Step: 200 +2023-04-17 14:27:26 - r - INFO: - Episode: 480/1000, Reward: 729.777, Step: 200 +2023-04-17 14:27:29 - r - INFO: - Episode: 481/1000, Reward: 1331.551, Step: 200 +2023-04-17 14:27:30 - r - INFO: - Episode: 482/1000, Reward: 1302.924, Step: 200 +2023-04-17 14:27:31 - r - INFO: - Episode: 483/1000, Reward: 1152.677, Step: 200 +2023-04-17 14:27:31 - r - INFO: - Episode: 484/1000, Reward: 1249.512, Step: 200 +2023-04-17 14:27:32 - r - INFO: - Episode: 485/1000, Reward: 428.394, Step: 200 +2023-04-17 14:27:36 - r - INFO: - Episode: 486/1000, Reward: 551.072, Step: 200 +2023-04-17 14:27:36 - r - INFO: - Episode: 487/1000, Reward: 1259.426, Step: 200 +2023-04-17 14:27:37 - r - INFO: - Episode: 488/1000, Reward: 1153.321, Step: 200 +2023-04-17 14:27:37 - r - INFO: - Episode: 489/1000, Reward: 1277.736, Step: 200 +2023-04-17 14:27:38 - r - INFO: - Episode: 490/1000, Reward: 1311.365, Step: 200 +2023-04-17 14:27:42 - r - INFO: - Episode: 491/1000, Reward: 1243.443, Step: 200 +2023-04-17 14:27:42 - r - INFO: - Episode: 492/1000, Reward: 1055.171, Step: 200 +2023-04-17 14:27:43 - r - INFO: - Episode: 493/1000, Reward: 1069.428, Step: 200 +2023-04-17 14:27:43 - r - INFO: - Episode: 494/1000, Reward: 1142.228, Step: 200 +2023-04-17 14:27:44 - r - INFO: - Episode: 495/1000, Reward: 1299.908, Step: 200 +2023-04-17 14:27:48 - r - INFO: - Episode: 496/1000, Reward: 1330.430, Step: 200 +2023-04-17 14:27:48 - r - INFO: - Episode: 497/1000, Reward: 1190.936, Step: 200 +2023-04-17 14:27:49 - r - INFO: - Episode: 498/1000, Reward: 1311.508, Step: 200 +2023-04-17 14:27:49 - r - INFO: - Episode: 499/1000, Reward: 238.201, Step: 200 +2023-04-17 14:27:50 - r - INFO: - Episode: 500/1000, Reward: 1317.070, Step: 200 +2023-04-17 14:27:54 - r - INFO: - Episode: 501/1000, Reward: 948.480, Step: 200 +2023-04-17 14:27:54 - r - INFO: - Episode: 502/1000, Reward: 711.125, Step: 200 +2023-04-17 14:27:55 - r - INFO: - Episode: 503/1000, Reward: -27.825, Step: 200 +2023-04-17 14:27:55 - r - INFO: - Episode: 504/1000, Reward: 228.798, Step: 200 +2023-04-17 14:27:56 - r - INFO: - Episode: 505/1000, Reward: 78.605, Step: 200 +2023-04-17 14:27:59 - r - INFO: - Episode: 506/1000, Reward: 1207.067, Step: 200 +2023-04-17 14:28:00 - r - INFO: - Episode: 507/1000, Reward: 1279.422, Step: 200 +2023-04-17 14:28:01 - r - INFO: - Episode: 508/1000, Reward: 1251.832, Step: 200 +2023-04-17 14:28:01 - r - INFO: - Episode: 509/1000, Reward: 975.879, Step: 200 +2023-04-17 14:28:02 - r - INFO: - Episode: 510/1000, Reward: 1295.005, Step: 200 +2023-04-17 14:28:05 - r - INFO: - Episode: 511/1000, Reward: 1140.732, Step: 200 +2023-04-17 14:28:06 - r - INFO: - Episode: 512/1000, Reward: 328.784, Step: 200 +2023-04-17 14:28:06 - r - INFO: - Episode: 513/1000, Reward: 1193.562, Step: 200 +2023-04-17 14:28:07 - r - INFO: - Episode: 514/1000, Reward: 537.291, Step: 200 +2023-04-17 14:28:08 - r - INFO: - Episode: 515/1000, Reward: 1206.355, Step: 200 +2023-04-17 14:28:11 - r - INFO: - Episode: 516/1000, Reward: 1303.571, Step: 200 +2023-04-17 14:28:12 - r - INFO: - Episode: 517/1000, Reward: 1323.482, Step: 200 +2023-04-17 14:28:12 - r - INFO: - Episode: 518/1000, Reward: 1242.277, Step: 200 +2023-04-17 14:28:13 - r - INFO: - Episode: 519/1000, Reward: 1068.176, Step: 200 +2023-04-17 14:28:14 - r - INFO: - Episode: 520/1000, Reward: 1293.309, Step: 200 +2023-04-17 14:28:17 - r - INFO: - Current episode 520 has the best eval reward: 1116.827 +2023-04-17 14:28:17 - r - INFO: - Episode: 521/1000, Reward: 1286.080, Step: 200 +2023-04-17 14:28:18 - r - INFO: - Episode: 522/1000, Reward: 1195.859, Step: 200 +2023-04-17 14:28:18 - r - INFO: - Episode: 523/1000, Reward: 1193.774, Step: 200 +2023-04-17 14:28:19 - r - INFO: - Episode: 524/1000, Reward: 360.607, Step: 200 +2023-04-17 14:28:20 - r - INFO: - Episode: 525/1000, Reward: 1191.235, Step: 200 +2023-04-17 14:28:23 - r - INFO: - Episode: 526/1000, Reward: 893.780, Step: 200 +2023-04-17 14:28:24 - r - INFO: - Episode: 527/1000, Reward: 1171.348, Step: 200 +2023-04-17 14:28:24 - r - INFO: - Episode: 528/1000, Reward: 995.855, Step: 200 +2023-04-17 14:28:25 - r - INFO: - Episode: 529/1000, Reward: 52.957, Step: 200 +2023-04-17 14:28:25 - r - INFO: - Episode: 530/1000, Reward: 1352.290, Step: 200 +2023-04-17 14:28:29 - r - INFO: - Episode: 531/1000, Reward: 1140.687, Step: 200 +2023-04-17 14:28:30 - r - INFO: - Episode: 532/1000, Reward: 1299.395, Step: 200 +2023-04-17 14:28:30 - r - INFO: - Episode: 533/1000, Reward: 809.787, Step: 200 +2023-04-17 14:28:31 - r - INFO: - Episode: 534/1000, Reward: 1198.862, Step: 200 +2023-04-17 14:28:31 - r - INFO: - Episode: 535/1000, Reward: 1112.083, Step: 200 +2023-04-17 14:28:35 - r - INFO: - Episode: 536/1000, Reward: 27.220, Step: 200 +2023-04-17 14:28:36 - r - INFO: - Episode: 537/1000, Reward: 923.530, Step: 200 +2023-04-17 14:28:36 - r - INFO: - Episode: 538/1000, Reward: 1116.382, Step: 200 +2023-04-17 14:28:37 - r - INFO: - Episode: 539/1000, Reward: 1089.475, Step: 200 +2023-04-17 14:28:37 - r - INFO: - Episode: 540/1000, Reward: 1401.341, Step: 200 +2023-04-17 14:28:41 - r - INFO: - Episode: 541/1000, Reward: 1138.145, Step: 200 +2023-04-17 14:28:41 - r - INFO: - Episode: 542/1000, Reward: 851.590, Step: 200 +2023-04-17 14:28:42 - r - INFO: - Episode: 543/1000, Reward: 288.912, Step: 200 +2023-04-17 14:28:43 - r - INFO: - Episode: 544/1000, Reward: 1086.558, Step: 200 +2023-04-17 14:28:43 - r - INFO: - Episode: 545/1000, Reward: 1005.923, Step: 200 +2023-04-17 14:28:47 - r - INFO: - Episode: 546/1000, Reward: 1130.195, Step: 200 +2023-04-17 14:28:47 - r - INFO: - Episode: 547/1000, Reward: 1265.428, Step: 200 +2023-04-17 14:28:48 - r - INFO: - Episode: 548/1000, Reward: 66.692, Step: 200 +2023-04-17 14:28:48 - r - INFO: - Episode: 549/1000, Reward: 1311.940, Step: 200 +2023-04-17 14:28:49 - r - INFO: - Episode: 550/1000, Reward: 946.601, Step: 200 +2023-04-17 14:28:52 - r - INFO: - Current episode 550 has the best eval reward: 1120.160 +2023-04-17 14:28:53 - r - INFO: - Episode: 551/1000, Reward: 1162.878, Step: 200 +2023-04-17 14:28:54 - r - INFO: - Episode: 552/1000, Reward: 1257.157, Step: 200 +2023-04-17 14:28:54 - r - INFO: - Episode: 553/1000, Reward: 1015.252, Step: 200 +2023-04-17 14:28:55 - r - INFO: - Episode: 554/1000, Reward: 1312.464, Step: 200 +2023-04-17 14:28:55 - r - INFO: - Episode: 555/1000, Reward: 1345.138, Step: 200 +2023-04-17 14:28:58 - r - INFO: - Current episode 555 has the best eval reward: 1237.822 +2023-04-17 14:28:59 - r - INFO: - Episode: 556/1000, Reward: 1231.974, Step: 200 +2023-04-17 14:29:00 - r - INFO: - Episode: 557/1000, Reward: 1315.708, Step: 200 +2023-04-17 14:29:00 - r - INFO: - Episode: 558/1000, Reward: 900.950, Step: 200 +2023-04-17 14:29:01 - r - INFO: - Episode: 559/1000, Reward: 1122.070, Step: 200 +2023-04-17 14:29:02 - r - INFO: - Episode: 560/1000, Reward: 585.558, Step: 200 +2023-04-17 14:29:05 - r - INFO: - Episode: 561/1000, Reward: 1220.437, Step: 200 +2023-04-17 14:29:06 - r - INFO: - Episode: 562/1000, Reward: 1190.981, Step: 200 +2023-04-17 14:29:07 - r - INFO: - Episode: 563/1000, Reward: 178.020, Step: 200 +2023-04-17 14:29:07 - r - INFO: - Episode: 564/1000, Reward: 1312.676, Step: 200 +2023-04-17 14:29:08 - r - INFO: - Episode: 565/1000, Reward: 1294.659, Step: 200 +2023-04-17 14:29:11 - r - INFO: - Current episode 565 has the best eval reward: 1267.254 +2023-04-17 14:29:12 - r - INFO: - Episode: 566/1000, Reward: 1331.174, Step: 200 +2023-04-17 14:29:12 - r - INFO: - Episode: 567/1000, Reward: 1261.742, Step: 200 +2023-04-17 14:29:13 - r - INFO: - Episode: 568/1000, Reward: 1201.538, Step: 200 +2023-04-17 14:29:14 - r - INFO: - Episode: 569/1000, Reward: 1196.742, Step: 200 +2023-04-17 14:29:14 - r - INFO: - Episode: 570/1000, Reward: 1263.599, Step: 200 +2023-04-17 14:29:18 - r - INFO: - Episode: 571/1000, Reward: 1081.410, Step: 200 +2023-04-17 14:29:18 - r - INFO: - Episode: 572/1000, Reward: 1227.263, Step: 200 +2023-04-17 14:29:19 - r - INFO: - Episode: 573/1000, Reward: 1089.583, Step: 200 +2023-04-17 14:29:20 - r - INFO: - Episode: 574/1000, Reward: 1280.784, Step: 200 +2023-04-17 14:29:20 - r - INFO: - Episode: 575/1000, Reward: 1328.005, Step: 200 +2023-04-17 14:29:24 - r - INFO: - Episode: 576/1000, Reward: 776.235, Step: 200 +2023-04-17 14:29:24 - r - INFO: - Episode: 577/1000, Reward: 1063.882, Step: 200 +2023-04-17 14:29:25 - r - INFO: - Episode: 578/1000, Reward: 863.966, Step: 200 +2023-04-17 14:29:25 - r - INFO: - Episode: 579/1000, Reward: 1268.379, Step: 200 +2023-04-17 14:29:26 - r - INFO: - Episode: 580/1000, Reward: 1329.069, Step: 200 +2023-04-17 14:29:30 - r - INFO: - Episode: 581/1000, Reward: 1224.770, Step: 200 +2023-04-17 14:29:30 - r - INFO: - Episode: 582/1000, Reward: 1161.005, Step: 200 +2023-04-17 14:29:31 - r - INFO: - Episode: 583/1000, Reward: 1161.707, Step: 200 +2023-04-17 14:29:31 - r - INFO: - Episode: 584/1000, Reward: 1255.891, Step: 200 +2023-04-17 14:29:32 - r - INFO: - Episode: 585/1000, Reward: 1070.103, Step: 200 +2023-04-17 14:29:35 - r - INFO: - Episode: 586/1000, Reward: 1024.018, Step: 200 +2023-04-17 14:29:36 - r - INFO: - Episode: 587/1000, Reward: 1227.930, Step: 200 +2023-04-17 14:29:36 - r - INFO: - Episode: 588/1000, Reward: 997.664, Step: 200 +2023-04-17 14:29:37 - r - INFO: - Episode: 589/1000, Reward: 1068.597, Step: 200 +2023-04-17 14:29:38 - r - INFO: - Episode: 590/1000, Reward: 1232.995, Step: 200 +2023-04-17 14:29:41 - r - INFO: - Current episode 590 has the best eval reward: 1281.434 +2023-04-17 14:29:41 - r - INFO: - Episode: 591/1000, Reward: 1132.430, Step: 200 +2023-04-17 14:29:42 - r - INFO: - Episode: 592/1000, Reward: 1351.904, Step: 200 +2023-04-17 14:29:42 - r - INFO: - Episode: 593/1000, Reward: 1216.381, Step: 200 +2023-04-17 14:29:43 - r - INFO: - Episode: 594/1000, Reward: 1087.879, Step: 200 +2023-04-17 14:29:44 - r - INFO: - Episode: 595/1000, Reward: 630.317, Step: 200 +2023-04-17 14:29:47 - r - INFO: - Episode: 596/1000, Reward: 990.984, Step: 200 +2023-04-17 14:29:48 - r - INFO: - Episode: 597/1000, Reward: 1309.481, Step: 200 +2023-04-17 14:29:48 - r - INFO: - Episode: 598/1000, Reward: 1180.740, Step: 200 +2023-04-17 14:29:49 - r - INFO: - Episode: 599/1000, Reward: 1301.051, Step: 200 +2023-04-17 14:29:49 - r - INFO: - Episode: 600/1000, Reward: 1221.446, Step: 200 +2023-04-17 14:29:53 - r - INFO: - Episode: 601/1000, Reward: 1274.307, Step: 200 +2023-04-17 14:29:53 - r - INFO: - Episode: 602/1000, Reward: 1101.141, Step: 200 +2023-04-17 14:29:54 - r - INFO: - Episode: 603/1000, Reward: 1287.806, Step: 200 +2023-04-17 14:29:54 - r - INFO: - Episode: 604/1000, Reward: 736.060, Step: 200 +2023-04-17 14:29:55 - r - INFO: - Episode: 605/1000, Reward: 780.333, Step: 200 +2023-04-17 14:29:58 - r - INFO: - Episode: 606/1000, Reward: 1232.103, Step: 200 +2023-04-17 14:29:59 - r - INFO: - Episode: 607/1000, Reward: 1196.191, Step: 200 +2023-04-17 14:30:00 - r - INFO: - Episode: 608/1000, Reward: 1256.484, Step: 200 +2023-04-17 14:30:00 - r - INFO: - Episode: 609/1000, Reward: 1275.764, Step: 200 +2023-04-17 14:30:01 - r - INFO: - Episode: 610/1000, Reward: 1255.036, Step: 200 +2023-04-17 14:30:04 - r - INFO: - Episode: 611/1000, Reward: 1205.183, Step: 200 +2023-04-17 14:30:05 - r - INFO: - Episode: 612/1000, Reward: 1290.039, Step: 200 +2023-04-17 14:30:06 - r - INFO: - Episode: 613/1000, Reward: 1093.536, Step: 200 +2023-04-17 14:30:06 - r - INFO: - Episode: 614/1000, Reward: 1235.028, Step: 200 +2023-04-17 14:30:07 - r - INFO: - Episode: 615/1000, Reward: 1323.850, Step: 200 +2023-04-17 14:30:10 - r - INFO: - Current episode 615 has the best eval reward: 1290.598 +2023-04-17 14:30:10 - r - INFO: - Episode: 616/1000, Reward: 1154.888, Step: 200 +2023-04-17 14:30:11 - r - INFO: - Episode: 617/1000, Reward: 1149.666, Step: 200 +2023-04-17 14:30:11 - r - INFO: - Episode: 618/1000, Reward: 1264.052, Step: 200 +2023-04-17 14:30:12 - r - INFO: - Episode: 619/1000, Reward: 1195.736, Step: 200 +2023-04-17 14:30:12 - r - INFO: - Episode: 620/1000, Reward: 1310.498, Step: 200 +2023-04-17 14:30:16 - r - INFO: - Episode: 621/1000, Reward: 1303.922, Step: 200 +2023-04-17 14:30:16 - r - INFO: - Episode: 622/1000, Reward: 1374.958, Step: 200 +2023-04-17 14:30:17 - r - INFO: - Episode: 623/1000, Reward: 93.046, Step: 200 +2023-04-17 14:30:18 - r - INFO: - Episode: 624/1000, Reward: 1265.441, Step: 200 +2023-04-17 14:30:18 - r - INFO: - Episode: 625/1000, Reward: 1199.816, Step: 200 +2023-04-17 14:30:22 - r - INFO: - Episode: 626/1000, Reward: 1136.972, Step: 200 +2023-04-17 14:30:22 - r - INFO: - Episode: 627/1000, Reward: 1249.220, Step: 200 +2023-04-17 14:30:23 - r - INFO: - Episode: 628/1000, Reward: 1274.604, Step: 200 +2023-04-17 14:30:23 - r - INFO: - Episode: 629/1000, Reward: 1135.944, Step: 200 +2023-04-17 14:30:24 - r - INFO: - Episode: 630/1000, Reward: 1068.062, Step: 200 +2023-04-17 14:30:28 - r - INFO: - Episode: 631/1000, Reward: 1184.754, Step: 200 +2023-04-17 14:30:29 - r - INFO: - Episode: 632/1000, Reward: 1286.905, Step: 200 +2023-04-17 14:30:29 - r - INFO: - Episode: 633/1000, Reward: 669.256, Step: 200 +2023-04-17 14:30:30 - r - INFO: - Episode: 634/1000, Reward: 1307.452, Step: 200 +2023-04-17 14:30:30 - r - INFO: - Episode: 635/1000, Reward: 1335.907, Step: 200 +2023-04-17 14:30:34 - r - INFO: - Episode: 636/1000, Reward: 1331.636, Step: 200 +2023-04-17 14:30:34 - r - INFO: - Episode: 637/1000, Reward: 1188.516, Step: 200 +2023-04-17 14:30:35 - r - INFO: - Episode: 638/1000, Reward: 1245.262, Step: 200 +2023-04-17 14:30:36 - r - INFO: - Episode: 639/1000, Reward: 1348.340, Step: 200 +2023-04-17 14:30:36 - r - INFO: - Episode: 640/1000, Reward: 857.471, Step: 200 +2023-04-17 14:30:40 - r - INFO: - Episode: 641/1000, Reward: 1209.847, Step: 200 +2023-04-17 14:30:40 - r - INFO: - Episode: 642/1000, Reward: 1240.901, Step: 200 +2023-04-17 14:30:41 - r - INFO: - Episode: 643/1000, Reward: 1109.122, Step: 200 +2023-04-17 14:30:41 - r - INFO: - Episode: 644/1000, Reward: 963.117, Step: 200 +2023-04-17 14:30:42 - r - INFO: - Episode: 645/1000, Reward: 1274.807, Step: 200 +2023-04-17 14:30:45 - r - INFO: - Episode: 646/1000, Reward: 1050.752, Step: 200 +2023-04-17 14:30:46 - r - INFO: - Episode: 647/1000, Reward: 1394.579, Step: 200 +2023-04-17 14:30:46 - r - INFO: - Episode: 648/1000, Reward: 260.696, Step: 200 +2023-04-17 14:30:47 - r - INFO: - Episode: 649/1000, Reward: 1221.685, Step: 200 +2023-04-17 14:30:47 - r - INFO: - Episode: 650/1000, Reward: 1349.005, Step: 200 +2023-04-17 14:30:50 - r - INFO: - Episode: 651/1000, Reward: 1148.666, Step: 200 +2023-04-17 14:30:51 - r - INFO: - Episode: 652/1000, Reward: 1307.231, Step: 200 +2023-04-17 14:30:52 - r - INFO: - Episode: 653/1000, Reward: 1217.638, Step: 200 +2023-04-17 14:30:52 - r - INFO: - Episode: 654/1000, Reward: 1298.523, Step: 200 +2023-04-17 14:30:53 - r - INFO: - Episode: 655/1000, Reward: 973.733, Step: 200 +2023-04-17 14:30:56 - r - INFO: - Episode: 656/1000, Reward: 1313.456, Step: 200 +2023-04-17 14:30:56 - r - INFO: - Episode: 657/1000, Reward: 1319.007, Step: 200 +2023-04-17 14:30:57 - r - INFO: - Episode: 658/1000, Reward: 1189.774, Step: 200 +2023-04-17 14:30:57 - r - INFO: - Episode: 659/1000, Reward: 1284.631, Step: 200 +2023-04-17 14:30:58 - r - INFO: - Episode: 660/1000, Reward: 1194.366, Step: 200 +2023-04-17 14:31:01 - r - INFO: - Episode: 661/1000, Reward: 1351.083, Step: 200 +2023-04-17 14:31:01 - r - INFO: - Episode: 662/1000, Reward: 1124.721, Step: 200 +2023-04-17 14:31:02 - r - INFO: - Episode: 663/1000, Reward: 1281.672, Step: 200 +2023-04-17 14:31:02 - r - INFO: - Episode: 664/1000, Reward: 17.134, Step: 200 +2023-04-17 14:31:03 - r - INFO: - Episode: 665/1000, Reward: 1307.462, Step: 200 +2023-04-17 14:31:06 - r - INFO: - Episode: 666/1000, Reward: 1270.906, Step: 200 +2023-04-17 14:31:06 - r - INFO: - Episode: 667/1000, Reward: 592.865, Step: 200 +2023-04-17 14:31:07 - r - INFO: - Episode: 668/1000, Reward: 1371.337, Step: 200 +2023-04-17 14:31:08 - r - INFO: - Episode: 669/1000, Reward: 1283.597, Step: 200 +2023-04-17 14:31:08 - r - INFO: - Episode: 670/1000, Reward: 1230.348, Step: 200 +2023-04-17 14:31:11 - r - INFO: - Episode: 671/1000, Reward: -14.820, Step: 200 +2023-04-17 14:31:12 - r - INFO: - Episode: 672/1000, Reward: 944.524, Step: 200 +2023-04-17 14:31:13 - r - INFO: - Episode: 673/1000, Reward: 955.326, Step: 200 +2023-04-17 14:31:13 - r - INFO: - Episode: 674/1000, Reward: 1201.171, Step: 200 +2023-04-17 14:31:14 - r - INFO: - Episode: 675/1000, Reward: 1182.224, Step: 200 +2023-04-17 14:31:17 - r - INFO: - Episode: 676/1000, Reward: 1288.239, Step: 200 +2023-04-17 14:31:18 - r - INFO: - Episode: 677/1000, Reward: 826.745, Step: 200 +2023-04-17 14:31:18 - r - INFO: - Episode: 678/1000, Reward: 1295.044, Step: 200 +2023-04-17 14:31:19 - r - INFO: - Episode: 679/1000, Reward: 867.348, Step: 200 +2023-04-17 14:31:19 - r - INFO: - Episode: 680/1000, Reward: 662.400, Step: 200 +2023-04-17 14:31:23 - r - INFO: - Episode: 681/1000, Reward: 1303.082, Step: 200 +2023-04-17 14:31:24 - r - INFO: - Episode: 682/1000, Reward: 492.349, Step: 200 +2023-04-17 14:31:24 - r - INFO: - Episode: 683/1000, Reward: 1276.593, Step: 200 +2023-04-17 14:31:25 - r - INFO: - Episode: 684/1000, Reward: 1084.811, Step: 200 +2023-04-17 14:31:26 - r - INFO: - Episode: 685/1000, Reward: 1373.639, Step: 200 +2023-04-17 14:31:29 - r - INFO: - Current episode 685 has the best eval reward: 1296.914 +2023-04-17 14:31:29 - r - INFO: - Episode: 686/1000, Reward: 1326.463, Step: 200 +2023-04-17 14:31:30 - r - INFO: - Episode: 687/1000, Reward: 1260.727, Step: 200 +2023-04-17 14:31:31 - r - INFO: - Episode: 688/1000, Reward: 1163.930, Step: 200 +2023-04-17 14:31:31 - r - INFO: - Episode: 689/1000, Reward: 1310.915, Step: 200 +2023-04-17 14:31:32 - r - INFO: - Episode: 690/1000, Reward: 1346.887, Step: 200 +2023-04-17 14:31:35 - r - INFO: - Episode: 691/1000, Reward: 1331.971, Step: 200 +2023-04-17 14:31:36 - r - INFO: - Episode: 692/1000, Reward: 1355.312, Step: 200 +2023-04-17 14:31:36 - r - INFO: - Episode: 693/1000, Reward: 216.866, Step: 200 +2023-04-17 14:31:37 - r - INFO: - Episode: 694/1000, Reward: 1220.595, Step: 200 +2023-04-17 14:31:37 - r - INFO: - Episode: 695/1000, Reward: 1257.603, Step: 200 +2023-04-17 14:31:41 - r - INFO: - Episode: 696/1000, Reward: 1279.505, Step: 200 +2023-04-17 14:31:42 - r - INFO: - Episode: 697/1000, Reward: 1243.701, Step: 200 +2023-04-17 14:31:42 - r - INFO: - Episode: 698/1000, Reward: 1301.858, Step: 200 +2023-04-17 14:31:43 - r - INFO: - Episode: 699/1000, Reward: 1319.853, Step: 200 +2023-04-17 14:31:43 - r - INFO: - Episode: 700/1000, Reward: 1315.377, Step: 200 +2023-04-17 14:31:47 - r - INFO: - Episode: 701/1000, Reward: 1343.651, Step: 200 +2023-04-17 14:31:47 - r - INFO: - Episode: 702/1000, Reward: 1298.581, Step: 200 +2023-04-17 14:31:48 - r - INFO: - Episode: 703/1000, Reward: 1390.105, Step: 200 +2023-04-17 14:31:48 - r - INFO: - Episode: 704/1000, Reward: 1081.106, Step: 200 +2023-04-17 14:31:49 - r - INFO: - Episode: 705/1000, Reward: 1274.281, Step: 200 +2023-04-17 14:31:52 - r - INFO: - Episode: 706/1000, Reward: 1301.908, Step: 200 +2023-04-17 14:31:53 - r - INFO: - Episode: 707/1000, Reward: 818.885, Step: 200 +2023-04-17 14:31:53 - r - INFO: - Episode: 708/1000, Reward: 1267.789, Step: 200 +2023-04-17 14:31:54 - r - INFO: - Episode: 709/1000, Reward: 1195.448, Step: 200 +2023-04-17 14:31:54 - r - INFO: - Episode: 710/1000, Reward: 1272.118, Step: 200 +2023-04-17 14:31:58 - r - INFO: - Episode: 711/1000, Reward: 1347.728, Step: 200 +2023-04-17 14:31:58 - r - INFO: - Episode: 712/1000, Reward: 1348.907, Step: 200 +2023-04-17 14:31:59 - r - INFO: - Episode: 713/1000, Reward: 1222.896, Step: 200 +2023-04-17 14:32:00 - r - INFO: - Episode: 714/1000, Reward: 1044.141, Step: 200 +2023-04-17 14:32:00 - r - INFO: - Episode: 715/1000, Reward: 1011.024, Step: 200 +2023-04-17 14:32:04 - r - INFO: - Episode: 716/1000, Reward: 1400.958, Step: 200 +2023-04-17 14:32:04 - r - INFO: - Episode: 717/1000, Reward: 1422.429, Step: 200 +2023-04-17 14:32:05 - r - INFO: - Episode: 718/1000, Reward: 1307.193, Step: 200 +2023-04-17 14:32:05 - r - INFO: - Episode: 719/1000, Reward: 1215.926, Step: 200 +2023-04-17 14:32:06 - r - INFO: - Episode: 720/1000, Reward: 571.403, Step: 200 +2023-04-17 14:32:09 - r - INFO: - Episode: 721/1000, Reward: 1188.454, Step: 200 +2023-04-17 14:32:10 - r - INFO: - Episode: 722/1000, Reward: 1255.989, Step: 200 +2023-04-17 14:32:11 - r - INFO: - Episode: 723/1000, Reward: 1302.865, Step: 200 +2023-04-17 14:32:11 - r - INFO: - Episode: 724/1000, Reward: 1368.529, Step: 200 +2023-04-17 14:32:12 - r - INFO: - Episode: 725/1000, Reward: 1344.853, Step: 200 +2023-04-17 14:32:15 - r - INFO: - Current episode 725 has the best eval reward: 1326.194 +2023-04-17 14:32:15 - r - INFO: - Episode: 726/1000, Reward: 1148.115, Step: 200 +2023-04-17 14:32:16 - r - INFO: - Episode: 727/1000, Reward: 1325.442, Step: 200 +2023-04-17 14:32:17 - r - INFO: - Episode: 728/1000, Reward: 904.671, Step: 200 +2023-04-17 14:32:17 - r - INFO: - Episode: 729/1000, Reward: 1289.060, Step: 200 +2023-04-17 14:32:18 - r - INFO: - Episode: 730/1000, Reward: 1253.961, Step: 200 +2023-04-17 14:32:22 - r - INFO: - Episode: 731/1000, Reward: 1263.881, Step: 200 +2023-04-17 14:32:22 - r - INFO: - Episode: 732/1000, Reward: 1308.113, Step: 200 +2023-04-17 14:32:23 - r - INFO: - Episode: 733/1000, Reward: 1282.323, Step: 200 +2023-04-17 14:32:23 - r - INFO: - Episode: 734/1000, Reward: 1333.625, Step: 200 +2023-04-17 14:32:24 - r - INFO: - Episode: 735/1000, Reward: 1295.069, Step: 200 +2023-04-17 14:32:27 - r - INFO: - Episode: 736/1000, Reward: 1246.314, Step: 200 +2023-04-17 14:32:28 - r - INFO: - Episode: 737/1000, Reward: 434.225, Step: 200 +2023-04-17 14:32:29 - r - INFO: - Episode: 738/1000, Reward: 1162.501, Step: 200 +2023-04-17 14:32:29 - r - INFO: - Episode: 739/1000, Reward: 1035.620, Step: 200 +2023-04-17 14:32:30 - r - INFO: - Episode: 740/1000, Reward: 205.288, Step: 200 +2023-04-17 14:32:33 - r - INFO: - Episode: 741/1000, Reward: 1264.967, Step: 200 +2023-04-17 14:32:33 - r - INFO: - Episode: 742/1000, Reward: 317.111, Step: 200 +2023-04-17 14:32:34 - r - INFO: - Episode: 743/1000, Reward: 1137.949, Step: 200 +2023-04-17 14:32:35 - r - INFO: - Episode: 744/1000, Reward: 1264.100, Step: 200 +2023-04-17 14:32:35 - r - INFO: - Episode: 745/1000, Reward: 1307.818, Step: 200 +2023-04-17 14:32:39 - r - INFO: - Episode: 746/1000, Reward: 1325.603, Step: 200 +2023-04-17 14:32:39 - r - INFO: - Episode: 747/1000, Reward: 1237.911, Step: 200 +2023-04-17 14:32:40 - r - INFO: - Episode: 748/1000, Reward: 769.344, Step: 200 +2023-04-17 14:32:40 - r - INFO: - Episode: 749/1000, Reward: 917.954, Step: 200 +2023-04-17 14:32:41 - r - INFO: - Episode: 750/1000, Reward: 1391.419, Step: 200 +2023-04-17 14:32:44 - r - INFO: - Episode: 751/1000, Reward: 1225.399, Step: 200 +2023-04-17 14:32:45 - r - INFO: - Episode: 752/1000, Reward: 1223.488, Step: 200 +2023-04-17 14:32:45 - r - INFO: - Episode: 753/1000, Reward: 1308.513, Step: 200 +2023-04-17 14:32:46 - r - INFO: - Episode: 754/1000, Reward: 1182.427, Step: 200 +2023-04-17 14:32:47 - r - INFO: - Episode: 755/1000, Reward: 1205.518, Step: 200 +2023-04-17 14:32:50 - r - INFO: - Episode: 756/1000, Reward: 1134.959, Step: 200 +2023-04-17 14:32:50 - r - INFO: - Episode: 757/1000, Reward: 1205.077, Step: 200 +2023-04-17 14:32:51 - r - INFO: - Episode: 758/1000, Reward: 28.444, Step: 200 +2023-04-17 14:32:52 - r - INFO: - Episode: 759/1000, Reward: 1170.101, Step: 200 +2023-04-17 14:32:52 - r - INFO: - Episode: 760/1000, Reward: 1073.834, Step: 200 +2023-04-17 14:32:56 - r - INFO: - Episode: 761/1000, Reward: 1209.624, Step: 200 +2023-04-17 14:32:56 - r - INFO: - Episode: 762/1000, Reward: 1332.656, Step: 200 +2023-04-17 14:32:57 - r - INFO: - Episode: 763/1000, Reward: 1324.009, Step: 200 +2023-04-17 14:32:57 - r - INFO: - Episode: 764/1000, Reward: 1300.299, Step: 200 +2023-04-17 14:32:58 - r - INFO: - Episode: 765/1000, Reward: 1471.591, Step: 200 +2023-04-17 14:33:01 - r - INFO: - Episode: 766/1000, Reward: 1102.395, Step: 200 +2023-04-17 14:33:02 - r - INFO: - Episode: 767/1000, Reward: 303.052, Step: 200 +2023-04-17 14:33:02 - r - INFO: - Episode: 768/1000, Reward: 1308.734, Step: 200 +2023-04-17 14:33:03 - r - INFO: - Episode: 769/1000, Reward: 1309.907, Step: 200 +2023-04-17 14:33:04 - r - INFO: - Episode: 770/1000, Reward: 1144.994, Step: 200 +2023-04-17 14:33:07 - r - INFO: - Episode: 771/1000, Reward: 1313.509, Step: 200 +2023-04-17 14:33:08 - r - INFO: - Episode: 772/1000, Reward: 1193.621, Step: 200 +2023-04-17 14:33:08 - r - INFO: - Episode: 773/1000, Reward: 1213.907, Step: 200 +2023-04-17 14:33:09 - r - INFO: - Episode: 774/1000, Reward: 1315.015, Step: 200 +2023-04-17 14:33:09 - r - INFO: - Episode: 775/1000, Reward: 1223.367, Step: 200 +2023-04-17 14:33:12 - r - INFO: - Episode: 776/1000, Reward: 1264.269, Step: 200 +2023-04-17 14:33:13 - r - INFO: - Episode: 777/1000, Reward: 1263.852, Step: 200 +2023-04-17 14:33:13 - r - INFO: - Episode: 778/1000, Reward: 1200.551, Step: 200 +2023-04-17 14:33:14 - r - INFO: - Episode: 779/1000, Reward: 1293.500, Step: 200 +2023-04-17 14:33:15 - r - INFO: - Episode: 780/1000, Reward: 1342.546, Step: 200 +2023-04-17 14:33:18 - r - INFO: - Episode: 781/1000, Reward: 878.710, Step: 200 +2023-04-17 14:33:18 - r - INFO: - Episode: 782/1000, Reward: 893.495, Step: 200 +2023-04-17 14:33:19 - r - INFO: - Episode: 783/1000, Reward: 1216.989, Step: 200 +2023-04-17 14:33:20 - r - INFO: - Episode: 784/1000, Reward: 1351.573, Step: 200 +2023-04-17 14:33:20 - r - INFO: - Episode: 785/1000, Reward: 1201.242, Step: 200 +2023-04-17 14:33:23 - r - INFO: - Episode: 786/1000, Reward: 1253.618, Step: 200 +2023-04-17 14:33:24 - r - INFO: - Episode: 787/1000, Reward: 1363.773, Step: 200 +2023-04-17 14:33:25 - r - INFO: - Episode: 788/1000, Reward: 1295.530, Step: 200 +2023-04-17 14:33:25 - r - INFO: - Episode: 789/1000, Reward: 830.562, Step: 200 +2023-04-17 14:33:26 - r - INFO: - Episode: 790/1000, Reward: 1273.959, Step: 200 +2023-04-17 14:33:29 - r - INFO: - Episode: 791/1000, Reward: 1363.947, Step: 200 +2023-04-17 14:33:30 - r - INFO: - Episode: 792/1000, Reward: 1424.507, Step: 200 +2023-04-17 14:33:30 - r - INFO: - Episode: 793/1000, Reward: 1228.585, Step: 200 +2023-04-17 14:33:31 - r - INFO: - Episode: 794/1000, Reward: 783.413, Step: 200 +2023-04-17 14:33:31 - r - INFO: - Episode: 795/1000, Reward: 1138.351, Step: 200 +2023-04-17 14:33:35 - r - INFO: - Episode: 796/1000, Reward: 1378.823, Step: 200 +2023-04-17 14:33:35 - r - INFO: - Episode: 797/1000, Reward: 1124.297, Step: 200 +2023-04-17 14:33:36 - r - INFO: - Episode: 798/1000, Reward: 1314.003, Step: 200 +2023-04-17 14:33:36 - r - INFO: - Episode: 799/1000, Reward: 1306.236, Step: 200 +2023-04-17 14:33:37 - r - INFO: - Episode: 800/1000, Reward: -8.030, Step: 200 +2023-04-17 14:33:40 - r - INFO: - Episode: 801/1000, Reward: 1261.650, Step: 200 +2023-04-17 14:33:41 - r - INFO: - Episode: 802/1000, Reward: 1204.928, Step: 200 +2023-04-17 14:33:42 - r - INFO: - Episode: 803/1000, Reward: 1317.253, Step: 200 +2023-04-17 14:33:42 - r - INFO: - Episode: 804/1000, Reward: 373.206, Step: 200 +2023-04-17 14:33:43 - r - INFO: - Episode: 805/1000, Reward: 1308.468, Step: 200 +2023-04-17 14:33:46 - r - INFO: - Episode: 806/1000, Reward: 1320.623, Step: 200 +2023-04-17 14:33:47 - r - INFO: - Episode: 807/1000, Reward: 1312.408, Step: 200 +2023-04-17 14:33:48 - r - INFO: - Episode: 808/1000, Reward: 218.550, Step: 200 +2023-04-17 14:33:48 - r - INFO: - Episode: 809/1000, Reward: 1361.010, Step: 200 +2023-04-17 14:33:49 - r - INFO: - Episode: 810/1000, Reward: 1312.133, Step: 200 +2023-04-17 14:33:52 - r - INFO: - Episode: 811/1000, Reward: 1285.328, Step: 200 +2023-04-17 14:33:53 - r - INFO: - Episode: 812/1000, Reward: 1349.134, Step: 200 +2023-04-17 14:33:53 - r - INFO: - Episode: 813/1000, Reward: 1313.963, Step: 200 +2023-04-17 14:33:54 - r - INFO: - Episode: 814/1000, Reward: 1278.172, Step: 200 +2023-04-17 14:33:54 - r - INFO: - Episode: 815/1000, Reward: 1387.267, Step: 200 +2023-04-17 14:33:58 - r - INFO: - Episode: 816/1000, Reward: 1358.309, Step: 200 +2023-04-17 14:33:58 - r - INFO: - Episode: 817/1000, Reward: 1291.117, Step: 200 +2023-04-17 14:33:59 - r - INFO: - Episode: 818/1000, Reward: 1364.458, Step: 200 +2023-04-17 14:34:00 - r - INFO: - Episode: 819/1000, Reward: 1285.375, Step: 200 +2023-04-17 14:34:00 - r - INFO: - Episode: 820/1000, Reward: 1237.397, Step: 200 +2023-04-17 14:34:04 - r - INFO: - Episode: 821/1000, Reward: 433.823, Step: 200 +2023-04-17 14:34:04 - r - INFO: - Episode: 822/1000, Reward: 1247.615, Step: 200 +2023-04-17 14:34:05 - r - INFO: - Episode: 823/1000, Reward: 1297.220, Step: 200 +2023-04-17 14:34:05 - r - INFO: - Episode: 824/1000, Reward: 1374.596, Step: 200 +2023-04-17 14:34:06 - r - INFO: - Episode: 825/1000, Reward: 1353.748, Step: 200 +2023-04-17 14:34:09 - r - INFO: - Episode: 826/1000, Reward: 1363.126, Step: 200 +2023-04-17 14:34:10 - r - INFO: - Episode: 827/1000, Reward: 1375.929, Step: 200 +2023-04-17 14:34:10 - r - INFO: - Episode: 828/1000, Reward: 1261.021, Step: 200 +2023-04-17 14:34:11 - r - INFO: - Episode: 829/1000, Reward: 1254.114, Step: 200 +2023-04-17 14:34:11 - r - INFO: - Episode: 830/1000, Reward: 1257.492, Step: 200 +2023-04-17 14:34:15 - r - INFO: - Episode: 831/1000, Reward: 1198.346, Step: 200 +2023-04-17 14:34:15 - r - INFO: - Episode: 832/1000, Reward: 1251.994, Step: 200 +2023-04-17 14:34:16 - r - INFO: - Episode: 833/1000, Reward: 1171.787, Step: 200 +2023-04-17 14:34:17 - r - INFO: - Episode: 834/1000, Reward: 1234.845, Step: 200 +2023-04-17 14:34:17 - r - INFO: - Episode: 835/1000, Reward: 1219.072, Step: 200 +2023-04-17 14:34:20 - r - INFO: - Episode: 836/1000, Reward: 1268.738, Step: 200 +2023-04-17 14:34:21 - r - INFO: - Episode: 837/1000, Reward: 1186.579, Step: 200 +2023-04-17 14:34:21 - r - INFO: - Episode: 838/1000, Reward: 1120.019, Step: 200 +2023-04-17 14:34:22 - r - INFO: - Episode: 839/1000, Reward: 1237.924, Step: 200 +2023-04-17 14:34:23 - r - INFO: - Episode: 840/1000, Reward: 1199.324, Step: 200 +2023-04-17 14:34:26 - r - INFO: - Episode: 841/1000, Reward: 1117.867, Step: 200 +2023-04-17 14:34:27 - r - INFO: - Episode: 842/1000, Reward: 1316.922, Step: 200 +2023-04-17 14:34:27 - r - INFO: - Episode: 843/1000, Reward: 1255.758, Step: 200 +2023-04-17 14:34:28 - r - INFO: - Episode: 844/1000, Reward: 1278.911, Step: 200 +2023-04-17 14:34:28 - r - INFO: - Episode: 845/1000, Reward: 1152.790, Step: 200 +2023-04-17 14:34:32 - r - INFO: - Episode: 846/1000, Reward: 1231.821, Step: 200 +2023-04-17 14:34:32 - r - INFO: - Episode: 847/1000, Reward: 1219.756, Step: 200 +2023-04-17 14:34:33 - r - INFO: - Episode: 848/1000, Reward: 1279.726, Step: 200 +2023-04-17 14:34:33 - r - INFO: - Episode: 849/1000, Reward: 647.142, Step: 200 +2023-04-17 14:34:34 - r - INFO: - Episode: 850/1000, Reward: 1220.165, Step: 200 +2023-04-17 14:34:38 - r - INFO: - Episode: 851/1000, Reward: 500.702, Step: 200 +2023-04-17 14:34:38 - r - INFO: - Episode: 852/1000, Reward: 1270.383, Step: 200 +2023-04-17 14:34:39 - r - INFO: - Episode: 853/1000, Reward: 1281.054, Step: 200 +2023-04-17 14:34:40 - r - INFO: - Episode: 854/1000, Reward: 1358.648, Step: 200 +2023-04-17 14:34:40 - r - INFO: - Episode: 855/1000, Reward: 1365.848, Step: 200 +2023-04-17 14:34:44 - r - INFO: - Episode: 856/1000, Reward: 1377.216, Step: 200 +2023-04-17 14:34:44 - r - INFO: - Episode: 857/1000, Reward: 1203.801, Step: 200 +2023-04-17 14:34:45 - r - INFO: - Episode: 858/1000, Reward: 1320.780, Step: 200 +2023-04-17 14:34:46 - r - INFO: - Episode: 859/1000, Reward: 1044.309, Step: 200 +2023-04-17 14:34:46 - r - INFO: - Episode: 860/1000, Reward: 1292.777, Step: 200 +2023-04-17 14:34:50 - r - INFO: - Episode: 861/1000, Reward: 1332.000, Step: 200 +2023-04-17 14:34:50 - r - INFO: - Episode: 862/1000, Reward: 1103.872, Step: 200 +2023-04-17 14:34:51 - r - INFO: - Episode: 863/1000, Reward: 1273.461, Step: 200 +2023-04-17 14:34:52 - r - INFO: - Episode: 864/1000, Reward: 1111.698, Step: 200 +2023-04-17 14:34:52 - r - INFO: - Episode: 865/1000, Reward: 1293.764, Step: 200 +2023-04-17 14:34:56 - r - INFO: - Episode: 866/1000, Reward: 1295.572, Step: 200 +2023-04-17 14:34:57 - r - INFO: - Episode: 867/1000, Reward: 1227.364, Step: 200 +2023-04-17 14:34:57 - r - INFO: - Episode: 868/1000, Reward: 850.904, Step: 200 +2023-04-17 14:34:58 - r - INFO: - Episode: 869/1000, Reward: 1205.854, Step: 200 +2023-04-17 14:34:58 - r - INFO: - Episode: 870/1000, Reward: 1139.253, Step: 200 +2023-04-17 14:35:02 - r - INFO: - Episode: 871/1000, Reward: 1245.345, Step: 200 +2023-04-17 14:35:03 - r - INFO: - Episode: 872/1000, Reward: 1053.338, Step: 200 +2023-04-17 14:35:03 - r - INFO: - Episode: 873/1000, Reward: 1317.996, Step: 200 +2023-04-17 14:35:04 - r - INFO: - Episode: 874/1000, Reward: 1334.646, Step: 200 +2023-04-17 14:35:04 - r - INFO: - Episode: 875/1000, Reward: 1283.764, Step: 200 +2023-04-17 14:35:08 - r - INFO: - Episode: 876/1000, Reward: 1414.084, Step: 200 +2023-04-17 14:35:08 - r - INFO: - Episode: 877/1000, Reward: 1305.049, Step: 200 +2023-04-17 14:35:09 - r - INFO: - Episode: 878/1000, Reward: 1212.828, Step: 200 +2023-04-17 14:35:09 - r - INFO: - Episode: 879/1000, Reward: 1297.015, Step: 200 +2023-04-17 14:35:10 - r - INFO: - Episode: 880/1000, Reward: 1343.653, Step: 200 +2023-04-17 14:35:14 - r - INFO: - Episode: 881/1000, Reward: 1292.702, Step: 200 +2023-04-17 14:35:14 - r - INFO: - Episode: 882/1000, Reward: 1415.727, Step: 200 +2023-04-17 14:35:15 - r - INFO: - Episode: 883/1000, Reward: 1264.775, Step: 200 +2023-04-17 14:35:15 - r - INFO: - Episode: 884/1000, Reward: 1024.746, Step: 200 +2023-04-17 14:35:16 - r - INFO: - Episode: 885/1000, Reward: 996.594, Step: 200 +2023-04-17 14:35:19 - r - INFO: - Episode: 886/1000, Reward: 1285.247, Step: 200 +2023-04-17 14:35:20 - r - INFO: - Episode: 887/1000, Reward: 1256.301, Step: 200 +2023-04-17 14:35:20 - r - INFO: - Episode: 888/1000, Reward: 1276.012, Step: 200 +2023-04-17 14:35:21 - r - INFO: - Episode: 889/1000, Reward: 1331.189, Step: 200 +2023-04-17 14:35:22 - r - INFO: - Episode: 890/1000, Reward: 1094.204, Step: 200 +2023-04-17 14:35:25 - r - INFO: - Episode: 891/1000, Reward: 1155.441, Step: 200 +2023-04-17 14:35:26 - r - INFO: - Episode: 892/1000, Reward: 1360.832, Step: 200 +2023-04-17 14:35:26 - r - INFO: - Episode: 893/1000, Reward: 1305.256, Step: 200 +2023-04-17 14:35:27 - r - INFO: - Episode: 894/1000, Reward: 1369.040, Step: 200 +2023-04-17 14:35:27 - r - INFO: - Episode: 895/1000, Reward: 1094.902, Step: 200 +2023-04-17 14:35:31 - r - INFO: - Episode: 896/1000, Reward: 260.662, Step: 200 +2023-04-17 14:35:32 - r - INFO: - Episode: 897/1000, Reward: 1292.906, Step: 200 +2023-04-17 14:35:32 - r - INFO: - Episode: 898/1000, Reward: 1140.988, Step: 200 +2023-04-17 14:35:33 - r - INFO: - Episode: 899/1000, Reward: 1228.286, Step: 200 +2023-04-17 14:35:34 - r - INFO: - Episode: 900/1000, Reward: 1314.350, Step: 200 +2023-04-17 14:35:37 - r - INFO: - Episode: 901/1000, Reward: 1335.637, Step: 200 +2023-04-17 14:35:38 - r - INFO: - Episode: 902/1000, Reward: 1347.558, Step: 200 +2023-04-17 14:35:38 - r - INFO: - Episode: 903/1000, Reward: 1340.510, Step: 200 +2023-04-17 14:35:39 - r - INFO: - Episode: 904/1000, Reward: 1327.608, Step: 200 +2023-04-17 14:35:40 - r - INFO: - Episode: 905/1000, Reward: 1294.383, Step: 200 +2023-04-17 14:35:43 - r - INFO: - Episode: 906/1000, Reward: 1255.816, Step: 200 +2023-04-17 14:35:44 - r - INFO: - Episode: 907/1000, Reward: 1376.404, Step: 200 +2023-04-17 14:35:44 - r - INFO: - Episode: 908/1000, Reward: 1343.875, Step: 200 +2023-04-17 14:35:45 - r - INFO: - Episode: 909/1000, Reward: 1349.540, Step: 200 +2023-04-17 14:35:46 - r - INFO: - Episode: 910/1000, Reward: 1298.711, Step: 200 +2023-04-17 14:35:49 - r - INFO: - Episode: 911/1000, Reward: 1295.301, Step: 200 +2023-04-17 14:35:50 - r - INFO: - Episode: 912/1000, Reward: 1356.655, Step: 200 +2023-04-17 14:35:50 - r - INFO: - Episode: 913/1000, Reward: 1410.249, Step: 200 +2023-04-17 14:35:51 - r - INFO: - Episode: 914/1000, Reward: 1265.445, Step: 200 +2023-04-17 14:35:51 - r - INFO: - Episode: 915/1000, Reward: 1418.927, Step: 200 +2023-04-17 14:35:55 - r - INFO: - Episode: 916/1000, Reward: 1363.484, Step: 200 +2023-04-17 14:35:55 - r - INFO: - Episode: 917/1000, Reward: 1439.892, Step: 200 +2023-04-17 14:35:56 - r - INFO: - Episode: 918/1000, Reward: 1300.727, Step: 200 +2023-04-17 14:35:57 - r - INFO: - Episode: 919/1000, Reward: 1238.133, Step: 200 +2023-04-17 14:35:57 - r - INFO: - Episode: 920/1000, Reward: 1358.927, Step: 200 +2023-04-17 14:36:01 - r - INFO: - Episode: 921/1000, Reward: 1365.792, Step: 200 +2023-04-17 14:36:01 - r - INFO: - Episode: 922/1000, Reward: 1352.012, Step: 200 +2023-04-17 14:36:02 - r - INFO: - Episode: 923/1000, Reward: 1079.368, Step: 200 +2023-04-17 14:36:02 - r - INFO: - Episode: 924/1000, Reward: 1288.186, Step: 200 +2023-04-17 14:36:03 - r - INFO: - Episode: 925/1000, Reward: 1315.143, Step: 200 +2023-04-17 14:36:06 - r - INFO: - Episode: 926/1000, Reward: 1214.652, Step: 200 +2023-04-17 14:36:07 - r - INFO: - Episode: 927/1000, Reward: 1358.633, Step: 200 +2023-04-17 14:36:07 - r - INFO: - Episode: 928/1000, Reward: 1166.570, Step: 200 +2023-04-17 14:36:08 - r - INFO: - Episode: 929/1000, Reward: 1281.574, Step: 200 +2023-04-17 14:36:09 - r - INFO: - Episode: 930/1000, Reward: 1258.010, Step: 200 +2023-04-17 14:36:12 - r - INFO: - Episode: 931/1000, Reward: 1278.305, Step: 200 +2023-04-17 14:36:13 - r - INFO: - Episode: 932/1000, Reward: 1333.524, Step: 200 +2023-04-17 14:36:14 - r - INFO: - Episode: 933/1000, Reward: 1127.434, Step: 200 +2023-04-17 14:36:14 - r - INFO: - Episode: 934/1000, Reward: 1371.973, Step: 200 +2023-04-17 14:36:15 - r - INFO: - Episode: 935/1000, Reward: 617.937, Step: 200 +2023-04-17 14:36:18 - r - INFO: - Episode: 936/1000, Reward: 1208.974, Step: 200 +2023-04-17 14:36:19 - r - INFO: - Episode: 937/1000, Reward: 1267.401, Step: 200 +2023-04-17 14:36:19 - r - INFO: - Episode: 938/1000, Reward: 1338.556, Step: 200 +2023-04-17 14:36:20 - r - INFO: - Episode: 939/1000, Reward: 1200.754, Step: 200 +2023-04-17 14:36:21 - r - INFO: - Episode: 940/1000, Reward: 1337.489, Step: 200 +2023-04-17 14:36:24 - r - INFO: - Episode: 941/1000, Reward: 1109.018, Step: 200 +2023-04-17 14:36:25 - r - INFO: - Episode: 942/1000, Reward: 946.854, Step: 200 +2023-04-17 14:36:26 - r - INFO: - Episode: 943/1000, Reward: 377.449, Step: 200 +2023-04-17 14:36:26 - r - INFO: - Episode: 944/1000, Reward: 1150.810, Step: 200 +2023-04-17 14:36:27 - r - INFO: - Episode: 945/1000, Reward: 1400.813, Step: 200 +2023-04-17 14:36:30 - r - INFO: - Current episode 945 has the best eval reward: 1328.293 +2023-04-17 14:36:30 - r - INFO: - Episode: 946/1000, Reward: 1274.516, Step: 200 +2023-04-17 14:36:31 - r - INFO: - Episode: 947/1000, Reward: 1225.964, Step: 200 +2023-04-17 14:36:32 - r - INFO: - Episode: 948/1000, Reward: 1360.490, Step: 200 +2023-04-17 14:36:32 - r - INFO: - Episode: 949/1000, Reward: 1207.276, Step: 200 +2023-04-17 14:36:33 - r - INFO: - Episode: 950/1000, Reward: 1248.215, Step: 200 +2023-04-17 14:36:37 - r - INFO: - Episode: 951/1000, Reward: 1419.342, Step: 200 +2023-04-17 14:36:37 - r - INFO: - Episode: 952/1000, Reward: 1369.437, Step: 200 +2023-04-17 14:36:38 - r - INFO: - Episode: 953/1000, Reward: 1273.212, Step: 200 +2023-04-17 14:36:38 - r - INFO: - Episode: 954/1000, Reward: 1348.288, Step: 200 +2023-04-17 14:36:39 - r - INFO: - Episode: 955/1000, Reward: 864.327, Step: 200 +2023-04-17 14:36:42 - r - INFO: - Episode: 956/1000, Reward: 1250.877, Step: 200 +2023-04-17 14:36:43 - r - INFO: - Episode: 957/1000, Reward: 1365.514, Step: 200 +2023-04-17 14:36:43 - r - INFO: - Episode: 958/1000, Reward: 1312.808, Step: 200 +2023-04-17 14:36:44 - r - INFO: - Episode: 959/1000, Reward: 1102.645, Step: 200 +2023-04-17 14:36:45 - r - INFO: - Episode: 960/1000, Reward: 1323.509, Step: 200 +2023-04-17 14:36:48 - r - INFO: - Episode: 961/1000, Reward: 1287.600, Step: 200 +2023-04-17 14:36:49 - r - INFO: - Episode: 962/1000, Reward: 1179.983, Step: 200 +2023-04-17 14:36:49 - r - INFO: - Episode: 963/1000, Reward: 1248.916, Step: 200 +2023-04-17 14:36:50 - r - INFO: - Episode: 964/1000, Reward: 1281.686, Step: 200 +2023-04-17 14:36:50 - r - INFO: - Episode: 965/1000, Reward: 1327.644, Step: 200 +2023-04-17 14:36:54 - r - INFO: - Episode: 966/1000, Reward: 1240.872, Step: 200 +2023-04-17 14:36:55 - r - INFO: - Episode: 967/1000, Reward: 1325.715, Step: 200 +2023-04-17 14:36:55 - r - INFO: - Episode: 968/1000, Reward: 1264.296, Step: 200 +2023-04-17 14:36:56 - r - INFO: - Episode: 969/1000, Reward: 1019.349, Step: 200 +2023-04-17 14:36:56 - r - INFO: - Episode: 970/1000, Reward: 1285.969, Step: 200 +2023-04-17 14:37:00 - r - INFO: - Episode: 971/1000, Reward: 1354.423, Step: 200 +2023-04-17 14:37:01 - r - INFO: - Episode: 972/1000, Reward: 1307.282, Step: 200 +2023-04-17 14:37:01 - r - INFO: - Episode: 973/1000, Reward: 1279.104, Step: 200 +2023-04-17 14:37:02 - r - INFO: - Episode: 974/1000, Reward: 1331.231, Step: 200 +2023-04-17 14:37:02 - r - INFO: - Episode: 975/1000, Reward: 1282.538, Step: 200 +2023-04-17 14:37:06 - r - INFO: - Episode: 976/1000, Reward: 1303.137, Step: 200 +2023-04-17 14:37:07 - r - INFO: - Episode: 977/1000, Reward: 1116.645, Step: 200 +2023-04-17 14:37:07 - r - INFO: - Episode: 978/1000, Reward: 1263.702, Step: 200 +2023-04-17 14:37:08 - r - INFO: - Episode: 979/1000, Reward: 1394.170, Step: 200 +2023-04-17 14:37:09 - r - INFO: - Episode: 980/1000, Reward: 1237.394, Step: 200 +2023-04-17 14:37:13 - r - INFO: - Episode: 981/1000, Reward: 986.832, Step: 200 +2023-04-17 14:37:13 - r - INFO: - Episode: 982/1000, Reward: 1324.843, Step: 200 +2023-04-17 14:37:14 - r - INFO: - Episode: 983/1000, Reward: 1297.139, Step: 200 +2023-04-17 14:37:14 - r - INFO: - Episode: 984/1000, Reward: 498.473, Step: 200 +2023-04-17 14:37:15 - r - INFO: - Episode: 985/1000, Reward: 1206.221, Step: 200 +2023-04-17 14:37:19 - r - INFO: - Episode: 986/1000, Reward: 1271.213, Step: 200 +2023-04-17 14:37:19 - r - INFO: - Episode: 987/1000, Reward: 1354.592, Step: 200 +2023-04-17 14:37:20 - r - INFO: - Episode: 988/1000, Reward: 1061.141, Step: 200 +2023-04-17 14:37:21 - r - INFO: - Episode: 989/1000, Reward: 1337.815, Step: 200 +2023-04-17 14:37:21 - r - INFO: - Episode: 990/1000, Reward: 1066.556, Step: 200 +2023-04-17 14:37:25 - r - INFO: - Episode: 991/1000, Reward: 1237.234, Step: 200 +2023-04-17 14:37:25 - r - INFO: - Episode: 992/1000, Reward: 1404.495, Step: 200 +2023-04-17 14:37:26 - r - INFO: - Episode: 993/1000, Reward: 1128.712, Step: 200 +2023-04-17 14:37:26 - r - INFO: - Episode: 994/1000, Reward: 1295.569, Step: 200 +2023-04-17 14:37:27 - r - INFO: - Episode: 995/1000, Reward: 1304.900, Step: 200 +2023-04-17 14:37:31 - r - INFO: - Episode: 996/1000, Reward: 1183.400, Step: 200 +2023-04-17 14:37:31 - r - INFO: - Episode: 997/1000, Reward: 1263.613, Step: 200 +2023-04-17 14:37:32 - r - INFO: - Episode: 998/1000, Reward: 1136.181, Step: 200 +2023-04-17 14:37:32 - r - INFO: - Episode: 999/1000, Reward: 1404.283, Step: 200 +2023-04-17 14:37:33 - r - INFO: - Episode: 1000/1000, Reward: 1171.637, Step: 200 +2023-04-17 14:37:36 - r - INFO: - Finish training! diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/action_bias.npy b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/action_bias.npy new file mode 100644 index 0000000000000000000000000000000000000000..6c6fe1c76066c93672fb5706a997a503b0673162 --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/action_bias.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c515444fcd73a5aa350c5bf219d2dc5eeaaff903072d1d8589fb51c62673d171 +size 152 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/action_scale.npy b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/action_scale.npy new file mode 100644 index 0000000000000000000000000000000000000000..451bf3e54e2dabc6c3b3b4a7997636eab067c68d --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/action_scale.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d5962f5d28bcd9bd926f0ec6edd6b85da90b0c02d79cec3c35d4591b3587fc +size 152 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/actor_checkpoint.pt b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c737e2fbebcdabfbe83e80b640f0f192d21cc02 --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc535a06c6b6773c6ebba205e0fb2671fed9823b09e60876d091dd6725af4c47 +size 969792 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/actor_target_checkpoint.pt b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/actor_target_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e670fa1c9f55f4353ee91ec852f6a01456a00703 --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/actor_target_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aad4bff7aa7ff81384429edb911a831ee74c50c4f988dc41268dfab92caf8874 +size 970658 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/critic_checkpoint.pt b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/critic_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdc526acb0ffe2c5794c990024f86df240e67bd2 --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/critic_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac57c694200476bed17edcbfc36efe70badc759545cb842b582a7aa11546a0e1 +size 1896308 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/critic_target_checkpoint.pt b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/critic_target_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a67dd2f492b466e272880ed5fa88bdaa75378a3 --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/critic_target_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cbf5b8bc2dad2d364450f1b19a8d7621a1a61f00cdd3e9d4b71759fb2c2af9f +size 1898169 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/vae_checkpoint.pt b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/vae_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b6bb0fd6f3d3d482f969ddcf4a6fb99fcf0cfba --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/models/vae_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce405d60071555ee6558572e9f81515da7b2a66deabdbbfa9c1a64c841e075e5 +size 8793863 diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/results/learning_curve.png b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..60a152d4f845eb36737616fb5b45b1be51f5c6f0 Binary files /dev/null and b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/results/learning_curve.png differ diff --git a/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/results/res.csv b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..fa9b5be40fd8dcbde555af2b705209cdb0dd7067 --- /dev/null +++ b/HalfCheetah-v3/Train_gym_BCQ_20230417-141811/results/res.csv @@ -0,0 +1,1001 @@ +episodes,rewards,steps +0,36.76739372924904,200 +1,21.782811530900425,200 +2,-60.802241604701045,200 +3,-30.031315363274846,200 +4,68.18557337042785,200 +5,434.88510203172876,200 +6,-39.49444660308185,200 +7,-8.455489769150272,200 +8,47.44555674497626,200 +9,-39.633645490963296,200 +10,7.466025278950358,200 +11,-26.82816771527468,200 +12,137.51404667310698,200 +13,-23.069428392789845,200 +14,76.06736098620857,200 +15,-3.251056509693662,200 +16,65.34830362605193,200 +17,-11.215389042526347,200 +18,11.327601154224695,200 +19,199.26528927056773,200 +20,408.7097819925073,200 +21,338.86930898909225,200 +22,-32.20059640326616,200 +23,367.2538368199918,200 +24,134.39879122260447,200 +25,28.196485159433077,200 +26,76.62912852459381,200 +27,193.26844425866568,200 +28,227.01805728671116,200 +29,276.6086640596659,200 +30,22.315628258035726,200 +31,94.40559938774585,200 +32,-31.563865677774977,200 +33,31.355328245528472,200 +34,403.36381775251164,200 +35,574.4585762636788,200 +36,280.11115948583637,200 +37,-94.24696850192927,200 +38,125.2885839893969,200 +39,775.171892866146,200 +40,13.563558717503676,200 +41,23.38501121469599,200 +42,348.05139226175237,200 +43,165.46617260481975,200 +44,113.78706430960354,200 +45,164.29266341305797,200 +46,192.77498258347327,200 +47,109.05508744204343,200 +48,-26.554384385186847,200 +49,321.20554612891897,200 +50,252.37892887455362,200 +51,53.37152256477139,200 +52,227.69062273967643,200 +53,99.68632079360194,200 +54,125.09259909066395,200 +55,637.9037788180719,200 +56,75.35024748060226,200 +57,157.2053925443866,200 +58,-80.93606683067317,200 +59,305.86769362430823,200 +60,358.85105028710797,200 +61,-73.47064182739582,200 +62,319.35270065641475,200 +63,82.72945570075083,200 +64,92.98842171533525,200 +65,145.37367488013626,200 +66,-12.395203030669522,200 +67,-52.08431986167721,200 +68,230.1912777760957,200 +69,157.50367841975896,200 +70,26.73413066185803,200 +71,1077.2510956671554,200 +72,316.20083818740545,200 +73,186.21031465677166,200 +74,330.8301613779432,200 +75,62.51256389890827,200 +76,94.11615302246636,200 +77,105.74207103290433,200 +78,318.6531613511205,200 +79,221.20547274359865,200 +80,394.3715103689688,200 +81,130.50841708725918,200 +82,352.31429250306593,200 +83,119.76297460602542,200 +84,197.98881796357574,200 +85,85.45959532507331,200 +86,363.02216637250376,200 +87,16.9030956541138,200 +88,391.96228500036347,200 +89,164.65665336009485,200 +90,551.2903469790793,200 +91,358.5401691322208,200 +92,233.18320661769096,200 +93,155.5926093200374,200 +94,220.0604776448689,200 +95,267.43708581523805,200 +96,99.22893293561752,200 +97,161.35899450416096,200 +98,-1.2406424612980473,200 +99,116.92043898658538,200 +100,188.3444163102192,200 +101,9.05631468280267,200 +102,928.3762923611087,200 +103,297.7358962285084,200 +104,122.11089205977923,200 +105,442.4253278458868,200 +106,67.1002701360931,200 +107,-47.06261296893477,200 +108,369.43965908241756,200 +109,66.8381294099503,200 +110,118.00692166174741,200 +111,289.2158489862109,200 +112,89.99541762852799,200 +113,99.79040953064606,200 +114,26.232026861457864,200 +115,375.3039125415709,200 +116,113.85223675123224,200 +117,236.1134472319439,200 +118,206.11889804167834,200 +119,78.33340232798321,200 +120,605.2355471630989,200 +121,16.319979724368633,200 +122,312.691339097712,200 +123,409.3266568778011,200 +124,301.53876369712157,200 +125,474.21344447291574,200 +126,63.95688966997369,200 +127,134.60504718859372,200 +128,180.44659076847884,200 +129,388.01306429411164,200 +130,125.37801400220226,200 +131,332.916580320436,200 +132,251.74450978022548,200 +133,787.6069171619899,200 +134,139.88842027284858,200 +135,224.8704307704015,200 +136,410.01736814843673,200 +137,224.0957343333755,200 +138,701.0131247536485,200 +139,960.259933690892,200 +140,-34.19325256504633,200 +141,63.261260331738086,200 +142,166.85881359594902,200 +143,-62.682970580857244,200 +144,213.82623291246733,200 +145,94.5389778921504,200 +146,117.66842968784988,200 +147,99.3631995444969,200 +148,480.80189923590586,200 +149,182.57169722361866,200 +150,3.1588333404124542,200 +151,146.94201165010475,200 +152,43.63649581547505,200 +153,33.39291010010789,200 +154,591.4823792726625,200 +155,169.5215458367996,200 +156,486.4115470511029,200 +157,429.2720222331244,200 +158,626.8502292948285,200 +159,193.04048981375007,200 +160,305.2801204216882,200 +161,537.0741858495585,200 +162,614.0443720358827,200 +163,16.9164509474371,200 +164,162.58141921076626,200 +165,187.6674671511663,200 +166,87.2352588035178,200 +167,-62.93745758424645,200 +168,23.23500597424871,200 +169,863.1235841348434,200 +170,-27.099476614298407,200 +171,323.27363195147586,200 +172,245.84635717878493,200 +173,27.393683334434883,200 +174,292.1282856021956,200 +175,225.70398241720503,200 +176,568.5165264324378,200 +177,277.1033426420881,200 +178,438.24587882460077,200 +179,130.91054311604262,200 +180,-31.32820066254305,200 +181,156.36080906023358,200 +182,-87.08686201617854,200 +183,26.352040366503054,200 +184,283.00879899133076,200 +185,131.23397310144853,200 +186,352.6718629628042,200 +187,-49.441639905194414,200 +188,261.8357090290728,200 +189,121.84889309296008,200 +190,259.22899068715236,200 +191,2.797011555346379,200 +192,88.65188682575747,200 +193,-112.53309145195158,200 +194,590.8222018376745,200 +195,246.96201404848165,200 +196,152.954085523833,200 +197,120.67948899365192,200 +198,673.1714223737249,200 +199,522.1254457908823,200 +200,340.3157755211404,200 +201,67.55424334140328,200 +202,134.18018728925273,200 +203,262.4432085896311,200 +204,233.21940709448262,200 +205,-81.75663683665859,200 +206,102.8851186080068,200 +207,347.45371871726235,200 +208,-32.42512132721816,200 +209,704.1772715209192,200 +210,227.64681448268334,200 +211,451.53361970448515,200 +212,676.2788844965098,200 +213,401.6666842951862,200 +214,377.5602892326344,200 +215,374.64971370689335,200 +216,6.813813895534382,200 +217,843.6966034996997,200 +218,930.7118693652529,200 +219,74.58479888683662,200 +220,-62.471275734284475,200 +221,91.4114609507963,200 +222,439.85821238932226,200 +223,828.925672560425,200 +224,49.01841441095343,200 +225,199.45693052762994,200 +226,108.96281887851718,200 +227,-73.16196430127127,200 +228,292.10766211491705,200 +229,74.09976646161483,200 +230,563.8714944945509,200 +231,-0.5935978840026409,200 +232,1001.4690747976032,200 +233,101.06527915410696,200 +234,334.9619708953843,200 +235,616.0735622482983,200 +236,534.5706646848772,200 +237,159.16587574415587,200 +238,1088.0356567437652,200 +239,818.4556598215164,200 +240,297.6884229361658,200 +241,678.415410714934,200 +242,77.52152564060223,200 +243,10.843339747480192,200 +244,14.062664257460252,200 +245,1165.726929574868,200 +246,473.0521491615995,200 +247,114.72216352954342,200 +248,98.67846174169037,200 +249,541.5355132986865,200 +250,170.18199399666872,200 +251,532.782300407852,200 +252,19.610762380412833,200 +253,1142.5336149086277,200 +254,1006.8308288201682,200 +255,1038.0007881620393,200 +256,291.7404628672199,200 +257,91.16973802554706,200 +258,325.68632425691175,200 +259,93.23962115336703,200 +260,28.747294292180438,200 +261,447.16220047522086,200 +262,292.820353827329,200 +263,51.00366750776926,200 +264,1314.8053990638864,200 +265,274.4917712781061,200 +266,242.0269161294635,200 +267,754.5696366410119,200 +268,76.26217446894789,200 +269,382.9319406494506,200 +270,183.57511727751216,200 +271,906.2850225090053,200 +272,663.8030636636046,200 +273,112.70196588300428,200 +274,163.3632610460068,200 +275,728.1860367835134,200 +276,654.4368612365998,200 +277,455.0330731271812,200 +278,789.5545040754009,200 +279,-71.62960605906038,200 +280,86.62900212868162,200 +281,502.36191589896026,200 +282,229.23508275949283,200 +283,39.4439971691847,200 +284,969.3895535765394,200 +285,-49.349463742276555,200 +286,557.668006249967,200 +287,180.88935493448497,200 +288,1270.017829039153,200 +289,-52.14531507919022,200 +290,338.98085332691414,200 +291,-10.907667018842389,200 +292,845.802967976387,200 +293,896.8345274302425,200 +294,403.89600139871646,200 +295,66.39565150052962,200 +296,207.75371853807323,200 +297,632.7792040445249,200 +298,361.87977031840944,200 +299,1098.7589152543912,200 +300,125.6978013477373,200 +301,105.35109530635613,200 +302,-51.80823523307184,200 +303,122.58596235309756,200 +304,131.5044001207421,200 +305,-54.18103625270173,200 +306,1209.181313515789,200 +307,-49.10918794805082,200 +308,229.58181742550389,200 +309,1087.9660665175104,200 +310,649.4884468717955,200 +311,-19.739938165282556,200 +312,588.8799952370739,200 +313,515.6711625299231,200 +314,47.393959036394016,200 +315,-27.627191695311,200 +316,1056.53741275132,200 +317,454.21628773272,200 +318,963.8421657315905,200 +319,-12.392350870048201,200 +320,-21.301281786559603,200 +321,905.0882767147057,200 +322,43.72725216634383,200 +323,1173.2651803746871,200 +324,148.0916877399422,200 +325,1195.8934062624037,200 +326,47.57426666690531,200 +327,-79.85403444624687,200 +328,28.465311468480206,200 +329,671.6978489319085,200 +330,-23.25034109042866,200 +331,451.18581604368,200 +332,592.1591737835588,200 +333,1009.0692210061612,200 +334,481.4999380333535,200 +335,992.4489104298838,200 +336,716.7023642585436,200 +337,697.7810222347489,200 +338,-34.708418253952075,200 +339,231.96060200966963,200 +340,798.4344109882891,200 +341,146.49847119066663,200 +342,1223.7393996211572,200 +343,326.79950117876535,200 +344,619.586342527402,200 +345,681.3827894203291,200 +346,118.49973001969843,200 +347,281.49019803529126,200 +348,101.6474706417024,200 +349,1102.036651124209,200 +350,470.4703368694899,200 +351,1139.7080697192303,200 +352,662.3967963365822,200 +353,704.0774409659232,200 +354,1056.7588847936213,200 +355,98.83368915108515,200 +356,465.3161405468699,200 +357,252.658080506748,200 +358,907.8685205437,200 +359,783.8534076193474,200 +360,1008.1767154054164,200 +361,550.1949588148342,200 +362,467.91011349555725,200 +363,80.38174634147069,200 +364,817.4591720790222,200 +365,760.7184013320087,200 +366,804.0092528980001,200 +367,1197.9918705778307,200 +368,730.0255570069837,200 +369,1089.5077267645474,200 +370,448.9713539044923,200 +371,788.667976424131,200 +372,880.6676700858413,200 +373,1206.5819290041418,200 +374,460.7445784604452,200 +375,331.94704315513434,200 +376,683.2610158549625,200 +377,612.4847535891954,200 +378,1333.8966411394424,200 +379,1174.567830511947,200 +380,216.48068997707793,200 +381,1095.8020638833705,200 +382,1235.7495248583932,200 +383,334.8649121916282,200 +384,467.62593285518403,200 +385,315.8884565642871,200 +386,116.420900938021,200 +387,18.114187213732478,200 +388,1196.0120595866897,200 +389,1231.0127941856717,200 +390,770.4811245582599,200 +391,703.6824498540799,200 +392,1135.027441359436,200 +393,158.62382925482177,200 +394,921.9136821809876,200 +395,1255.9496548085772,200 +396,1134.4132606734277,200 +397,1040.1125385909284,200 +398,176.98177006198605,200 +399,152.7160074476948,200 +400,1274.312285328552,200 +401,278.022778621336,200 +402,1282.7176593251788,200 +403,1035.864165818047,200 +404,1018.7272894890604,200 +405,691.9382358534921,200 +406,279.5300910797068,200 +407,749.0760385153271,200 +408,1310.2397112974402,200 +409,1023.311670159819,200 +410,328.8994723766308,200 +411,1060.7443765313028,200 +412,536.7368074406118,200 +413,959.2210413354354,200 +414,1066.3034318474672,200 +415,1148.9711990329442,200 +416,1274.5660161690487,200 +417,1257.9596925417404,200 +418,984.619124051061,200 +419,1159.0357126267095,200 +420,74.46521069723487,200 +421,10.670319151263358,200 +422,1274.7143869612858,200 +423,892.078936305285,200 +424,698.7157688528056,200 +425,1225.863603766104,200 +426,72.34621477253178,200 +427,962.214667107916,200 +428,14.878695896484492,200 +429,970.588311462387,200 +430,1158.3914047363594,200 +431,568.8670132682331,200 +432,1334.2520319781397,200 +433,1251.8422536545277,200 +434,827.3097266164995,200 +435,657.9880322358352,200 +436,34.56574825994723,200 +437,1077.559887145235,200 +438,35.43832778614842,200 +439,1220.9250202719588,200 +440,880.1398931044687,200 +441,1262.2426407593807,200 +442,375.21259236734545,200 +443,1335.945969072467,200 +444,-71.371661962711,200 +445,1068.7402312829493,200 +446,225.25920798533735,200 +447,1033.825301996415,200 +448,1154.3075616857548,200 +449,535.1407884107168,200 +450,883.8739430004354,200 +451,1253.951618430214,200 +452,949.9775571457307,200 +453,565.1153992520434,200 +454,1343.2199986448722,200 +455,1164.00509522989,200 +456,1324.9208668338872,200 +457,539.1947776672212,200 +458,972.3739700749883,200 +459,1019.5843019977289,200 +460,1355.1971724339633,200 +461,68.0876306846212,200 +462,604.8009416007412,200 +463,1186.1725683763968,200 +464,945.381045785621,200 +465,789.6037462245998,200 +466,1279.779548886788,200 +467,293.8027365895036,200 +468,805.2192055925134,200 +469,909.7639405327229,200 +470,331.2144816461096,200 +471,1313.8070069971359,200 +472,1155.0826684638087,200 +473,700.0476061976124,200 +474,1053.117036177466,200 +475,1165.5048748635636,200 +476,954.8942992378188,200 +477,1241.4155722865569,200 +478,1226.6568619499267,200 +479,729.7766208849698,200 +480,1331.5509405717248,200 +481,1302.92370469547,200 +482,1152.676594671519,200 +483,1249.512139023425,200 +484,428.3939632825898,200 +485,551.0715449260459,200 +486,1259.4261480449316,200 +487,1153.321426115467,200 +488,1277.735930222003,200 +489,1311.3651081605642,200 +490,1243.4431720631633,200 +491,1055.1708004585882,200 +492,1069.427590073286,200 +493,1142.2277585775676,200 +494,1299.9077353688067,200 +495,1330.429900753061,200 +496,1190.9356932688954,200 +497,1311.5081217665622,200 +498,238.2012726773845,200 +499,1317.069677180513,200 +500,948.4797142785034,200 +501,711.1245390646004,200 +502,-27.825445126009985,200 +503,228.7981861796212,200 +504,78.60519139832738,200 +505,1207.0673361441766,200 +506,1279.4223411284634,200 +507,1251.8319794235392,200 +508,975.8790683569836,200 +509,1295.0045178674357,200 +510,1140.732396425041,200 +511,328.7835817698428,200 +512,1193.5615947219953,200 +513,537.2913369606397,200 +514,1206.3546723956297,200 +515,1303.570648680148,200 +516,1323.4820895924945,200 +517,1242.2773006560546,200 +518,1068.176416320415,200 +519,1293.3086339161237,200 +520,1286.079688712263,200 +521,1195.8590549384771,200 +522,1193.7739596034835,200 +523,360.6069158883041,200 +524,1191.2349310322556,200 +525,893.7804841589525,200 +526,1171.3484589752136,200 +527,995.8550816175632,200 +528,52.95717862458184,200 +529,1352.2895433897486,200 +530,1140.6869752605348,200 +531,1299.395135369996,200 +532,809.7868390292864,200 +533,1198.8622175821442,200 +534,1112.0826444399117,200 +535,27.219928811971073,200 +536,923.5297687018281,200 +537,1116.382384887407,200 +538,1089.4749262758176,200 +539,1401.3409200567157,200 +540,1138.1447510114583,200 +541,851.5895660810451,200 +542,288.91191320454186,200 +543,1086.5583472912617,200 +544,1005.9233660438038,200 +545,1130.1954420164016,200 +546,1265.4284104283627,200 +547,66.69219805859301,200 +548,1311.9404240057606,200 +549,946.601363718449,200 +550,1162.8781007094383,200 +551,1257.1569755125752,200 +552,1015.2521066996956,200 +553,1312.4643518959235,200 +554,1345.1380326860042,200 +555,1231.974238983346,200 +556,1315.7081041078675,200 +557,900.9497602676565,200 +558,1122.0696128640434,200 +559,585.558317941235,200 +560,1220.4369528044126,200 +561,1190.9805007641817,200 +562,178.01964896167303,200 +563,1312.6763063757926,200 +564,1294.6590744075177,200 +565,1331.1744625562148,200 +566,1261.7416105451794,200 +567,1201.5379584550997,200 +568,1196.7417879580134,200 +569,1263.5989757342682,200 +570,1081.4098945162611,200 +571,1227.2628254748383,200 +572,1089.5833132391901,200 +573,1280.7837521282606,200 +574,1328.004611330183,200 +575,776.2351256527277,200 +576,1063.8816362683622,200 +577,863.9664101186374,200 +578,1268.3789414846894,200 +579,1329.06946664878,200 +580,1224.7703161098684,200 +581,1161.0046908965505,200 +582,1161.7071945762543,200 +583,1255.8908820946206,200 +584,1070.1026979717913,200 +585,1024.0176663832333,200 +586,1227.9300126303083,200 +587,997.6644948195258,200 +588,1068.5968157913394,200 +589,1232.9951145818916,200 +590,1132.4299814070853,200 +591,1351.9041041067449,200 +592,1216.3808581276433,200 +593,1087.8794373524854,200 +594,630.3168738055065,200 +595,990.9839847126733,200 +596,1309.4811632205822,200 +597,1180.7398314771942,200 +598,1301.0513176942454,200 +599,1221.4462123423577,200 +600,1274.306704420033,200 +601,1101.1411109929102,200 +602,1287.8057834685528,200 +603,736.0599262424686,200 +604,780.3331944514742,200 +605,1232.1034886095645,200 +606,1196.1911071668537,200 +607,1256.4841386077267,200 +608,1275.7640191613255,200 +609,1255.0361111649156,200 +610,1205.1826771927017,200 +611,1290.0388034723462,200 +612,1093.5362838250103,200 +613,1235.027893953363,200 +614,1323.8503719722055,200 +615,1154.8878966628251,200 +616,1149.666085773712,200 +617,1264.052134624296,200 +618,1195.7357818977919,200 +619,1310.497656608678,200 +620,1303.9215504654894,200 +621,1374.9578697595305,200 +622,93.04611688164071,200 +623,1265.441470353661,200 +624,1199.816055377981,200 +625,1136.9722777137029,200 +626,1249.2197247632014,200 +627,1274.6041221546577,200 +628,1135.943831426988,200 +629,1068.0615270241726,200 +630,1184.7544948064576,200 +631,1286.904908848446,200 +632,669.2557830842463,200 +633,1307.451811808104,200 +634,1335.9067958363855,200 +635,1331.6356743260944,200 +636,1188.5163214432596,200 +637,1245.2616452454513,200 +638,1348.3404773419425,200 +639,857.4710181994602,200 +640,1209.8470085286542,200 +641,1240.9013402306234,200 +642,1109.1224990229782,200 +643,963.1173744693345,200 +644,1274.8074450369515,200 +645,1050.7523863608849,200 +646,1394.5793631936226,200 +647,260.6955331136394,200 +648,1221.6853837608846,200 +649,1349.0053811668665,200 +650,1148.6660785198358,200 +651,1307.2312706068178,200 +652,1217.637976165473,200 +653,1298.5225827847335,200 +654,973.7326786534569,200 +655,1313.4557973099713,200 +656,1319.0070833415439,200 +657,1189.7735859872978,200 +658,1284.6311877177448,200 +659,1194.3664498893668,200 +660,1351.0830917374612,200 +661,1124.720948368334,200 +662,1281.6721417478263,200 +663,17.134309813314456,200 +664,1307.462275467669,200 +665,1270.90602024556,200 +666,592.865171405785,200 +667,1371.337088161201,200 +668,1283.5970262999776,200 +669,1230.3476221086369,200 +670,-14.82031804583265,200 +671,944.5238706605625,200 +672,955.3257470197433,200 +673,1201.1706098255377,200 +674,1182.2244647485325,200 +675,1288.2393179505243,200 +676,826.7453549741505,200 +677,1295.0443763528401,200 +678,867.3484095140675,200 +679,662.4004874532125,200 +680,1303.0824238780137,200 +681,492.34890938361974,200 +682,1276.5934012153339,200 +683,1084.8107024893113,200 +684,1373.639025272739,200 +685,1326.4629604597862,200 +686,1260.7274621984473,200 +687,1163.930212026234,200 +688,1310.9150907122876,200 +689,1346.8867650436846,200 +690,1331.9710987009767,200 +691,1355.311549936535,200 +692,216.86614285863948,200 +693,1220.5951909980097,200 +694,1257.6034679664795,200 +695,1279.505377043525,200 +696,1243.70054127802,200 +697,1301.8580115002621,200 +698,1319.8532027761023,200 +699,1315.3766627131727,200 +700,1343.650802784622,200 +701,1298.5813629433892,200 +702,1390.1054696762665,200 +703,1081.1063352238955,200 +704,1274.2813608195793,200 +705,1301.9081343624462,200 +706,818.8846137805339,200 +707,1267.7886795452068,200 +708,1195.447993252901,200 +709,1272.1178448411733,200 +710,1347.7278010904522,200 +711,1348.9069013565202,200 +712,1222.8961456906195,200 +713,1044.141115654651,200 +714,1011.0241480924523,200 +715,1400.9579364118817,200 +716,1422.4289443170696,200 +717,1307.1925814201632,200 +718,1215.9263978326057,200 +719,571.4030004069123,200 +720,1188.453722482976,200 +721,1255.9894534259868,200 +722,1302.864608729249,200 +723,1368.5288830572736,200 +724,1344.8526817356349,200 +725,1148.1154316682885,200 +726,1325.4419260541194,200 +727,904.6706929031338,200 +728,1289.0604197594662,200 +729,1253.9606742845208,200 +730,1263.8812807308038,200 +731,1308.1125844491103,200 +732,1282.3231784482398,200 +733,1333.6247744951095,200 +734,1295.0690756227095,200 +735,1246.3143141018274,200 +736,434.2254979118025,200 +737,1162.501459486045,200 +738,1035.6197324389934,200 +739,205.28750129810052,200 +740,1264.9667965867789,200 +741,317.1105555086431,200 +742,1137.949371816947,200 +743,1264.0997233755486,200 +744,1307.818327485623,200 +745,1325.6031212919008,200 +746,1237.9108910950108,200 +747,769.3441898065843,200 +748,917.9536090100722,200 +749,1391.4185808927714,200 +750,1225.3990992866243,200 +751,1223.4882486922504,200 +752,1308.5134157115217,200 +753,1182.4271793804307,200 +754,1205.5184802722458,200 +755,1134.959180080737,200 +756,1205.0769910961722,200 +757,28.44356597220449,200 +758,1170.1011481163841,200 +759,1073.8338274379394,200 +760,1209.6244861914126,200 +761,1332.6555158586157,200 +762,1324.0093061771508,200 +763,1300.2985061075358,200 +764,1471.5909670194824,200 +765,1102.3949663360522,200 +766,303.0523788573641,200 +767,1308.7342476811818,200 +768,1309.9067495079871,200 +769,1144.9937543709764,200 +770,1313.5087727623427,200 +771,1193.6213086974994,200 +772,1213.9073562455935,200 +773,1315.0151080817611,200 +774,1223.3673071454289,200 +775,1264.2694045094634,200 +776,1263.8519881971615,200 +777,1200.551498084575,200 +778,1293.4996639723588,200 +779,1342.5461608083458,200 +780,878.7096798708759,200 +781,893.4950137252579,200 +782,1216.9894947851428,200 +783,1351.5731551953827,200 +784,1201.2419647784725,200 +785,1253.6175157883736,200 +786,1363.7733671750038,200 +787,1295.5297211261511,200 +788,830.56186771396,200 +789,1273.9590671874387,200 +790,1363.9465570272143,200 +791,1424.5071602728992,200 +792,1228.5854612738613,200 +793,783.413217391935,200 +794,1138.3509139238133,200 +795,1378.8228168098858,200 +796,1124.296573583769,200 +797,1314.0031873521089,200 +798,1306.2364566094893,200 +799,-8.029680628837477,200 +800,1261.6503261801226,200 +801,1204.928372416024,200 +802,1317.253303374475,200 +803,373.2064664094397,200 +804,1308.4678991907015,200 +805,1320.6234492521419,200 +806,1312.4077660226437,200 +807,218.54955243461623,200 +808,1361.0097079173295,200 +809,1312.1329644440366,200 +810,1285.3275376469467,200 +811,1349.1341513693465,200 +812,1313.9634625398435,200 +813,1278.1724325214716,200 +814,1387.266797370428,200 +815,1358.3091928261617,200 +816,1291.116869123336,200 +817,1364.4581203229097,200 +818,1285.3751419605192,200 +819,1237.396512418198,200 +820,433.82330799232517,200 +821,1247.6154808303152,200 +822,1297.2195105844103,200 +823,1374.595760089741,200 +824,1353.7477146043213,200 +825,1363.126448574998,200 +826,1375.9285225299236,200 +827,1261.0207266733014,200 +828,1254.1143295388224,200 +829,1257.4923768027643,200 +830,1198.3459494184672,200 +831,1251.9940409317815,200 +832,1171.787009817163,200 +833,1234.8445006715922,200 +834,1219.0718910813728,200 +835,1268.738382382045,200 +836,1186.5786608441592,200 +837,1120.019208222791,200 +838,1237.9239566142412,200 +839,1199.3241748167886,200 +840,1117.8672799892038,200 +841,1316.9222648407372,200 +842,1255.757529460653,200 +843,1278.9112412078405,200 +844,1152.7904858641164,200 +845,1231.8207692696624,200 +846,1219.7558697200936,200 +847,1279.7262870708028,200 +848,647.1422578884601,200 +849,1220.1646436018152,200 +850,500.70211509824514,200 +851,1270.3832331981814,200 +852,1281.0543594750213,200 +853,1358.6479316744842,200 +854,1365.8475740149106,200 +855,1377.2158143257293,200 +856,1203.8012450423878,200 +857,1320.7795918357124,200 +858,1044.3087568453202,200 +859,1292.777467914661,200 +860,1332.0004518054975,200 +861,1103.871728155995,200 +862,1273.460563813086,200 +863,1111.6980608952938,200 +864,1293.764092794267,200 +865,1295.5723225069387,200 +866,1227.3636288655173,200 +867,850.9037994596573,200 +868,1205.8543933005615,200 +869,1139.2531248878793,200 +870,1245.3446099411055,200 +871,1053.3375096080836,200 +872,1317.995893177766,200 +873,1334.6456172955156,200 +874,1283.7635532520746,200 +875,1414.083726979151,200 +876,1305.0494246252292,200 +877,1212.8275304661422,200 +878,1297.0145096995675,200 +879,1343.652639221033,200 +880,1292.7019273236428,200 +881,1415.727400744613,200 +882,1264.7754286258885,200 +883,1024.7457363989672,200 +884,996.5941080765426,200 +885,1285.2469396614197,200 +886,1256.3006628258381,200 +887,1276.011784321513,200 +888,1331.1894373130015,200 +889,1094.2042641277926,200 +890,1155.440599439638,200 +891,1360.8317934498136,200 +892,1305.2559354686396,200 +893,1369.0397585683725,200 +894,1094.9016923329523,200 +895,260.6619183369852,200 +896,1292.9063178852998,200 +897,1140.988084315913,200 +898,1228.2861169194528,200 +899,1314.3496568380635,200 +900,1335.6373377012412,200 +901,1347.5577656300552,200 +902,1340.5096173030427,200 +903,1327.6083654250049,200 +904,1294.3826799262122,200 +905,1255.8162283120496,200 +906,1376.404408868546,200 +907,1343.8752653432273,200 +908,1349.5398819466586,200 +909,1298.7114433552345,200 +910,1295.300542328102,200 +911,1356.6551247506513,200 +912,1410.2491901664841,200 +913,1265.4453507594642,200 +914,1418.9270364428676,200 +915,1363.4843207784434,200 +916,1439.8918952448341,200 +917,1300.7266877790207,200 +918,1238.133166692546,200 +919,1358.9271954644455,200 +920,1365.7917793515485,200 +921,1352.0117093284218,200 +922,1079.368054275108,200 +923,1288.186401557541,200 +924,1315.1428739767975,200 +925,1214.6517293438303,200 +926,1358.6325819248038,200 +927,1166.5704985365217,200 +928,1281.5740786670451,200 +929,1258.010495815347,200 +930,1278.3051997446632,200 +931,1333.5242604054117,200 +932,1127.4338471445076,200 +933,1371.9734947664172,200 +934,617.9368278929227,200 +935,1208.9743632191994,200 +936,1267.4009122128182,200 +937,1338.5560819892214,200 +938,1200.7536536442049,200 +939,1337.4888629361203,200 +940,1109.0182022965341,200 +941,946.8538740568733,200 +942,377.44876347029793,200 +943,1150.8098668885473,200 +944,1400.812518778063,200 +945,1274.5160329585558,200 +946,1225.9642049248455,200 +947,1360.4903455553733,200 +948,1207.2757301151564,200 +949,1248.2154167833412,200 +950,1419.342391856144,200 +951,1369.4372230579088,200 +952,1273.2121089661161,200 +953,1348.2878133088916,200 +954,864.3272636016325,200 +955,1250.8774783483177,200 +956,1365.5141839266814,200 +957,1312.8082702658505,200 +958,1102.644625762553,200 +959,1323.5089850408172,200 +960,1287.5997508832418,200 +961,1179.9832943951235,200 +962,1248.9156413317166,200 +963,1281.686226283407,200 +964,1327.644210733243,200 +965,1240.872417827043,200 +966,1325.7152031327303,200 +967,1264.295665673612,200 +968,1019.3493454487582,200 +969,1285.968570410934,200 +970,1354.4229305849717,200 +971,1307.281930886449,200 +972,1279.1037644202245,200 +973,1331.2311461931506,200 +974,1282.538410631494,200 +975,1303.1372558301218,200 +976,1116.6452378614297,200 +977,1263.7021438711336,200 +978,1394.170193188739,200 +979,1237.3938257698514,200 +980,986.8324427757656,200 +981,1324.842577517942,200 +982,1297.1390849647132,200 +983,498.4727139540169,200 +984,1206.221190255387,200 +985,1271.2134557944337,200 +986,1354.5917814064571,200 +987,1061.14149202676,200 +988,1337.8146757310171,200 +989,1066.5556131830947,200 +990,1237.2338560205455,200 +991,1404.4952351032082,200 +992,1128.712255716023,200 +993,1295.5688970360743,200 +994,1304.900269403169,200 +995,1183.399925098816,200 +996,1263.613048988901,200 +997,1136.1809460059794,200 +998,1404.283450623993,200 +999,1171.636694235949,200 diff --git a/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/config.yaml b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c025e158c7dd25caef52c62c808ac6bd4f44aefe --- /dev/null +++ b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: PPO + device: cpu + env_name: LunarLander-v2 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_LunarLander-v2_PPO_20230402-223154 + max_steps: 1000 + mode: test + mp_backend: mp + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 20 + train_eps: 600 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + ppo_type: clip + sgd_batch_size: 32 + train_batch_size: 256 diff --git a/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/logs/log.txt b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4db542bd7dcac568ad20da1c5ab0754540c50848 --- /dev/null +++ b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/logs/log.txt @@ -0,0 +1,63 @@ +2023-04-02 23:19:50 - r - INFO: - Hyperparameters: +2023-04-02 23:19:50 - r - INFO: - ================================================================================ +2023-04-02 23:19:50 - r - INFO: - Name Value Type +2023-04-02 23:19:50 - r - INFO: - env_name LunarLander-v2 +2023-04-02 23:19:50 - r - INFO: - new_step_api 1 +2023-04-02 23:19:50 - r - INFO: - wrapper None +2023-04-02 23:19:50 - r - INFO: - render 0 +2023-04-02 23:19:50 - r - INFO: - algo_name PPO +2023-04-02 23:19:50 - r - INFO: - mode test +2023-04-02 23:19:50 - r - INFO: - mp_backend mp +2023-04-02 23:19:50 - r - INFO: - seed 1 +2023-04-02 23:19:50 - r - INFO: - device cpu +2023-04-02 23:19:50 - r - INFO: - train_eps 600 +2023-04-02 23:19:50 - r - INFO: - test_eps 20 +2023-04-02 23:19:50 - r - INFO: - eval_eps 10 +2023-04-02 23:19:50 - r - INFO: - eval_per_episode 5 +2023-04-02 23:19:50 - r - INFO: - max_steps 1000 +2023-04-02 23:19:50 - r - INFO: - load_checkpoint 1 +2023-04-02 23:19:50 - r - INFO: - load_path Train_LunarLander-v2_PPO_20230402-223154 +2023-04-02 23:19:50 - r - INFO: - show_fig 0 +2023-04-02 23:19:50 - r - INFO: - save_fig 1 +2023-04-02 23:19:50 - r - INFO: - ppo_type clip +2023-04-02 23:19:50 - r - INFO: - continuous 0 +2023-04-02 23:19:50 - r - INFO: - gamma 0.99 +2023-04-02 23:19:50 - r - INFO: - k_epochs 4 +2023-04-02 23:19:50 - r - INFO: - actor_lr 0.0003 +2023-04-02 23:19:50 - r - INFO: - critic_lr 0.001 +2023-04-02 23:19:50 - r - INFO: - eps_clip 0.2 +2023-04-02 23:19:50 - r - INFO: - entropy_coef 0.01 +2023-04-02 23:19:50 - r - INFO: - train_batch_size 256 +2023-04-02 23:19:50 - r - INFO: - sgd_batch_size 32 +2023-04-02 23:19:50 - r - INFO: - actor_hidden_dim 256 +2023-04-02 23:19:50 - r - INFO: - critic_hidden_dim 256 +2023-04-02 23:19:50 - r - INFO: - task_dir /home/zf/Documents/Python/joyrl-offline_off/tasks/Test_LunarLander-v2_PPO_20230402-231950 +2023-04-02 23:19:50 - r - INFO: - res_dir /home/zf/Documents/Python/joyrl-offline_off/tasks/Test_LunarLander-v2_PPO_20230402-231950/results +2023-04-02 23:19:50 - r - INFO: - log_dir /home/zf/Documents/Python/joyrl-offline_off/tasks/Test_LunarLander-v2_PPO_20230402-231950/logs +2023-04-02 23:19:50 - r - INFO: - traj_dir /home/zf/Documents/Python/joyrl-offline_off/tasks/Test_LunarLander-v2_PPO_20230402-231950/traj +2023-04-02 23:19:50 - r - INFO: - tb_dir /home/zf/Documents/Python/joyrl-offline_off/tasks/Test_LunarLander-v2_PPO_20230402-231950/tb_logs +2023-04-02 23:19:50 - r - INFO: - ================================================================================ +2023-04-02 23:19:50 - r - INFO: - n_states: 8, n_actions: 4 +2023-04-02 23:19:50 - r - INFO: - Start testing! +2023-04-02 23:19:50 - r - INFO: - Env: LunarLander-v2, Algorithm: PPO, Device: cpu +2023-04-02 23:19:50 - r - INFO: - Episode: 1/20, Reward: 252.001, Step: 220 +2023-04-02 23:19:50 - r - INFO: - Episode: 2/20, Reward: 248.030, Step: 255 +2023-04-02 23:19:51 - r - INFO: - Episode: 3/20, Reward: 270.691, Step: 234 +2023-04-02 23:19:51 - r - INFO: - Episode: 4/20, Reward: 251.967, Step: 237 +2023-04-02 23:19:51 - r - INFO: - Episode: 5/20, Reward: 243.157, Step: 235 +2023-04-02 23:19:51 - r - INFO: - Episode: 6/20, Reward: 249.370, Step: 229 +2023-04-02 23:19:51 - r - INFO: - Episode: 7/20, Reward: 249.592, Step: 227 +2023-04-02 23:19:51 - r - INFO: - Episode: 8/20, Reward: 250.621, Step: 229 +2023-04-02 23:19:51 - r - INFO: - Episode: 9/20, Reward: 252.866, Step: 225 +2023-04-02 23:19:51 - r - INFO: - Episode: 10/20, Reward: 266.123, Step: 237 +2023-04-02 23:19:51 - r - INFO: - Episode: 11/20, Reward: 269.425, Step: 247 +2023-04-02 23:19:51 - r - INFO: - Episode: 12/20, Reward: 260.002, Step: 209 +2023-04-02 23:19:51 - r - INFO: - Episode: 13/20, Reward: 270.213, Step: 244 +2023-04-02 23:19:51 - r - INFO: - Episode: 14/20, Reward: 38.579, Step: 158 +2023-04-02 23:19:52 - r - INFO: - Episode: 15/20, Reward: 42.728, Step: 181 +2023-04-02 23:19:52 - r - INFO: - Episode: 16/20, Reward: 262.007, Step: 209 +2023-04-02 23:19:52 - r - INFO: - Episode: 17/20, Reward: 267.853, Step: 236 +2023-04-02 23:19:52 - r - INFO: - Episode: 18/20, Reward: 46.030, Step: 169 +2023-04-02 23:19:52 - r - INFO: - Episode: 19/20, Reward: 254.678, Step: 277 +2023-04-02 23:19:52 - r - INFO: - Episode: 20/20, Reward: 18.323, Step: 158 +2023-04-02 23:19:52 - r - INFO: - Finish testing! diff --git a/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/models/actor.pth b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..2ac31119df0965876cff5dd2186dfa5e84e9ff90 --- /dev/null +++ b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6993835bd5028735d6192679250fc13cd0ccec60cfcfa6f99176c3242af3ebfc +size 278295 diff --git a/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/models/critic.pth b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..c380276fec7b3b8935b01ce68f678717a07469cc --- /dev/null +++ b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd3600cd04fce59c57db0cab66643b9385c927bcd40a6a1b415114ebab8f081f +size 275223 diff --git a/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/results/learning_curve.png b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..7316d9d3ca64d0d01325d16addcfb4691f615b70 Binary files /dev/null and b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/results/learning_curve.png differ diff --git a/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/results/res.csv b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..63edb1f82130f050be834d6376094e03b90de16d --- /dev/null +++ b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,252.00132352329445,220 +1,248.03029878869896,255 +2,270.69116893561363,234 +3,251.96742253953644,237 +4,243.15670333684696,235 +5,249.36998108772173,229 +6,249.59238649236136,227 +7,250.62076614707564,229 +8,252.86619536792665,225 +9,266.12317628639187,237 +10,269.4251149583948,247 +11,260.00199577747185,209 +12,270.213231810758,244 +13,38.57915893533237,158 +14,42.72804400273884,181 +15,262.0073808620749,209 +16,267.853389933307,236 +17,46.02973023128777,169 +18,254.67828868967476,277 +19,18.323484192388946,158 diff --git a/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/tb_logs/events.out.tfevents.1680448790.zf.223973.0 b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/tb_logs/events.out.tfevents.1680448790.zf.223973.0 new file mode 100644 index 0000000000000000000000000000000000000000..b3cfd872439d994ace670126196a75657fc0321d --- /dev/null +++ b/LunarLander-v2/Test_LunarLander-v2_PPO_20230402-231950/tb_logs/events.out.tfevents.1680448790.zf.223973.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54978dfb7e70a21619c10a0a3cce2825294b5648baf67c1867140775d1688f75 +size 40 diff --git a/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/config.yaml b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36622df9450c07d56a620c80131acd7dcb04d441 --- /dev/null +++ b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: PPO + device: cpu + env_name: LunarLander-v2 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: tasks + max_steps: 1000 + mode: train + mp_backend: mp + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 20 + train_eps: 600 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + continuous: false + critic_hidden_dim: 256 + critic_lr: 0.001 + entropy_coef: 0.01 + eps_clip: 0.2 + gamma: 0.99 + k_epochs: 4 + ppo_type: clip + sgd_batch_size: 32 + train_batch_size: 256 diff --git a/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/logs/log.txt b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3897d86ced31e68030599345f366ac21e7ff945c --- /dev/null +++ b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/logs/log.txt @@ -0,0 +1,661 @@ +2023-04-02 22:31:54 - r - INFO: - Hyperparameters: +2023-04-02 22:31:54 - r - INFO: - ================================================================================ +2023-04-02 22:31:54 - r - INFO: - Name Value Type +2023-04-02 22:31:54 - r - INFO: - env_name LunarLander-v2 +2023-04-02 22:31:54 - r - INFO: - new_step_api 1 +2023-04-02 22:31:54 - r - INFO: - wrapper None +2023-04-02 22:31:54 - r - INFO: - render 0 +2023-04-02 22:31:54 - r - INFO: - algo_name PPO +2023-04-02 22:31:54 - r - INFO: - mode train +2023-04-02 22:31:54 - r - INFO: - mp_backend mp +2023-04-02 22:31:54 - r - INFO: - seed 1 +2023-04-02 22:31:54 - r - INFO: - device cpu +2023-04-02 22:31:54 - r - INFO: - train_eps 600 +2023-04-02 22:31:54 - r - INFO: - test_eps 20 +2023-04-02 22:31:54 - r - INFO: - eval_eps 10 +2023-04-02 22:31:54 - r - INFO: - eval_per_episode 5 +2023-04-02 22:31:54 - r - INFO: - max_steps 1000 +2023-04-02 22:31:54 - r - INFO: - load_checkpoint 0 +2023-04-02 22:31:54 - r - INFO: - load_path tasks +2023-04-02 22:31:54 - r - INFO: - show_fig 0 +2023-04-02 22:31:54 - r - INFO: - save_fig 1 +2023-04-02 22:31:54 - r - INFO: - ppo_type clip +2023-04-02 22:31:54 - r - INFO: - continuous 0 +2023-04-02 22:31:54 - r - INFO: - gamma 0.99 +2023-04-02 22:31:54 - r - INFO: - k_epochs 4 +2023-04-02 22:31:54 - r - INFO: - actor_lr 0.0003 +2023-04-02 22:31:54 - r - INFO: - critic_lr 0.001 +2023-04-02 22:31:54 - r - INFO: - eps_clip 0.2 +2023-04-02 22:31:54 - r - INFO: - entropy_coef 0.01 +2023-04-02 22:31:54 - r - INFO: - train_batch_size 256 +2023-04-02 22:31:54 - r - INFO: - sgd_batch_size 32 +2023-04-02 22:31:54 - r - INFO: - actor_hidden_dim 256 +2023-04-02 22:31:54 - r - INFO: - critic_hidden_dim 256 +2023-04-02 22:31:54 - r - INFO: - task_dir /home/zf/Documents/Python/joyrl-offline/tasks/Train_LunarLander-v2_PPO_20230402-223154 +2023-04-02 22:31:54 - r - INFO: - res_dir /home/zf/Documents/Python/joyrl-offline/tasks/Train_LunarLander-v2_PPO_20230402-223154/results +2023-04-02 22:31:54 - r - INFO: - log_dir /home/zf/Documents/Python/joyrl-offline/tasks/Train_LunarLander-v2_PPO_20230402-223154/logs +2023-04-02 22:31:54 - r - INFO: - traj_dir /home/zf/Documents/Python/joyrl-offline/tasks/Train_LunarLander-v2_PPO_20230402-223154/traj +2023-04-02 22:31:54 - r - INFO: - tb_dir /home/zf/Documents/Python/joyrl-offline/tasks/Train_LunarLander-v2_PPO_20230402-223154/tb_logs +2023-04-02 22:31:54 - r - INFO: - ================================================================================ +2023-04-02 22:31:54 - r - INFO: - n_states: 8, n_actions: 4 +2023-04-02 22:31:54 - r - INFO: - Start training! +2023-04-02 22:31:54 - r - INFO: - Env: LunarLander-v2, Algorithm: PPO, Device: cpu +2023-04-02 22:31:54 - r - INFO: - Episode: 1/600, Reward: -121.444, Step: 114 +2023-04-02 22:31:54 - r - INFO: - Episode: 2/600, Reward: -277.234, Step: 118 +2023-04-02 22:31:55 - r - INFO: - Episode: 3/600, Reward: -115.896, Step: 118 +2023-04-02 22:31:55 - r - INFO: - Episode: 4/600, Reward: -369.750, Step: 103 +2023-04-02 22:31:55 - r - INFO: - Episode: 5/600, Reward: -337.845, Step: 104 +2023-04-02 22:31:55 - r - INFO: - Current episode 5 has the best eval reward: -133.460 +2023-04-02 22:31:55 - r - INFO: - Episode: 6/600, Reward: -153.433, Step: 104 +2023-04-02 22:31:55 - r - INFO: - Episode: 7/600, Reward: -97.777, Step: 106 +2023-04-02 22:31:55 - r - INFO: - Episode: 8/600, Reward: -14.116, Step: 106 +2023-04-02 22:31:56 - r - INFO: - Episode: 9/600, Reward: -281.984, Step: 119 +2023-04-02 22:31:56 - r - INFO: - Episode: 10/600, Reward: -115.022, Step: 113 +2023-04-02 22:31:56 - r - INFO: - Episode: 11/600, Reward: -104.564, Step: 117 +2023-04-02 22:31:56 - r - INFO: - Episode: 12/600, Reward: -96.841, Step: 105 +2023-04-02 22:31:56 - r - INFO: - Episode: 13/600, Reward: -174.983, Step: 104 +2023-04-02 22:31:56 - r - INFO: - Episode: 14/600, Reward: -107.475, Step: 107 +2023-04-02 22:31:56 - r - INFO: - Episode: 15/600, Reward: -196.714, Step: 109 +2023-04-02 22:31:57 - r - INFO: - Episode: 16/600, Reward: -216.498, Step: 98 +2023-04-02 22:31:57 - r - INFO: - Episode: 17/600, Reward: -255.592, Step: 115 +2023-04-02 22:31:57 - r - INFO: - Episode: 18/600, Reward: -241.319, Step: 116 +2023-04-02 22:31:57 - r - INFO: - Episode: 19/600, Reward: -135.709, Step: 129 +2023-04-02 22:31:57 - r - INFO: - Episode: 20/600, Reward: -239.036, Step: 118 +2023-04-02 22:31:58 - r - INFO: - Episode: 21/600, Reward: -35.755, Step: 135 +2023-04-02 22:31:58 - r - INFO: - Episode: 22/600, Reward: -323.152, Step: 114 +2023-04-02 22:31:58 - r - INFO: - Episode: 23/600, Reward: -219.333, Step: 147 +2023-04-02 22:31:58 - r - INFO: - Episode: 24/600, Reward: -88.332, Step: 124 +2023-04-02 22:31:58 - r - INFO: - Episode: 25/600, Reward: -203.773, Step: 132 +2023-04-02 22:31:59 - r - INFO: - Episode: 26/600, Reward: -294.652, Step: 152 +2023-04-02 22:31:59 - r - INFO: - Episode: 27/600, Reward: -296.472, Step: 128 +2023-04-02 22:31:59 - r - INFO: - Episode: 28/600, Reward: -79.829, Step: 146 +2023-04-02 22:31:59 - r - INFO: - Episode: 29/600, Reward: -540.169, Step: 116 +2023-04-02 22:31:59 - r - INFO: - Episode: 30/600, Reward: -581.587, Step: 98 +2023-04-02 22:32:00 - r - INFO: - Episode: 31/600, Reward: -517.071, Step: 110 +2023-04-02 22:32:00 - r - INFO: - Episode: 32/600, Reward: -421.562, Step: 111 +2023-04-02 22:32:00 - r - INFO: - Episode: 33/600, Reward: -314.088, Step: 120 +2023-04-02 22:32:00 - r - INFO: - Episode: 34/600, Reward: -207.563, Step: 114 +2023-04-02 22:32:00 - r - INFO: - Episode: 35/600, Reward: -100.580, Step: 101 +2023-04-02 22:32:00 - r - INFO: - Episode: 36/600, Reward: -316.158, Step: 108 +2023-04-02 22:32:00 - r - INFO: - Episode: 37/600, Reward: -312.437, Step: 107 +2023-04-02 22:32:00 - r - INFO: - Episode: 38/600, Reward: -358.396, Step: 110 +2023-04-02 22:32:00 - r - INFO: - Episode: 39/600, Reward: -124.137, Step: 108 +2023-04-02 22:32:01 - r - INFO: - Episode: 40/600, Reward: -156.087, Step: 109 +2023-04-02 22:32:02 - r - INFO: - Episode: 41/600, Reward: -201.888, Step: 127 +2023-04-02 22:32:02 - r - INFO: - Episode: 42/600, Reward: -75.112, Step: 134 +2023-04-02 22:32:02 - r - INFO: - Episode: 43/600, Reward: -294.268, Step: 125 +2023-04-02 22:32:02 - r - INFO: - Episode: 44/600, Reward: -449.123, Step: 147 +2023-04-02 22:32:02 - r - INFO: - Episode: 45/600, Reward: -251.930, Step: 146 +2023-04-02 22:32:03 - r - INFO: - Episode: 46/600, Reward: -363.396, Step: 151 +2023-04-02 22:32:03 - r - INFO: - Episode: 47/600, Reward: -121.840, Step: 135 +2023-04-02 22:32:03 - r - INFO: - Episode: 48/600, Reward: -586.513, Step: 173 +2023-04-02 22:32:03 - r - INFO: - Episode: 49/600, Reward: -430.392, Step: 138 +2023-04-02 22:32:03 - r - INFO: - Episode: 50/600, Reward: -322.294, Step: 140 +2023-04-02 22:32:04 - r - INFO: - Episode: 51/600, Reward: -266.473, Step: 134 +2023-04-02 22:32:04 - r - INFO: - Episode: 52/600, Reward: -290.379, Step: 109 +2023-04-02 22:32:04 - r - INFO: - Episode: 53/600, Reward: -564.944, Step: 115 +2023-04-02 22:32:04 - r - INFO: - Episode: 54/600, Reward: -161.722, Step: 138 +2023-04-02 22:32:04 - r - INFO: - Episode: 55/600, Reward: -218.362, Step: 132 +2023-04-02 22:32:05 - r - INFO: - Episode: 56/600, Reward: -90.506, Step: 135 +2023-04-02 22:32:05 - r - INFO: - Episode: 57/600, Reward: -48.795, Step: 115 +2023-04-02 22:32:05 - r - INFO: - Episode: 58/600, Reward: -239.578, Step: 110 +2023-04-02 22:32:05 - r - INFO: - Episode: 59/600, Reward: -148.223, Step: 113 +2023-04-02 22:32:05 - r - INFO: - Episode: 60/600, Reward: -102.341, Step: 107 +2023-04-02 22:32:05 - r - INFO: - Episode: 61/600, Reward: -90.720, Step: 116 +2023-04-02 22:32:06 - r - INFO: - Episode: 62/600, Reward: -126.898, Step: 119 +2023-04-02 22:32:06 - r - INFO: - Episode: 63/600, Reward: -405.098, Step: 140 +2023-04-02 22:32:06 - r - INFO: - Episode: 64/600, Reward: -409.668, Step: 114 +2023-04-02 22:32:06 - r - INFO: - Episode: 65/600, Reward: -59.205, Step: 115 +2023-04-02 22:32:06 - r - INFO: - Episode: 66/600, Reward: -212.760, Step: 120 +2023-04-02 22:32:07 - r - INFO: - Episode: 67/600, Reward: -83.135, Step: 121 +2023-04-02 22:32:07 - r - INFO: - Episode: 68/600, Reward: -93.160, Step: 113 +2023-04-02 22:32:07 - r - INFO: - Episode: 69/600, Reward: -115.493, Step: 116 +2023-04-02 22:32:07 - r - INFO: - Episode: 70/600, Reward: -101.286, Step: 107 +2023-04-02 22:32:07 - r - INFO: - Current episode 70 has the best eval reward: -112.992 +2023-04-02 22:32:07 - r - INFO: - Episode: 71/600, Reward: -6.531, Step: 161 +2023-04-02 22:32:08 - r - INFO: - Episode: 72/600, Reward: -195.514, Step: 127 +2023-04-02 22:32:08 - r - INFO: - Episode: 73/600, Reward: -105.655, Step: 117 +2023-04-02 22:32:08 - r - INFO: - Episode: 74/600, Reward: -208.326, Step: 131 +2023-04-02 22:32:08 - r - INFO: - Episode: 75/600, Reward: -92.113, Step: 126 +2023-04-02 22:32:08 - r - INFO: - Current episode 75 has the best eval reward: -74.040 +2023-04-02 22:32:08 - r - INFO: - Episode: 76/600, Reward: -108.639, Step: 135 +2023-04-02 22:32:09 - r - INFO: - Episode: 77/600, Reward: -235.689, Step: 141 +2023-04-02 22:32:09 - r - INFO: - Episode: 78/600, Reward: -115.349, Step: 130 +2023-04-02 22:32:09 - r - INFO: - Episode: 79/600, Reward: -66.639, Step: 117 +2023-04-02 22:32:09 - r - INFO: - Episode: 80/600, Reward: -84.450, Step: 124 +2023-04-02 22:32:09 - r - INFO: - Episode: 81/600, Reward: -75.365, Step: 110 +2023-04-02 22:32:09 - r - INFO: - Episode: 82/600, Reward: -70.921, Step: 132 +2023-04-02 22:32:10 - r - INFO: - Episode: 83/600, Reward: -87.512, Step: 113 +2023-04-02 22:32:10 - r - INFO: - Episode: 84/600, Reward: -78.195, Step: 111 +2023-04-02 22:32:10 - r - INFO: - Episode: 85/600, Reward: -200.101, Step: 117 +2023-04-02 22:32:10 - r - INFO: - Episode: 86/600, Reward: -310.614, Step: 134 +2023-04-02 22:32:10 - r - INFO: - Episode: 87/600, Reward: -315.094, Step: 122 +2023-04-02 22:32:10 - r - INFO: - Episode: 88/600, Reward: -76.644, Step: 137 +2023-04-02 22:32:11 - r - INFO: - Episode: 89/600, Reward: -116.261, Step: 123 +2023-04-02 22:32:11 - r - INFO: - Episode: 90/600, Reward: -34.352, Step: 140 +2023-04-02 22:32:11 - r - INFO: - Episode: 91/600, Reward: -80.778, Step: 153 +2023-04-02 22:32:11 - r - INFO: - Episode: 92/600, Reward: -217.031, Step: 142 +2023-04-02 22:32:12 - r - INFO: - Episode: 93/600, Reward: -24.960, Step: 162 +2023-04-02 22:32:12 - r - INFO: - Episode: 94/600, Reward: -60.824, Step: 158 +2023-04-02 22:32:12 - r - INFO: - Episode: 95/600, Reward: -340.600, Step: 136 +2023-04-02 22:32:13 - r - INFO: - Episode: 96/600, Reward: -277.215, Step: 126 +2023-04-02 22:32:13 - r - INFO: - Episode: 97/600, Reward: -125.626, Step: 131 +2023-04-02 22:32:13 - r - INFO: - Episode: 98/600, Reward: -45.416, Step: 124 +2023-04-02 22:32:13 - r - INFO: - Episode: 99/600, Reward: -67.034, Step: 142 +2023-04-02 22:32:13 - r - INFO: - Episode: 100/600, Reward: -88.193, Step: 130 +2023-04-02 22:32:14 - r - INFO: - Current episode 100 has the best eval reward: -50.244 +2023-04-02 22:32:14 - r - INFO: - Episode: 101/600, Reward: -58.403, Step: 133 +2023-04-02 22:32:14 - r - INFO: - Episode: 102/600, Reward: -76.792, Step: 122 +2023-04-02 22:32:14 - r - INFO: - Episode: 103/600, Reward: -79.530, Step: 122 +2023-04-02 22:32:14 - r - INFO: - Episode: 104/600, Reward: -76.829, Step: 138 +2023-04-02 22:32:14 - r - INFO: - Episode: 105/600, Reward: -74.573, Step: 162 +2023-04-02 22:32:15 - r - INFO: - Episode: 106/600, Reward: -94.756, Step: 126 +2023-04-02 22:32:15 - r - INFO: - Episode: 107/600, Reward: -57.446, Step: 135 +2023-04-02 22:32:15 - r - INFO: - Episode: 108/600, Reward: -30.263, Step: 153 +2023-04-02 22:32:15 - r - INFO: - Episode: 109/600, Reward: -15.612, Step: 126 +2023-04-02 22:32:15 - r - INFO: - Episode: 110/600, Reward: -42.464, Step: 112 +2023-04-02 22:32:16 - r - INFO: - Episode: 111/600, Reward: -53.891, Step: 131 +2023-04-02 22:32:16 - r - INFO: - Episode: 112/600, Reward: -47.279, Step: 135 +2023-04-02 22:32:16 - r - INFO: - Episode: 113/600, Reward: -28.310, Step: 171 +2023-04-02 22:32:16 - r - INFO: - Episode: 114/600, Reward: -71.555, Step: 161 +2023-04-02 22:32:16 - r - INFO: - Episode: 115/600, Reward: -46.645, Step: 165 +2023-04-02 22:32:17 - r - INFO: - Episode: 116/600, Reward: -110.328, Step: 176 +2023-04-02 22:32:17 - r - INFO: - Episode: 117/600, Reward: -1.405, Step: 193 +2023-04-02 22:32:17 - r - INFO: - Episode: 118/600, Reward: -80.805, Step: 228 +2023-04-02 22:32:19 - r - INFO: - Episode: 119/600, Reward: -431.340, Step: 790 +2023-04-02 22:32:19 - r - INFO: - Episode: 120/600, Reward: -148.612, Step: 161 +2023-04-02 22:32:20 - r - INFO: - Episode: 121/600, Reward: -176.688, Step: 170 +2023-04-02 22:32:20 - r - INFO: - Episode: 122/600, Reward: -75.354, Step: 159 +2023-04-02 22:32:21 - r - INFO: - Episode: 123/600, Reward: -377.815, Step: 350 +2023-04-02 22:32:21 - r - INFO: - Episode: 124/600, Reward: -226.834, Step: 395 +2023-04-02 22:32:21 - r - INFO: - Episode: 125/600, Reward: -55.800, Step: 198 +2023-04-02 22:32:23 - r - INFO: - Episode: 126/600, Reward: -145.402, Step: 236 +2023-04-02 22:32:23 - r - INFO: - Episode: 127/600, Reward: -132.791, Step: 168 +2023-04-02 22:32:23 - r - INFO: - Episode: 128/600, Reward: -363.342, Step: 259 +2023-04-02 22:32:23 - r - INFO: - Episode: 129/600, Reward: -1.090, Step: 155 +2023-04-02 22:32:23 - r - INFO: - Episode: 130/600, Reward: -67.628, Step: 193 +2023-04-02 22:32:24 - r - INFO: - Episode: 131/600, Reward: -117.421, Step: 138 +2023-04-02 22:32:24 - r - INFO: - Episode: 132/600, Reward: 9.361, Step: 167 +2023-04-02 22:32:24 - r - INFO: - Episode: 133/600, Reward: -418.884, Step: 161 +2023-04-02 22:32:24 - r - INFO: - Episode: 134/600, Reward: 7.493, Step: 161 +2023-04-02 22:32:25 - r - INFO: - Episode: 135/600, Reward: -271.478, Step: 536 +2023-04-02 22:32:28 - r - INFO: - Episode: 136/600, Reward: 34.584, Step: 1000 +2023-04-02 22:32:28 - r - INFO: - Episode: 137/600, Reward: -11.470, Step: 147 +2023-04-02 22:32:28 - r - INFO: - Episode: 138/600, Reward: -75.408, Step: 132 +2023-04-02 22:32:28 - r - INFO: - Episode: 139/600, Reward: -181.785, Step: 144 +2023-04-02 22:32:29 - r - INFO: - Episode: 140/600, Reward: -45.557, Step: 176 +2023-04-02 22:32:30 - r - INFO: - Current episode 140 has the best eval reward: -37.770 +2023-04-02 22:32:30 - r - INFO: - Episode: 141/600, Reward: -203.110, Step: 165 +2023-04-02 22:32:30 - r - INFO: - Episode: 142/600, Reward: -254.859, Step: 190 +2023-04-02 22:32:31 - r - INFO: - Episode: 143/600, Reward: 9.829, Step: 160 +2023-04-02 22:32:31 - r - INFO: - Episode: 144/600, Reward: 17.705, Step: 150 +2023-04-02 22:32:31 - r - INFO: - Episode: 145/600, Reward: 8.250, Step: 139 +2023-04-02 22:32:34 - r - INFO: - Episode: 146/600, Reward: -21.816, Step: 180 +2023-04-02 22:32:34 - r - INFO: - Episode: 147/600, Reward: -29.650, Step: 213 +2023-04-02 22:32:34 - r - INFO: - Episode: 148/600, Reward: 10.789, Step: 179 +2023-04-02 22:32:34 - r - INFO: - Episode: 149/600, Reward: -216.825, Step: 321 +2023-04-02 22:32:35 - r - INFO: - Episode: 150/600, Reward: -62.803, Step: 448 +2023-04-02 22:32:39 - r - INFO: - Episode: 151/600, Reward: 27.057, Step: 302 +2023-04-02 22:32:40 - r - INFO: - Episode: 152/600, Reward: -20.970, Step: 253 +2023-04-02 22:32:40 - r - INFO: - Episode: 153/600, Reward: -188.575, Step: 319 +2023-04-02 22:32:41 - r - INFO: - Episode: 154/600, Reward: -105.422, Step: 346 +2023-04-02 22:32:41 - r - INFO: - Episode: 155/600, Reward: -147.773, Step: 232 +2023-04-02 22:32:43 - r - INFO: - Episode: 156/600, Reward: -98.581, Step: 222 +2023-04-02 22:32:43 - r - INFO: - Episode: 157/600, Reward: -204.741, Step: 437 +2023-04-02 22:32:44 - r - INFO: - Episode: 158/600, Reward: 31.711, Step: 1000 +2023-04-02 22:32:46 - r - INFO: - Episode: 159/600, Reward: 33.317, Step: 1000 +2023-04-02 22:32:46 - r - INFO: - Episode: 160/600, Reward: 3.771, Step: 136 +2023-04-02 22:32:47 - r - INFO: - Current episode 160 has the best eval reward: -9.279 +2023-04-02 22:32:47 - r - INFO: - Episode: 161/600, Reward: 9.814, Step: 123 +2023-04-02 22:32:47 - r - INFO: - Episode: 162/600, Reward: 8.625, Step: 129 +2023-04-02 22:32:47 - r - INFO: - Episode: 163/600, Reward: -5.854, Step: 129 +2023-04-02 22:32:48 - r - INFO: - Episode: 164/600, Reward: 23.973, Step: 132 +2023-04-02 22:32:48 - r - INFO: - Episode: 165/600, Reward: -15.441, Step: 137 +2023-04-02 22:32:49 - r - INFO: - Current episode 165 has the best eval reward: 0.221 +2023-04-02 22:32:49 - r - INFO: - Episode: 166/600, Reward: -24.959, Step: 142 +2023-04-02 22:32:49 - r - INFO: - Episode: 167/600, Reward: -55.740, Step: 176 +2023-04-02 22:32:50 - r - INFO: - Episode: 168/600, Reward: 54.055, Step: 1000 +2023-04-02 22:32:52 - r - INFO: - Episode: 169/600, Reward: 60.755, Step: 1000 +2023-04-02 22:32:52 - r - INFO: - Episode: 170/600, Reward: -180.272, Step: 143 +2023-04-02 22:32:53 - r - INFO: - Episode: 171/600, Reward: -184.921, Step: 148 +2023-04-02 22:32:53 - r - INFO: - Episode: 172/600, Reward: -171.919, Step: 140 +2023-04-02 22:32:54 - r - INFO: - Episode: 173/600, Reward: -183.949, Step: 142 +2023-04-02 22:32:55 - r - INFO: - Episode: 174/600, Reward: 53.447, Step: 1000 +2023-04-02 22:32:56 - r - INFO: - Episode: 175/600, Reward: 79.824, Step: 1000 +2023-04-02 22:32:58 - r - INFO: - Episode: 176/600, Reward: -106.806, Step: 144 +2023-04-02 22:32:58 - r - INFO: - Episode: 177/600, Reward: -58.712, Step: 138 +2023-04-02 22:32:58 - r - INFO: - Episode: 178/600, Reward: -56.982, Step: 140 +2023-04-02 22:32:59 - r - INFO: - Episode: 179/600, Reward: 77.983, Step: 1000 +2023-04-02 22:33:00 - r - INFO: - Episode: 180/600, Reward: 18.476, Step: 166 +2023-04-02 22:33:01 - r - INFO: - Episode: 181/600, Reward: -17.921, Step: 153 +2023-04-02 22:33:01 - r - INFO: - Episode: 182/600, Reward: 4.517, Step: 149 +2023-04-02 22:33:02 - r - INFO: - Episode: 183/600, Reward: 93.140, Step: 1000 +2023-04-02 22:33:02 - r - INFO: - Episode: 184/600, Reward: -61.875, Step: 108 +2023-04-02 22:33:02 - r - INFO: - Episode: 185/600, Reward: -45.813, Step: 104 +2023-04-02 22:33:03 - r - INFO: - Episode: 186/600, Reward: -26.910, Step: 118 +2023-04-02 22:33:03 - r - INFO: - Episode: 187/600, Reward: -46.781, Step: 106 +2023-04-02 22:33:03 - r - INFO: - Episode: 188/600, Reward: -20.429, Step: 130 +2023-04-02 22:33:03 - r - INFO: - Episode: 189/600, Reward: -18.215, Step: 142 +2023-04-02 22:33:03 - r - INFO: - Episode: 190/600, Reward: -33.297, Step: 110 +2023-04-02 22:33:03 - r - INFO: - Episode: 191/600, Reward: -55.212, Step: 112 +2023-04-02 22:33:04 - r - INFO: - Episode: 192/600, Reward: -3.684, Step: 127 +2023-04-02 22:33:04 - r - INFO: - Episode: 193/600, Reward: -56.040, Step: 116 +2023-04-02 22:33:05 - r - INFO: - Episode: 194/600, Reward: 94.817, Step: 1000 +2023-04-02 22:33:05 - r - INFO: - Episode: 195/600, Reward: -39.891, Step: 130 +2023-04-02 22:33:08 - r - INFO: - Current episode 195 has the best eval reward: 3.858 +2023-04-02 22:33:09 - r - INFO: - Episode: 196/600, Reward: 68.629, Step: 1000 +2023-04-02 22:33:09 - r - INFO: - Episode: 197/600, Reward: -74.193, Step: 128 +2023-04-02 22:33:10 - r - INFO: - Episode: 198/600, Reward: 86.340, Step: 1000 +2023-04-02 22:33:11 - r - INFO: - Episode: 199/600, Reward: 89.574, Step: 1000 +2023-04-02 22:33:13 - r - INFO: - Episode: 200/600, Reward: 89.497, Step: 1000 +2023-04-02 22:33:15 - r - INFO: - Episode: 201/600, Reward: -37.336, Step: 136 +2023-04-02 22:33:15 - r - INFO: - Episode: 202/600, Reward: -99.698, Step: 131 +2023-04-02 22:33:15 - r - INFO: - Episode: 203/600, Reward: -63.996, Step: 162 +2023-04-02 22:33:16 - r - INFO: - Episode: 204/600, Reward: -74.190, Step: 155 +2023-04-02 22:33:16 - r - INFO: - Episode: 205/600, Reward: 209.337, Step: 736 +2023-04-02 22:33:19 - r - INFO: - Episode: 206/600, Reward: 188.765, Step: 757 +2023-04-02 22:33:19 - r - INFO: - Episode: 207/600, Reward: 5.051, Step: 148 +2023-04-02 22:33:19 - r - INFO: - Episode: 208/600, Reward: -40.342, Step: 179 +2023-04-02 22:33:19 - r - INFO: - Episode: 209/600, Reward: -70.096, Step: 244 +2023-04-02 22:33:21 - r - INFO: - Episode: 210/600, Reward: 75.045, Step: 1000 +2023-04-02 22:33:22 - r - INFO: - Episode: 211/600, Reward: -29.003, Step: 201 +2023-04-02 22:33:23 - r - INFO: - Episode: 212/600, Reward: 73.286, Step: 1000 +2023-04-02 22:33:23 - r - INFO: - Episode: 213/600, Reward: -216.103, Step: 168 +2023-04-02 22:33:24 - r - INFO: - Episode: 214/600, Reward: -86.516, Step: 221 +2023-04-02 22:33:24 - r - INFO: - Episode: 215/600, Reward: -143.634, Step: 182 +2023-04-02 22:33:27 - r - INFO: - Current episode 215 has the best eval reward: 18.498 +2023-04-02 22:33:27 - r - INFO: - Episode: 216/600, Reward: -146.850, Step: 169 +2023-04-02 22:33:27 - r - INFO: - Episode: 217/600, Reward: -43.835, Step: 188 +2023-04-02 22:33:27 - r - INFO: - Episode: 218/600, Reward: -40.004, Step: 207 +2023-04-02 22:33:28 - r - INFO: - Episode: 219/600, Reward: -44.306, Step: 189 +2023-04-02 22:33:28 - r - INFO: - Episode: 220/600, Reward: -3.133, Step: 135 +2023-04-02 22:33:31 - r - INFO: - Episode: 221/600, Reward: 85.608, Step: 1000 +2023-04-02 22:33:31 - r - INFO: - Episode: 222/600, Reward: -34.164, Step: 239 +2023-04-02 22:33:32 - r - INFO: - Episode: 223/600, Reward: 222.201, Step: 309 +2023-04-02 22:33:33 - r - INFO: - Episode: 224/600, Reward: 179.929, Step: 837 +2023-04-02 22:33:34 - r - INFO: - Episode: 225/600, Reward: 82.515, Step: 1000 +2023-04-02 22:33:36 - r - INFO: - Current episode 225 has the best eval reward: 24.103 +2023-04-02 22:33:38 - r - INFO: - Episode: 226/600, Reward: 72.611, Step: 1000 +2023-04-02 22:33:38 - r - INFO: - Episode: 227/600, Reward: 3.218, Step: 192 +2023-04-02 22:33:38 - r - INFO: - Episode: 228/600, Reward: 27.043, Step: 170 +2023-04-02 22:33:38 - r - INFO: - Episode: 229/600, Reward: 22.643, Step: 174 +2023-04-02 22:33:38 - r - INFO: - Episode: 230/600, Reward: -14.321, Step: 197 +2023-04-02 22:33:39 - r - INFO: - Current episode 230 has the best eval reward: 57.269 +2023-04-02 22:33:40 - r - INFO: - Episode: 231/600, Reward: 30.397, Step: 170 +2023-04-02 22:33:40 - r - INFO: - Episode: 232/600, Reward: 25.974, Step: 193 +2023-04-02 22:33:40 - r - INFO: - Episode: 233/600, Reward: 30.080, Step: 171 +2023-04-02 22:33:40 - r - INFO: - Episode: 234/600, Reward: -29.671, Step: 222 +2023-04-02 22:33:40 - r - INFO: - Episode: 235/600, Reward: 207.325, Step: 305 +2023-04-02 22:33:41 - r - INFO: - Episode: 236/600, Reward: 8.052, Step: 181 +2023-04-02 22:33:41 - r - INFO: - Episode: 237/600, Reward: 31.976, Step: 185 +2023-04-02 22:33:42 - r - INFO: - Episode: 238/600, Reward: 13.269, Step: 210 +2023-04-02 22:33:42 - r - INFO: - Episode: 239/600, Reward: 7.470, Step: 231 +2023-04-02 22:33:42 - r - INFO: - Episode: 240/600, Reward: -26.022, Step: 334 +2023-04-02 22:33:44 - r - INFO: - Episode: 241/600, Reward: 25.707, Step: 228 +2023-04-02 22:33:44 - r - INFO: - Episode: 242/600, Reward: -22.355, Step: 210 +2023-04-02 22:33:44 - r - INFO: - Episode: 243/600, Reward: 16.928, Step: 210 +2023-04-02 22:33:44 - r - INFO: - Episode: 244/600, Reward: 0.225, Step: 204 +2023-04-02 22:33:45 - r - INFO: - Episode: 245/600, Reward: 191.807, Step: 454 +2023-04-02 22:33:48 - r - INFO: - Episode: 246/600, Reward: -42.903, Step: 223 +2023-04-02 22:33:48 - r - INFO: - Episode: 247/600, Reward: -113.874, Step: 281 +2023-04-02 22:33:50 - r - INFO: - Episode: 248/600, Reward: 16.974, Step: 1000 +2023-04-02 22:33:50 - r - INFO: - Episode: 249/600, Reward: 214.347, Step: 310 +2023-04-02 22:33:50 - r - INFO: - Episode: 250/600, Reward: -156.822, Step: 288 +2023-04-02 22:33:53 - r - INFO: - Episode: 251/600, Reward: 182.684, Step: 412 +2023-04-02 22:33:54 - r - INFO: - Episode: 252/600, Reward: 174.696, Step: 959 +2023-04-02 22:33:55 - r - INFO: - Episode: 253/600, Reward: 199.880, Step: 314 +2023-04-02 22:33:55 - r - INFO: - Episode: 254/600, Reward: 189.431, Step: 395 +2023-04-02 22:33:56 - r - INFO: - Episode: 255/600, Reward: 189.496, Step: 369 +2023-04-02 22:33:57 - r - INFO: - Episode: 256/600, Reward: -34.829, Step: 216 +2023-04-02 22:33:57 - r - INFO: - Episode: 257/600, Reward: -63.171, Step: 241 +2023-04-02 22:33:57 - r - INFO: - Episode: 258/600, Reward: -62.713, Step: 239 +2023-04-02 22:33:58 - r - INFO: - Episode: 259/600, Reward: 188.234, Step: 485 +2023-04-02 22:33:58 - r - INFO: - Episode: 260/600, Reward: -40.356, Step: 194 +2023-04-02 22:33:59 - r - INFO: - Current episode 260 has the best eval reward: 80.901 +2023-04-02 22:33:59 - r - INFO: - Episode: 261/600, Reward: -8.431, Step: 197 +2023-04-02 22:34:00 - r - INFO: - Episode: 262/600, Reward: 164.231, Step: 385 +2023-04-02 22:34:00 - r - INFO: - Episode: 263/600, Reward: 179.970, Step: 350 +2023-04-02 22:34:01 - r - INFO: - Episode: 264/600, Reward: -172.766, Step: 298 +2023-04-02 22:34:01 - r - INFO: - Episode: 265/600, Reward: -148.916, Step: 257 +2023-04-02 22:34:03 - r - INFO: - Episode: 266/600, Reward: -184.392, Step: 377 +2023-04-02 22:34:03 - r - INFO: - Episode: 267/600, Reward: -148.969, Step: 396 +2023-04-02 22:34:03 - r - INFO: - Episode: 268/600, Reward: -109.250, Step: 314 +2023-04-02 22:34:04 - r - INFO: - Episode: 269/600, Reward: 13.637, Step: 256 +2023-04-02 22:34:04 - r - INFO: - Episode: 270/600, Reward: 28.715, Step: 256 +2023-04-02 22:34:06 - r - INFO: - Episode: 271/600, Reward: -46.103, Step: 284 +2023-04-02 22:34:07 - r - INFO: - Episode: 272/600, Reward: 249.831, Step: 339 +2023-04-02 22:34:07 - r - INFO: - Episode: 273/600, Reward: -8.305, Step: 262 +2023-04-02 22:34:07 - r - INFO: - Episode: 274/600, Reward: 244.855, Step: 399 +2023-04-02 22:34:08 - r - INFO: - Episode: 275/600, Reward: -23.199, Step: 229 +2023-04-02 22:34:09 - r - INFO: - Episode: 276/600, Reward: -18.049, Step: 200 +2023-04-02 22:34:09 - r - INFO: - Episode: 277/600, Reward: -132.946, Step: 248 +2023-04-02 22:34:09 - r - INFO: - Episode: 278/600, Reward: -29.598, Step: 206 +2023-04-02 22:34:10 - r - INFO: - Episode: 279/600, Reward: -126.089, Step: 193 +2023-04-02 22:34:10 - r - INFO: - Episode: 280/600, Reward: -80.033, Step: 217 +2023-04-02 22:34:12 - r - INFO: - Episode: 281/600, Reward: -37.706, Step: 250 +2023-04-02 22:34:12 - r - INFO: - Episode: 282/600, Reward: -15.304, Step: 254 +2023-04-02 22:34:12 - r - INFO: - Episode: 283/600, Reward: -47.292, Step: 270 +2023-04-02 22:34:13 - r - INFO: - Episode: 284/600, Reward: 224.679, Step: 397 +2023-04-02 22:34:13 - r - INFO: - Episode: 285/600, Reward: 13.596, Step: 283 +2023-04-02 22:34:16 - r - INFO: - Episode: 286/600, Reward: 250.095, Step: 344 +2023-04-02 22:34:16 - r - INFO: - Episode: 287/600, Reward: 5.143, Step: 235 +2023-04-02 22:34:16 - r - INFO: - Episode: 288/600, Reward: -213.223, Step: 484 +2023-04-02 22:34:17 - r - INFO: - Episode: 289/600, Reward: -33.505, Step: 229 +2023-04-02 22:34:17 - r - INFO: - Episode: 290/600, Reward: 144.377, Step: 480 +2023-04-02 22:34:20 - r - INFO: - Current episode 290 has the best eval reward: 208.309 +2023-04-02 22:34:20 - r - INFO: - Episode: 291/600, Reward: 186.786, Step: 474 +2023-04-02 22:34:20 - r - INFO: - Episode: 292/600, Reward: -8.561, Step: 272 +2023-04-02 22:34:21 - r - INFO: - Episode: 293/600, Reward: -38.962, Step: 264 +2023-04-02 22:34:21 - r - INFO: - Episode: 294/600, Reward: 156.429, Step: 393 +2023-04-02 22:34:21 - r - INFO: - Episode: 295/600, Reward: -11.140, Step: 266 +2023-04-02 22:34:24 - r - INFO: - Episode: 296/600, Reward: 165.346, Step: 487 +2023-04-02 22:34:25 - r - INFO: - Episode: 297/600, Reward: 152.515, Step: 773 +2023-04-02 22:34:26 - r - INFO: - Episode: 298/600, Reward: 136.377, Step: 531 +2023-04-02 22:34:27 - r - INFO: - Episode: 299/600, Reward: 126.601, Step: 763 +2023-04-02 22:34:28 - r - INFO: - Episode: 300/600, Reward: 126.580, Step: 791 +2023-04-02 22:34:32 - r - INFO: - Episode: 301/600, Reward: 147.956, Step: 704 +2023-04-02 22:34:33 - r - INFO: - Episode: 302/600, Reward: 227.980, Step: 411 +2023-04-02 22:34:33 - r - INFO: - Episode: 303/600, Reward: 235.315, Step: 437 +2023-04-02 22:34:34 - r - INFO: - Episode: 304/600, Reward: 5.183, Step: 263 +2023-04-02 22:34:34 - r - INFO: - Episode: 305/600, Reward: 16.238, Step: 267 +2023-04-02 22:34:36 - r - INFO: - Episode: 306/600, Reward: -3.761, Step: 272 +2023-04-02 22:34:36 - r - INFO: - Episode: 307/600, Reward: 250.964, Step: 342 +2023-04-02 22:34:36 - r - INFO: - Episode: 308/600, Reward: 245.651, Step: 314 +2023-04-02 22:34:37 - r - INFO: - Episode: 309/600, Reward: 255.293, Step: 306 +2023-04-02 22:34:37 - r - INFO: - Episode: 310/600, Reward: -62.482, Step: 371 +2023-04-02 22:34:39 - r - INFO: - Episode: 311/600, Reward: -24.992, Step: 338 +2023-04-02 22:34:41 - r - INFO: - Episode: 312/600, Reward: -173.280, Step: 891 +2023-04-02 22:34:41 - r - INFO: - Episode: 313/600, Reward: -86.008, Step: 510 +2023-04-02 22:34:42 - r - INFO: - Episode: 314/600, Reward: 251.627, Step: 365 +2023-04-02 22:34:42 - r - INFO: - Episode: 315/600, Reward: 235.291, Step: 359 +2023-04-02 22:34:44 - r - INFO: - Current episode 315 has the best eval reward: 209.267 +2023-04-02 22:34:44 - r - INFO: - Episode: 316/600, Reward: 242.802, Step: 384 +2023-04-02 22:34:45 - r - INFO: - Episode: 317/600, Reward: 238.875, Step: 400 +2023-04-02 22:34:45 - r - INFO: - Episode: 318/600, Reward: 13.820, Step: 246 +2023-04-02 22:34:46 - r - INFO: - Episode: 319/600, Reward: 243.893, Step: 366 +2023-04-02 22:34:46 - r - INFO: - Episode: 320/600, Reward: 247.083, Step: 329 +2023-04-02 22:34:49 - r - INFO: - Episode: 321/600, Reward: 149.406, Step: 550 +2023-04-02 22:34:50 - r - INFO: - Episode: 322/600, Reward: 239.891, Step: 407 +2023-04-02 22:34:50 - r - INFO: - Episode: 323/600, Reward: 234.083, Step: 413 +2023-04-02 22:34:51 - r - INFO: - Episode: 324/600, Reward: 246.263, Step: 363 +2023-04-02 22:34:51 - r - INFO: - Episode: 325/600, Reward: 246.975, Step: 343 +2023-04-02 22:34:53 - r - INFO: - Current episode 325 has the best eval reward: 214.499 +2023-04-02 22:34:53 - r - INFO: - Episode: 326/600, Reward: 181.409, Step: 390 +2023-04-02 22:34:53 - r - INFO: - Episode: 327/600, Reward: 245.080, Step: 344 +2023-04-02 22:34:54 - r - INFO: - Episode: 328/600, Reward: 233.944, Step: 455 +2023-04-02 22:34:55 - r - INFO: - Episode: 329/600, Reward: 213.536, Step: 461 +2023-04-02 22:34:55 - r - INFO: - Episode: 330/600, Reward: 224.410, Step: 397 +2023-04-02 22:34:57 - r - INFO: - Episode: 331/600, Reward: 233.005, Step: 364 +2023-04-02 22:34:58 - r - INFO: - Episode: 332/600, Reward: 8.840, Step: 217 +2023-04-02 22:34:58 - r - INFO: - Episode: 333/600, Reward: -3.008, Step: 218 +2023-04-02 22:34:58 - r - INFO: - Episode: 334/600, Reward: 12.239, Step: 250 +2023-04-02 22:34:58 - r - INFO: - Episode: 335/600, Reward: 8.395, Step: 235 +2023-04-02 22:35:00 - r - INFO: - Episode: 336/600, Reward: 226.148, Step: 291 +2023-04-02 22:35:00 - r - INFO: - Episode: 337/600, Reward: 247.587, Step: 320 +2023-04-02 22:35:01 - r - INFO: - Episode: 338/600, Reward: 244.383, Step: 390 +2023-04-02 22:35:01 - r - INFO: - Episode: 339/600, Reward: 240.439, Step: 399 +2023-04-02 22:35:02 - r - INFO: - Episode: 340/600, Reward: 176.350, Step: 407 +2023-04-02 22:35:05 - r - INFO: - Episode: 341/600, Reward: -37.625, Step: 348 +2023-04-02 22:35:05 - r - INFO: - Episode: 342/600, Reward: 239.805, Step: 386 +2023-04-02 22:35:06 - r - INFO: - Episode: 343/600, Reward: 236.588, Step: 375 +2023-04-02 22:35:06 - r - INFO: - Episode: 344/600, Reward: 212.320, Step: 420 +2023-04-02 22:35:07 - r - INFO: - Episode: 345/600, Reward: 239.345, Step: 327 +2023-04-02 22:35:09 - r - INFO: - Episode: 346/600, Reward: 245.970, Step: 357 +2023-04-02 22:35:09 - r - INFO: - Episode: 347/600, Reward: 243.653, Step: 352 +2023-04-02 22:35:10 - r - INFO: - Episode: 348/600, Reward: 225.001, Step: 333 +2023-04-02 22:35:10 - r - INFO: - Episode: 349/600, Reward: 242.180, Step: 305 +2023-04-02 22:35:10 - r - INFO: - Episode: 350/600, Reward: 13.432, Step: 265 +2023-04-02 22:35:13 - r - INFO: - Episode: 351/600, Reward: 240.129, Step: 398 +2023-04-02 22:35:13 - r - INFO: - Episode: 352/600, Reward: -9.537, Step: 353 +2023-04-02 22:35:14 - r - INFO: - Episode: 353/600, Reward: 142.987, Step: 795 +2023-04-02 22:35:15 - r - INFO: - Episode: 354/600, Reward: 229.011, Step: 381 +2023-04-02 22:35:15 - r - INFO: - Episode: 355/600, Reward: 239.234, Step: 394 +2023-04-02 22:35:17 - r - INFO: - Current episode 355 has the best eval reward: 231.954 +2023-04-02 22:35:17 - r - INFO: - Episode: 356/600, Reward: 244.898, Step: 346 +2023-04-02 22:35:18 - r - INFO: - Episode: 357/600, Reward: 245.168, Step: 352 +2023-04-02 22:35:18 - r - INFO: - Episode: 358/600, Reward: 247.098, Step: 319 +2023-04-02 22:35:19 - r - INFO: - Episode: 359/600, Reward: 242.512, Step: 419 +2023-04-02 22:35:19 - r - INFO: - Episode: 360/600, Reward: 24.758, Step: 265 +2023-04-02 22:35:21 - r - INFO: - Episode: 361/600, Reward: 184.121, Step: 324 +2023-04-02 22:35:22 - r - INFO: - Episode: 362/600, Reward: 209.176, Step: 395 +2023-04-02 22:35:22 - r - INFO: - Episode: 363/600, Reward: -12.265, Step: 221 +2023-04-02 22:35:22 - r - INFO: - Episode: 364/600, Reward: 32.635, Step: 239 +2023-04-02 22:35:22 - r - INFO: - Episode: 365/600, Reward: 7.066, Step: 286 +2023-04-02 22:35:25 - r - INFO: - Episode: 366/600, Reward: -57.704, Step: 356 +2023-04-02 22:35:26 - r - INFO: - Episode: 367/600, Reward: 119.182, Step: 654 +2023-04-02 22:35:27 - r - INFO: - Episode: 368/600, Reward: 156.614, Step: 717 +2023-04-02 22:35:27 - r - INFO: - Episode: 369/600, Reward: 241.747, Step: 423 +2023-04-02 22:35:28 - r - INFO: - Episode: 370/600, Reward: 234.864, Step: 366 +2023-04-02 22:35:31 - r - INFO: - Episode: 371/600, Reward: 222.284, Step: 447 +2023-04-02 22:35:31 - r - INFO: - Episode: 372/600, Reward: -19.122, Step: 335 +2023-04-02 22:35:32 - r - INFO: - Episode: 373/600, Reward: 225.110, Step: 428 +2023-04-02 22:35:33 - r - INFO: - Episode: 374/600, Reward: 240.094, Step: 415 +2023-04-02 22:35:33 - r - INFO: - Episode: 375/600, Reward: 174.815, Step: 545 +2023-04-02 22:35:37 - r - INFO: - Episode: 376/600, Reward: 192.837, Step: 461 +2023-04-02 22:35:37 - r - INFO: - Episode: 377/600, Reward: 242.377, Step: 361 +2023-04-02 22:35:38 - r - INFO: - Episode: 378/600, Reward: 241.636, Step: 388 +2023-04-02 22:35:38 - r - INFO: - Episode: 379/600, Reward: -4.094, Step: 318 +2023-04-02 22:35:39 - r - INFO: - Episode: 380/600, Reward: 224.748, Step: 349 +2023-04-02 22:35:42 - r - INFO: - Episode: 381/600, Reward: 237.187, Step: 367 +2023-04-02 22:35:42 - r - INFO: - Episode: 382/600, Reward: 241.407, Step: 397 +2023-04-02 22:35:43 - r - INFO: - Episode: 383/600, Reward: 239.651, Step: 367 +2023-04-02 22:35:43 - r - INFO: - Episode: 384/600, Reward: 241.749, Step: 362 +2023-04-02 22:35:44 - r - INFO: - Episode: 385/600, Reward: 191.455, Step: 415 +2023-04-02 22:35:47 - r - INFO: - Episode: 386/600, Reward: -17.334, Step: 312 +2023-04-02 22:35:47 - r - INFO: - Episode: 387/600, Reward: 252.577, Step: 356 +2023-04-02 22:35:47 - r - INFO: - Episode: 388/600, Reward: 251.911, Step: 314 +2023-04-02 22:35:48 - r - INFO: - Episode: 389/600, Reward: 252.396, Step: 341 +2023-04-02 22:35:48 - r - INFO: - Episode: 390/600, Reward: 248.863, Step: 311 +2023-04-02 22:35:51 - r - INFO: - Episode: 391/600, Reward: 202.274, Step: 395 +2023-04-02 22:35:51 - r - INFO: - Episode: 392/600, Reward: 241.861, Step: 318 +2023-04-02 22:35:52 - r - INFO: - Episode: 393/600, Reward: 255.668, Step: 322 +2023-04-02 22:35:52 - r - INFO: - Episode: 394/600, Reward: 248.078, Step: 298 +2023-04-02 22:35:52 - r - INFO: - Episode: 395/600, Reward: 255.693, Step: 296 +2023-04-02 22:35:54 - r - INFO: - Current episode 395 has the best eval reward: 252.299 +2023-04-02 22:35:54 - r - INFO: - Episode: 396/600, Reward: 253.226, Step: 318 +2023-04-02 22:35:55 - r - INFO: - Episode: 397/600, Reward: 236.214, Step: 491 +2023-04-02 22:35:55 - r - INFO: - Episode: 398/600, Reward: 244.976, Step: 324 +2023-04-02 22:35:55 - r - INFO: - Episode: 399/600, Reward: 249.893, Step: 301 +2023-04-02 22:35:56 - r - INFO: - Episode: 400/600, Reward: 242.053, Step: 348 +2023-04-02 22:35:58 - r - INFO: - Episode: 401/600, Reward: 237.409, Step: 340 +2023-04-02 22:35:59 - r - INFO: - Episode: 402/600, Reward: 248.976, Step: 322 +2023-04-02 22:35:59 - r - INFO: - Episode: 403/600, Reward: 240.694, Step: 333 +2023-04-02 22:36:00 - r - INFO: - Episode: 404/600, Reward: 236.968, Step: 380 +2023-04-02 22:36:00 - r - INFO: - Episode: 405/600, Reward: 225.875, Step: 320 +2023-04-02 22:36:03 - r - INFO: - Episode: 406/600, Reward: 220.762, Step: 735 +2023-04-02 22:36:03 - r - INFO: - Episode: 407/600, Reward: 234.115, Step: 497 +2023-04-02 22:36:04 - r - INFO: - Episode: 408/600, Reward: 228.883, Step: 397 +2023-04-02 22:36:05 - r - INFO: - Episode: 409/600, Reward: 237.333, Step: 386 +2023-04-02 22:36:05 - r - INFO: - Episode: 410/600, Reward: 235.915, Step: 354 +2023-04-02 22:36:07 - r - INFO: - Episode: 411/600, Reward: 238.182, Step: 343 +2023-04-02 22:36:08 - r - INFO: - Episode: 412/600, Reward: 157.924, Step: 518 +2023-04-02 22:36:08 - r - INFO: - Episode: 413/600, Reward: 15.345, Step: 270 +2023-04-02 22:36:08 - r - INFO: - Episode: 414/600, Reward: 242.835, Step: 304 +2023-04-02 22:36:09 - r - INFO: - Episode: 415/600, Reward: 10.263, Step: 241 +2023-04-02 22:36:11 - r - INFO: - Episode: 416/600, Reward: 240.273, Step: 301 +2023-04-02 22:36:11 - r - INFO: - Episode: 417/600, Reward: 247.024, Step: 301 +2023-04-02 22:36:12 - r - INFO: - Episode: 418/600, Reward: 248.339, Step: 308 +2023-04-02 22:36:12 - r - INFO: - Episode: 419/600, Reward: 250.754, Step: 294 +2023-04-02 22:36:12 - r - INFO: - Episode: 420/600, Reward: 260.319, Step: 272 +2023-04-02 22:36:14 - r - INFO: - Episode: 421/600, Reward: 258.923, Step: 269 +2023-04-02 22:36:14 - r - INFO: - Episode: 422/600, Reward: 10.691, Step: 229 +2023-04-02 22:36:15 - r - INFO: - Episode: 423/600, Reward: 248.147, Step: 263 +2023-04-02 22:36:15 - r - INFO: - Episode: 424/600, Reward: 251.983, Step: 295 +2023-04-02 22:36:15 - r - INFO: - Episode: 425/600, Reward: 258.080, Step: 284 +2023-04-02 22:36:17 - r - INFO: - Episode: 426/600, Reward: 259.407, Step: 268 +2023-04-02 22:36:17 - r - INFO: - Episode: 427/600, Reward: 47.025, Step: 219 +2023-04-02 22:36:18 - r - INFO: - Episode: 428/600, Reward: 260.112, Step: 249 +2023-04-02 22:36:18 - r - INFO: - Episode: 429/600, Reward: 262.850, Step: 272 +2023-04-02 22:36:18 - r - INFO: - Episode: 430/600, Reward: 258.350, Step: 275 +2023-04-02 22:36:20 - r - INFO: - Episode: 431/600, Reward: 16.468, Step: 245 +2023-04-02 22:36:20 - r - INFO: - Episode: 432/600, Reward: 234.909, Step: 275 +2023-04-02 22:36:20 - r - INFO: - Episode: 433/600, Reward: 223.706, Step: 297 +2023-04-02 22:36:21 - r - INFO: - Episode: 434/600, Reward: 247.155, Step: 293 +2023-04-02 22:36:21 - r - INFO: - Episode: 435/600, Reward: 249.757, Step: 288 +2023-04-02 22:36:23 - r - INFO: - Episode: 436/600, Reward: 239.382, Step: 282 +2023-04-02 22:36:23 - r - INFO: - Episode: 437/600, Reward: 262.216, Step: 261 +2023-04-02 22:36:23 - r - INFO: - Episode: 438/600, Reward: 260.980, Step: 280 +2023-04-02 22:36:24 - r - INFO: - Episode: 439/600, Reward: 262.113, Step: 282 +2023-04-02 22:36:24 - r - INFO: - Episode: 440/600, Reward: 259.490, Step: 264 +2023-04-02 22:36:25 - r - INFO: - Episode: 441/600, Reward: 257.432, Step: 297 +2023-04-02 22:36:26 - r - INFO: - Episode: 442/600, Reward: 264.151, Step: 263 +2023-04-02 22:36:26 - r - INFO: - Episode: 443/600, Reward: 193.695, Step: 415 +2023-04-02 22:36:26 - r - INFO: - Episode: 444/600, Reward: 264.278, Step: 240 +2023-04-02 22:36:27 - r - INFO: - Episode: 445/600, Reward: 262.986, Step: 250 +2023-04-02 22:36:28 - r - INFO: - Episode: 446/600, Reward: 261.351, Step: 238 +2023-04-02 22:36:28 - r - INFO: - Episode: 447/600, Reward: 45.025, Step: 219 +2023-04-02 22:36:29 - r - INFO: - Episode: 448/600, Reward: 261.015, Step: 267 +2023-04-02 22:36:29 - r - INFO: - Episode: 449/600, Reward: 263.976, Step: 263 +2023-04-02 22:36:29 - r - INFO: - Episode: 450/600, Reward: 243.877, Step: 296 +2023-04-02 22:36:31 - r - INFO: - Episode: 451/600, Reward: 260.750, Step: 273 +2023-04-02 22:36:31 - r - INFO: - Episode: 452/600, Reward: 256.636, Step: 258 +2023-04-02 22:36:31 - r - INFO: - Episode: 453/600, Reward: 256.439, Step: 254 +2023-04-02 22:36:32 - r - INFO: - Episode: 454/600, Reward: 265.349, Step: 255 +2023-04-02 22:36:32 - r - INFO: - Episode: 455/600, Reward: 264.665, Step: 266 +2023-04-02 22:36:33 - r - INFO: - Episode: 456/600, Reward: 253.014, Step: 253 +2023-04-02 22:36:33 - r - INFO: - Episode: 457/600, Reward: 32.262, Step: 194 +2023-04-02 22:36:34 - r - INFO: - Episode: 458/600, Reward: 264.128, Step: 232 +2023-04-02 22:36:34 - r - INFO: - Episode: 459/600, Reward: 265.752, Step: 255 +2023-04-02 22:36:34 - r - INFO: - Episode: 460/600, Reward: 263.110, Step: 237 +2023-04-02 22:36:35 - r - INFO: - Current episode 460 has the best eval reward: 266.538 +2023-04-02 22:36:35 - r - INFO: - Episode: 461/600, Reward: 30.295, Step: 162 +2023-04-02 22:36:35 - r - INFO: - Episode: 462/600, Reward: 271.226, Step: 214 +2023-04-02 22:36:36 - r - INFO: - Episode: 463/600, Reward: 13.384, Step: 181 +2023-04-02 22:36:36 - r - INFO: - Episode: 464/600, Reward: 174.842, Step: 395 +2023-04-02 22:36:36 - r - INFO: - Episode: 465/600, Reward: 1.168, Step: 187 +2023-04-02 22:36:37 - r - INFO: - Episode: 466/600, Reward: 13.686, Step: 206 +2023-04-02 22:36:38 - r - INFO: - Episode: 467/600, Reward: 257.603, Step: 271 +2023-04-02 22:36:38 - r - INFO: - Episode: 468/600, Reward: -6.680, Step: 203 +2023-04-02 22:36:38 - r - INFO: - Episode: 469/600, Reward: -28.242, Step: 229 +2023-04-02 22:36:39 - r - INFO: - Episode: 470/600, Reward: -25.970, Step: 237 +2023-04-02 22:36:40 - r - INFO: - Episode: 471/600, Reward: 196.133, Step: 301 +2023-04-02 22:36:40 - r - INFO: - Episode: 472/600, Reward: 206.873, Step: 298 +2023-04-02 22:36:40 - r - INFO: - Episode: 473/600, Reward: 11.522, Step: 197 +2023-04-02 22:36:41 - r - INFO: - Episode: 474/600, Reward: 0.357, Step: 201 +2023-04-02 22:36:41 - r - INFO: - Episode: 475/600, Reward: 195.312, Step: 344 +2023-04-02 22:36:42 - r - INFO: - Episode: 476/600, Reward: -54.742, Step: 194 +2023-04-02 22:36:42 - r - INFO: - Episode: 477/600, Reward: 4.921, Step: 173 +2023-04-02 22:36:43 - r - INFO: - Episode: 478/600, Reward: 196.359, Step: 253 +2023-04-02 22:36:43 - r - INFO: - Episode: 479/600, Reward: 175.233, Step: 287 +2023-04-02 22:36:43 - r - INFO: - Episode: 480/600, Reward: 178.848, Step: 287 +2023-04-02 22:36:44 - r - INFO: - Episode: 481/600, Reward: -48.507, Step: 179 +2023-04-02 22:36:44 - r - INFO: - Episode: 482/600, Reward: 6.498, Step: 174 +2023-04-02 22:36:44 - r - INFO: - Episode: 483/600, Reward: 261.715, Step: 216 +2023-04-02 22:36:45 - r - INFO: - Episode: 484/600, Reward: 203.306, Step: 366 +2023-04-02 22:36:45 - r - INFO: - Episode: 485/600, Reward: 261.610, Step: 212 +2023-04-02 22:36:46 - r - INFO: - Episode: 486/600, Reward: -60.526, Step: 200 +2023-04-02 22:36:47 - r - INFO: - Episode: 487/600, Reward: -50.410, Step: 246 +2023-04-02 22:36:47 - r - INFO: - Episode: 488/600, Reward: 262.397, Step: 274 +2023-04-02 22:36:47 - r - INFO: - Episode: 489/600, Reward: 257.958, Step: 265 +2023-04-02 22:36:48 - r - INFO: - Episode: 490/600, Reward: 259.685, Step: 293 +2023-04-02 22:36:49 - r - INFO: - Episode: 491/600, Reward: 259.445, Step: 242 +2023-04-02 22:36:50 - r - INFO: - Episode: 492/600, Reward: 246.089, Step: 326 +2023-04-02 22:36:50 - r - INFO: - Episode: 493/600, Reward: 250.200, Step: 372 +2023-04-02 22:36:51 - r - INFO: - Episode: 494/600, Reward: 162.818, Step: 496 +2023-04-02 22:36:51 - r - INFO: - Episode: 495/600, Reward: 253.942, Step: 298 +2023-04-02 22:36:53 - r - INFO: - Episode: 496/600, Reward: 258.923, Step: 255 +2023-04-02 22:36:54 - r - INFO: - Episode: 497/600, Reward: 260.284, Step: 265 +2023-04-02 22:36:54 - r - INFO: - Episode: 498/600, Reward: 251.429, Step: 285 +2023-04-02 22:36:54 - r - INFO: - Episode: 499/600, Reward: 263.885, Step: 257 +2023-04-02 22:36:54 - r - INFO: - Episode: 500/600, Reward: 260.733, Step: 234 +2023-04-02 22:36:56 - r - INFO: - Episode: 501/600, Reward: 249.684, Step: 235 +2023-04-02 22:36:56 - r - INFO: - Episode: 502/600, Reward: 260.315, Step: 234 +2023-04-02 22:36:56 - r - INFO: - Episode: 503/600, Reward: -11.048, Step: 214 +2023-04-02 22:36:57 - r - INFO: - Episode: 504/600, Reward: 258.626, Step: 241 +2023-04-02 22:36:57 - r - INFO: - Episode: 505/600, Reward: 15.664, Step: 219 +2023-04-02 22:36:59 - r - INFO: - Episode: 506/600, Reward: 258.389, Step: 295 +2023-04-02 22:36:59 - r - INFO: - Episode: 507/600, Reward: -186.030, Step: 412 +2023-04-02 22:37:00 - r - INFO: - Episode: 508/600, Reward: 188.378, Step: 480 +2023-04-02 22:37:00 - r - INFO: - Episode: 509/600, Reward: 256.534, Step: 294 +2023-04-02 22:37:02 - r - INFO: - Episode: 510/600, Reward: 187.044, Step: 922 +2023-04-02 22:37:09 - r - INFO: - Episode: 511/600, Reward: 263.852, Step: 232 +2023-04-02 22:37:09 - r - INFO: - Episode: 512/600, Reward: 248.015, Step: 241 +2023-04-02 22:37:10 - r - INFO: - Episode: 513/600, Reward: 249.355, Step: 227 +2023-04-02 22:37:10 - r - INFO: - Episode: 514/600, Reward: -22.340, Step: 176 +2023-04-02 22:37:11 - r - INFO: - Episode: 515/600, Reward: -5.522, Step: 1000 +2023-04-02 22:37:14 - r - INFO: - Episode: 516/600, Reward: -115.527, Step: 583 +2023-04-02 22:37:14 - r - INFO: - Episode: 517/600, Reward: 14.921, Step: 173 +2023-04-02 22:37:16 - r - INFO: - Episode: 518/600, Reward: -75.088, Step: 1000 +2023-04-02 22:37:16 - r - INFO: - Episode: 519/600, Reward: 243.402, Step: 322 +2023-04-02 22:37:18 - r - INFO: - Episode: 520/600, Reward: -92.877, Step: 1000 +2023-04-02 22:37:26 - r - INFO: - Episode: 521/600, Reward: -48.260, Step: 438 +2023-04-02 22:37:27 - r - INFO: - Episode: 522/600, Reward: 137.752, Step: 554 +2023-04-02 22:37:28 - r - INFO: - Episode: 523/600, Reward: -52.636, Step: 1000 +2023-04-02 22:37:30 - r - INFO: - Episode: 524/600, Reward: -85.157, Step: 1000 +2023-04-02 22:37:30 - r - INFO: - Episode: 525/600, Reward: -143.406, Step: 270 +2023-04-02 22:37:38 - r - INFO: - Episode: 526/600, Reward: -125.846, Step: 291 +2023-04-02 22:37:38 - r - INFO: - Episode: 527/600, Reward: -116.227, Step: 272 +2023-04-02 22:37:39 - r - INFO: - Episode: 528/600, Reward: -32.613, Step: 1000 +2023-04-02 22:37:40 - r - INFO: - Episode: 529/600, Reward: -26.567, Step: 216 +2023-04-02 22:37:40 - r - INFO: - Episode: 530/600, Reward: 219.599, Step: 645 +2023-04-02 22:37:46 - r - INFO: - Episode: 531/600, Reward: 244.029, Step: 498 +2023-04-02 22:37:46 - r - INFO: - Episode: 532/600, Reward: 255.433, Step: 257 +2023-04-02 22:37:46 - r - INFO: - Episode: 533/600, Reward: 254.793, Step: 323 +2023-04-02 22:37:47 - r - INFO: - Episode: 534/600, Reward: 256.517, Step: 268 +2023-04-02 22:37:47 - r - INFO: - Episode: 535/600, Reward: 260.777, Step: 257 +2023-04-02 22:37:49 - r - INFO: - Episode: 536/600, Reward: 236.733, Step: 304 +2023-04-02 22:37:51 - r - INFO: - Episode: 537/600, Reward: -73.740, Step: 1000 +2023-04-02 22:37:52 - r - INFO: - Episode: 538/600, Reward: -57.553, Step: 1000 +2023-04-02 22:37:54 - r - INFO: - Episode: 539/600, Reward: -77.619, Step: 1000 +2023-04-02 22:37:56 - r - INFO: - Episode: 540/600, Reward: -87.691, Step: 1000 +2023-04-02 22:37:58 - r - INFO: - Episode: 541/600, Reward: 104.132, Step: 812 +2023-04-02 22:37:59 - r - INFO: - Episode: 542/600, Reward: 256.767, Step: 275 +2023-04-02 22:37:59 - r - INFO: - Episode: 543/600, Reward: 8.147, Step: 232 +2023-04-02 22:38:00 - r - INFO: - Episode: 544/600, Reward: 155.294, Step: 615 +2023-04-02 22:38:00 - r - INFO: - Episode: 545/600, Reward: 229.551, Step: 337 +2023-04-02 22:38:04 - r - INFO: - Episode: 546/600, Reward: 156.560, Step: 466 +2023-04-02 22:38:04 - r - INFO: - Episode: 547/600, Reward: 238.245, Step: 243 +2023-04-02 22:38:04 - r - INFO: - Episode: 548/600, Reward: 248.843, Step: 274 +2023-04-02 22:38:05 - r - INFO: - Episode: 549/600, Reward: 254.527, Step: 251 +2023-04-02 22:38:05 - r - INFO: - Episode: 550/600, Reward: 267.225, Step: 228 +2023-04-02 22:38:06 - r - INFO: - Episode: 551/600, Reward: 264.262, Step: 265 +2023-04-02 22:38:06 - r - INFO: - Episode: 552/600, Reward: 243.735, Step: 269 +2023-04-02 22:38:07 - r - INFO: - Episode: 553/600, Reward: 42.273, Step: 165 +2023-04-02 22:38:07 - r - INFO: - Episode: 554/600, Reward: 248.316, Step: 246 +2023-04-02 22:38:07 - r - INFO: - Episode: 555/600, Reward: 40.931, Step: 173 +2023-04-02 22:38:08 - r - INFO: - Episode: 556/600, Reward: 16.317, Step: 161 +2023-04-02 22:38:08 - r - INFO: - Episode: 557/600, Reward: 46.279, Step: 171 +2023-04-02 22:38:08 - r - INFO: - Episode: 558/600, Reward: 258.548, Step: 245 +2023-04-02 22:38:08 - r - INFO: - Episode: 559/600, Reward: 256.210, Step: 227 +2023-04-02 22:38:09 - r - INFO: - Episode: 560/600, Reward: 24.411, Step: 159 +2023-04-02 22:38:09 - r - INFO: - Episode: 561/600, Reward: 269.466, Step: 204 +2023-04-02 22:38:10 - r - INFO: - Episode: 562/600, Reward: 45.629, Step: 165 +2023-04-02 22:38:10 - r - INFO: - Episode: 563/600, Reward: 267.713, Step: 215 +2023-04-02 22:38:10 - r - INFO: - Episode: 564/600, Reward: 261.260, Step: 254 +2023-04-02 22:38:10 - r - INFO: - Episode: 565/600, Reward: 28.404, Step: 152 +2023-04-02 22:38:11 - r - INFO: - Episode: 566/600, Reward: 29.313, Step: 149 +2023-04-02 22:38:12 - r - INFO: - Episode: 567/600, Reward: 261.192, Step: 504 +2023-04-02 22:38:12 - r - INFO: - Episode: 568/600, Reward: 258.287, Step: 217 +2023-04-02 22:38:12 - r - INFO: - Episode: 569/600, Reward: 269.920, Step: 240 +2023-04-02 22:38:12 - r - INFO: - Episode: 570/600, Reward: 37.344, Step: 167 +2023-04-02 22:38:14 - r - INFO: - Episode: 571/600, Reward: 268.270, Step: 244 +2023-04-02 22:38:14 - r - INFO: - Episode: 572/600, Reward: 264.803, Step: 274 +2023-04-02 22:38:14 - r - INFO: - Episode: 573/600, Reward: 270.073, Step: 223 +2023-04-02 22:38:14 - r - INFO: - Episode: 574/600, Reward: 270.384, Step: 237 +2023-04-02 22:38:15 - r - INFO: - Episode: 575/600, Reward: 259.619, Step: 250 +2023-04-02 22:38:16 - r - INFO: - Episode: 576/600, Reward: 270.013, Step: 213 +2023-04-02 22:38:16 - r - INFO: - Episode: 577/600, Reward: -2.147, Step: 160 +2023-04-02 22:38:17 - r - INFO: - Episode: 578/600, Reward: 243.889, Step: 229 +2023-04-02 22:38:17 - r - INFO: - Episode: 579/600, Reward: 202.207, Step: 488 +2023-04-02 22:38:18 - r - INFO: - Episode: 580/600, Reward: 269.916, Step: 281 +2023-04-02 22:38:20 - r - INFO: - Episode: 581/600, Reward: 241.754, Step: 263 +2023-04-02 22:38:20 - r - INFO: - Episode: 582/600, Reward: 250.112, Step: 209 +2023-04-02 22:38:20 - r - INFO: - Episode: 583/600, Reward: 251.388, Step: 324 +2023-04-02 22:38:21 - r - INFO: - Episode: 584/600, Reward: 234.244, Step: 368 +2023-04-02 22:38:21 - r - INFO: - Episode: 585/600, Reward: 252.416, Step: 245 +2023-04-02 22:38:22 - r - INFO: - Episode: 586/600, Reward: 237.838, Step: 311 +2023-04-02 22:38:23 - r - INFO: - Episode: 587/600, Reward: 266.629, Step: 270 +2023-04-02 22:38:23 - r - INFO: - Episode: 588/600, Reward: 240.826, Step: 256 +2023-04-02 22:38:23 - r - INFO: - Episode: 589/600, Reward: 236.912, Step: 239 +2023-04-02 22:38:23 - r - INFO: - Episode: 590/600, Reward: 32.451, Step: 216 +2023-04-02 22:38:25 - r - INFO: - Episode: 591/600, Reward: 259.219, Step: 247 +2023-04-02 22:38:25 - r - INFO: - Episode: 592/600, Reward: 248.622, Step: 226 +2023-04-02 22:38:25 - r - INFO: - Episode: 593/600, Reward: 267.273, Step: 219 +2023-04-02 22:38:26 - r - INFO: - Episode: 594/600, Reward: 268.889, Step: 225 +2023-04-02 22:38:26 - r - INFO: - Episode: 595/600, Reward: -9.316, Step: 223 +2023-04-02 22:38:27 - r - INFO: - Episode: 596/600, Reward: 267.592, Step: 233 +2023-04-02 22:38:28 - r - INFO: - Episode: 597/600, Reward: 253.772, Step: 287 +2023-04-02 22:38:28 - r - INFO: - Episode: 598/600, Reward: 270.646, Step: 280 +2023-04-02 22:38:28 - r - INFO: - Episode: 599/600, Reward: -12.607, Step: 218 +2023-04-02 22:38:28 - r - INFO: - Episode: 600/600, Reward: -33.086, Step: 172 +2023-04-02 22:38:29 - r - INFO: - Finish training! diff --git a/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/models/actor.pth b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..2ac31119df0965876cff5dd2186dfa5e84e9ff90 --- /dev/null +++ b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6993835bd5028735d6192679250fc13cd0ccec60cfcfa6f99176c3242af3ebfc +size 278295 diff --git a/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/models/critic.pth b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/models/critic.pth new file mode 100644 index 0000000000000000000000000000000000000000..c380276fec7b3b8935b01ce68f678717a07469cc --- /dev/null +++ b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/models/critic.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd3600cd04fce59c57db0cab66643b9385c927bcd40a6a1b415114ebab8f081f +size 275223 diff --git a/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/results/learning_curve.png b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..5fa280eae43acf93883f5e6b54a4c1c9990ad27c Binary files /dev/null and b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/results/learning_curve.png differ diff --git a/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/results/res.csv b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..be472f351266fb10a004dbebadfd5f5d3fa90b0e --- /dev/null +++ b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/results/res.csv @@ -0,0 +1,601 @@ +episodes,rewards,steps +0,-121.4435537010661,114 +1,-277.2337168439329,118 +2,-115.89584658653801,118 +3,-369.7503652717676,103 +4,-337.84515849816233,104 +5,-153.43349929213977,104 +6,-97.77684903785325,106 +7,-14.116394995798643,106 +8,-281.9837795819341,119 +9,-115.02198640142043,113 +10,-104.56350731698728,117 +11,-96.84051439180153,105 +12,-174.98341988072377,104 +13,-107.47455173561967,107 +14,-196.71405243942488,109 +15,-216.49781424947827,98 +16,-255.59229603211332,115 +17,-241.31871470021832,116 +18,-135.7087559489612,129 +19,-239.03582718723362,118 +20,-35.75462858180187,135 +21,-323.152415760807,114 +22,-219.33313173227987,147 +23,-88.33204842873221,124 +24,-203.77323601738453,132 +25,-294.6520033195416,152 +26,-296.47243945780286,128 +27,-79.82896211570062,146 +28,-540.1685714336504,116 +29,-581.5873482136951,98 +30,-517.0706211531556,110 +31,-421.5615195238158,111 +32,-314.08807098141074,120 +33,-207.56313572939666,114 +34,-100.57960197733945,101 +35,-316.1581953473273,108 +36,-312.4368938131997,107 +37,-358.39604177185885,110 +38,-124.13696163734706,108 +39,-156.08725526446395,109 +40,-201.88771826237542,127 +41,-75.11208676439486,134 +42,-294.26812549195483,125 +43,-449.12285123965194,147 +44,-251.92955846826626,146 +45,-363.3957688332977,151 +46,-121.84019809443278,135 +47,-586.5127192835472,173 +48,-430.39211688924473,138 +49,-322.2944779966817,140 +50,-266.47309315722157,134 +51,-290.37904985627057,109 +52,-564.943593934089,115 +53,-161.72227410560697,138 +54,-218.3617034996023,132 +55,-90.50602389975585,135 +56,-48.79471580854798,115 +57,-239.57805632887568,110 +58,-148.22301343223916,113 +59,-102.34136811568419,107 +60,-90.72019082582344,116 +61,-126.89790127525488,119 +62,-405.09751939682064,140 +63,-409.6682443182422,114 +64,-59.205396163284504,115 +65,-212.75955956879693,120 +66,-83.1353377288226,121 +67,-93.1597752144536,113 +68,-115.49343035374244,116 +69,-101.28561936910808,107 +70,-6.530652823416261,161 +71,-195.51437256760144,127 +72,-105.65542494129942,117 +73,-208.32633446544042,131 +74,-92.1128483594131,126 +75,-108.63864374357577,135 +76,-235.68949436096156,141 +77,-115.34907459572565,130 +78,-66.63914761846578,117 +79,-84.4499653429759,124 +80,-75.3646362536225,110 +81,-70.92097539699293,132 +82,-87.51195586150143,113 +83,-78.19459212110638,111 +84,-200.1007471210845,117 +85,-310.61361635430654,134 +86,-315.09392241722634,122 +87,-76.64369856956601,137 +88,-116.26080093817944,123 +89,-34.35228711816505,140 +90,-80.77835178853502,153 +91,-217.03095932734274,142 +92,-24.95982657022701,162 +93,-60.82370227759253,158 +94,-340.599837767738,136 +95,-277.21529189487103,126 +96,-125.6264174402502,131 +97,-45.416326285877645,124 +98,-67.0337891271648,142 +99,-88.19295404515915,130 +100,-58.4034314937311,133 +101,-76.79198138646171,122 +102,-79.5304699875941,122 +103,-76.82886738778868,138 +104,-74.57332253013061,162 +105,-94.75617669045961,126 +106,-57.44582945400073,135 +107,-30.263411948373147,153 +108,-15.612200978775576,126 +109,-42.46434184591685,112 +110,-53.89104734171288,131 +111,-47.278727789301286,135 +112,-28.309859024680335,171 +113,-71.55544847300304,161 +114,-46.64495088448577,165 +115,-110.32842229904796,176 +116,-1.405161956352174,193 +117,-80.80454674315676,228 +118,-431.3396660539486,790 +119,-148.6118639079431,161 +120,-176.68777256367952,170 +121,-75.35358922396269,159 +122,-377.81533842580734,350 +123,-226.83436662544193,395 +124,-55.7998530086288,198 +125,-145.4015272796019,236 +126,-132.79136219592533,168 +127,-363.3421559307926,259 +128,-1.0896506659831289,155 +129,-67.62829468261259,193 +130,-117.42133428774638,138 +131,9.361197947297214,167 +132,-418.8835414221543,161 +133,7.492992510362342,161 +134,-271.47757083383607,536 +135,34.58385190043157,1000 +136,-11.46963354229095,147 +137,-75.40815055821378,132 +138,-181.78529070417616,144 +139,-45.55724227166857,176 +140,-203.10995593480845,165 +141,-254.8590006788792,190 +142,9.828577830182425,160 +143,17.70539463900073,150 +144,8.25004372067771,139 +145,-21.815511498244447,180 +146,-29.650149431412572,213 +147,10.788718897807229,179 +148,-216.82467973154434,321 +149,-62.8025335663048,448 +150,27.057365641171998,302 +151,-20.9704211154839,253 +152,-188.57505681204438,319 +153,-105.42191367157646,346 +154,-147.7731657101151,232 +155,-98.5813316199688,222 +156,-204.74119269364996,437 +157,31.71130147323675,1000 +158,33.31651690278813,1000 +159,3.7707458157548928,136 +160,9.813931135823978,123 +161,8.625282236940478,129 +162,-5.854256252252696,129 +163,23.97284592028562,132 +164,-15.441232736008061,137 +165,-24.959178704764938,142 +166,-55.73955432286422,176 +167,54.0548499426338,1000 +168,60.755159667254034,1000 +169,-180.27219916426665,143 +170,-184.92116458682108,148 +171,-171.91877478330946,140 +172,-183.9494426683587,142 +173,53.44730309325153,1000 +174,79.82403951427798,1000 +175,-106.80597375587847,144 +176,-58.712435684264946,138 +177,-56.98173424453228,140 +178,77.98289493550237,1000 +179,18.476149904710567,166 +180,-17.92078842890362,153 +181,4.517193218793054,149 +182,93.13962399537732,1000 +183,-61.87478477664003,108 +184,-45.81301140117897,104 +185,-26.909707351052262,118 +186,-46.78056528366589,106 +187,-20.429447900968498,130 +188,-18.21479258361944,142 +189,-33.296532410638775,110 +190,-55.21233343310894,112 +191,-3.6836783279173773,127 +192,-56.03970158452347,116 +193,94.81744216247468,1000 +194,-39.89053618776789,130 +195,68.62866968902262,1000 +196,-74.19312568230035,128 +197,86.33986321327745,1000 +198,89.57423699982473,1000 +199,89.49749910176999,1000 +200,-37.33648966565038,136 +201,-99.69820842820494,131 +202,-63.99622140940182,162 +203,-74.19030533031196,155 +204,209.3365984974705,736 +205,188.76484798631014,757 +206,5.050895813648893,148 +207,-40.34203593616519,179 +208,-70.09613355045569,244 +209,75.04530327919545,1000 +210,-29.00283198426318,201 +211,73.28590275310603,1000 +212,-216.10319505840255,168 +213,-86.51624842928157,221 +214,-143.6343536952901,182 +215,-146.8496132059717,169 +216,-43.83499475364023,188 +217,-40.00440239256659,207 +218,-44.306122385227,189 +219,-3.133331927319091,135 +220,85.60847992204603,1000 +221,-34.163947921548356,239 +222,222.20125897956962,309 +223,179.92894733766195,837 +224,82.5146712695244,1000 +225,72.61108721734747,1000 +226,3.2184708616023556,192 +227,27.042915682095284,170 +228,22.642644935800746,174 +229,-14.321176405703852,197 +230,30.39744292646293,170 +231,25.97418619492565,193 +232,30.079935438613404,171 +233,-29.670957052133076,222 +234,207.32523992320486,305 +235,8.05180418000262,181 +236,31.975948967958175,185 +237,13.26908602748017,210 +238,7.469583361810592,231 +239,-26.021787170561396,334 +240,25.706721130981293,228 +241,-22.35455322064864,210 +242,16.92753792870819,210 +243,0.2245956225259249,204 +244,191.80689528095562,454 +245,-42.90291652885408,223 +246,-113.87449562882658,281 +247,16.97409449260437,1000 +248,214.34664771147473,310 +249,-156.82153074992206,288 +250,182.68390052696046,412 +251,174.69572048410726,959 +252,199.8804033773713,314 +253,189.43121183111793,395 +254,189.4963211927332,369 +255,-34.82937807083631,216 +256,-63.171330620816384,241 +257,-62.71308075399649,239 +258,188.23379419977778,485 +259,-40.35599300450741,194 +260,-8.431319198359049,197 +261,164.2306945978082,385 +262,179.9701517146237,350 +263,-172.76564683135345,298 +264,-148.916385899161,257 +265,-184.39226819153527,377 +266,-148.96880034534746,396 +267,-109.25001397930808,314 +268,13.636976753144708,256 +269,28.71526756392305,256 +270,-46.10290862068412,284 +271,249.8313238962968,339 +272,-8.304521959767882,262 +273,244.85500278648732,399 +274,-23.19857633822582,229 +275,-18.04905462123783,200 +276,-132.94580791813317,248 +277,-29.598369285291753,206 +278,-126.08853529644907,193 +279,-80.03291854428458,217 +280,-37.70634884887393,250 +281,-15.304042044378704,254 +282,-47.292337059817385,270 +283,224.67865487400897,397 +284,13.596204930635764,283 +285,250.09450683622148,344 +286,5.142803021454597,235 +287,-213.22328076130196,484 +288,-33.50467555858016,229 +289,144.37674432475413,480 +290,186.78605149811875,474 +291,-8.56102004401059,272 +292,-38.962464062761654,264 +293,156.42935007899825,393 +294,-11.140212613281946,266 +295,165.3455653833006,487 +296,152.51486908568046,773 +297,136.3765854079691,531 +298,126.60132092103643,763 +299,126.58005809334028,791 +300,147.95625308032598,704 +301,227.98029773836177,411 +302,235.31535750396364,437 +303,5.183221262271019,263 +304,16.237915368641694,267 +305,-3.760942031614718,272 +306,250.96393493567382,342 +307,245.65123101666558,314 +308,255.2926095576279,306 +309,-62.48158520671373,371 +310,-24.992286698630608,338 +311,-173.27961445372824,891 +312,-86.00762627706746,510 +313,251.6268627279285,365 +314,235.2905103209443,359 +315,242.80215837620614,384 +316,238.87456964787248,400 +317,13.819715784406895,246 +318,243.89316038881893,366 +319,247.08256522649128,329 +320,149.40625458856948,550 +321,239.89082904403716,407 +322,234.08349055138888,413 +323,246.2632048549079,363 +324,246.9748233398182,343 +325,181.4094906606236,390 +326,245.08042506047695,344 +327,233.94361813777635,455 +328,213.53550793190928,461 +329,224.41027116546283,397 +330,233.00506108045275,364 +331,8.8401160513787,217 +332,-3.008353609593513,218 +333,12.239476699762605,250 +334,8.395220318346063,235 +335,226.1478673658449,291 +336,247.5866877142129,320 +337,244.38305247975177,390 +338,240.4394967913221,399 +339,176.35015065021787,407 +340,-37.62450284038404,348 +341,239.80510585087262,386 +342,236.58842346484855,375 +343,212.31989873803053,420 +344,239.3449258703476,327 +345,245.9696566488755,357 +346,243.65319745483657,352 +347,225.00135489410454,333 +348,242.18039804072035,305 +349,13.432336257681456,265 +350,240.1286349902408,398 +351,-9.537037331061185,353 +352,142.98654565952688,795 +353,229.01100673785413,381 +354,239.2344745788054,394 +355,244.89830673583796,346 +356,245.16783771988463,352 +357,247.0977250253684,319 +358,242.5122577176075,419 +359,24.758102369591512,265 +360,184.12143860987965,324 +361,209.17550738272996,395 +362,-12.26536064566315,221 +363,32.63529817218236,239 +364,7.065505039179726,286 +365,-57.704238750678954,356 +366,119.18184810477676,654 +367,156.61410457895914,717 +368,241.7473170180333,423 +369,234.86445585892898,366 +370,222.28434911159508,447 +371,-19.12178762520466,335 +372,225.10960555107104,428 +373,240.0936821249997,415 +374,174.81525835989692,545 +375,192.83728006951307,461 +376,242.37726173860796,361 +377,241.63582766220318,388 +378,-4.094046655610057,318 +379,224.74816096978643,349 +380,237.18734361720965,367 +381,241.40667474130208,397 +382,239.6506534744606,367 +383,241.74904315508493,362 +384,191.4545628573082,415 +385,-17.33424970562622,312 +386,252.57651840733328,356 +387,251.9106243730417,314 +388,252.3955444345151,341 +389,248.86339096848147,311 +390,202.27429794368084,395 +391,241.86130517370322,318 +392,255.66809006190243,322 +393,248.07811691141382,298 +394,255.69311491365335,296 +395,253.2264609180931,318 +396,236.21386230627513,491 +397,244.97610926056728,324 +398,249.8931977102705,301 +399,242.05259295749354,348 +400,237.40922246659684,340 +401,248.9763553523382,322 +402,240.69445410186256,333 +403,236.96764617520896,380 +404,225.8751382806269,320 +405,220.76244191749078,735 +406,234.11525879182142,497 +407,228.8831001836119,397 +408,237.33273937208608,386 +409,235.91468093022144,354 +410,238.18236221533786,343 +411,157.9242992943984,518 +412,15.34534379545019,270 +413,242.8346482050788,304 +414,10.263182120870653,241 +415,240.2734179627465,301 +416,247.0240436654808,301 +417,248.33940960567213,308 +418,250.75442032025637,294 +419,260.31902113865317,272 +420,258.9229228079922,269 +421,10.690947345083842,229 +422,248.14707521736082,263 +423,251.98347575180352,295 +424,258.0797441761307,284 +425,259.40664435245327,268 +426,47.0249140193265,219 +427,260.1123252332866,249 +428,262.85014452941334,272 +429,258.3496911238782,275 +430,16.467512549585464,245 +431,234.90882915092544,275 +432,223.7056260239231,297 +433,247.15491640340286,293 +434,249.75710562162064,288 +435,239.3822273225357,282 +436,262.21616874566666,261 +437,260.9796364107036,280 +438,262.1133277433305,282 +439,259.49019669283155,264 +440,257.4317443431369,297 +441,264.1511217632742,263 +442,193.69454375687363,415 +443,264.27776682409547,240 +444,262.98631683233566,250 +445,261.3509058951816,238 +446,45.024974161372796,219 +447,261.01547851438096,267 +448,263.97574895580857,263 +449,243.87749695433868,296 +450,260.7499068974811,273 +451,256.6359537807431,258 +452,256.4392202495851,254 +453,265.3485011084516,255 +454,264.6649491060658,266 +455,253.01426487184213,253 +456,32.26185361526274,194 +457,264.12785415631424,232 +458,265.7517250885181,255 +459,263.1098048678994,237 +460,30.295480266561498,162 +461,271.2257875813036,214 +462,13.383906360491949,181 +463,174.84243160086717,395 +464,1.1683627009108477,187 +465,13.6861617842561,206 +466,257.6032417818615,271 +467,-6.679788780816949,203 +468,-28.241907365978363,229 +469,-25.96967031950598,237 +470,196.132561280647,301 +471,206.87284214287598,298 +472,11.522209756371865,197 +473,0.35713320994005926,201 +474,195.31222315614716,344 +475,-54.74222858013232,194 +476,4.921047654509863,173 +477,196.35851380996704,253 +478,175.23267447780157,287 +479,178.84781825391838,287 +480,-48.50724224396268,179 +481,6.49842627637824,174 +482,261.7150531075153,216 +483,203.30560029212717,366 +484,261.6103611096427,212 +485,-60.5258730275543,200 +486,-50.40995818676808,246 +487,262.3973602173586,274 +488,257.95755781576315,265 +489,259.6845359223705,293 +490,259.4445759781226,242 +491,246.0892037844935,326 +492,250.19985474455103,372 +493,162.81829356613014,496 +494,253.9416076134151,298 +495,258.92326893429413,255 +496,260.2836909050973,265 +497,251.42930029991106,285 +498,263.8850876851146,257 +499,260.7328107394893,234 +500,249.68364120758207,235 +501,260.3145305400507,234 +502,-11.04791273271482,214 +503,258.6263976138778,241 +504,15.663693952173233,219 +505,258.3891907439245,295 +506,-186.0298999238178,412 +507,188.3781923102294,480 +508,256.53414006086376,294 +509,187.04411001206608,922 +510,263.8516169210807,232 +511,248.01453155263266,241 +512,249.35454857673727,227 +513,-22.34042457092535,176 +514,-5.521626216730945,1000 +515,-115.52724669519574,583 +516,14.920636269058832,173 +517,-75.08783393424125,1000 +518,243.4021244365543,322 +519,-92.87676551229485,1000 +520,-48.25971780969998,438 +521,137.75198220881902,554 +522,-52.635790728959606,1000 +523,-85.15748088951378,1000 +524,-143.405744846395,270 +525,-125.84619032845781,291 +526,-116.2270183912975,272 +527,-32.61287342733353,1000 +528,-26.56719411293834,216 +529,219.59948999355066,645 +530,244.02903521483523,498 +531,255.43290781116505,257 +532,254.79270786040692,323 +533,256.5170413313458,268 +534,260.77656600820467,257 +535,236.73288260539525,304 +536,-73.7403435037866,1000 +537,-57.552847473157314,1000 +538,-77.61908170896807,1000 +539,-87.69069471360828,1000 +540,104.13188176959781,812 +541,256.7665340042805,275 +542,8.14659289882458,232 +543,155.29449730917855,615 +544,229.5508333140673,337 +545,156.5595545661251,466 +546,238.24466380078601,243 +547,248.84333770699223,274 +548,254.52667012312386,251 +549,267.2254334981493,228 +550,264.2620231295879,265 +551,243.73457511899542,269 +552,42.27294825629812,165 +553,248.31576317072654,246 +554,40.93090101445304,173 +555,16.31735148736901,161 +556,46.278882573689856,171 +557,258.5480475275368,245 +558,256.2098339642938,227 +559,24.410868631100442,159 +560,269.46594072256335,204 +561,45.6293070057059,165 +562,267.7129581351464,215 +563,261.26048805018684,254 +564,28.40397206785366,152 +565,29.31287350550005,149 +566,261.1923074578049,504 +567,258.28715168585427,217 +568,269.91959351306,240 +569,37.34357092816239,167 +570,268.2695593923595,244 +571,264.80267433817346,274 +572,270.07256340025765,223 +573,270.38405715643466,237 +574,259.61936359700235,250 +575,270.0132403465384,213 +576,-2.1467664634397607,160 +577,243.88877386994062,229 +578,202.20708268069205,488 +579,269.9155657133996,281 +580,241.75397593539407,263 +581,250.11169196501638,209 +582,251.38823453247963,324 +583,234.24375861629187,368 +584,252.4155945794129,245 +585,237.83831191379875,311 +586,266.6293667334187,270 +587,240.82636260625148,256 +588,236.91183731188158,239 +589,32.451331612090826,216 +590,259.2188546286159,247 +591,248.62212131780512,226 +592,267.27296300813936,219 +593,268.888913120205,225 +594,-9.315678121619413,223 +595,267.59239899240134,233 +596,253.7719285152298,287 +597,270.64595529483427,280 +598,-12.606739559191197,218 +599,-33.08598868077554,172 diff --git a/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/tb_logs/events.out.tfevents.1680445914.zf.220623.0 b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/tb_logs/events.out.tfevents.1680445914.zf.220623.0 new file mode 100644 index 0000000000000000000000000000000000000000..10c0184c1cfe9e3c3d45cc9c9f308f7908d82829 --- /dev/null +++ b/LunarLander-v2/Train_LunarLander-v2_PPO_20230402-223154/tb_logs/events.out.tfevents.1680445914.zf.220623.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:918047100dc40c9b8ae4897fde8dd757e38761e84dcff80bc91dc332a4379f1b +size 40 diff --git a/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/config.yaml b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89050710156ea6cabef5b112cb2a3ff7a90ad4b0 --- /dev/null +++ b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/config.yaml @@ -0,0 +1,34 @@ +general_cfg: + algo_name: SAC + continuous: true + device: cpu + env_name: LunarLanderContinuous-v2 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_LunarLanderContinuous-v2_SAC_20230402-170158 + max_steps: 500 + mode: test + mp_backend: mp + new_step_api: true + render: true + render_mode: human + save_fig: true + seed: 666 + show_fig: false + test_eps: 20 + train_eps: 2000 + wrapper: null +algo_cfg: + alpha: 0.2 + automatic_entropy_tuning: false + batch_size: 64 + buffer_size: 1000000 + gamma: 0.99 + hidden_dim: 256 + lr: 0.001 + n_epochs: 1 + policy_type: Gaussian + start_steps: 10000 + target_update_fre: 1 + tau: 0.005 diff --git a/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/logs/log.txt b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..dfd716b6fb244887d199a88474ffde22aa297ffa --- /dev/null +++ b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/logs/log.txt @@ -0,0 +1,66 @@ +2023-04-04 16:19:39 - r - INFO: - Hyperparameters: +2023-04-04 16:19:39 - r - INFO: - ================================================================================ +2023-04-04 16:19:39 - r - INFO: - Name Value Type +2023-04-04 16:19:39 - r - INFO: - env_name LunarLanderContinuous-v2 +2023-04-04 16:19:39 - r - INFO: - continuous 1 +2023-04-04 16:19:39 - r - INFO: - new_step_api 1 +2023-04-04 16:19:39 - r - INFO: - wrapper None +2023-04-04 16:19:39 - r - INFO: - render 1 +2023-04-04 16:19:39 - r - INFO: - algo_name SAC +2023-04-04 16:19:39 - r - INFO: - mode test +2023-04-04 16:19:39 - r - INFO: - render_mode human +2023-04-04 16:19:39 - r - INFO: - mp_backend mp +2023-04-04 16:19:39 - r - INFO: - seed 666 +2023-04-04 16:19:39 - r - INFO: - device cpu +2023-04-04 16:19:39 - r - INFO: - train_eps 2000 +2023-04-04 16:19:39 - r - INFO: - test_eps 20 +2023-04-04 16:19:39 - r - INFO: - eval_eps 10 +2023-04-04 16:19:39 - r - INFO: - eval_per_episode 5 +2023-04-04 16:19:39 - r - INFO: - max_steps 500 +2023-04-04 16:19:39 - r - INFO: - load_checkpoint 1 +2023-04-04 16:19:39 - r - INFO: - load_path Train_LunarLanderContinuous-v2_SAC_20230402-170158 +2023-04-04 16:19:39 - r - INFO: - show_fig 0 +2023-04-04 16:19:39 - r - INFO: - save_fig 1 +2023-04-04 16:19:39 - r - INFO: - policy_type Gaussian +2023-04-04 16:19:39 - r - INFO: - lr 0.001 +2023-04-04 16:19:39 - r - INFO: - gamma 0.99 +2023-04-04 16:19:39 - r - INFO: - tau 0.005 +2023-04-04 16:19:39 - r - INFO: - alpha 0.2 +2023-04-04 16:19:39 - r - INFO: - automatic_entropy_tuning 0 +2023-04-04 16:19:39 - r - INFO: - batch_size 64 +2023-04-04 16:19:39 - r - INFO: - hidden_dim 256 +2023-04-04 16:19:39 - r - INFO: - n_epochs 1 +2023-04-04 16:19:39 - r - INFO: - start_steps 10000 +2023-04-04 16:19:39 - r - INFO: - target_update_fre 1 +2023-04-04 16:19:39 - r - INFO: - buffer_size 1000000 +2023-04-04 16:19:39 - r - INFO: - task_dir G:\Python3\RL\Pytorch\joyrl-offline-main/tasks/Test_LunarLanderContinuous-v2_SAC_20230404-161934 +2023-04-04 16:19:39 - r - INFO: - res_dir G:\Python3\RL\Pytorch\joyrl-offline-main/tasks/Test_LunarLanderContinuous-v2_SAC_20230404-161934/results +2023-04-04 16:19:39 - r - INFO: - log_dir G:\Python3\RL\Pytorch\joyrl-offline-main/tasks/Test_LunarLanderContinuous-v2_SAC_20230404-161934/logs +2023-04-04 16:19:39 - r - INFO: - traj_dir G:\Python3\RL\Pytorch\joyrl-offline-main/tasks/Test_LunarLanderContinuous-v2_SAC_20230404-161934/traj +2023-04-04 16:19:39 - r - INFO: - tb_dir G:\Python3\RL\Pytorch\joyrl-offline-main/tasks/Test_LunarLanderContinuous-v2_SAC_20230404-161934/tb_logs +2023-04-04 16:19:39 - r - INFO: - ================================================================================ +2023-04-04 16:19:39 - r - INFO: - action_bound: 1.0 +2023-04-04 16:19:39 - r - INFO: - n_states: 8, n_actions: 2 +2023-04-04 16:19:39 - r - INFO: - Start testing! +2023-04-04 16:19:39 - r - INFO: - Env: LunarLanderContinuous-v2, Algorithm: SAC, Device: cpu +2023-04-04 16:19:45 - r - INFO: - Episode: 1/20, Reward: 247.478, Step: 230 +2023-04-04 16:19:49 - r - INFO: - Episode: 2/20, Reward: 260.547, Step: 195 +2023-04-04 16:19:53 - r - INFO: - Episode: 3/20, Reward: 259.074, Step: 178 +2023-04-04 16:19:57 - r - INFO: - Episode: 4/20, Reward: 296.493, Step: 192 +2023-04-04 16:20:01 - r - INFO: - Episode: 5/20, Reward: 292.460, Step: 225 +2023-04-04 16:20:07 - r - INFO: - Episode: 6/20, Reward: 282.687, Step: 294 +2023-04-04 16:20:12 - r - INFO: - Episode: 7/20, Reward: 239.822, Step: 242 +2023-04-04 16:20:16 - r - INFO: - Episode: 8/20, Reward: 270.872, Step: 196 +2023-04-04 16:20:21 - r - INFO: - Episode: 9/20, Reward: 276.446, Step: 213 +2023-04-04 16:20:24 - r - INFO: - Episode: 10/20, Reward: 284.918, Step: 186 +2023-04-04 16:20:31 - r - INFO: - Episode: 11/20, Reward: 278.167, Step: 305 +2023-04-04 16:20:34 - r - INFO: - Episode: 12/20, Reward: 294.946, Step: 180 +2023-04-04 16:20:39 - r - INFO: - Episode: 13/20, Reward: 277.009, Step: 211 +2023-04-04 16:20:43 - r - INFO: - Episode: 14/20, Reward: 275.229, Step: 199 +2023-04-04 16:20:48 - r - INFO: - Episode: 15/20, Reward: 283.965, Step: 237 +2023-04-04 16:20:52 - r - INFO: - Episode: 16/20, Reward: 268.659, Step: 226 +2023-04-04 16:20:57 - r - INFO: - Episode: 17/20, Reward: 277.079, Step: 235 +2023-04-04 16:21:02 - r - INFO: - Episode: 18/20, Reward: 263.336, Step: 245 +2023-04-04 16:21:07 - r - INFO: - Episode: 19/20, Reward: 238.160, Step: 221 +2023-04-04 16:21:12 - r - INFO: - Episode: 20/20, Reward: 293.206, Step: 257 +2023-04-04 16:21:12 - r - INFO: - Finish testing! diff --git a/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/models/checkpoint.pt b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf0709d1d5d2a65b0c739031f193eb4c6f8ce7aa --- /dev/null +++ b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92fd11443697b011d143a0fbd8df18e32f50821e5a0d3224ea57eeb636b376ad +size 3053965 diff --git a/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/results/learning_curve.png b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..e59f415978b49a892b60ba4493d866ea0b574a58 Binary files /dev/null and b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/results/learning_curve.png differ diff --git a/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/results/res.csv b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..baf159f30705f0d111bf974b271640336996cede --- /dev/null +++ b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,247.4777506669992,230 +1,260.5471488405748,195 +2,259.07383565977284,178 +3,296.49265989627963,192 +4,292.46007763772684,225 +5,282.6871119541543,294 +6,239.8224313725888,242 +7,270.8720028007499,196 +8,276.4455183730331,213 +9,284.9177361872782,186 +10,278.1668900670394,305 +11,294.94603703971507,180 +12,277.00855628365036,211 +13,275.2288665844296,199 +14,283.96500101467626,237 +15,268.6593087235675,226 +16,277.07937244869163,235 +17,263.33613970419935,245 +18,238.1601944417381,221 +19,293.2062954228968,257 diff --git a/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/tb_logs/events.out.tfevents.1680596379.LAPTOP-VAQ5K17P.14872.0 b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/tb_logs/events.out.tfevents.1680596379.LAPTOP-VAQ5K17P.14872.0 new file mode 100644 index 0000000000000000000000000000000000000000..26fe26f4b2b981ec1d90c9e35852c1a74abc504c --- /dev/null +++ b/LunarLanderContinuous-v2/Test_LunarLanderContinuous-v2_SAC_20230404-161934/tb_logs/events.out.tfevents.1680596379.LAPTOP-VAQ5K17P.14872.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eaeaf0459a2b34db3d55c1a23511c47af597fd5450c7e5fc90040f77fdd10a3 +size 40 diff --git a/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/config.yaml b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72d1b380339794cf12f3b3738a58aea9eb3f8026 --- /dev/null +++ b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: SAC + continuous: true + device: cuda + env_name: LunarLanderContinuous-v2 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_LunarLanderContinuous-v2_PPO_20230401-102521 + max_steps: 500 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 666 + show_fig: false + test_eps: 20 + train_eps: 2000 + wrapper: null +algo_cfg: + alpha: 0.1 + automatic_entropy_tuning: false + batch_size: 64 + buffer_size: 1000000 + gamma: 0.99 + hidden_dim: 256 + lr: 0.001 + n_epochs: 1 + policy_type: Gaussian + start_steps: 10000 + target_update_fre: 1 + tau: 0.005 diff --git a/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/logs/log.txt b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e6d0bf0caeff13a1541a5200a8101a2771f4c9a --- /dev/null +++ b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/logs/log.txt @@ -0,0 +1,2064 @@ +2023-04-02 17:01:58 - r - INFO: - Hyperparameters: +2023-04-02 17:01:58 - r - INFO: - ================================================================================ +2023-04-02 17:01:58 - r - INFO: - Name Value Type +2023-04-02 17:01:58 - r - INFO: - env_name LunarLanderContinuous-v2 +2023-04-02 17:01:58 - r - INFO: - continuous 1 +2023-04-02 17:01:58 - r - INFO: - new_step_api 1 +2023-04-02 17:01:58 - r - INFO: - wrapper None +2023-04-02 17:01:58 - r - INFO: - render 0 +2023-04-02 17:01:58 - r - INFO: - algo_name SAC +2023-04-02 17:01:58 - r - INFO: - mode train +2023-04-02 17:01:58 - r - INFO: - seed 666 +2023-04-02 17:01:58 - r - INFO: - device cuda +2023-04-02 17:01:58 - r - INFO: - train_eps 2000 +2023-04-02 17:01:58 - r - INFO: - test_eps 20 +2023-04-02 17:01:58 - r - INFO: - eval_eps 10 +2023-04-02 17:01:58 - r - INFO: - eval_per_episode 5 +2023-04-02 17:01:58 - r - INFO: - max_steps 500 +2023-04-02 17:01:58 - r - INFO: - load_checkpoint 0 +2023-04-02 17:01:58 - r - INFO: - load_path Train_LunarLanderContinuous-v2_PPO_20230401-102521 +2023-04-02 17:01:58 - r - INFO: - show_fig 0 +2023-04-02 17:01:58 - r - INFO: - save_fig 1 +2023-04-02 17:01:58 - r - INFO: - policy_type Gaussian +2023-04-02 17:01:58 - r - INFO: - lr 0.001 +2023-04-02 17:01:58 - r - INFO: - gamma 0.99 +2023-04-02 17:01:58 - r - INFO: - tau 0.005 +2023-04-02 17:01:58 - r - INFO: - alpha 0.1 +2023-04-02 17:01:58 - r - INFO: - automatic_entropy_tuning 0 +2023-04-02 17:01:58 - r - INFO: - batch_size 64 +2023-04-02 17:01:58 - r - INFO: - hidden_dim 256 +2023-04-02 17:01:58 - r - INFO: - n_epochs 1 +2023-04-02 17:01:58 - r - INFO: - start_steps 10000 +2023-04-02 17:01:58 - r - INFO: - target_update_fre 1 +2023-04-02 17:01:58 - r - INFO: - buffer_size 1000000 +2023-04-02 17:01:58 - r - INFO: - task_dir /home/mekeypan/code/joyrl-offline-main/tasks/Train_LunarLanderContinuous-v2_SAC_20230402-170158 +2023-04-02 17:01:58 - r - INFO: - res_dir /home/mekeypan/code/joyrl-offline-main/tasks/Train_LunarLanderContinuous-v2_SAC_20230402-170158/results +2023-04-02 17:01:58 - r - INFO: - log_dir /home/mekeypan/code/joyrl-offline-main/tasks/Train_LunarLanderContinuous-v2_SAC_20230402-170158/logs +2023-04-02 17:01:58 - r - INFO: - traj_dir /home/mekeypan/code/joyrl-offline-main/tasks/Train_LunarLanderContinuous-v2_SAC_20230402-170158/traj +2023-04-02 17:01:58 - r - INFO: - tb_dir /home/mekeypan/code/joyrl-offline-main/tasks/Train_LunarLanderContinuous-v2_SAC_20230402-170158/tb_logs +2023-04-02 17:01:58 - r - INFO: - ================================================================================ +2023-04-02 17:01:58 - r - INFO: - action_bound: 1.0 +2023-04-02 17:01:58 - r - INFO: - n_states: 8, n_actions: 2 +2023-04-02 17:02:00 - r - INFO: - Start training! +2023-04-02 17:02:00 - r - INFO: - Env: LunarLanderContinuous-v2, Algorithm: SAC, Device: cuda +2023-04-02 17:02:00 - r - INFO: - Episode: 1/2000, Reward: -335.186, Step: 123 +2023-04-02 17:02:01 - r - INFO: - Episode: 2/2000, Reward: -146.332, Step: 117 +2023-04-02 17:02:01 - r - INFO: - Episode: 3/2000, Reward: -49.347, Step: 86 +2023-04-02 17:02:01 - r - INFO: - Episode: 4/2000, Reward: -7.132, Step: 134 +2023-04-02 17:02:02 - r - INFO: - Episode: 5/2000, Reward: -136.546, Step: 173 +2023-04-02 17:02:02 - r - INFO: - Current episode 5 has the best eval reward: -551.957 +2023-04-02 17:02:02 - r - INFO: - Episode: 6/2000, Reward: -271.141, Step: 84 +2023-04-02 17:02:03 - r - INFO: - Episode: 7/2000, Reward: -389.103, Step: 97 +2023-04-02 17:02:03 - r - INFO: - Episode: 8/2000, Reward: -148.466, Step: 91 +2023-04-02 17:02:03 - r - INFO: - Episode: 9/2000, Reward: -333.847, Step: 101 +2023-04-02 17:02:03 - r - INFO: - Episode: 10/2000, Reward: -362.735, Step: 121 +2023-04-02 17:02:04 - r - INFO: - Episode: 11/2000, Reward: -375.207, Step: 128 +2023-04-02 17:02:04 - r - INFO: - Episode: 12/2000, Reward: -109.791, Step: 122 +2023-04-02 17:02:05 - r - INFO: - Episode: 13/2000, Reward: -87.356, Step: 94 +2023-04-02 17:02:05 - r - INFO: - Episode: 14/2000, Reward: -81.820, Step: 105 +2023-04-02 17:02:05 - r - INFO: - Episode: 15/2000, Reward: -202.145, Step: 211 +2023-04-02 17:02:06 - r - INFO: - Episode: 16/2000, Reward: -188.199, Step: 90 +2023-04-02 17:02:06 - r - INFO: - Episode: 17/2000, Reward: -467.215, Step: 111 +2023-04-02 17:02:07 - r - INFO: - Episode: 18/2000, Reward: -219.569, Step: 144 +2023-04-02 17:02:07 - r - INFO: - Episode: 19/2000, Reward: -91.690, Step: 76 +2023-04-02 17:02:07 - r - INFO: - Episode: 20/2000, Reward: -138.183, Step: 112 +2023-04-02 17:02:09 - r - INFO: - Current episode 20 has the best eval reward: -297.426 +2023-04-02 17:02:09 - r - INFO: - Episode: 21/2000, Reward: -397.217, Step: 96 +2023-04-02 17:02:09 - r - INFO: - Episode: 22/2000, Reward: -84.067, Step: 95 +2023-04-02 17:02:09 - r - INFO: - Episode: 23/2000, Reward: -92.998, Step: 119 +2023-04-02 17:02:10 - r - INFO: - Episode: 24/2000, Reward: -71.454, Step: 118 +2023-04-02 17:02:10 - r - INFO: - Episode: 25/2000, Reward: -311.551, Step: 94 +2023-04-02 17:02:12 - r - INFO: - Current episode 25 has the best eval reward: -190.733 +2023-04-02 17:02:12 - r - INFO: - Episode: 26/2000, Reward: -70.260, Step: 66 +2023-04-02 17:02:13 - r - INFO: - Episode: 27/2000, Reward: -282.444, Step: 115 +2023-04-02 17:02:13 - r - INFO: - Episode: 28/2000, Reward: -250.240, Step: 96 +2023-04-02 17:02:13 - r - INFO: - Episode: 29/2000, Reward: -125.759, Step: 67 +2023-04-02 17:02:13 - r - INFO: - Episode: 30/2000, Reward: -102.451, Step: 95 +2023-04-02 17:02:14 - r - INFO: - Episode: 31/2000, Reward: -147.842, Step: 118 +2023-04-02 17:02:15 - r - INFO: - Episode: 32/2000, Reward: -93.586, Step: 98 +2023-04-02 17:02:15 - r - INFO: - Episode: 33/2000, Reward: -137.025, Step: 143 +2023-04-02 17:02:15 - r - INFO: - Episode: 34/2000, Reward: -128.482, Step: 82 +2023-04-02 17:02:15 - r - INFO: - Episode: 35/2000, Reward: -109.860, Step: 90 +2023-04-02 17:02:17 - r - INFO: - Episode: 36/2000, Reward: -227.679, Step: 179 +2023-04-02 17:02:17 - r - INFO: - Episode: 37/2000, Reward: -106.314, Step: 122 +2023-04-02 17:02:17 - r - INFO: - Episode: 38/2000, Reward: -69.046, Step: 76 +2023-04-02 17:02:18 - r - INFO: - Episode: 39/2000, Reward: -290.349, Step: 147 +2023-04-02 17:02:18 - r - INFO: - Episode: 40/2000, Reward: -264.111, Step: 99 +2023-04-02 17:02:20 - r - INFO: - Current episode 40 has the best eval reward: -116.300 +2023-04-02 17:02:20 - r - INFO: - Episode: 41/2000, Reward: -327.212, Step: 83 +2023-04-02 17:02:21 - r - INFO: - Episode: 42/2000, Reward: -71.324, Step: 74 +2023-04-02 17:02:21 - r - INFO: - Episode: 43/2000, Reward: -102.273, Step: 94 +2023-04-02 17:02:21 - r - INFO: - Episode: 44/2000, Reward: -78.855, Step: 98 +2023-04-02 17:02:21 - r - INFO: - Episode: 45/2000, Reward: -339.199, Step: 115 +2023-04-02 17:02:26 - r - INFO: - Current episode 45 has the best eval reward: 4.163 +2023-04-02 17:02:27 - r - INFO: - Episode: 46/2000, Reward: -131.524, Step: 144 +2023-04-02 17:02:27 - r - INFO: - Episode: 47/2000, Reward: -252.219, Step: 98 +2023-04-02 17:02:28 - r - INFO: - Episode: 48/2000, Reward: -211.692, Step: 172 +2023-04-02 17:02:28 - r - INFO: - Episode: 49/2000, Reward: -56.506, Step: 163 +2023-04-02 17:02:28 - r - INFO: - Episode: 50/2000, Reward: -398.424, Step: 100 +2023-04-02 17:02:32 - r - INFO: - Episode: 51/2000, Reward: -224.203, Step: 104 +2023-04-02 17:02:33 - r - INFO: - Episode: 52/2000, Reward: -130.316, Step: 120 +2023-04-02 17:02:33 - r - INFO: - Episode: 53/2000, Reward: -339.995, Step: 89 +2023-04-02 17:02:33 - r - INFO: - Episode: 54/2000, Reward: -83.408, Step: 89 +2023-04-02 17:02:33 - r - INFO: - Episode: 55/2000, Reward: -199.259, Step: 151 +2023-04-02 17:02:38 - r - INFO: - Current episode 55 has the best eval reward: 12.573 +2023-04-02 17:02:38 - r - INFO: - Episode: 56/2000, Reward: -109.943, Step: 70 +2023-04-02 17:02:39 - r - INFO: - Episode: 57/2000, Reward: -401.644, Step: 152 +2023-04-02 17:02:39 - r - INFO: - Episode: 58/2000, Reward: -364.894, Step: 137 +2023-04-02 17:02:39 - r - INFO: - Episode: 59/2000, Reward: -138.212, Step: 79 +2023-04-02 17:02:39 - r - INFO: - Episode: 60/2000, Reward: -64.858, Step: 105 +2023-04-02 17:02:44 - r - INFO: - Episode: 61/2000, Reward: -375.989, Step: 88 +2023-04-02 17:02:44 - r - INFO: - Episode: 62/2000, Reward: -92.479, Step: 98 +2023-04-02 17:02:45 - r - INFO: - Episode: 63/2000, Reward: -134.209, Step: 144 +2023-04-02 17:02:45 - r - INFO: - Episode: 64/2000, Reward: -431.685, Step: 112 +2023-04-02 17:02:45 - r - INFO: - Episode: 65/2000, Reward: -109.817, Step: 117 +2023-04-02 17:02:49 - r - INFO: - Current episode 65 has the best eval reward: 35.696 +2023-04-02 17:02:51 - r - INFO: - Episode: 66/2000, Reward: -8.706, Step: 500 +2023-04-02 17:02:51 - r - INFO: - Episode: 67/2000, Reward: -231.906, Step: 177 +2023-04-02 17:02:52 - r - INFO: - Episode: 68/2000, Reward: -105.437, Step: 73 +2023-04-02 17:02:52 - r - INFO: - Episode: 69/2000, Reward: -168.650, Step: 112 +2023-04-02 17:02:52 - r - INFO: - Episode: 70/2000, Reward: -330.786, Step: 121 +2023-04-02 17:02:56 - r - INFO: - Episode: 71/2000, Reward: -290.083, Step: 146 +2023-04-02 17:02:57 - r - INFO: - Episode: 72/2000, Reward: -160.008, Step: 83 +2023-04-02 17:02:57 - r - INFO: - Episode: 73/2000, Reward: -181.910, Step: 101 +2023-04-02 17:02:57 - r - INFO: - Episode: 74/2000, Reward: -351.980, Step: 99 +2023-04-02 17:02:57 - r - INFO: - Episode: 75/2000, Reward: -251.505, Step: 115 +2023-04-02 17:03:02 - r - INFO: - Episode: 76/2000, Reward: -213.521, Step: 109 +2023-04-02 17:03:02 - r - INFO: - Episode: 77/2000, Reward: 32.147, Step: 80 +2023-04-02 17:03:02 - r - INFO: - Episode: 78/2000, Reward: -63.694, Step: 68 +2023-04-02 17:03:03 - r - INFO: - Episode: 79/2000, Reward: -94.629, Step: 81 +2023-04-02 17:03:03 - r - INFO: - Episode: 80/2000, Reward: -323.123, Step: 95 +2023-04-02 17:03:07 - r - INFO: - Episode: 81/2000, Reward: -185.847, Step: 86 +2023-04-02 17:03:07 - r - INFO: - Episode: 82/2000, Reward: 17.534, Step: 97 +2023-04-02 17:03:07 - r - INFO: - Episode: 83/2000, Reward: -352.611, Step: 124 +2023-04-02 17:03:08 - r - INFO: - Episode: 84/2000, Reward: -144.591, Step: 125 +2023-04-02 17:03:08 - r - INFO: - Episode: 85/2000, Reward: -81.377, Step: 127 +2023-04-02 17:03:12 - r - INFO: - Episode: 86/2000, Reward: -449.579, Step: 137 +2023-04-02 17:03:12 - r - INFO: - Episode: 87/2000, Reward: -296.892, Step: 96 +2023-04-02 17:03:14 - r - INFO: - Episode: 88/2000, Reward: 216.328, Step: 416 +2023-04-02 17:03:15 - r - INFO: - Episode: 89/2000, Reward: -128.510, Step: 429 +2023-04-02 17:03:16 - r - INFO: - Episode: 90/2000, Reward: -22.118, Step: 170 +2023-04-02 17:03:19 - r - INFO: - Episode: 91/2000, Reward: -118.992, Step: 363 +2023-04-02 17:03:19 - r - INFO: - Episode: 92/2000, Reward: -222.075, Step: 162 +2023-04-02 17:03:21 - r - INFO: - Episode: 93/2000, Reward: 67.734, Step: 500 +2023-04-02 17:03:22 - r - INFO: - Episode: 94/2000, Reward: -76.770, Step: 135 +2023-04-02 17:03:23 - r - INFO: - Episode: 95/2000, Reward: 7.710, Step: 500 +2023-04-02 17:03:25 - r - INFO: - Episode: 96/2000, Reward: -111.437, Step: 123 +2023-04-02 17:03:26 - r - INFO: - Episode: 97/2000, Reward: -207.811, Step: 185 +2023-04-02 17:03:28 - r - INFO: - Episode: 98/2000, Reward: 26.642, Step: 500 +2023-04-02 17:03:29 - r - INFO: - Episode: 99/2000, Reward: -187.697, Step: 415 +2023-04-02 17:03:31 - r - INFO: - Episode: 100/2000, Reward: -153.783, Step: 478 +2023-04-02 17:03:34 - r - INFO: - Episode: 101/2000, Reward: -146.702, Step: 500 +2023-04-02 17:03:36 - r - INFO: - Episode: 102/2000, Reward: -44.651, Step: 500 +2023-04-02 17:03:37 - r - INFO: - Episode: 103/2000, Reward: 93.267, Step: 500 +2023-04-02 17:03:39 - r - INFO: - Episode: 104/2000, Reward: -124.245, Step: 500 +2023-04-02 17:03:41 - r - INFO: - Episode: 105/2000, Reward: -157.671, Step: 436 +2023-04-02 17:03:44 - r - INFO: - Episode: 106/2000, Reward: -132.946, Step: 408 +2023-04-02 17:03:46 - r - INFO: - Episode: 107/2000, Reward: 67.286, Step: 500 +2023-04-02 17:03:48 - r - INFO: - Episode: 108/2000, Reward: 25.657, Step: 500 +2023-04-02 17:03:50 - r - INFO: - Episode: 109/2000, Reward: 139.037, Step: 500 +2023-04-02 17:03:51 - r - INFO: - Episode: 110/2000, Reward: -120.712, Step: 320 +2023-04-02 17:03:54 - r - INFO: - Episode: 111/2000, Reward: -119.316, Step: 243 +2023-04-02 17:03:56 - r - INFO: - Episode: 112/2000, Reward: 5.972, Step: 500 +2023-04-02 17:03:57 - r - INFO: - Episode: 113/2000, Reward: 0.549, Step: 263 +2023-04-02 17:03:58 - r - INFO: - Episode: 114/2000, Reward: -0.512, Step: 500 +2023-04-02 17:04:00 - r - INFO: - Episode: 115/2000, Reward: -6.916, Step: 500 +2023-04-02 17:04:06 - r - INFO: - Episode: 116/2000, Reward: 43.148, Step: 500 +2023-04-02 17:04:08 - r - INFO: - Episode: 117/2000, Reward: 58.623, Step: 500 +2023-04-02 17:04:09 - r - INFO: - Episode: 118/2000, Reward: -59.779, Step: 367 +2023-04-02 17:04:11 - r - INFO: - Episode: 119/2000, Reward: 27.162, Step: 500 +2023-04-02 17:04:12 - r - INFO: - Episode: 120/2000, Reward: -331.298, Step: 288 +2023-04-02 17:04:17 - r - INFO: - Episode: 121/2000, Reward: 66.016, Step: 500 +2023-04-02 17:04:19 - r - INFO: - Episode: 122/2000, Reward: 98.263, Step: 500 +2023-04-02 17:04:20 - r - INFO: - Episode: 123/2000, Reward: 85.698, Step: 500 +2023-04-02 17:04:22 - r - INFO: - Episode: 124/2000, Reward: 52.121, Step: 500 +2023-04-02 17:04:24 - r - INFO: - Episode: 125/2000, Reward: 44.965, Step: 500 +2023-04-02 17:04:28 - r - INFO: - Current episode 125 has the best eval reward: 55.250 +2023-04-02 17:04:30 - r - INFO: - Episode: 126/2000, Reward: 67.252, Step: 500 +2023-04-02 17:04:32 - r - INFO: - Episode: 127/2000, Reward: 44.393, Step: 500 +2023-04-02 17:04:34 - r - INFO: - Episode: 128/2000, Reward: 48.001, Step: 500 +2023-04-02 17:04:36 - r - INFO: - Episode: 129/2000, Reward: 60.078, Step: 500 +2023-04-02 17:04:37 - r - INFO: - Episode: 130/2000, Reward: 83.935, Step: 500 +2023-04-02 17:04:42 - r - INFO: - Current episode 130 has the best eval reward: 60.505 +2023-04-02 17:04:44 - r - INFO: - Episode: 131/2000, Reward: 4.328, Step: 500 +2023-04-02 17:04:46 - r - INFO: - Episode: 132/2000, Reward: 107.038, Step: 500 +2023-04-02 17:04:47 - r - INFO: - Episode: 133/2000, Reward: 125.021, Step: 500 +2023-04-02 17:04:49 - r - INFO: - Episode: 134/2000, Reward: 40.747, Step: 500 +2023-04-02 17:04:51 - r - INFO: - Episode: 135/2000, Reward: 50.921, Step: 500 +2023-04-02 17:04:57 - r - INFO: - Episode: 136/2000, Reward: 68.341, Step: 500 +2023-04-02 17:04:58 - r - INFO: - Episode: 137/2000, Reward: -25.494, Step: 296 +2023-04-02 17:05:00 - r - INFO: - Episode: 138/2000, Reward: 79.880, Step: 500 +2023-04-02 17:05:02 - r - INFO: - Episode: 139/2000, Reward: 85.216, Step: 500 +2023-04-02 17:05:04 - r - INFO: - Episode: 140/2000, Reward: 64.893, Step: 500 +2023-04-02 17:05:09 - r - INFO: - Episode: 141/2000, Reward: 52.419, Step: 500 +2023-04-02 17:05:11 - r - INFO: - Episode: 142/2000, Reward: 65.888, Step: 500 +2023-04-02 17:05:12 - r - INFO: - Episode: 143/2000, Reward: 74.977, Step: 500 +2023-04-02 17:05:14 - r - INFO: - Episode: 144/2000, Reward: 81.173, Step: 500 +2023-04-02 17:05:16 - r - INFO: - Episode: 145/2000, Reward: 72.121, Step: 500 +2023-04-02 17:05:20 - r - INFO: - Current episode 145 has the best eval reward: 93.267 +2023-04-02 17:05:22 - r - INFO: - Episode: 146/2000, Reward: 68.567, Step: 500 +2023-04-02 17:05:24 - r - INFO: - Episode: 147/2000, Reward: 70.244, Step: 500 +2023-04-02 17:05:25 - r - INFO: - Episode: 148/2000, Reward: 80.281, Step: 500 +2023-04-02 17:05:26 - r - INFO: - Episode: 149/2000, Reward: -53.250, Step: 193 +2023-04-02 17:05:28 - r - INFO: - Episode: 150/2000, Reward: 59.549, Step: 500 +2023-04-02 17:05:34 - r - INFO: - Episode: 151/2000, Reward: 62.839, Step: 500 +2023-04-02 17:05:36 - r - INFO: - Episode: 152/2000, Reward: 68.573, Step: 500 +2023-04-02 17:05:38 - r - INFO: - Episode: 153/2000, Reward: 57.284, Step: 500 +2023-04-02 17:05:40 - r - INFO: - Episode: 154/2000, Reward: 22.237, Step: 500 +2023-04-02 17:05:41 - r - INFO: - Episode: 155/2000, Reward: 49.399, Step: 500 +2023-04-02 17:05:47 - r - INFO: - Episode: 156/2000, Reward: 99.905, Step: 500 +2023-04-02 17:05:49 - r - INFO: - Episode: 157/2000, Reward: 48.116, Step: 500 +2023-04-02 17:05:51 - r - INFO: - Episode: 158/2000, Reward: 47.771, Step: 500 +2023-04-02 17:05:52 - r - INFO: - Episode: 159/2000, Reward: 62.650, Step: 500 +2023-04-02 17:05:54 - r - INFO: - Episode: 160/2000, Reward: 42.427, Step: 500 +2023-04-02 17:06:00 - r - INFO: - Episode: 161/2000, Reward: 40.485, Step: 500 +2023-04-02 17:06:02 - r - INFO: - Episode: 162/2000, Reward: 67.384, Step: 500 +2023-04-02 17:06:04 - r - INFO: - Episode: 163/2000, Reward: 72.459, Step: 500 +2023-04-02 17:06:06 - r - INFO: - Episode: 164/2000, Reward: 54.667, Step: 500 +2023-04-02 17:06:07 - r - INFO: - Episode: 165/2000, Reward: 41.189, Step: 500 +2023-04-02 17:06:14 - r - INFO: - Episode: 166/2000, Reward: 59.122, Step: 500 +2023-04-02 17:06:15 - r - INFO: - Episode: 167/2000, Reward: 73.414, Step: 500 +2023-04-02 17:06:17 - r - INFO: - Episode: 168/2000, Reward: 30.800, Step: 500 +2023-04-02 17:06:19 - r - INFO: - Episode: 169/2000, Reward: 49.296, Step: 500 +2023-04-02 17:06:21 - r - INFO: - Episode: 170/2000, Reward: 22.689, Step: 500 +2023-04-02 17:06:27 - r - INFO: - Episode: 171/2000, Reward: 84.012, Step: 500 +2023-04-02 17:06:29 - r - INFO: - Episode: 172/2000, Reward: 47.430, Step: 500 +2023-04-02 17:06:31 - r - INFO: - Episode: 173/2000, Reward: 50.824, Step: 500 +2023-04-02 17:06:32 - r - INFO: - Episode: 174/2000, Reward: 0.843, Step: 500 +2023-04-02 17:06:34 - r - INFO: - Episode: 175/2000, Reward: 83.856, Step: 500 +2023-04-02 17:06:40 - r - INFO: - Episode: 176/2000, Reward: 72.401, Step: 500 +2023-04-02 17:06:42 - r - INFO: - Episode: 177/2000, Reward: 59.342, Step: 500 +2023-04-02 17:06:44 - r - INFO: - Episode: 178/2000, Reward: 55.131, Step: 500 +2023-04-02 17:06:45 - r - INFO: - Episode: 179/2000, Reward: 5.335, Step: 500 +2023-04-02 17:06:47 - r - INFO: - Episode: 180/2000, Reward: 13.718, Step: 500 +2023-04-02 17:06:52 - r - INFO: - Episode: 181/2000, Reward: -6.817, Step: 500 +2023-04-02 17:06:54 - r - INFO: - Episode: 182/2000, Reward: 65.057, Step: 500 +2023-04-02 17:06:56 - r - INFO: - Episode: 183/2000, Reward: 27.472, Step: 500 +2023-04-02 17:06:57 - r - INFO: - Episode: 184/2000, Reward: 52.487, Step: 500 +2023-04-02 17:06:59 - r - INFO: - Episode: 185/2000, Reward: 61.231, Step: 500 +2023-04-02 17:07:05 - r - INFO: - Episode: 186/2000, Reward: 37.880, Step: 500 +2023-04-02 17:07:07 - r - INFO: - Episode: 187/2000, Reward: -1.011, Step: 500 +2023-04-02 17:07:09 - r - INFO: - Episode: 188/2000, Reward: 63.352, Step: 500 +2023-04-02 17:07:11 - r - INFO: - Episode: 189/2000, Reward: 63.433, Step: 500 +2023-04-02 17:07:12 - r - INFO: - Episode: 190/2000, Reward: 46.328, Step: 500 +2023-04-02 17:07:19 - r - INFO: - Episode: 191/2000, Reward: 63.838, Step: 500 +2023-04-02 17:07:21 - r - INFO: - Episode: 192/2000, Reward: 73.876, Step: 500 +2023-04-02 17:07:22 - r - INFO: - Episode: 193/2000, Reward: 89.817, Step: 500 +2023-04-02 17:07:24 - r - INFO: - Episode: 194/2000, Reward: 74.416, Step: 500 +2023-04-02 17:07:25 - r - INFO: - Episode: 195/2000, Reward: 220.807, Step: 392 +2023-04-02 17:07:31 - r - INFO: - Episode: 196/2000, Reward: 96.706, Step: 500 +2023-04-02 17:07:33 - r - INFO: - Episode: 197/2000, Reward: 66.968, Step: 500 +2023-04-02 17:07:35 - r - INFO: - Episode: 198/2000, Reward: 100.056, Step: 500 +2023-04-02 17:07:36 - r - INFO: - Episode: 199/2000, Reward: 62.518, Step: 500 +2023-04-02 17:07:38 - r - INFO: - Episode: 200/2000, Reward: 85.498, Step: 500 +2023-04-02 17:07:44 - r - INFO: - Episode: 201/2000, Reward: 80.493, Step: 500 +2023-04-02 17:07:46 - r - INFO: - Episode: 202/2000, Reward: 75.365, Step: 500 +2023-04-02 17:07:47 - r - INFO: - Episode: 203/2000, Reward: 241.564, Step: 391 +2023-04-02 17:07:49 - r - INFO: - Episode: 204/2000, Reward: 66.481, Step: 500 +2023-04-02 17:07:51 - r - INFO: - Episode: 205/2000, Reward: 47.221, Step: 500 +2023-04-02 17:07:57 - r - INFO: - Episode: 206/2000, Reward: 67.232, Step: 500 +2023-04-02 17:07:59 - r - INFO: - Episode: 207/2000, Reward: 21.196, Step: 500 +2023-04-02 17:08:01 - r - INFO: - Episode: 208/2000, Reward: 29.922, Step: 500 +2023-04-02 17:08:03 - r - INFO: - Episode: 209/2000, Reward: -219.808, Step: 500 +2023-04-02 17:08:04 - r - INFO: - Episode: 210/2000, Reward: 28.148, Step: 500 +2023-04-02 17:08:10 - r - INFO: - Episode: 211/2000, Reward: 119.306, Step: 500 +2023-04-02 17:08:12 - r - INFO: - Episode: 212/2000, Reward: 41.218, Step: 500 +2023-04-02 17:08:13 - r - INFO: - Episode: 213/2000, Reward: 48.157, Step: 500 +2023-04-02 17:08:15 - r - INFO: - Episode: 214/2000, Reward: 40.825, Step: 500 +2023-04-02 17:08:17 - r - INFO: - Episode: 215/2000, Reward: 36.237, Step: 500 +2023-04-02 17:08:23 - r - INFO: - Episode: 216/2000, Reward: 55.477, Step: 500 +2023-04-02 17:08:25 - r - INFO: - Episode: 217/2000, Reward: 91.114, Step: 500 +2023-04-02 17:08:27 - r - INFO: - Episode: 218/2000, Reward: 49.977, Step: 500 +2023-04-02 17:08:29 - r - INFO: - Episode: 219/2000, Reward: 50.971, Step: 500 +2023-04-02 17:08:30 - r - INFO: - Episode: 220/2000, Reward: 11.904, Step: 500 +2023-04-02 17:08:36 - r - INFO: - Episode: 221/2000, Reward: 40.030, Step: 500 +2023-04-02 17:08:38 - r - INFO: - Episode: 222/2000, Reward: 48.166, Step: 500 +2023-04-02 17:08:40 - r - INFO: - Episode: 223/2000, Reward: 65.464, Step: 500 +2023-04-02 17:08:41 - r - INFO: - Episode: 224/2000, Reward: 59.396, Step: 500 +2023-04-02 17:08:43 - r - INFO: - Episode: 225/2000, Reward: 52.248, Step: 500 +2023-04-02 17:08:49 - r - INFO: - Episode: 226/2000, Reward: 49.615, Step: 500 +2023-04-02 17:08:51 - r - INFO: - Episode: 227/2000, Reward: 81.173, Step: 500 +2023-04-02 17:08:52 - r - INFO: - Episode: 228/2000, Reward: 87.680, Step: 500 +2023-04-02 17:08:54 - r - INFO: - Episode: 229/2000, Reward: 113.539, Step: 500 +2023-04-02 17:08:56 - r - INFO: - Episode: 230/2000, Reward: 96.736, Step: 500 +2023-04-02 17:09:02 - r - INFO: - Episode: 231/2000, Reward: 55.291, Step: 500 +2023-04-02 17:09:04 - r - INFO: - Episode: 232/2000, Reward: 69.914, Step: 500 +2023-04-02 17:09:06 - r - INFO: - Episode: 233/2000, Reward: 62.676, Step: 500 +2023-04-02 17:09:07 - r - INFO: - Episode: 234/2000, Reward: 95.236, Step: 500 +2023-04-02 17:09:09 - r - INFO: - Episode: 235/2000, Reward: 75.096, Step: 500 +2023-04-02 17:09:15 - r - INFO: - Episode: 236/2000, Reward: 54.479, Step: 500 +2023-04-02 17:09:17 - r - INFO: - Episode: 237/2000, Reward: 96.796, Step: 500 +2023-04-02 17:09:19 - r - INFO: - Episode: 238/2000, Reward: 61.501, Step: 500 +2023-04-02 17:09:20 - r - INFO: - Episode: 239/2000, Reward: 108.869, Step: 500 +2023-04-02 17:09:22 - r - INFO: - Episode: 240/2000, Reward: 132.813, Step: 500 +2023-04-02 17:09:26 - r - INFO: - Current episode 240 has the best eval reward: 126.009 +2023-04-02 17:09:28 - r - INFO: - Episode: 241/2000, Reward: 168.310, Step: 500 +2023-04-02 17:09:29 - r - INFO: - Episode: 242/2000, Reward: 207.370, Step: 382 +2023-04-02 17:09:31 - r - INFO: - Episode: 243/2000, Reward: 87.304, Step: 500 +2023-04-02 17:09:33 - r - INFO: - Episode: 244/2000, Reward: 76.859, Step: 500 +2023-04-02 17:09:33 - r - INFO: - Episode: 245/2000, Reward: 261.096, Step: 245 +2023-04-02 17:09:37 - r - INFO: - Current episode 245 has the best eval reward: 177.732 +2023-04-02 17:09:39 - r - INFO: - Episode: 246/2000, Reward: 107.399, Step: 500 +2023-04-02 17:09:40 - r - INFO: - Episode: 247/2000, Reward: 53.493, Step: 500 +2023-04-02 17:09:42 - r - INFO: - Episode: 248/2000, Reward: 103.035, Step: 500 +2023-04-02 17:09:43 - r - INFO: - Episode: 249/2000, Reward: 244.069, Step: 283 +2023-04-02 17:09:45 - r - INFO: - Episode: 250/2000, Reward: 70.083, Step: 500 +2023-04-02 17:09:49 - r - INFO: - Episode: 251/2000, Reward: 245.596, Step: 421 +2023-04-02 17:09:51 - r - INFO: - Episode: 252/2000, Reward: 51.372, Step: 500 +2023-04-02 17:09:53 - r - INFO: - Episode: 253/2000, Reward: 102.817, Step: 500 +2023-04-02 17:09:54 - r - INFO: - Episode: 254/2000, Reward: 65.061, Step: 500 +2023-04-02 17:09:56 - r - INFO: - Episode: 255/2000, Reward: 68.420, Step: 500 +2023-04-02 17:10:02 - r - INFO: - Episode: 256/2000, Reward: 241.908, Step: 272 +2023-04-02 17:10:03 - r - INFO: - Episode: 257/2000, Reward: 111.012, Step: 500 +2023-04-02 17:10:04 - r - INFO: - Episode: 258/2000, Reward: 240.694, Step: 247 +2023-04-02 17:10:06 - r - INFO: - Episode: 259/2000, Reward: 84.152, Step: 500 +2023-04-02 17:10:08 - r - INFO: - Episode: 260/2000, Reward: 238.469, Step: 444 +2023-04-02 17:10:14 - r - INFO: - Episode: 261/2000, Reward: 118.716, Step: 500 +2023-04-02 17:10:16 - r - INFO: - Episode: 262/2000, Reward: 78.818, Step: 500 +2023-04-02 17:10:18 - r - INFO: - Episode: 263/2000, Reward: 89.858, Step: 500 +2023-04-02 17:10:19 - r - INFO: - Episode: 264/2000, Reward: 87.142, Step: 500 +2023-04-02 17:10:21 - r - INFO: - Episode: 265/2000, Reward: 115.955, Step: 500 +2023-04-02 17:10:28 - r - INFO: - Episode: 266/2000, Reward: 48.376, Step: 500 +2023-04-02 17:10:28 - r - INFO: - Episode: 267/2000, Reward: 236.271, Step: 218 +2023-04-02 17:10:30 - r - INFO: - Episode: 268/2000, Reward: 86.927, Step: 500 +2023-04-02 17:10:32 - r - INFO: - Episode: 269/2000, Reward: 75.460, Step: 500 +2023-04-02 17:10:34 - r - INFO: - Episode: 270/2000, Reward: 55.031, Step: 500 +2023-04-02 17:10:40 - r - INFO: - Episode: 271/2000, Reward: 104.025, Step: 500 +2023-04-02 17:10:41 - r - INFO: - Episode: 272/2000, Reward: 221.703, Step: 314 +2023-04-02 17:10:42 - r - INFO: - Episode: 273/2000, Reward: 77.342, Step: 500 +2023-04-02 17:10:44 - r - INFO: - Episode: 274/2000, Reward: 108.010, Step: 500 +2023-04-02 17:10:46 - r - INFO: - Episode: 275/2000, Reward: 50.224, Step: 500 +2023-04-02 17:10:52 - r - INFO: - Episode: 276/2000, Reward: 82.222, Step: 500 +2023-04-02 17:10:54 - r - INFO: - Episode: 277/2000, Reward: 151.524, Step: 500 +2023-04-02 17:10:56 - r - INFO: - Episode: 278/2000, Reward: 74.857, Step: 500 +2023-04-02 17:10:57 - r - INFO: - Episode: 279/2000, Reward: 49.077, Step: 500 +2023-04-02 17:10:59 - r - INFO: - Episode: 280/2000, Reward: 275.258, Step: 375 +2023-04-02 17:11:03 - r - INFO: - Episode: 281/2000, Reward: 230.595, Step: 437 +2023-04-02 17:11:05 - r - INFO: - Episode: 282/2000, Reward: 67.872, Step: 500 +2023-04-02 17:11:07 - r - INFO: - Episode: 283/2000, Reward: 19.102, Step: 500 +2023-04-02 17:11:08 - r - INFO: - Episode: 284/2000, Reward: 224.036, Step: 319 +2023-04-02 17:11:09 - r - INFO: - Episode: 285/2000, Reward: 221.912, Step: 466 +2023-04-02 17:11:14 - r - INFO: - Episode: 286/2000, Reward: 61.454, Step: 500 +2023-04-02 17:11:15 - r - INFO: - Episode: 287/2000, Reward: 249.158, Step: 223 +2023-04-02 17:11:16 - r - INFO: - Episode: 288/2000, Reward: 149.407, Step: 500 +2023-04-02 17:11:18 - r - INFO: - Episode: 289/2000, Reward: 80.134, Step: 500 +2023-04-02 17:11:19 - r - INFO: - Episode: 290/2000, Reward: 274.078, Step: 425 +2023-04-02 17:11:24 - r - INFO: - Episode: 291/2000, Reward: 232.336, Step: 367 +2023-04-02 17:11:25 - r - INFO: - Episode: 292/2000, Reward: 239.740, Step: 481 +2023-04-02 17:11:27 - r - INFO: - Episode: 293/2000, Reward: 248.379, Step: 460 +2023-04-02 17:11:28 - r - INFO: - Episode: 294/2000, Reward: 252.630, Step: 329 +2023-04-02 17:11:30 - r - INFO: - Episode: 295/2000, Reward: 123.938, Step: 500 +2023-04-02 17:11:34 - r - INFO: - Episode: 296/2000, Reward: 283.002, Step: 424 +2023-04-02 17:11:35 - r - INFO: - Episode: 297/2000, Reward: 242.529, Step: 416 +2023-04-02 17:11:37 - r - INFO: - Episode: 298/2000, Reward: 284.932, Step: 499 +2023-04-02 17:11:38 - r - INFO: - Episode: 299/2000, Reward: 231.752, Step: 318 +2023-04-02 17:11:39 - r - INFO: - Episode: 300/2000, Reward: 185.982, Step: 324 +2023-04-02 17:11:44 - r - INFO: - Episode: 301/2000, Reward: 122.701, Step: 500 +2023-04-02 17:11:46 - r - INFO: - Episode: 302/2000, Reward: 233.977, Step: 404 +2023-04-02 17:11:47 - r - INFO: - Episode: 303/2000, Reward: 275.742, Step: 444 +2023-04-02 17:11:48 - r - INFO: - Episode: 304/2000, Reward: 266.109, Step: 367 +2023-04-02 17:11:50 - r - INFO: - Episode: 305/2000, Reward: 219.937, Step: 466 +2023-04-02 17:11:53 - r - INFO: - Current episode 305 has the best eval reward: 189.878 +2023-04-02 17:11:54 - r - INFO: - Episode: 306/2000, Reward: 5.923, Step: 500 +2023-04-02 17:11:55 - r - INFO: - Episode: 307/2000, Reward: 235.918, Step: 355 +2023-04-02 17:11:57 - r - INFO: - Episode: 308/2000, Reward: 203.818, Step: 432 +2023-04-02 17:11:59 - r - INFO: - Episode: 309/2000, Reward: 57.377, Step: 500 +2023-04-02 17:12:00 - r - INFO: - Episode: 310/2000, Reward: 247.665, Step: 346 +2023-04-02 17:12:05 - r - INFO: - Episode: 311/2000, Reward: 273.225, Step: 290 +2023-04-02 17:12:06 - r - INFO: - Episode: 312/2000, Reward: 132.607, Step: 500 +2023-04-02 17:12:08 - r - INFO: - Episode: 313/2000, Reward: 49.316, Step: 500 +2023-04-02 17:12:10 - r - INFO: - Episode: 314/2000, Reward: 220.877, Step: 443 +2023-04-02 17:12:11 - r - INFO: - Episode: 315/2000, Reward: -70.148, Step: 500 +2023-04-02 17:12:16 - r - INFO: - Episode: 316/2000, Reward: 60.215, Step: 500 +2023-04-02 17:12:17 - r - INFO: - Episode: 317/2000, Reward: 94.787, Step: 500 +2023-04-02 17:12:19 - r - INFO: - Episode: 318/2000, Reward: 66.432, Step: 500 +2023-04-02 17:12:21 - r - INFO: - Episode: 319/2000, Reward: 231.036, Step: 361 +2023-04-02 17:12:22 - r - INFO: - Episode: 320/2000, Reward: 70.284, Step: 500 +2023-04-02 17:12:27 - r - INFO: - Episode: 321/2000, Reward: 142.774, Step: 500 +2023-04-02 17:12:28 - r - INFO: - Episode: 322/2000, Reward: 238.742, Step: 245 +2023-04-02 17:12:30 - r - INFO: - Episode: 323/2000, Reward: 68.940, Step: 500 +2023-04-02 17:12:31 - r - INFO: - Episode: 324/2000, Reward: 41.174, Step: 500 +2023-04-02 17:12:33 - r - INFO: - Episode: 325/2000, Reward: 229.247, Step: 460 +2023-04-02 17:12:38 - r - INFO: - Episode: 326/2000, Reward: 243.423, Step: 495 +2023-04-02 17:12:40 - r - INFO: - Episode: 327/2000, Reward: 176.549, Step: 500 +2023-04-02 17:12:42 - r - INFO: - Episode: 328/2000, Reward: 83.894, Step: 500 +2023-04-02 17:12:43 - r - INFO: - Episode: 329/2000, Reward: 270.525, Step: 326 +2023-04-02 17:12:44 - r - INFO: - Episode: 330/2000, Reward: 210.721, Step: 423 +2023-04-02 17:12:49 - r - INFO: - Episode: 331/2000, Reward: 90.853, Step: 500 +2023-04-02 17:12:50 - r - INFO: - Episode: 332/2000, Reward: 238.088, Step: 409 +2023-04-02 17:12:51 - r - INFO: - Episode: 333/2000, Reward: 19.871, Step: 268 +2023-04-02 17:12:53 - r - INFO: - Episode: 334/2000, Reward: 78.112, Step: 500 +2023-04-02 17:12:54 - r - INFO: - Episode: 335/2000, Reward: 257.919, Step: 463 +2023-04-02 17:12:59 - r - INFO: - Episode: 336/2000, Reward: 255.656, Step: 222 +2023-04-02 17:13:01 - r - INFO: - Episode: 337/2000, Reward: 71.970, Step: 500 +2023-04-02 17:13:02 - r - INFO: - Episode: 338/2000, Reward: 249.309, Step: 411 +2023-04-02 17:13:03 - r - INFO: - Episode: 339/2000, Reward: 261.781, Step: 318 +2023-04-02 17:13:05 - r - INFO: - Episode: 340/2000, Reward: 74.793, Step: 500 +2023-04-02 17:13:11 - r - INFO: - Episode: 341/2000, Reward: 47.848, Step: 500 +2023-04-02 17:13:12 - r - INFO: - Episode: 342/2000, Reward: 164.869, Step: 500 +2023-04-02 17:13:14 - r - INFO: - Episode: 343/2000, Reward: 166.293, Step: 500 +2023-04-02 17:13:15 - r - INFO: - Episode: 344/2000, Reward: 299.449, Step: 320 +2023-04-02 17:13:17 - r - INFO: - Episode: 345/2000, Reward: 107.061, Step: 500 +2023-04-02 17:13:23 - r - INFO: - Episode: 346/2000, Reward: 69.110, Step: 500 +2023-04-02 17:13:24 - r - INFO: - Episode: 347/2000, Reward: 250.032, Step: 314 +2023-04-02 17:13:25 - r - INFO: - Episode: 348/2000, Reward: 114.938, Step: 500 +2023-04-02 17:13:27 - r - INFO: - Episode: 349/2000, Reward: 91.270, Step: 500 +2023-04-02 17:13:29 - r - INFO: - Episode: 350/2000, Reward: 61.315, Step: 500 +2023-04-02 17:13:33 - r - INFO: - Episode: 351/2000, Reward: 237.919, Step: 278 +2023-04-02 17:13:35 - r - INFO: - Episode: 352/2000, Reward: -192.674, Step: 454 +2023-04-02 17:13:36 - r - INFO: - Episode: 353/2000, Reward: 235.424, Step: 405 +2023-04-02 17:13:38 - r - INFO: - Episode: 354/2000, Reward: 56.253, Step: 500 +2023-04-02 17:13:40 - r - INFO: - Episode: 355/2000, Reward: 110.435, Step: 500 +2023-04-02 17:13:42 - r - INFO: - Episode: 356/2000, Reward: 300.861, Step: 231 +2023-04-02 17:13:44 - r - INFO: - Episode: 357/2000, Reward: 291.987, Step: 425 +2023-04-02 17:13:45 - r - INFO: - Episode: 358/2000, Reward: 275.443, Step: 216 +2023-04-02 17:13:46 - r - INFO: - Episode: 359/2000, Reward: -2.769, Step: 500 +2023-04-02 17:13:48 - r - INFO: - Episode: 360/2000, Reward: 93.315, Step: 500 +2023-04-02 17:13:53 - r - INFO: - Episode: 361/2000, Reward: 63.734, Step: 500 +2023-04-02 17:13:54 - r - INFO: - Episode: 362/2000, Reward: 251.989, Step: 201 +2023-04-02 17:13:55 - r - INFO: - Episode: 363/2000, Reward: 102.283, Step: 500 +2023-04-02 17:13:56 - r - INFO: - Episode: 364/2000, Reward: 316.788, Step: 263 +2023-04-02 17:13:57 - r - INFO: - Episode: 365/2000, Reward: 292.444, Step: 175 +2023-04-02 17:14:00 - r - INFO: - Episode: 366/2000, Reward: 278.672, Step: 186 +2023-04-02 17:14:01 - r - INFO: - Episode: 367/2000, Reward: 218.511, Step: 496 +2023-04-02 17:14:02 - r - INFO: - Episode: 368/2000, Reward: 234.831, Step: 229 +2023-04-02 17:14:03 - r - INFO: - Episode: 369/2000, Reward: 275.847, Step: 393 +2023-04-02 17:14:05 - r - INFO: - Episode: 370/2000, Reward: 142.143, Step: 500 +2023-04-02 17:14:06 - r - INFO: - Current episode 370 has the best eval reward: 251.393 +2023-04-02 17:14:08 - r - INFO: - Episode: 371/2000, Reward: 196.221, Step: 425 +2023-04-02 17:14:09 - r - INFO: - Episode: 372/2000, Reward: 234.269, Step: 310 +2023-04-02 17:14:10 - r - INFO: - Episode: 373/2000, Reward: 248.375, Step: 225 +2023-04-02 17:14:11 - r - INFO: - Episode: 374/2000, Reward: 68.945, Step: 500 +2023-04-02 17:14:13 - r - INFO: - Episode: 375/2000, Reward: 70.377, Step: 500 +2023-04-02 17:14:18 - r - INFO: - Episode: 376/2000, Reward: 98.785, Step: 500 +2023-04-02 17:14:19 - r - INFO: - Episode: 377/2000, Reward: 154.427, Step: 500 +2023-04-02 17:14:20 - r - INFO: - Episode: 378/2000, Reward: 272.539, Step: 253 +2023-04-02 17:14:22 - r - INFO: - Episode: 379/2000, Reward: 91.071, Step: 500 +2023-04-02 17:14:22 - r - INFO: - Episode: 380/2000, Reward: 290.926, Step: 240 +2023-04-02 17:14:26 - r - INFO: - Episode: 381/2000, Reward: 262.292, Step: 245 +2023-04-02 17:14:26 - r - INFO: - Episode: 382/2000, Reward: 25.823, Step: 146 +2023-04-02 17:14:28 - r - INFO: - Episode: 383/2000, Reward: 194.548, Step: 500 +2023-04-02 17:14:29 - r - INFO: - Episode: 384/2000, Reward: 269.672, Step: 346 +2023-04-02 17:14:31 - r - INFO: - Episode: 385/2000, Reward: 134.076, Step: 500 +2023-04-02 17:14:35 - r - INFO: - Episode: 386/2000, Reward: 173.045, Step: 500 +2023-04-02 17:14:37 - r - INFO: - Episode: 387/2000, Reward: 224.589, Step: 446 +2023-04-02 17:14:38 - r - INFO: - Episode: 388/2000, Reward: 240.812, Step: 255 +2023-04-02 17:14:39 - r - INFO: - Episode: 389/2000, Reward: 173.898, Step: 500 +2023-04-02 17:14:41 - r - INFO: - Episode: 390/2000, Reward: 108.280, Step: 500 +2023-04-02 17:14:44 - r - INFO: - Episode: 391/2000, Reward: 284.382, Step: 404 +2023-04-02 17:14:46 - r - INFO: - Episode: 392/2000, Reward: 136.216, Step: 500 +2023-04-02 17:14:47 - r - INFO: - Episode: 393/2000, Reward: 235.233, Step: 255 +2023-04-02 17:14:48 - r - INFO: - Episode: 394/2000, Reward: 147.380, Step: 500 +2023-04-02 17:14:50 - r - INFO: - Episode: 395/2000, Reward: 80.427, Step: 500 +2023-04-02 17:14:55 - r - INFO: - Episode: 396/2000, Reward: 192.683, Step: 500 +2023-04-02 17:14:56 - r - INFO: - Episode: 397/2000, Reward: 158.313, Step: 500 +2023-04-02 17:14:58 - r - INFO: - Episode: 398/2000, Reward: 95.542, Step: 500 +2023-04-02 17:14:59 - r - INFO: - Episode: 399/2000, Reward: 288.452, Step: 336 +2023-04-02 17:15:01 - r - INFO: - Episode: 400/2000, Reward: 59.046, Step: 500 +2023-04-02 17:15:07 - r - INFO: - Episode: 401/2000, Reward: 74.897, Step: 500 +2023-04-02 17:15:09 - r - INFO: - Episode: 402/2000, Reward: 163.009, Step: 500 +2023-04-02 17:15:10 - r - INFO: - Episode: 403/2000, Reward: 256.089, Step: 350 +2023-04-02 17:15:11 - r - INFO: - Episode: 404/2000, Reward: 279.462, Step: 302 +2023-04-02 17:15:12 - r - INFO: - Episode: 405/2000, Reward: 125.194, Step: 500 +2023-04-02 17:15:17 - r - INFO: - Episode: 406/2000, Reward: 103.790, Step: 500 +2023-04-02 17:15:17 - r - INFO: - Episode: 407/2000, Reward: 227.769, Step: 240 +2023-04-02 17:15:19 - r - INFO: - Episode: 408/2000, Reward: 197.261, Step: 472 +2023-04-02 17:15:20 - r - INFO: - Episode: 409/2000, Reward: 228.685, Step: 245 +2023-04-02 17:15:21 - r - INFO: - Episode: 410/2000, Reward: 257.539, Step: 244 +2023-04-02 17:15:23 - r - INFO: - Current episode 410 has the best eval reward: 262.057 +2023-04-02 17:15:23 - r - INFO: - Episode: 411/2000, Reward: 255.272, Step: 176 +2023-04-02 17:15:25 - r - INFO: - Episode: 412/2000, Reward: 150.731, Step: 500 +2023-04-02 17:15:26 - r - INFO: - Episode: 413/2000, Reward: 131.959, Step: 500 +2023-04-02 17:15:28 - r - INFO: - Episode: 414/2000, Reward: 115.706, Step: 500 +2023-04-02 17:15:30 - r - INFO: - Episode: 415/2000, Reward: 154.295, Step: 500 +2023-04-02 17:15:34 - r - INFO: - Episode: 416/2000, Reward: 126.888, Step: 500 +2023-04-02 17:15:35 - r - INFO: - Episode: 417/2000, Reward: 240.250, Step: 369 +2023-04-02 17:15:37 - r - INFO: - Episode: 418/2000, Reward: 109.040, Step: 500 +2023-04-02 17:15:38 - r - INFO: - Episode: 419/2000, Reward: 168.347, Step: 500 +2023-04-02 17:15:40 - r - INFO: - Episode: 420/2000, Reward: 222.260, Step: 370 +2023-04-02 17:15:43 - r - INFO: - Episode: 421/2000, Reward: 258.399, Step: 439 +2023-04-02 17:15:45 - r - INFO: - Episode: 422/2000, Reward: 129.854, Step: 500 +2023-04-02 17:15:47 - r - INFO: - Episode: 423/2000, Reward: 94.458, Step: 500 +2023-04-02 17:15:48 - r - INFO: - Episode: 424/2000, Reward: 113.403, Step: 500 +2023-04-02 17:15:50 - r - INFO: - Episode: 425/2000, Reward: 136.925, Step: 500 +2023-04-02 17:15:56 - r - INFO: - Episode: 426/2000, Reward: -94.726, Step: 469 +2023-04-02 17:15:58 - r - INFO: - Episode: 427/2000, Reward: 115.162, Step: 500 +2023-04-02 17:16:00 - r - INFO: - Episode: 428/2000, Reward: 82.106, Step: 500 +2023-04-02 17:16:02 - r - INFO: - Episode: 429/2000, Reward: 81.885, Step: 500 +2023-04-02 17:16:04 - r - INFO: - Episode: 430/2000, Reward: 60.603, Step: 500 +2023-04-02 17:16:09 - r - INFO: - Episode: 431/2000, Reward: -143.100, Step: 412 +2023-04-02 17:16:11 - r - INFO: - Episode: 432/2000, Reward: 230.568, Step: 478 +2023-04-02 17:16:13 - r - INFO: - Episode: 433/2000, Reward: 120.338, Step: 500 +2023-04-02 17:16:14 - r - INFO: - Episode: 434/2000, Reward: 44.079, Step: 500 +2023-04-02 17:16:16 - r - INFO: - Episode: 435/2000, Reward: 152.759, Step: 500 +2023-04-02 17:16:22 - r - INFO: - Episode: 436/2000, Reward: 144.575, Step: 500 +2023-04-02 17:16:24 - r - INFO: - Episode: 437/2000, Reward: 145.322, Step: 500 +2023-04-02 17:16:26 - r - INFO: - Episode: 438/2000, Reward: 79.266, Step: 500 +2023-04-02 17:16:27 - r - INFO: - Episode: 439/2000, Reward: 94.467, Step: 500 +2023-04-02 17:16:29 - r - INFO: - Episode: 440/2000, Reward: 60.343, Step: 500 +2023-04-02 17:16:35 - r - INFO: - Episode: 441/2000, Reward: 104.834, Step: 500 +2023-04-02 17:16:37 - r - INFO: - Episode: 442/2000, Reward: 124.206, Step: 500 +2023-04-02 17:16:38 - r - INFO: - Episode: 443/2000, Reward: 268.521, Step: 324 +2023-04-02 17:16:39 - r - INFO: - Episode: 444/2000, Reward: 131.413, Step: 500 +2023-04-02 17:16:41 - r - INFO: - Episode: 445/2000, Reward: 144.072, Step: 500 +2023-04-02 17:16:46 - r - INFO: - Episode: 446/2000, Reward: 171.705, Step: 500 +2023-04-02 17:16:48 - r - INFO: - Episode: 447/2000, Reward: 165.030, Step: 500 +2023-04-02 17:16:50 - r - INFO: - Episode: 448/2000, Reward: 163.711, Step: 500 +2023-04-02 17:16:51 - r - INFO: - Episode: 449/2000, Reward: 148.218, Step: 500 +2023-04-02 17:16:53 - r - INFO: - Episode: 450/2000, Reward: 148.257, Step: 500 +2023-04-02 17:16:59 - r - INFO: - Episode: 451/2000, Reward: 90.484, Step: 500 +2023-04-02 17:17:01 - r - INFO: - Episode: 452/2000, Reward: 135.404, Step: 500 +2023-04-02 17:17:03 - r - INFO: - Episode: 453/2000, Reward: 144.511, Step: 500 +2023-04-02 17:17:04 - r - INFO: - Episode: 454/2000, Reward: 234.631, Step: 352 +2023-04-02 17:17:06 - r - INFO: - Episode: 455/2000, Reward: 88.209, Step: 500 +2023-04-02 17:17:12 - r - INFO: - Episode: 456/2000, Reward: 111.254, Step: 500 +2023-04-02 17:17:13 - r - INFO: - Episode: 457/2000, Reward: 199.102, Step: 444 +2023-04-02 17:17:15 - r - INFO: - Episode: 458/2000, Reward: 68.839, Step: 500 +2023-04-02 17:17:17 - r - INFO: - Episode: 459/2000, Reward: 141.112, Step: 500 +2023-04-02 17:17:18 - r - INFO: - Episode: 460/2000, Reward: 173.353, Step: 500 +2023-04-02 17:17:24 - r - INFO: - Episode: 461/2000, Reward: 95.637, Step: 500 +2023-04-02 17:17:26 - r - INFO: - Episode: 462/2000, Reward: 121.918, Step: 500 +2023-04-02 17:17:27 - r - INFO: - Episode: 463/2000, Reward: 193.360, Step: 500 +2023-04-02 17:17:29 - r - INFO: - Episode: 464/2000, Reward: 152.985, Step: 500 +2023-04-02 17:17:31 - r - INFO: - Episode: 465/2000, Reward: 119.458, Step: 500 +2023-04-02 17:17:37 - r - INFO: - Episode: 466/2000, Reward: 99.698, Step: 500 +2023-04-02 17:17:39 - r - INFO: - Episode: 467/2000, Reward: 80.067, Step: 500 +2023-04-02 17:17:41 - r - INFO: - Episode: 468/2000, Reward: 41.525, Step: 500 +2023-04-02 17:17:43 - r - INFO: - Episode: 469/2000, Reward: 62.501, Step: 500 +2023-04-02 17:17:45 - r - INFO: - Episode: 470/2000, Reward: 179.439, Step: 500 +2023-04-02 17:17:50 - r - INFO: - Episode: 471/2000, Reward: 131.549, Step: 500 +2023-04-02 17:17:52 - r - INFO: - Episode: 472/2000, Reward: 194.722, Step: 500 +2023-04-02 17:17:54 - r - INFO: - Episode: 473/2000, Reward: 105.245, Step: 500 +2023-04-02 17:17:56 - r - INFO: - Episode: 474/2000, Reward: 44.400, Step: 500 +2023-04-02 17:17:57 - r - INFO: - Episode: 475/2000, Reward: 113.217, Step: 500 +2023-04-02 17:18:01 - r - INFO: - Episode: 476/2000, Reward: 227.567, Step: 469 +2023-04-02 17:18:03 - r - INFO: - Episode: 477/2000, Reward: 139.589, Step: 500 +2023-04-02 17:18:05 - r - INFO: - Episode: 478/2000, Reward: 157.529, Step: 500 +2023-04-02 17:18:06 - r - INFO: - Episode: 479/2000, Reward: 179.548, Step: 500 +2023-04-02 17:18:08 - r - INFO: - Episode: 480/2000, Reward: 189.190, Step: 500 +2023-04-02 17:18:12 - r - INFO: - Episode: 481/2000, Reward: 174.706, Step: 500 +2023-04-02 17:18:14 - r - INFO: - Episode: 482/2000, Reward: 80.299, Step: 500 +2023-04-02 17:18:15 - r - INFO: - Episode: 483/2000, Reward: 97.300, Step: 500 +2023-04-02 17:18:17 - r - INFO: - Episode: 484/2000, Reward: 120.445, Step: 500 +2023-04-02 17:18:19 - r - INFO: - Episode: 485/2000, Reward: 39.956, Step: 500 +2023-04-02 17:18:24 - r - INFO: - Episode: 486/2000, Reward: 124.274, Step: 500 +2023-04-02 17:18:26 - r - INFO: - Episode: 487/2000, Reward: 173.572, Step: 500 +2023-04-02 17:18:27 - r - INFO: - Episode: 488/2000, Reward: 156.925, Step: 500 +2023-04-02 17:18:29 - r - INFO: - Episode: 489/2000, Reward: 136.897, Step: 500 +2023-04-02 17:18:31 - r - INFO: - Episode: 490/2000, Reward: 125.316, Step: 500 +2023-04-02 17:18:35 - r - INFO: - Episode: 491/2000, Reward: 60.583, Step: 500 +2023-04-02 17:18:37 - r - INFO: - Episode: 492/2000, Reward: 42.151, Step: 500 +2023-04-02 17:18:38 - r - INFO: - Episode: 493/2000, Reward: 36.822, Step: 500 +2023-04-02 17:18:40 - r - INFO: - Episode: 494/2000, Reward: 201.235, Step: 500 +2023-04-02 17:18:42 - r - INFO: - Episode: 495/2000, Reward: 214.009, Step: 404 +2023-04-02 17:18:46 - r - INFO: - Episode: 496/2000, Reward: 116.190, Step: 500 +2023-04-02 17:18:48 - r - INFO: - Episode: 497/2000, Reward: 178.809, Step: 500 +2023-04-02 17:18:50 - r - INFO: - Episode: 498/2000, Reward: 147.585, Step: 500 +2023-04-02 17:18:52 - r - INFO: - Episode: 499/2000, Reward: 140.599, Step: 500 +2023-04-02 17:18:53 - r - INFO: - Episode: 500/2000, Reward: 147.714, Step: 500 +2023-04-02 17:18:59 - r - INFO: - Episode: 501/2000, Reward: 165.817, Step: 500 +2023-04-02 17:19:01 - r - INFO: - Episode: 502/2000, Reward: 139.851, Step: 500 +2023-04-02 17:19:03 - r - INFO: - Episode: 503/2000, Reward: 127.953, Step: 500 +2023-04-02 17:19:05 - r - INFO: - Episode: 504/2000, Reward: 153.347, Step: 500 +2023-04-02 17:19:05 - r - INFO: - Episode: 505/2000, Reward: 48.806, Step: 127 +2023-04-02 17:19:10 - r - INFO: - Episode: 506/2000, Reward: 138.314, Step: 500 +2023-04-02 17:19:12 - r - INFO: - Episode: 507/2000, Reward: 124.921, Step: 500 +2023-04-02 17:19:14 - r - INFO: - Episode: 508/2000, Reward: 140.218, Step: 500 +2023-04-02 17:19:15 - r - INFO: - Episode: 509/2000, Reward: 119.559, Step: 500 +2023-04-02 17:19:17 - r - INFO: - Episode: 510/2000, Reward: 154.350, Step: 500 +2023-04-02 17:19:23 - r - INFO: - Episode: 511/2000, Reward: 143.242, Step: 500 +2023-04-02 17:19:25 - r - INFO: - Episode: 512/2000, Reward: 167.694, Step: 500 +2023-04-02 17:19:27 - r - INFO: - Episode: 513/2000, Reward: 169.052, Step: 500 +2023-04-02 17:19:28 - r - INFO: - Episode: 514/2000, Reward: 168.836, Step: 500 +2023-04-02 17:19:30 - r - INFO: - Episode: 515/2000, Reward: 179.314, Step: 500 +2023-04-02 17:19:35 - r - INFO: - Episode: 516/2000, Reward: 166.207, Step: 500 +2023-04-02 17:19:37 - r - INFO: - Episode: 517/2000, Reward: 144.854, Step: 500 +2023-04-02 17:19:37 - r - INFO: - Episode: 518/2000, Reward: 253.817, Step: 267 +2023-04-02 17:19:38 - r - INFO: - Episode: 519/2000, Reward: 242.519, Step: 284 +2023-04-02 17:19:40 - r - INFO: - Episode: 520/2000, Reward: 170.780, Step: 500 +2023-04-02 17:19:44 - r - INFO: - Episode: 521/2000, Reward: 231.058, Step: 294 +2023-04-02 17:19:46 - r - INFO: - Episode: 522/2000, Reward: 113.638, Step: 500 +2023-04-02 17:19:47 - r - INFO: - Episode: 523/2000, Reward: 154.746, Step: 500 +2023-04-02 17:19:49 - r - INFO: - Episode: 524/2000, Reward: 199.392, Step: 500 +2023-04-02 17:19:51 - r - INFO: - Episode: 525/2000, Reward: 125.146, Step: 500 +2023-04-02 17:19:55 - r - INFO: - Episode: 526/2000, Reward: 176.619, Step: 500 +2023-04-02 17:19:57 - r - INFO: - Episode: 527/2000, Reward: 128.680, Step: 500 +2023-04-02 17:19:58 - r - INFO: - Episode: 528/2000, Reward: 163.289, Step: 500 +2023-04-02 17:20:00 - r - INFO: - Episode: 529/2000, Reward: 186.398, Step: 500 +2023-04-02 17:20:02 - r - INFO: - Episode: 530/2000, Reward: 73.892, Step: 500 +2023-04-02 17:20:07 - r - INFO: - Episode: 531/2000, Reward: 244.725, Step: 497 +2023-04-02 17:20:08 - r - INFO: - Episode: 532/2000, Reward: 253.071, Step: 448 +2023-04-02 17:20:10 - r - INFO: - Episode: 533/2000, Reward: 55.163, Step: 500 +2023-04-02 17:20:12 - r - INFO: - Episode: 534/2000, Reward: 145.117, Step: 500 +2023-04-02 17:20:14 - r - INFO: - Episode: 535/2000, Reward: 145.650, Step: 500 +2023-04-02 17:20:20 - r - INFO: - Episode: 536/2000, Reward: 78.817, Step: 500 +2023-04-02 17:20:22 - r - INFO: - Episode: 537/2000, Reward: 169.905, Step: 500 +2023-04-02 17:20:23 - r - INFO: - Episode: 538/2000, Reward: 140.678, Step: 500 +2023-04-02 17:20:25 - r - INFO: - Episode: 539/2000, Reward: 161.709, Step: 500 +2023-04-02 17:20:26 - r - INFO: - Episode: 540/2000, Reward: 143.762, Step: 500 +2023-04-02 17:20:32 - r - INFO: - Episode: 541/2000, Reward: 137.740, Step: 500 +2023-04-02 17:20:33 - r - INFO: - Episode: 542/2000, Reward: 159.340, Step: 500 +2023-04-02 17:20:35 - r - INFO: - Episode: 543/2000, Reward: 155.035, Step: 500 +2023-04-02 17:20:37 - r - INFO: - Episode: 544/2000, Reward: 206.490, Step: 500 +2023-04-02 17:20:39 - r - INFO: - Episode: 545/2000, Reward: 118.116, Step: 500 +2023-04-02 17:20:42 - r - INFO: - Episode: 546/2000, Reward: 163.790, Step: 500 +2023-04-02 17:20:44 - r - INFO: - Episode: 547/2000, Reward: 45.738, Step: 500 +2023-04-02 17:20:45 - r - INFO: - Episode: 548/2000, Reward: 173.185, Step: 500 +2023-04-02 17:20:47 - r - INFO: - Episode: 549/2000, Reward: 152.525, Step: 500 +2023-04-02 17:20:49 - r - INFO: - Episode: 550/2000, Reward: 45.611, Step: 500 +2023-04-02 17:20:53 - r - INFO: - Episode: 551/2000, Reward: 161.860, Step: 500 +2023-04-02 17:20:54 - r - INFO: - Episode: 552/2000, Reward: 143.837, Step: 500 +2023-04-02 17:20:56 - r - INFO: - Episode: 553/2000, Reward: 160.317, Step: 500 +2023-04-02 17:20:58 - r - INFO: - Episode: 554/2000, Reward: 140.131, Step: 500 +2023-04-02 17:20:59 - r - INFO: - Episode: 555/2000, Reward: 144.395, Step: 500 +2023-04-02 17:21:03 - r - INFO: - Episode: 556/2000, Reward: 174.230, Step: 500 +2023-04-02 17:21:04 - r - INFO: - Episode: 557/2000, Reward: 243.935, Step: 437 +2023-04-02 17:21:06 - r - INFO: - Episode: 558/2000, Reward: 106.565, Step: 500 +2023-04-02 17:21:07 - r - INFO: - Episode: 559/2000, Reward: 131.281, Step: 500 +2023-04-02 17:21:09 - r - INFO: - Episode: 560/2000, Reward: 147.749, Step: 500 +2023-04-02 17:21:12 - r - INFO: - Episode: 561/2000, Reward: 136.367, Step: 500 +2023-04-02 17:21:14 - r - INFO: - Episode: 562/2000, Reward: 135.084, Step: 500 +2023-04-02 17:21:15 - r - INFO: - Episode: 563/2000, Reward: 141.138, Step: 500 +2023-04-02 17:21:17 - r - INFO: - Episode: 564/2000, Reward: 191.163, Step: 500 +2023-04-02 17:21:19 - r - INFO: - Episode: 565/2000, Reward: 145.375, Step: 500 +2023-04-02 17:21:23 - r - INFO: - Episode: 566/2000, Reward: 154.434, Step: 500 +2023-04-02 17:21:25 - r - INFO: - Episode: 567/2000, Reward: 140.494, Step: 500 +2023-04-02 17:21:26 - r - INFO: - Episode: 568/2000, Reward: 161.736, Step: 500 +2023-04-02 17:21:28 - r - INFO: - Episode: 569/2000, Reward: 146.859, Step: 500 +2023-04-02 17:21:30 - r - INFO: - Episode: 570/2000, Reward: 130.828, Step: 500 +2023-04-02 17:21:33 - r - INFO: - Episode: 571/2000, Reward: 135.798, Step: 500 +2023-04-02 17:21:34 - r - INFO: - Episode: 572/2000, Reward: 236.415, Step: 202 +2023-04-02 17:21:36 - r - INFO: - Episode: 573/2000, Reward: 171.209, Step: 500 +2023-04-02 17:21:37 - r - INFO: - Episode: 574/2000, Reward: 152.714, Step: 500 +2023-04-02 17:21:39 - r - INFO: - Episode: 575/2000, Reward: 96.333, Step: 500 +2023-04-02 17:21:43 - r - INFO: - Episode: 576/2000, Reward: 195.220, Step: 500 +2023-04-02 17:21:44 - r - INFO: - Episode: 577/2000, Reward: 156.134, Step: 500 +2023-04-02 17:21:46 - r - INFO: - Episode: 578/2000, Reward: 85.455, Step: 500 +2023-04-02 17:21:48 - r - INFO: - Episode: 579/2000, Reward: 165.539, Step: 500 +2023-04-02 17:21:49 - r - INFO: - Episode: 580/2000, Reward: 104.339, Step: 500 +2023-04-02 17:21:54 - r - INFO: - Episode: 581/2000, Reward: 168.184, Step: 500 +2023-04-02 17:21:56 - r - INFO: - Episode: 582/2000, Reward: 70.338, Step: 500 +2023-04-02 17:21:57 - r - INFO: - Episode: 583/2000, Reward: 157.880, Step: 500 +2023-04-02 17:21:58 - r - INFO: - Episode: 584/2000, Reward: 270.935, Step: 333 +2023-04-02 17:22:00 - r - INFO: - Episode: 585/2000, Reward: 147.926, Step: 500 +2023-04-02 17:22:02 - r - INFO: - Current episode 585 has the best eval reward: 274.543 +2023-04-02 17:22:03 - r - INFO: - Episode: 586/2000, Reward: 143.246, Step: 500 +2023-04-02 17:22:05 - r - INFO: - Episode: 587/2000, Reward: 157.737, Step: 500 +2023-04-02 17:22:06 - r - INFO: - Episode: 588/2000, Reward: 169.830, Step: 500 +2023-04-02 17:22:08 - r - INFO: - Episode: 589/2000, Reward: 152.453, Step: 500 +2023-04-02 17:22:10 - r - INFO: - Episode: 590/2000, Reward: 175.007, Step: 500 +2023-04-02 17:22:14 - r - INFO: - Episode: 591/2000, Reward: 159.316, Step: 500 +2023-04-02 17:22:15 - r - INFO: - Episode: 592/2000, Reward: 288.784, Step: 371 +2023-04-02 17:22:16 - r - INFO: - Episode: 593/2000, Reward: 272.622, Step: 476 +2023-04-02 17:22:18 - r - INFO: - Episode: 594/2000, Reward: 159.225, Step: 500 +2023-04-02 17:22:20 - r - INFO: - Episode: 595/2000, Reward: 147.642, Step: 500 +2023-04-02 17:22:22 - r - INFO: - Episode: 596/2000, Reward: 173.861, Step: 500 +2023-04-02 17:22:24 - r - INFO: - Episode: 597/2000, Reward: 191.687, Step: 500 +2023-04-02 17:22:25 - r - INFO: - Episode: 598/2000, Reward: 160.518, Step: 500 +2023-04-02 17:22:27 - r - INFO: - Episode: 599/2000, Reward: 184.454, Step: 500 +2023-04-02 17:22:29 - r - INFO: - Episode: 600/2000, Reward: 159.431, Step: 500 +2023-04-02 17:22:31 - r - INFO: - Episode: 601/2000, Reward: 171.571, Step: 500 +2023-04-02 17:22:33 - r - INFO: - Episode: 602/2000, Reward: 130.022, Step: 500 +2023-04-02 17:22:35 - r - INFO: - Episode: 603/2000, Reward: 171.697, Step: 500 +2023-04-02 17:22:36 - r - INFO: - Episode: 604/2000, Reward: 167.206, Step: 500 +2023-04-02 17:22:38 - r - INFO: - Episode: 605/2000, Reward: 115.098, Step: 500 +2023-04-02 17:22:41 - r - INFO: - Episode: 606/2000, Reward: 147.046, Step: 500 +2023-04-02 17:22:43 - r - INFO: - Episode: 607/2000, Reward: 162.688, Step: 500 +2023-04-02 17:22:45 - r - INFO: - Episode: 608/2000, Reward: 162.249, Step: 500 +2023-04-02 17:22:46 - r - INFO: - Episode: 609/2000, Reward: 147.710, Step: 500 +2023-04-02 17:22:47 - r - INFO: - Episode: 610/2000, Reward: 243.151, Step: 288 +2023-04-02 17:22:50 - r - INFO: - Episode: 611/2000, Reward: 177.253, Step: 500 +2023-04-02 17:22:52 - r - INFO: - Episode: 612/2000, Reward: 165.700, Step: 500 +2023-04-02 17:22:54 - r - INFO: - Episode: 613/2000, Reward: 148.889, Step: 500 +2023-04-02 17:22:54 - r - INFO: - Episode: 614/2000, Reward: 226.879, Step: 200 +2023-04-02 17:22:56 - r - INFO: - Episode: 615/2000, Reward: 154.365, Step: 500 +2023-04-02 17:22:59 - r - INFO: - Episode: 616/2000, Reward: 173.454, Step: 500 +2023-04-02 17:23:01 - r - INFO: - Episode: 617/2000, Reward: 175.111, Step: 500 +2023-04-02 17:23:03 - r - INFO: - Episode: 618/2000, Reward: 179.468, Step: 500 +2023-04-02 17:23:04 - r - INFO: - Episode: 619/2000, Reward: 141.792, Step: 500 +2023-04-02 17:23:06 - r - INFO: - Episode: 620/2000, Reward: 166.798, Step: 500 +2023-04-02 17:23:09 - r - INFO: - Episode: 621/2000, Reward: 160.329, Step: 500 +2023-04-02 17:23:11 - r - INFO: - Episode: 622/2000, Reward: 60.514, Step: 500 +2023-04-02 17:23:12 - r - INFO: - Episode: 623/2000, Reward: 142.302, Step: 500 +2023-04-02 17:23:14 - r - INFO: - Episode: 624/2000, Reward: 159.234, Step: 500 +2023-04-02 17:23:16 - r - INFO: - Episode: 625/2000, Reward: 148.049, Step: 500 +2023-04-02 17:23:20 - r - INFO: - Episode: 626/2000, Reward: 147.050, Step: 500 +2023-04-02 17:23:21 - r - INFO: - Episode: 627/2000, Reward: 160.085, Step: 500 +2023-04-02 17:23:22 - r - INFO: - Episode: 628/2000, Reward: 246.940, Step: 237 +2023-04-02 17:23:24 - r - INFO: - Episode: 629/2000, Reward: 140.569, Step: 500 +2023-04-02 17:23:24 - r - INFO: - Episode: 630/2000, Reward: 220.179, Step: 254 +2023-04-02 17:23:27 - r - INFO: - Episode: 631/2000, Reward: 146.772, Step: 500 +2023-04-02 17:23:29 - r - INFO: - Episode: 632/2000, Reward: 160.990, Step: 500 +2023-04-02 17:23:31 - r - INFO: - Episode: 633/2000, Reward: 45.443, Step: 500 +2023-04-02 17:23:33 - r - INFO: - Episode: 634/2000, Reward: 157.219, Step: 500 +2023-04-02 17:23:34 - r - INFO: - Episode: 635/2000, Reward: 170.002, Step: 500 +2023-04-02 17:23:37 - r - INFO: - Episode: 636/2000, Reward: 138.159, Step: 500 +2023-04-02 17:23:39 - r - INFO: - Episode: 637/2000, Reward: 141.842, Step: 500 +2023-04-02 17:23:40 - r - INFO: - Episode: 638/2000, Reward: 68.448, Step: 500 +2023-04-02 17:23:42 - r - INFO: - Episode: 639/2000, Reward: 172.118, Step: 500 +2023-04-02 17:23:44 - r - INFO: - Episode: 640/2000, Reward: 210.900, Step: 500 +2023-04-02 17:23:47 - r - INFO: - Episode: 641/2000, Reward: 190.504, Step: 500 +2023-04-02 17:23:49 - r - INFO: - Episode: 642/2000, Reward: 138.360, Step: 500 +2023-04-02 17:23:51 - r - INFO: - Episode: 643/2000, Reward: 146.220, Step: 500 +2023-04-02 17:23:53 - r - INFO: - Episode: 644/2000, Reward: 166.868, Step: 500 +2023-04-02 17:23:54 - r - INFO: - Episode: 645/2000, Reward: 168.485, Step: 500 +2023-04-02 17:23:57 - r - INFO: - Episode: 646/2000, Reward: 187.705, Step: 500 +2023-04-02 17:23:59 - r - INFO: - Episode: 647/2000, Reward: 268.588, Step: 477 +2023-04-02 17:24:00 - r - INFO: - Episode: 648/2000, Reward: 194.176, Step: 500 +2023-04-02 17:24:02 - r - INFO: - Episode: 649/2000, Reward: 177.343, Step: 500 +2023-04-02 17:24:04 - r - INFO: - Episode: 650/2000, Reward: 158.419, Step: 500 +2023-04-02 17:24:08 - r - INFO: - Episode: 651/2000, Reward: 83.667, Step: 500 +2023-04-02 17:24:09 - r - INFO: - Episode: 652/2000, Reward: 194.884, Step: 500 +2023-04-02 17:24:11 - r - INFO: - Episode: 653/2000, Reward: 170.189, Step: 500 +2023-04-02 17:24:12 - r - INFO: - Episode: 654/2000, Reward: 210.277, Step: 268 +2023-04-02 17:24:13 - r - INFO: - Episode: 655/2000, Reward: 257.631, Step: 300 +2023-04-02 17:24:16 - r - INFO: - Episode: 656/2000, Reward: 172.851, Step: 500 +2023-04-02 17:24:17 - r - INFO: - Episode: 657/2000, Reward: 79.618, Step: 500 +2023-04-02 17:24:19 - r - INFO: - Episode: 658/2000, Reward: 174.706, Step: 500 +2023-04-02 17:24:21 - r - INFO: - Episode: 659/2000, Reward: 155.652, Step: 500 +2023-04-02 17:24:23 - r - INFO: - Episode: 660/2000, Reward: 171.951, Step: 500 +2023-04-02 17:24:26 - r - INFO: - Episode: 661/2000, Reward: 64.393, Step: 500 +2023-04-02 17:24:28 - r - INFO: - Episode: 662/2000, Reward: 143.965, Step: 500 +2023-04-02 17:24:29 - r - INFO: - Episode: 663/2000, Reward: 120.038, Step: 500 +2023-04-02 17:24:31 - r - INFO: - Episode: 664/2000, Reward: 159.388, Step: 500 +2023-04-02 17:24:33 - r - INFO: - Episode: 665/2000, Reward: 171.713, Step: 500 +2023-04-02 17:24:36 - r - INFO: - Episode: 666/2000, Reward: 146.707, Step: 500 +2023-04-02 17:24:38 - r - INFO: - Episode: 667/2000, Reward: 200.714, Step: 500 +2023-04-02 17:24:40 - r - INFO: - Episode: 668/2000, Reward: 136.026, Step: 500 +2023-04-02 17:24:41 - r - INFO: - Episode: 669/2000, Reward: 150.452, Step: 500 +2023-04-02 17:24:43 - r - INFO: - Episode: 670/2000, Reward: 72.897, Step: 500 +2023-04-02 17:24:48 - r - INFO: - Episode: 671/2000, Reward: 170.243, Step: 500 +2023-04-02 17:24:50 - r - INFO: - Episode: 672/2000, Reward: 114.410, Step: 500 +2023-04-02 17:24:51 - r - INFO: - Episode: 673/2000, Reward: 167.822, Step: 500 +2023-04-02 17:24:53 - r - INFO: - Episode: 674/2000, Reward: 116.936, Step: 500 +2023-04-02 17:24:55 - r - INFO: - Episode: 675/2000, Reward: 173.136, Step: 500 +2023-04-02 17:24:58 - r - INFO: - Episode: 676/2000, Reward: 253.531, Step: 401 +2023-04-02 17:24:59 - r - INFO: - Episode: 677/2000, Reward: 152.846, Step: 500 +2023-04-02 17:25:01 - r - INFO: - Episode: 678/2000, Reward: 58.615, Step: 500 +2023-04-02 17:25:03 - r - INFO: - Episode: 679/2000, Reward: 132.049, Step: 500 +2023-04-02 17:25:05 - r - INFO: - Episode: 680/2000, Reward: 172.062, Step: 500 +2023-04-02 17:25:08 - r - INFO: - Episode: 681/2000, Reward: 154.862, Step: 500 +2023-04-02 17:25:10 - r - INFO: - Episode: 682/2000, Reward: 161.948, Step: 500 +2023-04-02 17:25:11 - r - INFO: - Episode: 683/2000, Reward: 152.773, Step: 500 +2023-04-02 17:25:13 - r - INFO: - Episode: 684/2000, Reward: 162.977, Step: 500 +2023-04-02 17:25:15 - r - INFO: - Episode: 685/2000, Reward: 189.376, Step: 500 +2023-04-02 17:25:18 - r - INFO: - Episode: 686/2000, Reward: 161.614, Step: 500 +2023-04-02 17:25:20 - r - INFO: - Episode: 687/2000, Reward: 149.134, Step: 500 +2023-04-02 17:25:21 - r - INFO: - Episode: 688/2000, Reward: 66.386, Step: 500 +2023-04-02 17:25:23 - r - INFO: - Episode: 689/2000, Reward: 192.734, Step: 500 +2023-04-02 17:25:25 - r - INFO: - Episode: 690/2000, Reward: 165.886, Step: 500 +2023-04-02 17:25:30 - r - INFO: - Episode: 691/2000, Reward: 114.343, Step: 500 +2023-04-02 17:25:32 - r - INFO: - Episode: 692/2000, Reward: 149.717, Step: 500 +2023-04-02 17:25:34 - r - INFO: - Episode: 693/2000, Reward: 54.851, Step: 500 +2023-04-02 17:25:35 - r - INFO: - Episode: 694/2000, Reward: 264.542, Step: 484 +2023-04-02 17:25:36 - r - INFO: - Episode: 695/2000, Reward: 233.194, Step: 197 +2023-04-02 17:25:39 - r - INFO: - Episode: 696/2000, Reward: 127.616, Step: 500 +2023-04-02 17:25:41 - r - INFO: - Episode: 697/2000, Reward: 157.027, Step: 500 +2023-04-02 17:25:43 - r - INFO: - Episode: 698/2000, Reward: 129.940, Step: 500 +2023-04-02 17:25:45 - r - INFO: - Episode: 699/2000, Reward: 145.463, Step: 500 +2023-04-02 17:25:46 - r - INFO: - Episode: 700/2000, Reward: 253.007, Step: 345 +2023-04-02 17:25:51 - r - INFO: - Episode: 701/2000, Reward: 102.389, Step: 500 +2023-04-02 17:25:53 - r - INFO: - Episode: 702/2000, Reward: 156.285, Step: 500 +2023-04-02 17:25:54 - r - INFO: - Episode: 703/2000, Reward: 145.216, Step: 500 +2023-04-02 17:25:56 - r - INFO: - Episode: 704/2000, Reward: 154.976, Step: 500 +2023-04-02 17:25:57 - r - INFO: - Episode: 705/2000, Reward: 270.957, Step: 324 +2023-04-02 17:26:02 - r - INFO: - Episode: 706/2000, Reward: 161.416, Step: 500 +2023-04-02 17:26:04 - r - INFO: - Episode: 707/2000, Reward: 140.496, Step: 500 +2023-04-02 17:26:05 - r - INFO: - Episode: 708/2000, Reward: 147.496, Step: 500 +2023-04-02 17:26:07 - r - INFO: - Episode: 709/2000, Reward: 164.891, Step: 500 +2023-04-02 17:26:09 - r - INFO: - Episode: 710/2000, Reward: 191.660, Step: 500 +2023-04-02 17:26:12 - r - INFO: - Episode: 711/2000, Reward: 178.549, Step: 500 +2023-04-02 17:26:13 - r - INFO: - Episode: 712/2000, Reward: 254.518, Step: 275 +2023-04-02 17:26:15 - r - INFO: - Episode: 713/2000, Reward: 150.219, Step: 500 +2023-04-02 17:26:16 - r - INFO: - Episode: 714/2000, Reward: 139.653, Step: 500 +2023-04-02 17:26:18 - r - INFO: - Episode: 715/2000, Reward: 161.001, Step: 500 +2023-04-02 17:26:19 - r - INFO: - Current episode 715 has the best eval reward: 276.698 +2023-04-02 17:26:21 - r - INFO: - Episode: 716/2000, Reward: 158.855, Step: 500 +2023-04-02 17:26:23 - r - INFO: - Episode: 717/2000, Reward: 150.751, Step: 500 +2023-04-02 17:26:24 - r - INFO: - Episode: 718/2000, Reward: 142.357, Step: 500 +2023-04-02 17:26:26 - r - INFO: - Episode: 719/2000, Reward: 180.246, Step: 500 +2023-04-02 17:26:27 - r - INFO: - Episode: 720/2000, Reward: 172.591, Step: 500 +2023-04-02 17:26:30 - r - INFO: - Episode: 721/2000, Reward: 161.319, Step: 500 +2023-04-02 17:26:32 - r - INFO: - Episode: 722/2000, Reward: 276.863, Step: 466 +2023-04-02 17:26:34 - r - INFO: - Episode: 723/2000, Reward: 170.251, Step: 500 +2023-04-02 17:26:35 - r - INFO: - Episode: 724/2000, Reward: 226.631, Step: 350 +2023-04-02 17:26:36 - r - INFO: - Episode: 725/2000, Reward: 161.949, Step: 500 +2023-04-02 17:26:38 - r - INFO: - Current episode 725 has the best eval reward: 277.832 +2023-04-02 17:26:39 - r - INFO: - Episode: 726/2000, Reward: 256.410, Step: 371 +2023-04-02 17:26:40 - r - INFO: - Episode: 727/2000, Reward: 131.853, Step: 500 +2023-04-02 17:26:42 - r - INFO: - Episode: 728/2000, Reward: 159.934, Step: 500 +2023-04-02 17:26:44 - r - INFO: - Episode: 729/2000, Reward: 192.149, Step: 500 +2023-04-02 17:26:46 - r - INFO: - Episode: 730/2000, Reward: 162.630, Step: 500 +2023-04-02 17:26:50 - r - INFO: - Episode: 731/2000, Reward: 178.456, Step: 500 +2023-04-02 17:26:51 - r - INFO: - Episode: 732/2000, Reward: 240.379, Step: 322 +2023-04-02 17:26:53 - r - INFO: - Episode: 733/2000, Reward: 145.998, Step: 500 +2023-04-02 17:26:54 - r - INFO: - Episode: 734/2000, Reward: 199.958, Step: 500 +2023-04-02 17:26:56 - r - INFO: - Episode: 735/2000, Reward: 255.747, Step: 392 +2023-04-02 17:26:59 - r - INFO: - Episode: 736/2000, Reward: 187.919, Step: 500 +2023-04-02 17:27:00 - r - INFO: - Episode: 737/2000, Reward: 255.449, Step: 155 +2023-04-02 17:27:01 - r - INFO: - Episode: 738/2000, Reward: 187.973, Step: 500 +2023-04-02 17:27:03 - r - INFO: - Episode: 739/2000, Reward: 166.175, Step: 500 +2023-04-02 17:27:03 - r - INFO: - Episode: 740/2000, Reward: 267.826, Step: 174 +2023-04-02 17:27:06 - r - INFO: - Episode: 741/2000, Reward: 254.587, Step: 409 +2023-04-02 17:27:08 - r - INFO: - Episode: 742/2000, Reward: 149.955, Step: 500 +2023-04-02 17:27:09 - r - INFO: - Episode: 743/2000, Reward: 266.430, Step: 364 +2023-04-02 17:27:11 - r - INFO: - Episode: 744/2000, Reward: 143.219, Step: 500 +2023-04-02 17:27:12 - r - INFO: - Episode: 745/2000, Reward: 140.025, Step: 500 +2023-04-02 17:27:16 - r - INFO: - Episode: 746/2000, Reward: 137.100, Step: 500 +2023-04-02 17:27:18 - r - INFO: - Episode: 747/2000, Reward: 146.920, Step: 500 +2023-04-02 17:27:20 - r - INFO: - Episode: 748/2000, Reward: 131.326, Step: 500 +2023-04-02 17:27:21 - r - INFO: - Episode: 749/2000, Reward: 153.067, Step: 500 +2023-04-02 17:27:23 - r - INFO: - Episode: 750/2000, Reward: 146.854, Step: 500 +2023-04-02 17:27:26 - r - INFO: - Episode: 751/2000, Reward: 156.110, Step: 500 +2023-04-02 17:27:28 - r - INFO: - Episode: 752/2000, Reward: 144.649, Step: 500 +2023-04-02 17:27:30 - r - INFO: - Episode: 753/2000, Reward: 143.316, Step: 500 +2023-04-02 17:27:31 - r - INFO: - Episode: 754/2000, Reward: 165.287, Step: 500 +2023-04-02 17:27:33 - r - INFO: - Episode: 755/2000, Reward: 194.332, Step: 500 +2023-04-02 17:27:37 - r - INFO: - Episode: 756/2000, Reward: 180.081, Step: 500 +2023-04-02 17:27:38 - r - INFO: - Episode: 757/2000, Reward: 170.398, Step: 500 +2023-04-02 17:27:40 - r - INFO: - Episode: 758/2000, Reward: 148.453, Step: 500 +2023-04-02 17:27:41 - r - INFO: - Episode: 759/2000, Reward: 274.943, Step: 287 +2023-04-02 17:27:42 - r - INFO: - Episode: 760/2000, Reward: 188.348, Step: 500 +2023-04-02 17:27:46 - r - INFO: - Episode: 761/2000, Reward: 136.494, Step: 500 +2023-04-02 17:27:48 - r - INFO: - Episode: 762/2000, Reward: 173.845, Step: 500 +2023-04-02 17:27:49 - r - INFO: - Episode: 763/2000, Reward: 166.229, Step: 500 +2023-04-02 17:27:51 - r - INFO: - Episode: 764/2000, Reward: 167.808, Step: 500 +2023-04-02 17:27:53 - r - INFO: - Episode: 765/2000, Reward: 163.159, Step: 500 +2023-04-02 17:27:58 - r - INFO: - Episode: 766/2000, Reward: 127.002, Step: 500 +2023-04-02 17:27:59 - r - INFO: - Episode: 767/2000, Reward: 157.314, Step: 500 +2023-04-02 17:28:01 - r - INFO: - Episode: 768/2000, Reward: 130.207, Step: 500 +2023-04-02 17:28:03 - r - INFO: - Episode: 769/2000, Reward: 131.434, Step: 500 +2023-04-02 17:28:04 - r - INFO: - Episode: 770/2000, Reward: 170.445, Step: 500 +2023-04-02 17:28:08 - r - INFO: - Episode: 771/2000, Reward: 178.418, Step: 500 +2023-04-02 17:28:10 - r - INFO: - Episode: 772/2000, Reward: 128.247, Step: 500 +2023-04-02 17:28:11 - r - INFO: - Episode: 773/2000, Reward: 143.580, Step: 500 +2023-04-02 17:28:13 - r - INFO: - Episode: 774/2000, Reward: 190.579, Step: 500 +2023-04-02 17:28:14 - r - INFO: - Episode: 775/2000, Reward: 303.212, Step: 339 +2023-04-02 17:28:17 - r - INFO: - Episode: 776/2000, Reward: 259.582, Step: 327 +2023-04-02 17:28:19 - r - INFO: - Episode: 777/2000, Reward: 199.346, Step: 500 +2023-04-02 17:28:20 - r - INFO: - Episode: 778/2000, Reward: 176.061, Step: 500 +2023-04-02 17:28:22 - r - INFO: - Episode: 779/2000, Reward: 110.993, Step: 500 +2023-04-02 17:28:24 - r - INFO: - Episode: 780/2000, Reward: 141.705, Step: 500 +2023-04-02 17:28:28 - r - INFO: - Episode: 781/2000, Reward: 150.007, Step: 500 +2023-04-02 17:28:29 - r - INFO: - Episode: 782/2000, Reward: 61.286, Step: 234 +2023-04-02 17:28:30 - r - INFO: - Episode: 783/2000, Reward: 224.278, Step: 356 +2023-04-02 17:28:31 - r - INFO: - Episode: 784/2000, Reward: 178.359, Step: 500 +2023-04-02 17:28:33 - r - INFO: - Episode: 785/2000, Reward: 259.007, Step: 349 +2023-04-02 17:28:37 - r - INFO: - Episode: 786/2000, Reward: 229.029, Step: 425 +2023-04-02 17:28:38 - r - INFO: - Episode: 787/2000, Reward: 148.319, Step: 500 +2023-04-02 17:28:40 - r - INFO: - Episode: 788/2000, Reward: 133.494, Step: 500 +2023-04-02 17:28:41 - r - INFO: - Episode: 789/2000, Reward: 294.850, Step: 469 +2023-04-02 17:28:43 - r - INFO: - Episode: 790/2000, Reward: 133.065, Step: 500 +2023-04-02 17:28:47 - r - INFO: - Episode: 791/2000, Reward: 162.026, Step: 500 +2023-04-02 17:28:48 - r - INFO: - Episode: 792/2000, Reward: 286.286, Step: 425 +2023-04-02 17:28:50 - r - INFO: - Episode: 793/2000, Reward: 121.962, Step: 500 +2023-04-02 17:28:51 - r - INFO: - Episode: 794/2000, Reward: 128.718, Step: 500 +2023-04-02 17:28:53 - r - INFO: - Episode: 795/2000, Reward: 155.282, Step: 500 +2023-04-02 17:28:57 - r - INFO: - Episode: 796/2000, Reward: 145.682, Step: 500 +2023-04-02 17:28:58 - r - INFO: - Episode: 797/2000, Reward: 154.759, Step: 500 +2023-04-02 17:29:00 - r - INFO: - Episode: 798/2000, Reward: 190.051, Step: 500 +2023-04-02 17:29:02 - r - INFO: - Episode: 799/2000, Reward: 149.315, Step: 500 +2023-04-02 17:29:03 - r - INFO: - Episode: 800/2000, Reward: 175.007, Step: 500 +2023-04-02 17:29:07 - r - INFO: - Episode: 801/2000, Reward: 169.720, Step: 500 +2023-04-02 17:29:08 - r - INFO: - Episode: 802/2000, Reward: 203.459, Step: 500 +2023-04-02 17:29:10 - r - INFO: - Episode: 803/2000, Reward: 188.197, Step: 500 +2023-04-02 17:29:11 - r - INFO: - Episode: 804/2000, Reward: 183.191, Step: 500 +2023-04-02 17:29:12 - r - INFO: - Episode: 805/2000, Reward: 275.051, Step: 190 +2023-04-02 17:29:14 - r - INFO: - Episode: 806/2000, Reward: 267.075, Step: 342 +2023-04-02 17:29:15 - r - INFO: - Episode: 807/2000, Reward: 279.022, Step: 249 +2023-04-02 17:29:16 - r - INFO: - Episode: 808/2000, Reward: 181.366, Step: 500 +2023-04-02 17:29:17 - r - INFO: - Episode: 809/2000, Reward: 259.603, Step: 292 +2023-04-02 17:29:19 - r - INFO: - Episode: 810/2000, Reward: 156.134, Step: 500 +2023-04-02 17:29:22 - r - INFO: - Episode: 811/2000, Reward: 264.251, Step: 308 +2023-04-02 17:29:23 - r - INFO: - Episode: 812/2000, Reward: 147.985, Step: 500 +2023-04-02 17:29:25 - r - INFO: - Episode: 813/2000, Reward: 188.485, Step: 500 +2023-04-02 17:29:27 - r - INFO: - Episode: 814/2000, Reward: 162.784, Step: 500 +2023-04-02 17:29:27 - r - INFO: - Episode: 815/2000, Reward: 46.947, Step: 101 +2023-04-02 17:29:31 - r - INFO: - Episode: 816/2000, Reward: 127.797, Step: 500 +2023-04-02 17:29:33 - r - INFO: - Episode: 817/2000, Reward: 134.618, Step: 500 +2023-04-02 17:29:34 - r - INFO: - Episode: 818/2000, Reward: 265.647, Step: 426 +2023-04-02 17:29:36 - r - INFO: - Episode: 819/2000, Reward: 279.430, Step: 410 +2023-04-02 17:29:36 - r - INFO: - Episode: 820/2000, Reward: 266.516, Step: 218 +2023-04-02 17:29:39 - r - INFO: - Episode: 821/2000, Reward: 266.212, Step: 375 +2023-04-02 17:29:40 - r - INFO: - Episode: 822/2000, Reward: 267.092, Step: 391 +2023-04-02 17:29:41 - r - INFO: - Episode: 823/2000, Reward: 274.119, Step: 348 +2023-04-02 17:29:42 - r - INFO: - Episode: 824/2000, Reward: 264.546, Step: 314 +2023-04-02 17:29:43 - r - INFO: - Episode: 825/2000, Reward: 283.044, Step: 177 +2023-04-02 17:29:46 - r - INFO: - Episode: 826/2000, Reward: 166.427, Step: 500 +2023-04-02 17:29:48 - r - INFO: - Episode: 827/2000, Reward: 172.521, Step: 500 +2023-04-02 17:29:49 - r - INFO: - Episode: 828/2000, Reward: 86.291, Step: 500 +2023-04-02 17:29:51 - r - INFO: - Episode: 829/2000, Reward: 173.811, Step: 500 +2023-04-02 17:29:52 - r - INFO: - Episode: 830/2000, Reward: 285.639, Step: 295 +2023-04-02 17:29:55 - r - INFO: - Episode: 831/2000, Reward: 285.836, Step: 223 +2023-04-02 17:29:56 - r - INFO: - Episode: 832/2000, Reward: 239.115, Step: 414 +2023-04-02 17:29:58 - r - INFO: - Episode: 833/2000, Reward: 165.277, Step: 500 +2023-04-02 17:29:59 - r - INFO: - Episode: 834/2000, Reward: 172.914, Step: 500 +2023-04-02 17:30:01 - r - INFO: - Episode: 835/2000, Reward: 152.487, Step: 500 +2023-04-02 17:30:05 - r - INFO: - Episode: 836/2000, Reward: 173.240, Step: 500 +2023-04-02 17:30:06 - r - INFO: - Episode: 837/2000, Reward: 201.654, Step: 500 +2023-04-02 17:30:08 - r - INFO: - Episode: 838/2000, Reward: 143.447, Step: 500 +2023-04-02 17:30:10 - r - INFO: - Episode: 839/2000, Reward: 183.327, Step: 500 +2023-04-02 17:30:12 - r - INFO: - Episode: 840/2000, Reward: 192.257, Step: 500 +2023-04-02 17:30:15 - r - INFO: - Episode: 841/2000, Reward: 160.902, Step: 500 +2023-04-02 17:30:17 - r - INFO: - Episode: 842/2000, Reward: 106.158, Step: 500 +2023-04-02 17:30:18 - r - INFO: - Episode: 843/2000, Reward: 203.447, Step: 500 +2023-04-02 17:30:20 - r - INFO: - Episode: 844/2000, Reward: 183.282, Step: 500 +2023-04-02 17:30:22 - r - INFO: - Episode: 845/2000, Reward: 151.221, Step: 500 +2023-04-02 17:30:25 - r - INFO: - Episode: 846/2000, Reward: 121.937, Step: 500 +2023-04-02 17:30:27 - r - INFO: - Episode: 847/2000, Reward: 183.455, Step: 500 +2023-04-02 17:30:28 - r - INFO: - Episode: 848/2000, Reward: 274.001, Step: 404 +2023-04-02 17:30:30 - r - INFO: - Episode: 849/2000, Reward: 151.770, Step: 500 +2023-04-02 17:30:32 - r - INFO: - Episode: 850/2000, Reward: 67.192, Step: 500 +2023-04-02 17:30:35 - r - INFO: - Episode: 851/2000, Reward: 157.259, Step: 500 +2023-04-02 17:30:37 - r - INFO: - Episode: 852/2000, Reward: 171.409, Step: 500 +2023-04-02 17:30:38 - r - INFO: - Episode: 853/2000, Reward: 174.164, Step: 500 +2023-04-02 17:30:39 - r - INFO: - Episode: 854/2000, Reward: 267.083, Step: 293 +2023-04-02 17:30:41 - r - INFO: - Episode: 855/2000, Reward: 178.866, Step: 500 +2023-04-02 17:30:45 - r - INFO: - Episode: 856/2000, Reward: 122.243, Step: 500 +2023-04-02 17:30:47 - r - INFO: - Episode: 857/2000, Reward: 157.172, Step: 500 +2023-04-02 17:30:47 - r - INFO: - Episode: 858/2000, Reward: 240.833, Step: 285 +2023-04-02 17:30:49 - r - INFO: - Episode: 859/2000, Reward: 191.624, Step: 500 +2023-04-02 17:30:51 - r - INFO: - Episode: 860/2000, Reward: 173.357, Step: 500 +2023-04-02 17:30:55 - r - INFO: - Episode: 861/2000, Reward: 133.482, Step: 500 +2023-04-02 17:30:56 - r - INFO: - Episode: 862/2000, Reward: 260.151, Step: 333 +2023-04-02 17:30:58 - r - INFO: - Episode: 863/2000, Reward: 179.186, Step: 500 +2023-04-02 17:30:59 - r - INFO: - Episode: 864/2000, Reward: 285.075, Step: 347 +2023-04-02 17:31:00 - r - INFO: - Episode: 865/2000, Reward: 260.444, Step: 328 +2023-04-02 17:31:04 - r - INFO: - Episode: 866/2000, Reward: 133.926, Step: 500 +2023-04-02 17:31:05 - r - INFO: - Episode: 867/2000, Reward: 181.061, Step: 500 +2023-04-02 17:31:07 - r - INFO: - Episode: 868/2000, Reward: 192.243, Step: 500 +2023-04-02 17:31:09 - r - INFO: - Episode: 869/2000, Reward: 186.076, Step: 500 +2023-04-02 17:31:10 - r - INFO: - Episode: 870/2000, Reward: 186.251, Step: 500 +2023-04-02 17:31:13 - r - INFO: - Episode: 871/2000, Reward: 196.658, Step: 500 +2023-04-02 17:31:15 - r - INFO: - Episode: 872/2000, Reward: 117.935, Step: 500 +2023-04-02 17:31:17 - r - INFO: - Episode: 873/2000, Reward: 113.048, Step: 500 +2023-04-02 17:31:18 - r - INFO: - Episode: 874/2000, Reward: 239.121, Step: 248 +2023-04-02 17:31:19 - r - INFO: - Episode: 875/2000, Reward: 166.100, Step: 500 +2023-04-02 17:31:24 - r - INFO: - Episode: 876/2000, Reward: 150.886, Step: 500 +2023-04-02 17:31:25 - r - INFO: - Episode: 877/2000, Reward: 150.958, Step: 500 +2023-04-02 17:31:27 - r - INFO: - Episode: 878/2000, Reward: 140.960, Step: 500 +2023-04-02 17:31:29 - r - INFO: - Episode: 879/2000, Reward: 157.251, Step: 500 +2023-04-02 17:31:30 - r - INFO: - Episode: 880/2000, Reward: 163.327, Step: 500 +2023-04-02 17:31:34 - r - INFO: - Episode: 881/2000, Reward: 149.283, Step: 500 +2023-04-02 17:31:35 - r - INFO: - Episode: 882/2000, Reward: 124.972, Step: 500 +2023-04-02 17:31:37 - r - INFO: - Episode: 883/2000, Reward: 140.226, Step: 500 +2023-04-02 17:31:39 - r - INFO: - Episode: 884/2000, Reward: 170.306, Step: 500 +2023-04-02 17:31:40 - r - INFO: - Episode: 885/2000, Reward: 154.929, Step: 500 +2023-04-02 17:31:44 - r - INFO: - Episode: 886/2000, Reward: 171.043, Step: 500 +2023-04-02 17:31:45 - r - INFO: - Episode: 887/2000, Reward: 184.299, Step: 500 +2023-04-02 17:31:47 - r - INFO: - Episode: 888/2000, Reward: 144.331, Step: 500 +2023-04-02 17:31:49 - r - INFO: - Episode: 889/2000, Reward: 186.998, Step: 500 +2023-04-02 17:31:50 - r - INFO: - Episode: 890/2000, Reward: 142.256, Step: 500 +2023-04-02 17:31:55 - r - INFO: - Episode: 891/2000, Reward: 69.780, Step: 500 +2023-04-02 17:31:56 - r - INFO: - Episode: 892/2000, Reward: 143.724, Step: 500 +2023-04-02 17:31:58 - r - INFO: - Episode: 893/2000, Reward: 157.743, Step: 500 +2023-04-02 17:32:00 - r - INFO: - Episode: 894/2000, Reward: 124.009, Step: 500 +2023-04-02 17:32:01 - r - INFO: - Episode: 895/2000, Reward: 174.477, Step: 500 +2023-04-02 17:32:06 - r - INFO: - Episode: 896/2000, Reward: 181.664, Step: 500 +2023-04-02 17:32:07 - r - INFO: - Episode: 897/2000, Reward: 142.503, Step: 500 +2023-04-02 17:32:08 - r - INFO: - Episode: 898/2000, Reward: 266.200, Step: 276 +2023-04-02 17:32:10 - r - INFO: - Episode: 899/2000, Reward: 162.973, Step: 500 +2023-04-02 17:32:11 - r - INFO: - Episode: 900/2000, Reward: 245.063, Step: 383 +2023-04-02 17:32:14 - r - INFO: - Episode: 901/2000, Reward: 158.161, Step: 500 +2023-04-02 17:32:15 - r - INFO: - Episode: 902/2000, Reward: 220.955, Step: 422 +2023-04-02 17:32:17 - r - INFO: - Episode: 903/2000, Reward: 169.392, Step: 500 +2023-04-02 17:32:19 - r - INFO: - Episode: 904/2000, Reward: 168.990, Step: 500 +2023-04-02 17:32:19 - r - INFO: - Episode: 905/2000, Reward: 277.377, Step: 230 +2023-04-02 17:32:23 - r - INFO: - Episode: 906/2000, Reward: 75.529, Step: 500 +2023-04-02 17:32:25 - r - INFO: - Episode: 907/2000, Reward: 46.474, Step: 500 +2023-04-02 17:32:26 - r - INFO: - Episode: 908/2000, Reward: 174.568, Step: 500 +2023-04-02 17:32:28 - r - INFO: - Episode: 909/2000, Reward: 62.746, Step: 500 +2023-04-02 17:32:30 - r - INFO: - Episode: 910/2000, Reward: 71.022, Step: 500 +2023-04-02 17:32:34 - r - INFO: - Episode: 911/2000, Reward: 175.022, Step: 500 +2023-04-02 17:32:35 - r - INFO: - Episode: 912/2000, Reward: 154.787, Step: 500 +2023-04-02 17:32:37 - r - INFO: - Episode: 913/2000, Reward: 160.732, Step: 500 +2023-04-02 17:32:39 - r - INFO: - Episode: 914/2000, Reward: 42.584, Step: 500 +2023-04-02 17:32:41 - r - INFO: - Episode: 915/2000, Reward: 55.691, Step: 500 +2023-04-02 17:32:44 - r - INFO: - Episode: 916/2000, Reward: 169.247, Step: 500 +2023-04-02 17:32:45 - r - INFO: - Episode: 917/2000, Reward: 255.888, Step: 366 +2023-04-02 17:32:47 - r - INFO: - Episode: 918/2000, Reward: 172.844, Step: 500 +2023-04-02 17:32:48 - r - INFO: - Episode: 919/2000, Reward: 183.663, Step: 500 +2023-04-02 17:32:50 - r - INFO: - Episode: 920/2000, Reward: 163.106, Step: 500 +2023-04-02 17:32:54 - r - INFO: - Episode: 921/2000, Reward: 53.330, Step: 500 +2023-04-02 17:32:56 - r - INFO: - Episode: 922/2000, Reward: 155.913, Step: 500 +2023-04-02 17:32:56 - r - INFO: - Episode: 923/2000, Reward: 264.506, Step: 244 +2023-04-02 17:32:58 - r - INFO: - Episode: 924/2000, Reward: 281.857, Step: 348 +2023-04-02 17:32:59 - r - INFO: - Episode: 925/2000, Reward: 240.650, Step: 464 +2023-04-02 17:33:03 - r - INFO: - Episode: 926/2000, Reward: 105.517, Step: 500 +2023-04-02 17:33:04 - r - INFO: - Episode: 927/2000, Reward: 115.797, Step: 500 +2023-04-02 17:33:06 - r - INFO: - Episode: 928/2000, Reward: 174.732, Step: 500 +2023-04-02 17:33:07 - r - INFO: - Episode: 929/2000, Reward: 33.513, Step: 121 +2023-04-02 17:33:08 - r - INFO: - Episode: 930/2000, Reward: 167.905, Step: 500 +2023-04-02 17:33:13 - r - INFO: - Episode: 931/2000, Reward: 90.085, Step: 500 +2023-04-02 17:33:14 - r - INFO: - Episode: 932/2000, Reward: 156.062, Step: 500 +2023-04-02 17:33:16 - r - INFO: - Episode: 933/2000, Reward: 150.206, Step: 500 +2023-04-02 17:33:18 - r - INFO: - Episode: 934/2000, Reward: 255.026, Step: 390 +2023-04-02 17:33:19 - r - INFO: - Episode: 935/2000, Reward: 141.997, Step: 500 +2023-04-02 17:33:23 - r - INFO: - Episode: 936/2000, Reward: 178.906, Step: 500 +2023-04-02 17:33:25 - r - INFO: - Episode: 937/2000, Reward: 191.184, Step: 500 +2023-04-02 17:33:25 - r - INFO: - Episode: 938/2000, Reward: 272.604, Step: 223 +2023-04-02 17:33:26 - r - INFO: - Episode: 939/2000, Reward: 285.087, Step: 211 +2023-04-02 17:33:27 - r - INFO: - Episode: 940/2000, Reward: 305.660, Step: 475 +2023-04-02 17:33:31 - r - INFO: - Episode: 941/2000, Reward: 192.648, Step: 500 +2023-04-02 17:33:32 - r - INFO: - Episode: 942/2000, Reward: 162.220, Step: 500 +2023-04-02 17:33:34 - r - INFO: - Episode: 943/2000, Reward: 181.104, Step: 500 +2023-04-02 17:33:35 - r - INFO: - Episode: 944/2000, Reward: 178.521, Step: 500 +2023-04-02 17:33:37 - r - INFO: - Episode: 945/2000, Reward: 184.128, Step: 500 +2023-04-02 17:33:41 - r - INFO: - Episode: 946/2000, Reward: 266.842, Step: 234 +2023-04-02 17:33:42 - r - INFO: - Episode: 947/2000, Reward: 289.756, Step: 388 +2023-04-02 17:33:44 - r - INFO: - Episode: 948/2000, Reward: 178.633, Step: 500 +2023-04-02 17:33:45 - r - INFO: - Episode: 949/2000, Reward: 249.792, Step: 242 +2023-04-02 17:33:46 - r - INFO: - Episode: 950/2000, Reward: 184.056, Step: 500 +2023-04-02 17:33:49 - r - INFO: - Episode: 951/2000, Reward: 238.345, Step: 286 +2023-04-02 17:33:50 - r - INFO: - Episode: 952/2000, Reward: 297.867, Step: 319 +2023-04-02 17:33:51 - r - INFO: - Episode: 953/2000, Reward: 283.501, Step: 220 +2023-04-02 17:33:52 - r - INFO: - Episode: 954/2000, Reward: 149.895, Step: 500 +2023-04-02 17:33:54 - r - INFO: - Episode: 955/2000, Reward: 168.552, Step: 500 +2023-04-02 17:33:57 - r - INFO: - Episode: 956/2000, Reward: 183.185, Step: 500 +2023-04-02 17:33:59 - r - INFO: - Episode: 957/2000, Reward: 277.095, Step: 488 +2023-04-02 17:34:00 - r - INFO: - Episode: 958/2000, Reward: 186.480, Step: 500 +2023-04-02 17:34:02 - r - INFO: - Episode: 959/2000, Reward: 209.114, Step: 500 +2023-04-02 17:34:04 - r - INFO: - Episode: 960/2000, Reward: 156.289, Step: 500 +2023-04-02 17:34:08 - r - INFO: - Episode: 961/2000, Reward: 178.345, Step: 500 +2023-04-02 17:34:09 - r - INFO: - Episode: 962/2000, Reward: 159.125, Step: 500 +2023-04-02 17:34:11 - r - INFO: - Episode: 963/2000, Reward: 273.302, Step: 444 +2023-04-02 17:34:12 - r - INFO: - Episode: 964/2000, Reward: 161.629, Step: 500 +2023-04-02 17:34:14 - r - INFO: - Episode: 965/2000, Reward: 181.979, Step: 500 +2023-04-02 17:34:17 - r - INFO: - Episode: 966/2000, Reward: 154.721, Step: 500 +2023-04-02 17:34:18 - r - INFO: - Episode: 967/2000, Reward: 175.988, Step: 500 +2023-04-02 17:34:20 - r - INFO: - Episode: 968/2000, Reward: 140.375, Step: 500 +2023-04-02 17:34:22 - r - INFO: - Episode: 969/2000, Reward: 179.081, Step: 500 +2023-04-02 17:34:23 - r - INFO: - Episode: 970/2000, Reward: 179.021, Step: 500 +2023-04-02 17:34:26 - r - INFO: - Episode: 971/2000, Reward: 165.741, Step: 500 +2023-04-02 17:34:28 - r - INFO: - Episode: 972/2000, Reward: 147.483, Step: 500 +2023-04-02 17:34:29 - r - INFO: - Episode: 973/2000, Reward: 161.143, Step: 500 +2023-04-02 17:34:31 - r - INFO: - Episode: 974/2000, Reward: 175.403, Step: 500 +2023-04-02 17:34:33 - r - INFO: - Episode: 975/2000, Reward: 149.650, Step: 500 +2023-04-02 17:34:36 - r - INFO: - Episode: 976/2000, Reward: 154.568, Step: 500 +2023-04-02 17:34:37 - r - INFO: - Episode: 977/2000, Reward: 193.001, Step: 500 +2023-04-02 17:34:39 - r - INFO: - Episode: 978/2000, Reward: 145.853, Step: 500 +2023-04-02 17:34:41 - r - INFO: - Episode: 979/2000, Reward: 150.487, Step: 500 +2023-04-02 17:34:42 - r - INFO: - Episode: 980/2000, Reward: 105.903, Step: 500 +2023-04-02 17:34:45 - r - INFO: - Episode: 981/2000, Reward: 167.522, Step: 500 +2023-04-02 17:34:47 - r - INFO: - Episode: 982/2000, Reward: 174.352, Step: 500 +2023-04-02 17:34:49 - r - INFO: - Episode: 983/2000, Reward: 158.827, Step: 500 +2023-04-02 17:34:51 - r - INFO: - Episode: 984/2000, Reward: 159.924, Step: 500 +2023-04-02 17:34:52 - r - INFO: - Episode: 985/2000, Reward: 177.726, Step: 500 +2023-04-02 17:34:55 - r - INFO: - Episode: 986/2000, Reward: 176.153, Step: 500 +2023-04-02 17:34:56 - r - INFO: - Episode: 987/2000, Reward: 275.485, Step: 224 +2023-04-02 17:34:57 - r - INFO: - Episode: 988/2000, Reward: 263.478, Step: 351 +2023-04-02 17:34:58 - r - INFO: - Episode: 989/2000, Reward: 166.661, Step: 500 +2023-04-02 17:35:00 - r - INFO: - Episode: 990/2000, Reward: 183.750, Step: 500 +2023-04-02 17:35:03 - r - INFO: - Episode: 991/2000, Reward: 148.426, Step: 500 +2023-04-02 17:35:05 - r - INFO: - Episode: 992/2000, Reward: 259.901, Step: 499 +2023-04-02 17:35:07 - r - INFO: - Episode: 993/2000, Reward: 186.979, Step: 500 +2023-04-02 17:35:08 - r - INFO: - Episode: 994/2000, Reward: 305.413, Step: 304 +2023-04-02 17:35:09 - r - INFO: - Episode: 995/2000, Reward: 180.928, Step: 500 +2023-04-02 17:35:13 - r - INFO: - Episode: 996/2000, Reward: 208.026, Step: 500 +2023-04-02 17:35:13 - r - INFO: - Episode: 997/2000, Reward: -30.511, Step: 122 +2023-04-02 17:35:13 - r - INFO: - Episode: 998/2000, Reward: -50.063, Step: 146 +2023-04-02 17:35:15 - r - INFO: - Episode: 999/2000, Reward: 164.258, Step: 500 +2023-04-02 17:35:16 - r - INFO: - Episode: 1000/2000, Reward: 281.223, Step: 376 +2023-04-02 17:35:18 - r - INFO: - Episode: 1001/2000, Reward: -30.392, Step: 76 +2023-04-02 17:35:20 - r - INFO: - Episode: 1002/2000, Reward: 139.700, Step: 500 +2023-04-02 17:35:20 - r - INFO: - Episode: 1003/2000, Reward: -99.942, Step: 106 +2023-04-02 17:35:20 - r - INFO: - Episode: 1004/2000, Reward: -75.559, Step: 99 +2023-04-02 17:35:22 - r - INFO: - Episode: 1005/2000, Reward: 199.968, Step: 500 +2023-04-02 17:35:25 - r - INFO: - Episode: 1006/2000, Reward: 163.050, Step: 500 +2023-04-02 17:35:26 - r - INFO: - Episode: 1007/2000, Reward: -121.822, Step: 120 +2023-04-02 17:35:26 - r - INFO: - Episode: 1008/2000, Reward: -76.382, Step: 102 +2023-04-02 17:35:26 - r - INFO: - Episode: 1009/2000, Reward: -4.493, Step: 112 +2023-04-02 17:35:28 - r - INFO: - Episode: 1010/2000, Reward: 152.346, Step: 500 +2023-04-02 17:35:30 - r - INFO: - Episode: 1011/2000, Reward: 0.071, Step: 124 +2023-04-02 17:35:30 - r - INFO: - Episode: 1012/2000, Reward: -48.288, Step: 116 +2023-04-02 17:35:31 - r - INFO: - Episode: 1013/2000, Reward: 257.538, Step: 276 +2023-04-02 17:35:33 - r - INFO: - Episode: 1014/2000, Reward: 290.490, Step: 487 +2023-04-02 17:35:33 - r - INFO: - Episode: 1015/2000, Reward: -88.330, Step: 127 +2023-04-02 17:35:36 - r - INFO: - Episode: 1016/2000, Reward: -34.103, Step: 114 +2023-04-02 17:35:37 - r - INFO: - Episode: 1017/2000, Reward: 304.458, Step: 397 +2023-04-02 17:35:38 - r - INFO: - Episode: 1018/2000, Reward: -113.067, Step: 155 +2023-04-02 17:35:40 - r - INFO: - Episode: 1019/2000, Reward: 135.402, Step: 500 +2023-04-02 17:35:41 - r - INFO: - Episode: 1020/2000, Reward: 149.666, Step: 500 +2023-04-02 17:35:44 - r - INFO: - Episode: 1021/2000, Reward: 313.447, Step: 247 +2023-04-02 17:35:44 - r - INFO: - Episode: 1022/2000, Reward: 296.569, Step: 238 +2023-04-02 17:35:45 - r - INFO: - Episode: 1023/2000, Reward: -118.769, Step: 117 +2023-04-02 17:35:45 - r - INFO: - Episode: 1024/2000, Reward: -75.869, Step: 166 +2023-04-02 17:35:47 - r - INFO: - Episode: 1025/2000, Reward: 132.888, Step: 500 +2023-04-02 17:35:50 - r - INFO: - Episode: 1026/2000, Reward: 185.757, Step: 500 +2023-04-02 17:35:52 - r - INFO: - Episode: 1027/2000, Reward: 181.194, Step: 500 +2023-04-02 17:35:52 - r - INFO: - Episode: 1028/2000, Reward: -75.124, Step: 139 +2023-04-02 17:35:54 - r - INFO: - Episode: 1029/2000, Reward: 129.974, Step: 500 +2023-04-02 17:35:55 - r - INFO: - Episode: 1030/2000, Reward: -124.818, Step: 328 +2023-04-02 17:35:58 - r - INFO: - Episode: 1031/2000, Reward: -122.139, Step: 321 +2023-04-02 17:35:58 - r - INFO: - Episode: 1032/2000, Reward: -87.381, Step: 142 +2023-04-02 17:36:00 - r - INFO: - Episode: 1033/2000, Reward: 145.167, Step: 500 +2023-04-02 17:36:00 - r - INFO: - Episode: 1034/2000, Reward: -78.508, Step: 154 +2023-04-02 17:36:01 - r - INFO: - Episode: 1035/2000, Reward: 256.198, Step: 375 +2023-04-02 17:36:04 - r - INFO: - Episode: 1036/2000, Reward: 252.597, Step: 302 +2023-04-02 17:36:06 - r - INFO: - Episode: 1037/2000, Reward: 279.286, Step: 416 +2023-04-02 17:36:07 - r - INFO: - Episode: 1038/2000, Reward: 172.630, Step: 500 +2023-04-02 17:36:08 - r - INFO: - Episode: 1039/2000, Reward: 234.566, Step: 289 +2023-04-02 17:36:10 - r - INFO: - Episode: 1040/2000, Reward: 151.148, Step: 500 +2023-04-02 17:36:14 - r - INFO: - Episode: 1041/2000, Reward: 186.234, Step: 500 +2023-04-02 17:36:14 - r - INFO: - Episode: 1042/2000, Reward: -75.100, Step: 178 +2023-04-02 17:36:16 - r - INFO: - Episode: 1043/2000, Reward: 180.630, Step: 500 +2023-04-02 17:36:18 - r - INFO: - Episode: 1044/2000, Reward: 132.605, Step: 500 +2023-04-02 17:36:18 - r - INFO: - Episode: 1045/2000, Reward: 246.566, Step: 249 +2023-04-02 17:36:21 - r - INFO: - Episode: 1046/2000, Reward: -76.381, Step: 190 +2023-04-02 17:36:23 - r - INFO: - Episode: 1047/2000, Reward: 167.671, Step: 500 +2023-04-02 17:36:25 - r - INFO: - Episode: 1048/2000, Reward: 124.886, Step: 500 +2023-04-02 17:36:26 - r - INFO: - Episode: 1049/2000, Reward: 232.969, Step: 476 +2023-04-02 17:36:27 - r - INFO: - Episode: 1050/2000, Reward: -98.094, Step: 195 +2023-04-02 17:36:31 - r - INFO: - Episode: 1051/2000, Reward: 155.045, Step: 500 +2023-04-02 17:36:32 - r - INFO: - Episode: 1052/2000, Reward: -115.995, Step: 189 +2023-04-02 17:36:32 - r - INFO: - Episode: 1053/2000, Reward: -121.250, Step: 227 +2023-04-02 17:36:34 - r - INFO: - Episode: 1054/2000, Reward: 273.595, Step: 451 +2023-04-02 17:36:36 - r - INFO: - Episode: 1055/2000, Reward: 145.701, Step: 500 +2023-04-02 17:36:41 - r - INFO: - Episode: 1056/2000, Reward: 142.444, Step: 500 +2023-04-02 17:36:42 - r - INFO: - Episode: 1057/2000, Reward: 150.843, Step: 500 +2023-04-02 17:36:44 - r - INFO: - Episode: 1058/2000, Reward: 172.868, Step: 500 +2023-04-02 17:36:46 - r - INFO: - Episode: 1059/2000, Reward: 18.657, Step: 500 +2023-04-02 17:36:47 - r - INFO: - Episode: 1060/2000, Reward: 115.206, Step: 500 +2023-04-02 17:36:52 - r - INFO: - Episode: 1061/2000, Reward: 173.103, Step: 500 +2023-04-02 17:36:54 - r - INFO: - Episode: 1062/2000, Reward: 96.790, Step: 500 +2023-04-02 17:36:55 - r - INFO: - Episode: 1063/2000, Reward: 287.109, Step: 298 +2023-04-02 17:36:56 - r - INFO: - Episode: 1064/2000, Reward: 272.055, Step: 434 +2023-04-02 17:36:57 - r - INFO: - Episode: 1065/2000, Reward: 267.309, Step: 279 +2023-04-02 17:37:02 - r - INFO: - Episode: 1066/2000, Reward: 177.608, Step: 500 +2023-04-02 17:37:03 - r - INFO: - Episode: 1067/2000, Reward: 298.982, Step: 259 +2023-04-02 17:37:04 - r - INFO: - Episode: 1068/2000, Reward: 183.426, Step: 500 +2023-04-02 17:37:06 - r - INFO: - Episode: 1069/2000, Reward: 166.563, Step: 500 +2023-04-02 17:37:07 - r - INFO: - Episode: 1070/2000, Reward: 235.370, Step: 254 +2023-04-02 17:37:10 - r - INFO: - Episode: 1071/2000, Reward: 119.212, Step: 500 +2023-04-02 17:37:12 - r - INFO: - Episode: 1072/2000, Reward: 182.639, Step: 500 +2023-04-02 17:37:14 - r - INFO: - Episode: 1073/2000, Reward: 187.752, Step: 500 +2023-04-02 17:37:15 - r - INFO: - Episode: 1074/2000, Reward: 172.069, Step: 500 +2023-04-02 17:37:17 - r - INFO: - Episode: 1075/2000, Reward: 167.828, Step: 500 +2023-04-02 17:37:22 - r - INFO: - Episode: 1076/2000, Reward: 218.587, Step: 414 +2023-04-02 17:37:23 - r - INFO: - Episode: 1077/2000, Reward: 264.740, Step: 402 +2023-04-02 17:37:24 - r - INFO: - Episode: 1078/2000, Reward: 262.205, Step: 185 +2023-04-02 17:37:25 - r - INFO: - Episode: 1079/2000, Reward: 299.799, Step: 253 +2023-04-02 17:37:26 - r - INFO: - Episode: 1080/2000, Reward: 269.094, Step: 358 +2023-04-02 17:37:30 - r - INFO: - Episode: 1081/2000, Reward: 145.161, Step: 500 +2023-04-02 17:37:31 - r - INFO: - Episode: 1082/2000, Reward: 304.743, Step: 368 +2023-04-02 17:37:32 - r - INFO: - Episode: 1083/2000, Reward: 267.463, Step: 348 +2023-04-02 17:37:34 - r - INFO: - Episode: 1084/2000, Reward: 258.779, Step: 361 +2023-04-02 17:37:35 - r - INFO: - Episode: 1085/2000, Reward: 139.699, Step: 500 +2023-04-02 17:37:40 - r - INFO: - Episode: 1086/2000, Reward: 154.753, Step: 500 +2023-04-02 17:37:41 - r - INFO: - Episode: 1087/2000, Reward: 271.497, Step: 393 +2023-04-02 17:37:43 - r - INFO: - Episode: 1088/2000, Reward: 171.776, Step: 500 +2023-04-02 17:37:45 - r - INFO: - Episode: 1089/2000, Reward: 177.549, Step: 500 +2023-04-02 17:37:46 - r - INFO: - Episode: 1090/2000, Reward: 150.011, Step: 500 +2023-04-02 17:37:50 - r - INFO: - Episode: 1091/2000, Reward: 125.967, Step: 500 +2023-04-02 17:37:52 - r - INFO: - Episode: 1092/2000, Reward: 143.697, Step: 500 +2023-04-02 17:37:52 - r - INFO: - Episode: 1093/2000, Reward: 246.979, Step: 208 +2023-04-02 17:37:54 - r - INFO: - Episode: 1094/2000, Reward: 151.975, Step: 500 +2023-04-02 17:37:56 - r - INFO: - Episode: 1095/2000, Reward: 133.599, Step: 500 +2023-04-02 17:38:00 - r - INFO: - Episode: 1096/2000, Reward: 126.423, Step: 500 +2023-04-02 17:38:01 - r - INFO: - Episode: 1097/2000, Reward: 298.164, Step: 353 +2023-04-02 17:38:03 - r - INFO: - Episode: 1098/2000, Reward: 134.719, Step: 500 +2023-04-02 17:38:04 - r - INFO: - Episode: 1099/2000, Reward: 253.402, Step: 392 +2023-04-02 17:38:06 - r - INFO: - Episode: 1100/2000, Reward: 244.190, Step: 420 +2023-04-02 17:38:09 - r - INFO: - Episode: 1101/2000, Reward: 276.029, Step: 310 +2023-04-02 17:38:10 - r - INFO: - Episode: 1102/2000, Reward: 251.959, Step: 388 +2023-04-02 17:38:12 - r - INFO: - Episode: 1103/2000, Reward: 173.289, Step: 500 +2023-04-02 17:38:13 - r - INFO: - Episode: 1104/2000, Reward: 235.758, Step: 401 +2023-04-02 17:38:14 - r - INFO: - Episode: 1105/2000, Reward: 266.514, Step: 366 +2023-04-02 17:38:18 - r - INFO: - Episode: 1106/2000, Reward: 141.774, Step: 500 +2023-04-02 17:38:19 - r - INFO: - Episode: 1107/2000, Reward: 155.952, Step: 500 +2023-04-02 17:38:21 - r - INFO: - Episode: 1108/2000, Reward: 141.462, Step: 500 +2023-04-02 17:38:23 - r - INFO: - Episode: 1109/2000, Reward: 188.198, Step: 500 +2023-04-02 17:38:24 - r - INFO: - Episode: 1110/2000, Reward: 141.096, Step: 500 +2023-04-02 17:38:27 - r - INFO: - Episode: 1111/2000, Reward: 131.210, Step: 500 +2023-04-02 17:38:29 - r - INFO: - Episode: 1112/2000, Reward: 171.605, Step: 500 +2023-04-02 17:38:31 - r - INFO: - Episode: 1113/2000, Reward: 194.109, Step: 500 +2023-04-02 17:38:32 - r - INFO: - Episode: 1114/2000, Reward: 149.131, Step: 500 +2023-04-02 17:38:34 - r - INFO: - Episode: 1115/2000, Reward: 183.034, Step: 500 +2023-04-02 17:38:37 - r - INFO: - Episode: 1116/2000, Reward: 230.023, Step: 434 +2023-04-02 17:38:39 - r - INFO: - Episode: 1117/2000, Reward: 182.478, Step: 500 +2023-04-02 17:38:41 - r - INFO: - Episode: 1118/2000, Reward: 122.206, Step: 500 +2023-04-02 17:38:42 - r - INFO: - Episode: 1119/2000, Reward: 135.091, Step: 500 +2023-04-02 17:38:44 - r - INFO: - Episode: 1120/2000, Reward: 159.138, Step: 500 +2023-04-02 17:38:47 - r - INFO: - Episode: 1121/2000, Reward: 14.307, Step: 500 +2023-04-02 17:38:48 - r - INFO: - Episode: 1122/2000, Reward: 267.837, Step: 364 +2023-04-02 17:38:50 - r - INFO: - Episode: 1123/2000, Reward: 167.063, Step: 500 +2023-04-02 17:38:52 - r - INFO: - Episode: 1124/2000, Reward: 154.840, Step: 500 +2023-04-02 17:38:52 - r - INFO: - Episode: 1125/2000, Reward: 259.395, Step: 288 +2023-04-02 17:38:56 - r - INFO: - Episode: 1126/2000, Reward: 175.438, Step: 500 +2023-04-02 17:38:57 - r - INFO: - Episode: 1127/2000, Reward: 300.866, Step: 481 +2023-04-02 17:38:59 - r - INFO: - Episode: 1128/2000, Reward: 177.353, Step: 500 +2023-04-02 17:39:01 - r - INFO: - Episode: 1129/2000, Reward: 188.255, Step: 500 +2023-04-02 17:39:02 - r - INFO: - Episode: 1130/2000, Reward: 147.822, Step: 500 +2023-04-02 17:39:06 - r - INFO: - Episode: 1131/2000, Reward: 157.936, Step: 500 +2023-04-02 17:39:07 - r - INFO: - Episode: 1132/2000, Reward: 285.717, Step: 416 +2023-04-02 17:39:09 - r - INFO: - Episode: 1133/2000, Reward: 41.542, Step: 500 +2023-04-02 17:39:10 - r - INFO: - Episode: 1134/2000, Reward: 279.474, Step: 484 +2023-04-02 17:39:12 - r - INFO: - Episode: 1135/2000, Reward: 278.445, Step: 433 +2023-04-02 17:39:16 - r - INFO: - Episode: 1136/2000, Reward: 143.951, Step: 500 +2023-04-02 17:39:17 - r - INFO: - Episode: 1137/2000, Reward: 240.379, Step: 340 +2023-04-02 17:39:19 - r - INFO: - Episode: 1138/2000, Reward: 117.687, Step: 500 +2023-04-02 17:39:21 - r - INFO: - Episode: 1139/2000, Reward: 128.302, Step: 500 +2023-04-02 17:39:22 - r - INFO: - Episode: 1140/2000, Reward: 97.758, Step: 500 +2023-04-02 17:39:26 - r - INFO: - Episode: 1141/2000, Reward: 182.289, Step: 500 +2023-04-02 17:39:28 - r - INFO: - Episode: 1142/2000, Reward: 155.897, Step: 500 +2023-04-02 17:39:29 - r - INFO: - Episode: 1143/2000, Reward: -25.210, Step: 500 +2023-04-02 17:39:31 - r - INFO: - Episode: 1144/2000, Reward: 147.928, Step: 500 +2023-04-02 17:39:33 - r - INFO: - Episode: 1145/2000, Reward: 38.864, Step: 500 +2023-04-02 17:39:36 - r - INFO: - Episode: 1146/2000, Reward: 188.418, Step: 500 +2023-04-02 17:39:38 - r - INFO: - Episode: 1147/2000, Reward: 112.534, Step: 500 +2023-04-02 17:39:39 - r - INFO: - Episode: 1148/2000, Reward: 131.917, Step: 500 +2023-04-02 17:39:41 - r - INFO: - Episode: 1149/2000, Reward: 158.129, Step: 500 +2023-04-02 17:39:43 - r - INFO: - Episode: 1150/2000, Reward: 115.475, Step: 500 +2023-04-02 17:39:46 - r - INFO: - Episode: 1151/2000, Reward: 167.573, Step: 500 +2023-04-02 17:39:48 - r - INFO: - Episode: 1152/2000, Reward: 171.881, Step: 500 +2023-04-02 17:39:49 - r - INFO: - Episode: 1153/2000, Reward: 193.253, Step: 500 +2023-04-02 17:39:51 - r - INFO: - Episode: 1154/2000, Reward: 161.494, Step: 500 +2023-04-02 17:39:53 - r - INFO: - Episode: 1155/2000, Reward: 104.528, Step: 500 +2023-04-02 17:39:56 - r - INFO: - Episode: 1156/2000, Reward: 223.049, Step: 464 +2023-04-02 17:39:58 - r - INFO: - Episode: 1157/2000, Reward: 184.173, Step: 500 +2023-04-02 17:40:00 - r - INFO: - Episode: 1158/2000, Reward: 179.396, Step: 500 +2023-04-02 17:40:01 - r - INFO: - Episode: 1159/2000, Reward: 185.304, Step: 500 +2023-04-02 17:40:03 - r - INFO: - Episode: 1160/2000, Reward: 166.373, Step: 500 +2023-04-02 17:40:07 - r - INFO: - Episode: 1161/2000, Reward: 169.662, Step: 500 +2023-04-02 17:40:08 - r - INFO: - Episode: 1162/2000, Reward: 49.670, Step: 500 +2023-04-02 17:40:10 - r - INFO: - Episode: 1163/2000, Reward: 49.801, Step: 500 +2023-04-02 17:40:12 - r - INFO: - Episode: 1164/2000, Reward: 166.130, Step: 500 +2023-04-02 17:40:14 - r - INFO: - Episode: 1165/2000, Reward: 60.247, Step: 500 +2023-04-02 17:40:18 - r - INFO: - Episode: 1166/2000, Reward: 120.505, Step: 500 +2023-04-02 17:40:19 - r - INFO: - Episode: 1167/2000, Reward: 130.130, Step: 500 +2023-04-02 17:40:21 - r - INFO: - Episode: 1168/2000, Reward: 162.645, Step: 500 +2023-04-02 17:40:23 - r - INFO: - Episode: 1169/2000, Reward: 142.341, Step: 500 +2023-04-02 17:40:25 - r - INFO: - Episode: 1170/2000, Reward: 185.427, Step: 500 +2023-04-02 17:40:28 - r - INFO: - Episode: 1171/2000, Reward: 167.660, Step: 500 +2023-04-02 17:40:30 - r - INFO: - Episode: 1172/2000, Reward: 164.412, Step: 500 +2023-04-02 17:40:32 - r - INFO: - Episode: 1173/2000, Reward: 142.007, Step: 500 +2023-04-02 17:40:34 - r - INFO: - Episode: 1174/2000, Reward: 198.548, Step: 500 +2023-04-02 17:40:35 - r - INFO: - Episode: 1175/2000, Reward: 227.541, Step: 306 +2023-04-02 17:40:38 - r - INFO: - Episode: 1176/2000, Reward: 165.052, Step: 500 +2023-04-02 17:40:40 - r - INFO: - Episode: 1177/2000, Reward: 150.327, Step: 500 +2023-04-02 17:40:42 - r - INFO: - Episode: 1178/2000, Reward: 147.362, Step: 500 +2023-04-02 17:40:43 - r - INFO: - Episode: 1179/2000, Reward: 146.729, Step: 500 +2023-04-02 17:40:45 - r - INFO: - Episode: 1180/2000, Reward: 152.631, Step: 500 +2023-04-02 17:40:49 - r - INFO: - Episode: 1181/2000, Reward: 160.197, Step: 500 +2023-04-02 17:40:51 - r - INFO: - Episode: 1182/2000, Reward: 169.733, Step: 500 +2023-04-02 17:40:52 - r - INFO: - Episode: 1183/2000, Reward: 150.627, Step: 500 +2023-04-02 17:40:54 - r - INFO: - Episode: 1184/2000, Reward: 159.977, Step: 500 +2023-04-02 17:40:56 - r - INFO: - Episode: 1185/2000, Reward: 150.264, Step: 500 +2023-04-02 17:40:59 - r - INFO: - Episode: 1186/2000, Reward: 184.854, Step: 500 +2023-04-02 17:41:01 - r - INFO: - Episode: 1187/2000, Reward: 181.854, Step: 500 +2023-04-02 17:41:02 - r - INFO: - Episode: 1188/2000, Reward: 41.085, Step: 500 +2023-04-02 17:41:04 - r - INFO: - Episode: 1189/2000, Reward: 164.381, Step: 500 +2023-04-02 17:41:05 - r - INFO: - Episode: 1190/2000, Reward: 186.024, Step: 500 +2023-04-02 17:41:10 - r - INFO: - Episode: 1191/2000, Reward: 113.067, Step: 500 +2023-04-02 17:41:11 - r - INFO: - Episode: 1192/2000, Reward: 173.953, Step: 500 +2023-04-02 17:41:13 - r - INFO: - Episode: 1193/2000, Reward: 158.378, Step: 500 +2023-04-02 17:41:15 - r - INFO: - Episode: 1194/2000, Reward: 155.998, Step: 500 +2023-04-02 17:41:17 - r - INFO: - Episode: 1195/2000, Reward: 163.770, Step: 500 +2023-04-02 17:41:23 - r - INFO: - Episode: 1196/2000, Reward: 152.690, Step: 500 +2023-04-02 17:41:25 - r - INFO: - Episode: 1197/2000, Reward: 70.567, Step: 500 +2023-04-02 17:41:26 - r - INFO: - Episode: 1198/2000, Reward: 178.096, Step: 500 +2023-04-02 17:41:28 - r - INFO: - Episode: 1199/2000, Reward: 183.259, Step: 500 +2023-04-02 17:41:30 - r - INFO: - Episode: 1200/2000, Reward: 138.459, Step: 500 +2023-04-02 17:41:36 - r - INFO: - Episode: 1201/2000, Reward: 127.479, Step: 500 +2023-04-02 17:41:38 - r - INFO: - Episode: 1202/2000, Reward: 184.172, Step: 500 +2023-04-02 17:41:39 - r - INFO: - Episode: 1203/2000, Reward: 90.241, Step: 500 +2023-04-02 17:41:41 - r - INFO: - Episode: 1204/2000, Reward: 180.416, Step: 500 +2023-04-02 17:41:43 - r - INFO: - Episode: 1205/2000, Reward: 145.855, Step: 500 +2023-04-02 17:41:46 - r - INFO: - Episode: 1206/2000, Reward: 235.454, Step: 370 +2023-04-02 17:41:47 - r - INFO: - Episode: 1207/2000, Reward: 156.513, Step: 500 +2023-04-02 17:41:49 - r - INFO: - Episode: 1208/2000, Reward: 122.444, Step: 500 +2023-04-02 17:41:51 - r - INFO: - Episode: 1209/2000, Reward: 159.142, Step: 500 +2023-04-02 17:41:52 - r - INFO: - Episode: 1210/2000, Reward: 167.363, Step: 500 +2023-04-02 17:41:57 - r - INFO: - Episode: 1211/2000, Reward: 170.874, Step: 500 +2023-04-02 17:41:59 - r - INFO: - Episode: 1212/2000, Reward: 117.087, Step: 500 +2023-04-02 17:42:01 - r - INFO: - Episode: 1213/2000, Reward: 145.857, Step: 500 +2023-04-02 17:42:02 - r - INFO: - Episode: 1214/2000, Reward: 149.083, Step: 500 +2023-04-02 17:42:03 - r - INFO: - Episode: 1215/2000, Reward: 54.686, Step: 97 +2023-04-02 17:42:08 - r - INFO: - Episode: 1216/2000, Reward: 156.117, Step: 500 +2023-04-02 17:42:10 - r - INFO: - Episode: 1217/2000, Reward: 188.935, Step: 500 +2023-04-02 17:42:11 - r - INFO: - Episode: 1218/2000, Reward: 98.865, Step: 500 +2023-04-02 17:42:13 - r - INFO: - Episode: 1219/2000, Reward: 94.269, Step: 500 +2023-04-02 17:42:15 - r - INFO: - Episode: 1220/2000, Reward: 94.905, Step: 500 +2023-04-02 17:42:20 - r - INFO: - Episode: 1221/2000, Reward: 72.687, Step: 500 +2023-04-02 17:42:21 - r - INFO: - Episode: 1222/2000, Reward: 85.988, Step: 500 +2023-04-02 17:42:23 - r - INFO: - Episode: 1223/2000, Reward: 81.263, Step: 500 +2023-04-02 17:42:25 - r - INFO: - Episode: 1224/2000, Reward: 112.453, Step: 500 +2023-04-02 17:42:26 - r - INFO: - Episode: 1225/2000, Reward: 208.577, Step: 339 +2023-04-02 17:42:30 - r - INFO: - Episode: 1226/2000, Reward: 71.908, Step: 500 +2023-04-02 17:42:32 - r - INFO: - Episode: 1227/2000, Reward: 77.995, Step: 500 +2023-04-02 17:42:34 - r - INFO: - Episode: 1228/2000, Reward: 132.131, Step: 500 +2023-04-02 17:42:36 - r - INFO: - Episode: 1229/2000, Reward: 37.142, Step: 500 +2023-04-02 17:42:37 - r - INFO: - Episode: 1230/2000, Reward: 149.749, Step: 500 +2023-04-02 17:42:42 - r - INFO: - Episode: 1231/2000, Reward: 240.923, Step: 245 +2023-04-02 17:42:43 - r - INFO: - Episode: 1232/2000, Reward: 268.596, Step: 383 +2023-04-02 17:42:44 - r - INFO: - Episode: 1233/2000, Reward: 250.738, Step: 263 +2023-04-02 17:42:46 - r - INFO: - Episode: 1234/2000, Reward: 141.810, Step: 500 +2023-04-02 17:42:47 - r - INFO: - Episode: 1235/2000, Reward: 141.225, Step: 500 +2023-04-02 17:42:53 - r - INFO: - Episode: 1236/2000, Reward: 85.074, Step: 500 +2023-04-02 17:42:55 - r - INFO: - Episode: 1237/2000, Reward: 109.810, Step: 500 +2023-04-02 17:42:56 - r - INFO: - Episode: 1238/2000, Reward: 188.897, Step: 500 +2023-04-02 17:42:58 - r - INFO: - Episode: 1239/2000, Reward: 259.421, Step: 486 +2023-04-02 17:43:00 - r - INFO: - Episode: 1240/2000, Reward: 95.789, Step: 500 +2023-04-02 17:43:05 - r - INFO: - Episode: 1241/2000, Reward: 269.054, Step: 402 +2023-04-02 17:43:06 - r - INFO: - Episode: 1242/2000, Reward: 243.106, Step: 410 +2023-04-02 17:43:08 - r - INFO: - Episode: 1243/2000, Reward: 140.128, Step: 500 +2023-04-02 17:43:09 - r - INFO: - Episode: 1244/2000, Reward: 134.958, Step: 500 +2023-04-02 17:43:11 - r - INFO: - Episode: 1245/2000, Reward: 105.355, Step: 500 +2023-04-02 17:43:18 - r - INFO: - Episode: 1246/2000, Reward: 77.880, Step: 500 +2023-04-02 17:43:20 - r - INFO: - Episode: 1247/2000, Reward: 165.125, Step: 500 +2023-04-02 17:43:21 - r - INFO: - Episode: 1248/2000, Reward: 206.350, Step: 500 +2023-04-02 17:43:23 - r - INFO: - Episode: 1249/2000, Reward: 168.429, Step: 500 +2023-04-02 17:43:25 - r - INFO: - Episode: 1250/2000, Reward: 84.374, Step: 500 +2023-04-02 17:43:30 - r - INFO: - Episode: 1251/2000, Reward: 263.253, Step: 362 +2023-04-02 17:43:31 - r - INFO: - Episode: 1252/2000, Reward: 124.819, Step: 500 +2023-04-02 17:43:33 - r - INFO: - Episode: 1253/2000, Reward: 150.899, Step: 500 +2023-04-02 17:43:35 - r - INFO: - Episode: 1254/2000, Reward: 162.644, Step: 500 +2023-04-02 17:43:36 - r - INFO: - Episode: 1255/2000, Reward: 162.676, Step: 500 +2023-04-02 17:43:39 - r - INFO: - Episode: 1256/2000, Reward: 167.083, Step: 500 +2023-04-02 17:43:41 - r - INFO: - Episode: 1257/2000, Reward: 149.358, Step: 500 +2023-04-02 17:43:43 - r - INFO: - Episode: 1258/2000, Reward: 158.817, Step: 500 +2023-04-02 17:43:44 - r - INFO: - Episode: 1259/2000, Reward: 152.741, Step: 500 +2023-04-02 17:43:46 - r - INFO: - Episode: 1260/2000, Reward: 101.432, Step: 500 +2023-04-02 17:43:50 - r - INFO: - Episode: 1261/2000, Reward: 87.130, Step: 500 +2023-04-02 17:43:51 - r - INFO: - Episode: 1262/2000, Reward: 138.959, Step: 500 +2023-04-02 17:43:53 - r - INFO: - Episode: 1263/2000, Reward: 152.794, Step: 500 +2023-04-02 17:43:55 - r - INFO: - Episode: 1264/2000, Reward: 139.968, Step: 500 +2023-04-02 17:43:56 - r - INFO: - Episode: 1265/2000, Reward: 97.061, Step: 500 +2023-04-02 17:43:59 - r - INFO: - Episode: 1266/2000, Reward: 151.447, Step: 500 +2023-04-02 17:44:01 - r - INFO: - Episode: 1267/2000, Reward: 142.344, Step: 500 +2023-04-02 17:44:03 - r - INFO: - Episode: 1268/2000, Reward: 165.640, Step: 500 +2023-04-02 17:44:04 - r - INFO: - Episode: 1269/2000, Reward: 169.712, Step: 500 +2023-04-02 17:44:06 - r - INFO: - Episode: 1270/2000, Reward: 173.098, Step: 500 +2023-04-02 17:44:11 - r - INFO: - Episode: 1271/2000, Reward: 167.294, Step: 500 +2023-04-02 17:44:12 - r - INFO: - Episode: 1272/2000, Reward: 235.788, Step: 437 +2023-04-02 17:44:14 - r - INFO: - Episode: 1273/2000, Reward: 153.693, Step: 500 +2023-04-02 17:44:14 - r - INFO: - Episode: 1274/2000, Reward: 0.690, Step: 119 +2023-04-02 17:44:16 - r - INFO: - Episode: 1275/2000, Reward: 183.330, Step: 500 +2023-04-02 17:44:19 - r - INFO: - Episode: 1276/2000, Reward: 157.130, Step: 500 +2023-04-02 17:44:20 - r - INFO: - Episode: 1277/2000, Reward: 203.678, Step: 500 +2023-04-02 17:44:22 - r - INFO: - Episode: 1278/2000, Reward: 160.707, Step: 500 +2023-04-02 17:44:24 - r - INFO: - Episode: 1279/2000, Reward: 167.325, Step: 500 +2023-04-02 17:44:25 - r - INFO: - Episode: 1280/2000, Reward: 170.344, Step: 500 +2023-04-02 17:44:29 - r - INFO: - Episode: 1281/2000, Reward: 196.588, Step: 500 +2023-04-02 17:44:31 - r - INFO: - Episode: 1282/2000, Reward: 138.214, Step: 500 +2023-04-02 17:44:33 - r - INFO: - Episode: 1283/2000, Reward: 135.387, Step: 500 +2023-04-02 17:44:35 - r - INFO: - Episode: 1284/2000, Reward: 99.431, Step: 500 +2023-04-02 17:44:36 - r - INFO: - Episode: 1285/2000, Reward: 175.205, Step: 500 +2023-04-02 17:44:40 - r - INFO: - Episode: 1286/2000, Reward: 178.080, Step: 500 +2023-04-02 17:44:42 - r - INFO: - Episode: 1287/2000, Reward: 144.115, Step: 500 +2023-04-02 17:44:43 - r - INFO: - Episode: 1288/2000, Reward: 160.716, Step: 500 +2023-04-02 17:44:45 - r - INFO: - Episode: 1289/2000, Reward: 154.906, Step: 500 +2023-04-02 17:44:47 - r - INFO: - Episode: 1290/2000, Reward: 160.290, Step: 500 +2023-04-02 17:44:49 - r - INFO: - Episode: 1291/2000, Reward: 62.903, Step: 185 +2023-04-02 17:44:50 - r - INFO: - Episode: 1292/2000, Reward: 191.615, Step: 500 +2023-04-02 17:44:52 - r - INFO: - Episode: 1293/2000, Reward: 185.429, Step: 500 +2023-04-02 17:44:53 - r - INFO: - Episode: 1294/2000, Reward: 148.850, Step: 500 +2023-04-02 17:44:55 - r - INFO: - Episode: 1295/2000, Reward: 153.101, Step: 500 +2023-04-02 17:44:58 - r - INFO: - Episode: 1296/2000, Reward: 166.471, Step: 500 +2023-04-02 17:45:00 - r - INFO: - Episode: 1297/2000, Reward: 113.925, Step: 500 +2023-04-02 17:45:02 - r - INFO: - Episode: 1298/2000, Reward: 142.432, Step: 500 +2023-04-02 17:45:04 - r - INFO: - Episode: 1299/2000, Reward: 189.822, Step: 500 +2023-04-02 17:45:05 - r - INFO: - Episode: 1300/2000, Reward: 177.857, Step: 500 +2023-04-02 17:45:08 - r - INFO: - Episode: 1301/2000, Reward: 177.271, Step: 500 +2023-04-02 17:45:09 - r - INFO: - Episode: 1302/2000, Reward: 176.168, Step: 500 +2023-04-02 17:45:11 - r - INFO: - Episode: 1303/2000, Reward: 156.805, Step: 500 +2023-04-02 17:45:13 - r - INFO: - Episode: 1304/2000, Reward: 152.077, Step: 500 +2023-04-02 17:45:14 - r - INFO: - Episode: 1305/2000, Reward: 176.287, Step: 500 +2023-04-02 17:45:17 - r - INFO: - Episode: 1306/2000, Reward: 147.637, Step: 500 +2023-04-02 17:45:19 - r - INFO: - Episode: 1307/2000, Reward: 173.574, Step: 500 +2023-04-02 17:45:21 - r - INFO: - Episode: 1308/2000, Reward: 161.945, Step: 500 +2023-04-02 17:45:22 - r - INFO: - Episode: 1309/2000, Reward: 183.707, Step: 500 +2023-04-02 17:45:23 - r - INFO: - Episode: 1310/2000, Reward: 290.132, Step: 208 +2023-04-02 17:45:26 - r - INFO: - Episode: 1311/2000, Reward: 191.501, Step: 500 +2023-04-02 17:45:27 - r - INFO: - Episode: 1312/2000, Reward: 164.635, Step: 500 +2023-04-02 17:45:29 - r - INFO: - Episode: 1313/2000, Reward: 193.185, Step: 500 +2023-04-02 17:45:31 - r - INFO: - Episode: 1314/2000, Reward: 177.105, Step: 500 +2023-04-02 17:45:32 - r - INFO: - Episode: 1315/2000, Reward: 136.990, Step: 500 +2023-04-02 17:45:36 - r - INFO: - Episode: 1316/2000, Reward: 151.840, Step: 500 +2023-04-02 17:45:38 - r - INFO: - Episode: 1317/2000, Reward: 171.022, Step: 500 +2023-04-02 17:45:39 - r - INFO: - Episode: 1318/2000, Reward: 179.947, Step: 500 +2023-04-02 17:45:41 - r - INFO: - Episode: 1319/2000, Reward: 182.342, Step: 500 +2023-04-02 17:45:43 - r - INFO: - Episode: 1320/2000, Reward: 175.897, Step: 500 +2023-04-02 17:45:45 - r - INFO: - Episode: 1321/2000, Reward: 174.352, Step: 500 +2023-04-02 17:45:47 - r - INFO: - Episode: 1322/2000, Reward: 193.203, Step: 500 +2023-04-02 17:45:49 - r - INFO: - Episode: 1323/2000, Reward: 167.973, Step: 500 +2023-04-02 17:45:49 - r - INFO: - Episode: 1324/2000, Reward: 72.734, Step: 105 +2023-04-02 17:45:50 - r - INFO: - Episode: 1325/2000, Reward: 182.584, Step: 500 +2023-04-02 17:45:52 - r - INFO: - Episode: 1326/2000, Reward: 34.199, Step: 101 +2023-04-02 17:45:53 - r - INFO: - Episode: 1327/2000, Reward: 206.361, Step: 500 +2023-04-02 17:45:55 - r - INFO: - Episode: 1328/2000, Reward: 166.399, Step: 500 +2023-04-02 17:45:55 - r - INFO: - Episode: 1329/2000, Reward: 17.409, Step: 120 +2023-04-02 17:45:57 - r - INFO: - Episode: 1330/2000, Reward: 195.660, Step: 500 +2023-04-02 17:46:00 - r - INFO: - Episode: 1331/2000, Reward: 165.263, Step: 500 +2023-04-02 17:46:02 - r - INFO: - Episode: 1332/2000, Reward: 184.225, Step: 500 +2023-04-02 17:46:03 - r - INFO: - Episode: 1333/2000, Reward: 159.056, Step: 500 +2023-04-02 17:46:05 - r - INFO: - Episode: 1334/2000, Reward: 185.419, Step: 500 +2023-04-02 17:46:06 - r - INFO: - Episode: 1335/2000, Reward: 160.004, Step: 500 +2023-04-02 17:46:09 - r - INFO: - Episode: 1336/2000, Reward: 162.423, Step: 500 +2023-04-02 17:46:11 - r - INFO: - Episode: 1337/2000, Reward: 142.442, Step: 500 +2023-04-02 17:46:13 - r - INFO: - Episode: 1338/2000, Reward: 158.439, Step: 500 +2023-04-02 17:46:14 - r - INFO: - Episode: 1339/2000, Reward: 175.065, Step: 500 +2023-04-02 17:46:16 - r - INFO: - Episode: 1340/2000, Reward: 209.095, Step: 500 +2023-04-02 17:46:19 - r - INFO: - Episode: 1341/2000, Reward: 186.046, Step: 500 +2023-04-02 17:46:19 - r - INFO: - Episode: 1342/2000, Reward: 68.520, Step: 89 +2023-04-02 17:46:21 - r - INFO: - Episode: 1343/2000, Reward: 187.935, Step: 500 +2023-04-02 17:46:21 - r - INFO: - Episode: 1344/2000, Reward: 29.421, Step: 112 +2023-04-02 17:46:22 - r - INFO: - Episode: 1345/2000, Reward: 269.824, Step: 421 +2023-04-02 17:46:25 - r - INFO: - Episode: 1346/2000, Reward: 166.662, Step: 500 +2023-04-02 17:46:27 - r - INFO: - Episode: 1347/2000, Reward: 201.283, Step: 500 +2023-04-02 17:46:28 - r - INFO: - Episode: 1348/2000, Reward: 256.846, Step: 475 +2023-04-02 17:46:30 - r - INFO: - Episode: 1349/2000, Reward: 145.378, Step: 500 +2023-04-02 17:46:30 - r - INFO: - Episode: 1350/2000, Reward: 39.482, Step: 168 +2023-04-02 17:46:32 - r - INFO: - Episode: 1351/2000, Reward: 288.251, Step: 286 +2023-04-02 17:46:34 - r - INFO: - Episode: 1352/2000, Reward: 170.464, Step: 500 +2023-04-02 17:46:35 - r - INFO: - Episode: 1353/2000, Reward: 172.216, Step: 500 +2023-04-02 17:46:37 - r - INFO: - Episode: 1354/2000, Reward: 170.642, Step: 500 +2023-04-02 17:46:38 - r - INFO: - Episode: 1355/2000, Reward: 262.935, Step: 390 +2023-04-02 17:46:41 - r - INFO: - Episode: 1356/2000, Reward: 158.680, Step: 500 +2023-04-02 17:46:43 - r - INFO: - Episode: 1357/2000, Reward: 159.873, Step: 500 +2023-04-02 17:46:44 - r - INFO: - Episode: 1358/2000, Reward: 157.179, Step: 500 +2023-04-02 17:46:46 - r - INFO: - Episode: 1359/2000, Reward: 160.195, Step: 500 +2023-04-02 17:46:47 - r - INFO: - Episode: 1360/2000, Reward: 149.022, Step: 500 +2023-04-02 17:46:50 - r - INFO: - Episode: 1361/2000, Reward: 159.186, Step: 500 +2023-04-02 17:46:51 - r - INFO: - Episode: 1362/2000, Reward: 287.863, Step: 317 +2023-04-02 17:46:53 - r - INFO: - Episode: 1363/2000, Reward: 180.563, Step: 500 +2023-04-02 17:46:54 - r - INFO: - Episode: 1364/2000, Reward: 109.404, Step: 500 +2023-04-02 17:46:56 - r - INFO: - Episode: 1365/2000, Reward: 265.565, Step: 400 +2023-04-02 17:46:59 - r - INFO: - Episode: 1366/2000, Reward: 244.588, Step: 472 +2023-04-02 17:47:01 - r - INFO: - Episode: 1367/2000, Reward: 134.947, Step: 500 +2023-04-02 17:47:02 - r - INFO: - Episode: 1368/2000, Reward: 142.878, Step: 500 +2023-04-02 17:47:04 - r - INFO: - Episode: 1369/2000, Reward: 248.503, Step: 477 +2023-04-02 17:47:06 - r - INFO: - Episode: 1370/2000, Reward: 148.096, Step: 500 +2023-04-02 17:47:09 - r - INFO: - Episode: 1371/2000, Reward: 246.353, Step: 282 +2023-04-02 17:47:10 - r - INFO: - Episode: 1372/2000, Reward: 188.405, Step: 500 +2023-04-02 17:47:12 - r - INFO: - Episode: 1373/2000, Reward: 167.211, Step: 500 +2023-04-02 17:47:14 - r - INFO: - Episode: 1374/2000, Reward: 138.966, Step: 500 +2023-04-02 17:47:16 - r - INFO: - Episode: 1375/2000, Reward: 115.428, Step: 500 +2023-04-02 17:47:19 - r - INFO: - Episode: 1376/2000, Reward: 225.605, Step: 333 +2023-04-02 17:47:20 - r - INFO: - Episode: 1377/2000, Reward: 75.100, Step: 500 +2023-04-02 17:47:22 - r - INFO: - Episode: 1378/2000, Reward: 171.631, Step: 500 +2023-04-02 17:47:23 - r - INFO: - Episode: 1379/2000, Reward: 277.781, Step: 359 +2023-04-02 17:47:24 - r - INFO: - Episode: 1380/2000, Reward: 281.211, Step: 230 +2023-04-02 17:47:28 - r - INFO: - Episode: 1381/2000, Reward: 159.957, Step: 500 +2023-04-02 17:47:30 - r - INFO: - Episode: 1382/2000, Reward: 77.420, Step: 500 +2023-04-02 17:47:31 - r - INFO: - Episode: 1383/2000, Reward: 263.979, Step: 237 +2023-04-02 17:47:32 - r - INFO: - Episode: 1384/2000, Reward: 235.847, Step: 464 +2023-04-02 17:47:33 - r - INFO: - Episode: 1385/2000, Reward: 239.454, Step: 340 +2023-04-02 17:47:36 - r - INFO: - Episode: 1386/2000, Reward: 160.009, Step: 500 +2023-04-02 17:47:38 - r - INFO: - Episode: 1387/2000, Reward: 163.490, Step: 500 +2023-04-02 17:47:40 - r - INFO: - Episode: 1388/2000, Reward: 270.877, Step: 419 +2023-04-02 17:47:41 - r - INFO: - Episode: 1389/2000, Reward: -169.019, Step: 389 +2023-04-02 17:47:42 - r - INFO: - Episode: 1390/2000, Reward: -149.658, Step: 419 +2023-04-02 17:47:45 - r - INFO: - Episode: 1391/2000, Reward: 203.292, Step: 500 +2023-04-02 17:47:47 - r - INFO: - Episode: 1392/2000, Reward: 152.405, Step: 500 +2023-04-02 17:47:47 - r - INFO: - Episode: 1393/2000, Reward: 28.149, Step: 120 +2023-04-02 17:47:49 - r - INFO: - Episode: 1394/2000, Reward: 179.677, Step: 500 +2023-04-02 17:47:50 - r - INFO: - Episode: 1395/2000, Reward: 158.915, Step: 500 +2023-04-02 17:47:54 - r - INFO: - Episode: 1396/2000, Reward: 172.779, Step: 500 +2023-04-02 17:47:55 - r - INFO: - Episode: 1397/2000, Reward: 176.154, Step: 500 +2023-04-02 17:47:57 - r - INFO: - Episode: 1398/2000, Reward: 170.595, Step: 500 +2023-04-02 17:47:59 - r - INFO: - Episode: 1399/2000, Reward: 122.235, Step: 500 +2023-04-02 17:48:00 - r - INFO: - Episode: 1400/2000, Reward: 172.671, Step: 500 +2023-04-02 17:48:04 - r - INFO: - Episode: 1401/2000, Reward: 147.412, Step: 500 +2023-04-02 17:48:05 - r - INFO: - Episode: 1402/2000, Reward: 239.050, Step: 272 +2023-04-02 17:48:06 - r - INFO: - Episode: 1403/2000, Reward: 241.072, Step: 371 +2023-04-02 17:48:08 - r - INFO: - Episode: 1404/2000, Reward: 138.645, Step: 500 +2023-04-02 17:48:09 - r - INFO: - Episode: 1405/2000, Reward: 242.967, Step: 458 +2023-04-02 17:48:12 - r - INFO: - Episode: 1406/2000, Reward: 176.487, Step: 500 +2023-04-02 17:48:13 - r - INFO: - Episode: 1407/2000, Reward: 163.541, Step: 500 +2023-04-02 17:48:15 - r - INFO: - Episode: 1408/2000, Reward: 148.696, Step: 500 +2023-04-02 17:48:17 - r - INFO: - Episode: 1409/2000, Reward: 163.794, Step: 500 +2023-04-02 17:48:18 - r - INFO: - Episode: 1410/2000, Reward: 174.072, Step: 500 +2023-04-02 17:48:22 - r - INFO: - Episode: 1411/2000, Reward: 199.271, Step: 500 +2023-04-02 17:48:23 - r - INFO: - Episode: 1412/2000, Reward: 175.115, Step: 500 +2023-04-02 17:48:25 - r - INFO: - Episode: 1413/2000, Reward: 173.692, Step: 500 +2023-04-02 17:48:26 - r - INFO: - Episode: 1414/2000, Reward: 272.252, Step: 315 +2023-04-02 17:48:28 - r - INFO: - Episode: 1415/2000, Reward: 153.087, Step: 500 +2023-04-02 17:48:32 - r - INFO: - Episode: 1416/2000, Reward: 172.008, Step: 500 +2023-04-02 17:48:34 - r - INFO: - Episode: 1417/2000, Reward: 81.994, Step: 500 +2023-04-02 17:48:35 - r - INFO: - Episode: 1418/2000, Reward: 175.501, Step: 500 +2023-04-02 17:48:37 - r - INFO: - Episode: 1419/2000, Reward: 178.412, Step: 500 +2023-04-02 17:48:39 - r - INFO: - Episode: 1420/2000, Reward: 193.365, Step: 500 +2023-04-02 17:48:41 - r - INFO: - Episode: 1421/2000, Reward: 264.565, Step: 237 +2023-04-02 17:48:42 - r - INFO: - Episode: 1422/2000, Reward: 163.526, Step: 500 +2023-04-02 17:48:44 - r - INFO: - Episode: 1423/2000, Reward: 177.584, Step: 500 +2023-04-02 17:48:45 - r - INFO: - Episode: 1424/2000, Reward: 264.544, Step: 287 +2023-04-02 17:48:46 - r - INFO: - Episode: 1425/2000, Reward: 302.510, Step: 439 +2023-04-02 17:48:50 - r - INFO: - Episode: 1426/2000, Reward: 148.903, Step: 500 +2023-04-02 17:48:52 - r - INFO: - Episode: 1427/2000, Reward: 162.556, Step: 500 +2023-04-02 17:48:53 - r - INFO: - Episode: 1428/2000, Reward: 265.088, Step: 448 +2023-04-02 17:48:55 - r - INFO: - Episode: 1429/2000, Reward: 260.662, Step: 427 +2023-04-02 17:48:56 - r - INFO: - Episode: 1430/2000, Reward: 171.946, Step: 500 +2023-04-02 17:49:02 - r - INFO: - Episode: 1431/2000, Reward: 111.226, Step: 500 +2023-04-02 17:49:03 - r - INFO: - Episode: 1432/2000, Reward: 228.726, Step: 471 +2023-04-02 17:49:05 - r - INFO: - Episode: 1433/2000, Reward: 163.916, Step: 500 +2023-04-02 17:49:07 - r - INFO: - Episode: 1434/2000, Reward: 147.293, Step: 500 +2023-04-02 17:49:08 - r - INFO: - Episode: 1435/2000, Reward: 141.017, Step: 500 +2023-04-02 17:49:14 - r - INFO: - Episode: 1436/2000, Reward: 100.285, Step: 500 +2023-04-02 17:49:15 - r - INFO: - Episode: 1437/2000, Reward: 125.923, Step: 500 +2023-04-02 17:49:16 - r - INFO: - Episode: 1438/2000, Reward: 252.563, Step: 397 +2023-04-02 17:49:18 - r - INFO: - Episode: 1439/2000, Reward: 92.732, Step: 500 +2023-04-02 17:49:20 - r - INFO: - Episode: 1440/2000, Reward: 144.810, Step: 500 +2023-04-02 17:49:26 - r - INFO: - Episode: 1441/2000, Reward: 173.472, Step: 500 +2023-04-02 17:49:27 - r - INFO: - Episode: 1442/2000, Reward: 148.279, Step: 500 +2023-04-02 17:49:29 - r - INFO: - Episode: 1443/2000, Reward: 115.538, Step: 500 +2023-04-02 17:49:31 - r - INFO: - Episode: 1444/2000, Reward: 181.985, Step: 500 +2023-04-02 17:49:33 - r - INFO: - Episode: 1445/2000, Reward: 113.001, Step: 500 +2023-04-02 17:49:38 - r - INFO: - Episode: 1446/2000, Reward: 141.357, Step: 500 +2023-04-02 17:49:39 - r - INFO: - Episode: 1447/2000, Reward: 239.632, Step: 500 +2023-04-02 17:49:41 - r - INFO: - Episode: 1448/2000, Reward: 155.704, Step: 500 +2023-04-02 17:49:43 - r - INFO: - Episode: 1449/2000, Reward: 153.911, Step: 500 +2023-04-02 17:49:44 - r - INFO: - Episode: 1450/2000, Reward: 145.376, Step: 500 +2023-04-02 17:49:50 - r - INFO: - Episode: 1451/2000, Reward: 142.364, Step: 500 +2023-04-02 17:49:52 - r - INFO: - Episode: 1452/2000, Reward: 167.891, Step: 500 +2023-04-02 17:49:54 - r - INFO: - Episode: 1453/2000, Reward: 143.955, Step: 500 +2023-04-02 17:49:56 - r - INFO: - Episode: 1454/2000, Reward: 180.379, Step: 500 +2023-04-02 17:49:57 - r - INFO: - Episode: 1455/2000, Reward: 130.874, Step: 500 +2023-04-02 17:50:01 - r - INFO: - Episode: 1456/2000, Reward: 123.120, Step: 500 +2023-04-02 17:50:02 - r - INFO: - Episode: 1457/2000, Reward: 164.369, Step: 500 +2023-04-02 17:50:03 - r - INFO: - Episode: 1458/2000, Reward: 233.703, Step: 322 +2023-04-02 17:50:05 - r - INFO: - Episode: 1459/2000, Reward: 143.107, Step: 500 +2023-04-02 17:50:07 - r - INFO: - Episode: 1460/2000, Reward: 160.406, Step: 500 +2023-04-02 17:50:11 - r - INFO: - Episode: 1461/2000, Reward: 87.888, Step: 500 +2023-04-02 17:50:13 - r - INFO: - Episode: 1462/2000, Reward: 180.026, Step: 500 +2023-04-02 17:50:15 - r - INFO: - Episode: 1463/2000, Reward: 127.501, Step: 500 +2023-04-02 17:50:17 - r - INFO: - Episode: 1464/2000, Reward: 165.281, Step: 500 +2023-04-02 17:50:18 - r - INFO: - Episode: 1465/2000, Reward: 152.744, Step: 500 +2023-04-02 17:50:23 - r - INFO: - Episode: 1466/2000, Reward: 99.813, Step: 500 +2023-04-02 17:50:25 - r - INFO: - Episode: 1467/2000, Reward: 62.346, Step: 500 +2023-04-02 17:50:27 - r - INFO: - Episode: 1468/2000, Reward: 180.240, Step: 500 +2023-04-02 17:50:28 - r - INFO: - Episode: 1469/2000, Reward: 154.080, Step: 500 +2023-04-02 17:50:30 - r - INFO: - Episode: 1470/2000, Reward: 168.433, Step: 500 +2023-04-02 17:50:34 - r - INFO: - Episode: 1471/2000, Reward: 140.522, Step: 500 +2023-04-02 17:50:36 - r - INFO: - Episode: 1472/2000, Reward: 179.114, Step: 500 +2023-04-02 17:50:38 - r - INFO: - Episode: 1473/2000, Reward: 160.097, Step: 500 +2023-04-02 17:50:40 - r - INFO: - Episode: 1474/2000, Reward: 181.557, Step: 500 +2023-04-02 17:50:42 - r - INFO: - Episode: 1475/2000, Reward: 139.528, Step: 500 +2023-04-02 17:50:46 - r - INFO: - Episode: 1476/2000, Reward: 147.358, Step: 500 +2023-04-02 17:50:48 - r - INFO: - Episode: 1477/2000, Reward: 170.898, Step: 500 +2023-04-02 17:50:50 - r - INFO: - Episode: 1478/2000, Reward: 141.257, Step: 500 +2023-04-02 17:50:51 - r - INFO: - Episode: 1479/2000, Reward: 271.105, Step: 324 +2023-04-02 17:50:53 - r - INFO: - Episode: 1480/2000, Reward: 194.653, Step: 500 +2023-04-02 17:50:56 - r - INFO: - Episode: 1481/2000, Reward: 197.728, Step: 500 +2023-04-02 17:50:58 - r - INFO: - Episode: 1482/2000, Reward: 154.363, Step: 500 +2023-04-02 17:50:59 - r - INFO: - Episode: 1483/2000, Reward: 108.326, Step: 500 +2023-04-02 17:51:01 - r - INFO: - Episode: 1484/2000, Reward: 184.293, Step: 500 +2023-04-02 17:51:03 - r - INFO: - Episode: 1485/2000, Reward: 198.835, Step: 500 +2023-04-02 17:51:08 - r - INFO: - Episode: 1486/2000, Reward: 247.366, Step: 418 +2023-04-02 17:51:10 - r - INFO: - Episode: 1487/2000, Reward: 191.860, Step: 500 +2023-04-02 17:51:12 - r - INFO: - Episode: 1488/2000, Reward: 137.711, Step: 500 +2023-04-02 17:51:13 - r - INFO: - Episode: 1489/2000, Reward: 145.699, Step: 500 +2023-04-02 17:51:15 - r - INFO: - Episode: 1490/2000, Reward: 146.479, Step: 500 +2023-04-02 17:51:19 - r - INFO: - Episode: 1491/2000, Reward: 161.982, Step: 500 +2023-04-02 17:51:20 - r - INFO: - Episode: 1492/2000, Reward: 143.884, Step: 500 +2023-04-02 17:51:22 - r - INFO: - Episode: 1493/2000, Reward: 183.709, Step: 500 +2023-04-02 17:51:24 - r - INFO: - Episode: 1494/2000, Reward: 269.529, Step: 491 +2023-04-02 17:51:25 - r - INFO: - Episode: 1495/2000, Reward: 160.461, Step: 500 +2023-04-02 17:51:30 - r - INFO: - Episode: 1496/2000, Reward: 164.515, Step: 500 +2023-04-02 17:51:32 - r - INFO: - Episode: 1497/2000, Reward: 159.015, Step: 500 +2023-04-02 17:51:33 - r - INFO: - Episode: 1498/2000, Reward: 157.130, Step: 500 +2023-04-02 17:51:35 - r - INFO: - Episode: 1499/2000, Reward: 149.830, Step: 500 +2023-04-02 17:51:36 - r - INFO: - Episode: 1500/2000, Reward: 256.634, Step: 418 +2023-04-02 17:51:41 - r - INFO: - Episode: 1501/2000, Reward: 297.830, Step: 473 +2023-04-02 17:51:43 - r - INFO: - Episode: 1502/2000, Reward: 134.635, Step: 500 +2023-04-02 17:51:44 - r - INFO: - Episode: 1503/2000, Reward: 287.946, Step: 445 +2023-04-02 17:51:45 - r - INFO: - Episode: 1504/2000, Reward: 257.349, Step: 442 +2023-04-02 17:51:47 - r - INFO: - Episode: 1505/2000, Reward: 136.070, Step: 500 +2023-04-02 17:51:51 - r - INFO: - Episode: 1506/2000, Reward: 165.582, Step: 500 +2023-04-02 17:51:53 - r - INFO: - Episode: 1507/2000, Reward: 141.830, Step: 500 +2023-04-02 17:51:55 - r - INFO: - Episode: 1508/2000, Reward: 297.612, Step: 425 +2023-04-02 17:51:56 - r - INFO: - Episode: 1509/2000, Reward: 260.675, Step: 403 +2023-04-02 17:51:58 - r - INFO: - Episode: 1510/2000, Reward: 167.450, Step: 500 +2023-04-02 17:52:02 - r - INFO: - Episode: 1511/2000, Reward: 308.962, Step: 257 +2023-04-02 17:52:03 - r - INFO: - Episode: 1512/2000, Reward: 260.136, Step: 173 +2023-04-02 17:52:05 - r - INFO: - Episode: 1513/2000, Reward: 174.980, Step: 500 +2023-04-02 17:52:06 - r - INFO: - Episode: 1514/2000, Reward: 226.258, Step: 482 +2023-04-02 17:52:08 - r - INFO: - Episode: 1515/2000, Reward: 231.533, Step: 404 +2023-04-02 17:52:12 - r - INFO: - Episode: 1516/2000, Reward: 251.236, Step: 441 +2023-04-02 17:52:14 - r - INFO: - Episode: 1517/2000, Reward: 125.375, Step: 500 +2023-04-02 17:52:15 - r - INFO: - Episode: 1518/2000, Reward: 187.719, Step: 500 +2023-04-02 17:52:17 - r - INFO: - Episode: 1519/2000, Reward: 254.037, Step: 411 +2023-04-02 17:52:18 - r - INFO: - Episode: 1520/2000, Reward: 154.291, Step: 500 +2023-04-02 17:52:21 - r - INFO: - Episode: 1521/2000, Reward: 255.917, Step: 301 +2023-04-02 17:52:23 - r - INFO: - Episode: 1522/2000, Reward: 160.472, Step: 500 +2023-04-02 17:52:25 - r - INFO: - Episode: 1523/2000, Reward: 126.540, Step: 500 +2023-04-02 17:52:26 - r - INFO: - Episode: 1524/2000, Reward: 270.497, Step: 390 +2023-04-02 17:52:27 - r - INFO: - Episode: 1525/2000, Reward: 248.310, Step: 296 +2023-04-02 17:52:32 - r - INFO: - Episode: 1526/2000, Reward: 195.446, Step: 500 +2023-04-02 17:52:33 - r - INFO: - Episode: 1527/2000, Reward: 245.962, Step: 474 +2023-04-02 17:52:35 - r - INFO: - Episode: 1528/2000, Reward: 254.262, Step: 455 +2023-04-02 17:52:36 - r - INFO: - Episode: 1529/2000, Reward: 241.758, Step: 245 +2023-04-02 17:52:37 - r - INFO: - Episode: 1530/2000, Reward: 274.820, Step: 348 +2023-04-02 17:52:39 - r - INFO: - Episode: 1531/2000, Reward: 269.070, Step: 266 +2023-04-02 17:52:40 - r - INFO: - Episode: 1532/2000, Reward: 259.111, Step: 252 +2023-04-02 17:52:41 - r - INFO: - Episode: 1533/2000, Reward: 268.200, Step: 251 +2023-04-02 17:52:41 - r - INFO: - Episode: 1534/2000, Reward: 277.413, Step: 209 +2023-04-02 17:52:42 - r - INFO: - Episode: 1535/2000, Reward: 304.464, Step: 261 +2023-04-02 17:52:46 - r - INFO: - Episode: 1536/2000, Reward: 280.544, Step: 200 +2023-04-02 17:52:47 - r - INFO: - Episode: 1537/2000, Reward: 264.827, Step: 212 +2023-04-02 17:52:48 - r - INFO: - Episode: 1538/2000, Reward: 295.931, Step: 382 +2023-04-02 17:52:49 - r - INFO: - Episode: 1539/2000, Reward: 275.130, Step: 218 +2023-04-02 17:52:50 - r - INFO: - Episode: 1540/2000, Reward: 280.089, Step: 337 +2023-04-02 17:52:55 - r - INFO: - Episode: 1541/2000, Reward: 215.621, Step: 443 +2023-04-02 17:52:57 - r - INFO: - Episode: 1542/2000, Reward: 135.738, Step: 500 +2023-04-02 17:52:58 - r - INFO: - Episode: 1543/2000, Reward: 256.197, Step: 433 +2023-04-02 17:52:59 - r - INFO: - Episode: 1544/2000, Reward: 269.540, Step: 286 +2023-04-02 17:53:00 - r - INFO: - Episode: 1545/2000, Reward: 290.937, Step: 273 +2023-04-02 17:53:02 - r - INFO: - Episode: 1546/2000, Reward: 268.157, Step: 269 +2023-04-02 17:53:03 - r - INFO: - Episode: 1547/2000, Reward: 258.270, Step: 194 +2023-04-02 17:53:05 - r - INFO: - Episode: 1548/2000, Reward: 177.280, Step: 500 +2023-04-02 17:53:06 - r - INFO: - Episode: 1549/2000, Reward: 177.025, Step: 500 +2023-04-02 17:53:08 - r - INFO: - Episode: 1550/2000, Reward: 134.999, Step: 500 +2023-04-02 17:53:13 - r - INFO: - Episode: 1551/2000, Reward: 132.792, Step: 500 +2023-04-02 17:53:14 - r - INFO: - Episode: 1552/2000, Reward: 253.549, Step: 250 +2023-04-02 17:53:16 - r - INFO: - Episode: 1553/2000, Reward: 69.108, Step: 500 +2023-04-02 17:53:17 - r - INFO: - Episode: 1554/2000, Reward: 246.183, Step: 344 +2023-04-02 17:53:18 - r - INFO: - Episode: 1555/2000, Reward: 297.572, Step: 213 +2023-04-02 17:53:20 - r - INFO: - Episode: 1556/2000, Reward: 257.402, Step: 296 +2023-04-02 17:53:21 - r - INFO: - Episode: 1557/2000, Reward: 276.011, Step: 278 +2023-04-02 17:53:22 - r - INFO: - Episode: 1558/2000, Reward: 199.503, Step: 500 +2023-04-02 17:53:24 - r - INFO: - Episode: 1559/2000, Reward: 249.809, Step: 339 +2023-04-02 17:53:25 - r - INFO: - Episode: 1560/2000, Reward: 279.447, Step: 316 +2023-04-02 17:53:27 - r - INFO: - Episode: 1561/2000, Reward: 277.081, Step: 392 +2023-04-02 17:53:29 - r - INFO: - Episode: 1562/2000, Reward: 180.215, Step: 500 +2023-04-02 17:53:31 - r - INFO: - Episode: 1563/2000, Reward: 157.545, Step: 500 +2023-04-02 17:53:32 - r - INFO: - Episode: 1564/2000, Reward: 304.317, Step: 375 +2023-04-02 17:53:33 - r - INFO: - Episode: 1565/2000, Reward: 126.022, Step: 500 +2023-04-02 17:53:38 - r - INFO: - Episode: 1566/2000, Reward: 166.360, Step: 500 +2023-04-02 17:53:40 - r - INFO: - Episode: 1567/2000, Reward: 279.415, Step: 441 +2023-04-02 17:53:41 - r - INFO: - Episode: 1568/2000, Reward: 256.601, Step: 249 +2023-04-02 17:53:42 - r - INFO: - Episode: 1569/2000, Reward: 293.092, Step: 365 +2023-04-02 17:53:43 - r - INFO: - Episode: 1570/2000, Reward: 162.797, Step: 500 +2023-04-02 17:53:49 - r - INFO: - Episode: 1571/2000, Reward: 290.390, Step: 341 +2023-04-02 17:53:51 - r - INFO: - Episode: 1572/2000, Reward: 188.523, Step: 500 +2023-04-02 17:53:53 - r - INFO: - Episode: 1573/2000, Reward: 161.847, Step: 500 +2023-04-02 17:53:54 - r - INFO: - Episode: 1574/2000, Reward: 179.732, Step: 500 +2023-04-02 17:53:55 - r - INFO: - Episode: 1575/2000, Reward: 277.798, Step: 332 +2023-04-02 17:53:58 - r - INFO: - Episode: 1576/2000, Reward: 170.946, Step: 500 +2023-04-02 17:54:00 - r - INFO: - Episode: 1577/2000, Reward: 273.102, Step: 464 +2023-04-02 17:54:01 - r - INFO: - Episode: 1578/2000, Reward: 267.928, Step: 297 +2023-04-02 17:54:02 - r - INFO: - Episode: 1579/2000, Reward: 166.099, Step: 500 +2023-04-02 17:54:04 - r - INFO: - Episode: 1580/2000, Reward: 162.873, Step: 500 +2023-04-02 17:54:07 - r - INFO: - Episode: 1581/2000, Reward: 184.086, Step: 500 +2023-04-02 17:54:08 - r - INFO: - Episode: 1582/2000, Reward: 185.087, Step: 500 +2023-04-02 17:54:09 - r - INFO: - Episode: 1583/2000, Reward: 262.456, Step: 281 +2023-04-02 17:54:11 - r - INFO: - Episode: 1584/2000, Reward: 159.684, Step: 500 +2023-04-02 17:54:13 - r - INFO: - Episode: 1585/2000, Reward: 182.608, Step: 500 +2023-04-02 17:54:16 - r - INFO: - Episode: 1586/2000, Reward: 162.400, Step: 500 +2023-04-02 17:54:17 - r - INFO: - Episode: 1587/2000, Reward: 190.636, Step: 500 +2023-04-02 17:54:18 - r - INFO: - Episode: 1588/2000, Reward: 301.820, Step: 246 +2023-04-02 17:54:20 - r - INFO: - Episode: 1589/2000, Reward: 137.549, Step: 500 +2023-04-02 17:54:21 - r - INFO: - Episode: 1590/2000, Reward: 169.945, Step: 500 +2023-04-02 17:54:23 - r - INFO: - Current episode 1590 has the best eval reward: 278.389 +2023-04-02 17:54:25 - r - INFO: - Episode: 1591/2000, Reward: 154.069, Step: 500 +2023-04-02 17:54:26 - r - INFO: - Episode: 1592/2000, Reward: 175.888, Step: 500 +2023-04-02 17:54:28 - r - INFO: - Episode: 1593/2000, Reward: 159.327, Step: 500 +2023-04-02 17:54:30 - r - INFO: - Episode: 1594/2000, Reward: 169.499, Step: 500 +2023-04-02 17:54:31 - r - INFO: - Episode: 1595/2000, Reward: 175.837, Step: 500 +2023-04-02 17:54:33 - r - INFO: - Current episode 1595 has the best eval reward: 287.446 +2023-04-02 17:54:34 - r - INFO: - Episode: 1596/2000, Reward: 193.754, Step: 500 +2023-04-02 17:54:36 - r - INFO: - Episode: 1597/2000, Reward: 292.903, Step: 493 +2023-04-02 17:54:38 - r - INFO: - Episode: 1598/2000, Reward: 153.628, Step: 500 +2023-04-02 17:54:39 - r - INFO: - Episode: 1599/2000, Reward: 287.230, Step: 272 +2023-04-02 17:54:41 - r - INFO: - Episode: 1600/2000, Reward: 177.525, Step: 500 +2023-04-02 17:54:45 - r - INFO: - Episode: 1601/2000, Reward: 158.693, Step: 500 +2023-04-02 17:54:47 - r - INFO: - Episode: 1602/2000, Reward: 163.456, Step: 500 +2023-04-02 17:54:48 - r - INFO: - Episode: 1603/2000, Reward: 187.724, Step: 500 +2023-04-02 17:54:50 - r - INFO: - Episode: 1604/2000, Reward: 161.325, Step: 500 +2023-04-02 17:54:52 - r - INFO: - Episode: 1605/2000, Reward: 159.551, Step: 500 +2023-04-02 17:54:56 - r - INFO: - Episode: 1606/2000, Reward: 164.171, Step: 500 +2023-04-02 17:54:58 - r - INFO: - Episode: 1607/2000, Reward: 321.030, Step: 446 +2023-04-02 17:54:59 - r - INFO: - Episode: 1608/2000, Reward: 185.278, Step: 500 +2023-04-02 17:55:01 - r - INFO: - Episode: 1609/2000, Reward: 169.570, Step: 500 +2023-04-02 17:55:03 - r - INFO: - Episode: 1610/2000, Reward: 171.501, Step: 500 +2023-04-02 17:55:06 - r - INFO: - Episode: 1611/2000, Reward: 264.626, Step: 402 +2023-04-02 17:55:08 - r - INFO: - Episode: 1612/2000, Reward: 305.294, Step: 405 +2023-04-02 17:55:09 - r - INFO: - Episode: 1613/2000, Reward: 31.805, Step: 500 +2023-04-02 17:55:11 - r - INFO: - Episode: 1614/2000, Reward: -138.779, Step: 475 +2023-04-02 17:55:13 - r - INFO: - Episode: 1615/2000, Reward: 269.342, Step: 471 +2023-04-02 17:55:17 - r - INFO: - Episode: 1616/2000, Reward: 159.831, Step: 500 +2023-04-02 17:55:18 - r - INFO: - Episode: 1617/2000, Reward: 257.118, Step: 337 +2023-04-02 17:55:19 - r - INFO: - Episode: 1618/2000, Reward: 270.663, Step: 328 +2023-04-02 17:55:21 - r - INFO: - Episode: 1619/2000, Reward: 166.302, Step: 500 +2023-04-02 17:55:22 - r - INFO: - Episode: 1620/2000, Reward: 167.357, Step: 500 +2023-04-02 17:55:25 - r - INFO: - Episode: 1621/2000, Reward: 255.992, Step: 414 +2023-04-02 17:55:26 - r - INFO: - Episode: 1622/2000, Reward: 267.240, Step: 208 +2023-04-02 17:55:27 - r - INFO: - Episode: 1623/2000, Reward: 249.122, Step: 286 +2023-04-02 17:55:28 - r - INFO: - Episode: 1624/2000, Reward: 209.906, Step: 500 +2023-04-02 17:55:30 - r - INFO: - Episode: 1625/2000, Reward: 148.311, Step: 500 +2023-04-02 17:55:33 - r - INFO: - Episode: 1626/2000, Reward: 177.464, Step: 500 +2023-04-02 17:55:34 - r - INFO: - Episode: 1627/2000, Reward: 252.619, Step: 337 +2023-04-02 17:55:36 - r - INFO: - Episode: 1628/2000, Reward: 245.124, Step: 475 +2023-04-02 17:55:37 - r - INFO: - Episode: 1629/2000, Reward: 140.021, Step: 500 +2023-04-02 17:55:39 - r - INFO: - Episode: 1630/2000, Reward: 293.650, Step: 461 +2023-04-02 17:55:41 - r - INFO: - Episode: 1631/2000, Reward: 248.079, Step: 236 +2023-04-02 17:55:43 - r - INFO: - Episode: 1632/2000, Reward: 195.727, Step: 500 +2023-04-02 17:55:43 - r - INFO: - Episode: 1633/2000, Reward: 227.635, Step: 208 +2023-04-02 17:55:45 - r - INFO: - Episode: 1634/2000, Reward: 162.580, Step: 500 +2023-04-02 17:55:47 - r - INFO: - Episode: 1635/2000, Reward: 238.886, Step: 452 +2023-04-02 17:55:50 - r - INFO: - Episode: 1636/2000, Reward: 176.263, Step: 500 +2023-04-02 17:55:51 - r - INFO: - Episode: 1637/2000, Reward: 158.475, Step: 500 +2023-04-02 17:55:53 - r - INFO: - Episode: 1638/2000, Reward: 164.808, Step: 500 +2023-04-02 17:55:55 - r - INFO: - Episode: 1639/2000, Reward: 162.811, Step: 500 +2023-04-02 17:55:57 - r - INFO: - Episode: 1640/2000, Reward: 158.956, Step: 500 +2023-04-02 17:56:00 - r - INFO: - Episode: 1641/2000, Reward: 171.176, Step: 500 +2023-04-02 17:56:01 - r - INFO: - Episode: 1642/2000, Reward: 241.916, Step: 291 +2023-04-02 17:56:02 - r - INFO: - Episode: 1643/2000, Reward: 189.970, Step: 500 +2023-04-02 17:56:03 - r - INFO: - Episode: 1644/2000, Reward: 269.123, Step: 272 +2023-04-02 17:56:05 - r - INFO: - Episode: 1645/2000, Reward: 153.222, Step: 500 +2023-04-02 17:56:08 - r - INFO: - Episode: 1646/2000, Reward: 268.097, Step: 426 +2023-04-02 17:56:10 - r - INFO: - Episode: 1647/2000, Reward: 192.730, Step: 500 +2023-04-02 17:56:11 - r - INFO: - Episode: 1648/2000, Reward: 179.734, Step: 500 +2023-04-02 17:56:13 - r - INFO: - Episode: 1649/2000, Reward: 151.816, Step: 500 +2023-04-02 17:56:15 - r - INFO: - Episode: 1650/2000, Reward: 267.519, Step: 459 +2023-04-02 17:56:18 - r - INFO: - Episode: 1651/2000, Reward: 227.440, Step: 397 +2023-04-02 17:56:19 - r - INFO: - Episode: 1652/2000, Reward: 289.028, Step: 325 +2023-04-02 17:56:21 - r - INFO: - Episode: 1653/2000, Reward: 161.341, Step: 500 +2023-04-02 17:56:22 - r - INFO: - Episode: 1654/2000, Reward: 289.043, Step: 276 +2023-04-02 17:56:23 - r - INFO: - Episode: 1655/2000, Reward: 270.718, Step: 361 +2023-04-02 17:56:26 - r - INFO: - Episode: 1656/2000, Reward: 287.222, Step: 389 +2023-04-02 17:56:28 - r - INFO: - Episode: 1657/2000, Reward: 169.972, Step: 500 +2023-04-02 17:56:29 - r - INFO: - Episode: 1658/2000, Reward: 189.025, Step: 500 +2023-04-02 17:56:31 - r - INFO: - Episode: 1659/2000, Reward: 253.558, Step: 473 +2023-04-02 17:56:33 - r - INFO: - Episode: 1660/2000, Reward: 149.325, Step: 500 +2023-04-02 17:56:36 - r - INFO: - Episode: 1661/2000, Reward: 143.250, Step: 500 +2023-04-02 17:56:37 - r - INFO: - Episode: 1662/2000, Reward: 264.552, Step: 210 +2023-04-02 17:56:38 - r - INFO: - Episode: 1663/2000, Reward: 250.782, Step: 322 +2023-04-02 17:56:38 - r - INFO: - Episode: 1664/2000, Reward: 258.504, Step: 251 +2023-04-02 17:56:40 - r - INFO: - Episode: 1665/2000, Reward: 164.167, Step: 500 +2023-04-02 17:56:43 - r - INFO: - Episode: 1666/2000, Reward: 66.992, Step: 500 +2023-04-02 17:56:45 - r - INFO: - Episode: 1667/2000, Reward: 144.657, Step: 500 +2023-04-02 17:56:46 - r - INFO: - Episode: 1668/2000, Reward: 231.816, Step: 359 +2023-04-02 17:56:48 - r - INFO: - Episode: 1669/2000, Reward: 234.655, Step: 407 +2023-04-02 17:56:49 - r - INFO: - Episode: 1670/2000, Reward: 241.850, Step: 304 +2023-04-02 17:56:54 - r - INFO: - Episode: 1671/2000, Reward: 133.874, Step: 500 +2023-04-02 17:56:54 - r - INFO: - Episode: 1672/2000, Reward: 290.671, Step: 190 +2023-04-02 17:56:55 - r - INFO: - Episode: 1673/2000, Reward: 288.557, Step: 204 +2023-04-02 17:56:55 - r - INFO: - Episode: 1674/2000, Reward: 230.423, Step: 256 +2023-04-02 17:56:57 - r - INFO: - Episode: 1675/2000, Reward: 164.687, Step: 500 +2023-04-02 17:57:00 - r - INFO: - Episode: 1676/2000, Reward: 244.798, Step: 346 +2023-04-02 17:57:01 - r - INFO: - Episode: 1677/2000, Reward: 248.814, Step: 291 +2023-04-02 17:57:02 - r - INFO: - Episode: 1678/2000, Reward: 154.744, Step: 500 +2023-04-02 17:57:04 - r - INFO: - Episode: 1679/2000, Reward: 259.817, Step: 494 +2023-04-02 17:57:05 - r - INFO: - Episode: 1680/2000, Reward: 283.426, Step: 314 +2023-04-02 17:57:08 - r - INFO: - Episode: 1681/2000, Reward: 261.645, Step: 259 +2023-04-02 17:57:09 - r - INFO: - Episode: 1682/2000, Reward: 263.012, Step: 184 +2023-04-02 17:57:10 - r - INFO: - Episode: 1683/2000, Reward: 131.874, Step: 500 +2023-04-02 17:57:12 - r - INFO: - Episode: 1684/2000, Reward: 246.283, Step: 444 +2023-04-02 17:57:13 - r - INFO: - Episode: 1685/2000, Reward: 284.506, Step: 220 +2023-04-02 17:57:14 - r - INFO: - Episode: 1686/2000, Reward: -3.433, Step: 155 +2023-04-02 17:57:16 - r - INFO: - Episode: 1687/2000, Reward: 268.403, Step: 393 +2023-04-02 17:57:17 - r - INFO: - Episode: 1688/2000, Reward: 263.658, Step: 283 +2023-04-02 17:57:18 - r - INFO: - Episode: 1689/2000, Reward: 170.772, Step: 500 +2023-04-02 17:57:20 - r - INFO: - Episode: 1690/2000, Reward: 258.251, Step: 399 +2023-04-02 17:57:23 - r - INFO: - Episode: 1691/2000, Reward: 152.823, Step: 500 +2023-04-02 17:57:25 - r - INFO: - Episode: 1692/2000, Reward: 255.562, Step: 379 +2023-04-02 17:57:26 - r - INFO: - Episode: 1693/2000, Reward: 240.843, Step: 486 +2023-04-02 17:57:28 - r - INFO: - Episode: 1694/2000, Reward: 125.214, Step: 500 +2023-04-02 17:57:30 - r - INFO: - Episode: 1695/2000, Reward: 161.391, Step: 500 +2023-04-02 17:57:34 - r - INFO: - Episode: 1696/2000, Reward: 276.130, Step: 493 +2023-04-02 17:57:36 - r - INFO: - Episode: 1697/2000, Reward: 110.547, Step: 500 +2023-04-02 17:57:37 - r - INFO: - Episode: 1698/2000, Reward: 286.769, Step: 323 +2023-04-02 17:57:38 - r - INFO: - Episode: 1699/2000, Reward: 262.766, Step: 465 +2023-04-02 17:57:40 - r - INFO: - Episode: 1700/2000, Reward: 154.838, Step: 500 +2023-04-02 17:57:44 - r - INFO: - Episode: 1701/2000, Reward: 190.777, Step: 500 +2023-04-02 17:57:45 - r - INFO: - Episode: 1702/2000, Reward: 181.068, Step: 500 +2023-04-02 17:57:46 - r - INFO: - Episode: 1703/2000, Reward: 234.492, Step: 302 +2023-04-02 17:57:47 - r - INFO: - Episode: 1704/2000, Reward: 289.479, Step: 293 +2023-04-02 17:57:49 - r - INFO: - Episode: 1705/2000, Reward: 180.898, Step: 500 +2023-04-02 17:57:52 - r - INFO: - Episode: 1706/2000, Reward: 158.409, Step: 500 +2023-04-02 17:57:54 - r - INFO: - Episode: 1707/2000, Reward: 198.389, Step: 500 +2023-04-02 17:57:56 - r - INFO: - Episode: 1708/2000, Reward: 175.622, Step: 500 +2023-04-02 17:57:57 - r - INFO: - Episode: 1709/2000, Reward: 254.574, Step: 295 +2023-04-02 17:57:58 - r - INFO: - Episode: 1710/2000, Reward: 270.174, Step: 375 +2023-04-02 17:58:01 - r - INFO: - Episode: 1711/2000, Reward: 230.631, Step: 454 +2023-04-02 17:58:02 - r - INFO: - Episode: 1712/2000, Reward: 252.910, Step: 367 +2023-04-02 17:58:04 - r - INFO: - Episode: 1713/2000, Reward: 280.203, Step: 498 +2023-04-02 17:58:05 - r - INFO: - Episode: 1714/2000, Reward: 292.604, Step: 340 +2023-04-02 17:58:05 - r - INFO: - Episode: 1715/2000, Reward: 225.013, Step: 264 +2023-04-02 17:58:09 - r - INFO: - Episode: 1716/2000, Reward: 166.422, Step: 500 +2023-04-02 17:58:10 - r - INFO: - Episode: 1717/2000, Reward: 144.085, Step: 500 +2023-04-02 17:58:12 - r - INFO: - Episode: 1718/2000, Reward: 157.688, Step: 500 +2023-04-02 17:58:14 - r - INFO: - Episode: 1719/2000, Reward: 279.304, Step: 494 +2023-04-02 17:58:14 - r - INFO: - Episode: 1720/2000, Reward: 272.746, Step: 218 +2023-04-02 17:58:18 - r - INFO: - Episode: 1721/2000, Reward: 183.062, Step: 500 +2023-04-02 17:58:20 - r - INFO: - Episode: 1722/2000, Reward: 267.190, Step: 433 +2023-04-02 17:58:21 - r - INFO: - Episode: 1723/2000, Reward: 183.114, Step: 500 +2023-04-02 17:58:23 - r - INFO: - Episode: 1724/2000, Reward: 170.956, Step: 500 +2023-04-02 17:58:25 - r - INFO: - Episode: 1725/2000, Reward: 168.951, Step: 500 +2023-04-02 17:58:29 - r - INFO: - Episode: 1726/2000, Reward: 148.860, Step: 500 +2023-04-02 17:58:30 - r - INFO: - Episode: 1727/2000, Reward: 278.877, Step: 214 +2023-04-02 17:58:31 - r - INFO: - Episode: 1728/2000, Reward: 250.939, Step: 306 +2023-04-02 17:58:31 - r - INFO: - Episode: 1729/2000, Reward: 273.982, Step: 159 +2023-04-02 17:58:32 - r - INFO: - Episode: 1730/2000, Reward: 284.726, Step: 190 +2023-04-02 17:58:35 - r - INFO: - Episode: 1731/2000, Reward: 287.713, Step: 289 +2023-04-02 17:58:37 - r - INFO: - Episode: 1732/2000, Reward: 281.706, Step: 412 +2023-04-02 17:58:38 - r - INFO: - Episode: 1733/2000, Reward: 264.377, Step: 432 +2023-04-02 17:58:39 - r - INFO: - Episode: 1734/2000, Reward: 274.669, Step: 236 +2023-04-02 17:58:40 - r - INFO: - Episode: 1735/2000, Reward: 265.417, Step: 259 +2023-04-02 17:58:42 - r - INFO: - Episode: 1736/2000, Reward: 289.193, Step: 202 +2023-04-02 17:58:43 - r - INFO: - Episode: 1737/2000, Reward: 274.647, Step: 294 +2023-04-02 17:58:45 - r - INFO: - Episode: 1738/2000, Reward: 115.570, Step: 500 +2023-04-02 17:58:45 - r - INFO: - Episode: 1739/2000, Reward: 255.916, Step: 200 +2023-04-02 17:58:47 - r - INFO: - Episode: 1740/2000, Reward: 68.571, Step: 500 +2023-04-02 17:58:50 - r - INFO: - Episode: 1741/2000, Reward: 244.077, Step: 237 +2023-04-02 17:58:51 - r - INFO: - Episode: 1742/2000, Reward: 248.023, Step: 325 +2023-04-02 17:58:52 - r - INFO: - Episode: 1743/2000, Reward: 285.805, Step: 232 +2023-04-02 17:58:54 - r - INFO: - Episode: 1744/2000, Reward: 197.225, Step: 500 +2023-04-02 17:58:54 - r - INFO: - Episode: 1745/2000, Reward: 242.958, Step: 212 +2023-04-02 17:58:57 - r - INFO: - Episode: 1746/2000, Reward: 256.659, Step: 186 +2023-04-02 17:58:59 - r - INFO: - Episode: 1747/2000, Reward: 56.267, Step: 500 +2023-04-02 17:59:00 - r - INFO: - Episode: 1748/2000, Reward: 262.751, Step: 449 +2023-04-02 17:59:02 - r - INFO: - Episode: 1749/2000, Reward: 140.384, Step: 500 +2023-04-02 17:59:04 - r - INFO: - Episode: 1750/2000, Reward: 47.951, Step: 500 +2023-04-02 17:59:07 - r - INFO: - Episode: 1751/2000, Reward: 189.210, Step: 500 +2023-04-02 17:59:08 - r - INFO: - Episode: 1752/2000, Reward: 40.117, Step: 500 +2023-04-02 17:59:09 - r - INFO: - Episode: 1753/2000, Reward: 267.690, Step: 212 +2023-04-02 17:59:10 - r - INFO: - Episode: 1754/2000, Reward: 274.262, Step: 215 +2023-04-02 17:59:10 - r - INFO: - Episode: 1755/2000, Reward: 271.929, Step: 188 +2023-04-02 17:59:13 - r - INFO: - Episode: 1756/2000, Reward: 145.653, Step: 500 +2023-04-02 17:59:15 - r - INFO: - Episode: 1757/2000, Reward: 144.720, Step: 500 +2023-04-02 17:59:16 - r - INFO: - Episode: 1758/2000, Reward: 248.142, Step: 333 +2023-04-02 17:59:17 - r - INFO: - Episode: 1759/2000, Reward: 238.232, Step: 357 +2023-04-02 17:59:19 - r - INFO: - Episode: 1760/2000, Reward: 37.041, Step: 500 +2023-04-02 17:59:22 - r - INFO: - Episode: 1761/2000, Reward: 246.442, Step: 283 +2023-04-02 17:59:23 - r - INFO: - Episode: 1762/2000, Reward: 270.697, Step: 340 +2023-04-02 17:59:25 - r - INFO: - Episode: 1763/2000, Reward: 46.915, Step: 500 +2023-04-02 17:59:27 - r - INFO: - Episode: 1764/2000, Reward: -124.170, Step: 500 +2023-04-02 17:59:27 - r - INFO: - Episode: 1765/2000, Reward: 264.985, Step: 176 +2023-04-02 17:59:30 - r - INFO: - Episode: 1766/2000, Reward: 272.432, Step: 335 +2023-04-02 17:59:32 - r - INFO: - Episode: 1767/2000, Reward: 184.680, Step: 500 +2023-04-02 17:59:34 - r - INFO: - Episode: 1768/2000, Reward: 169.249, Step: 500 +2023-04-02 17:59:35 - r - INFO: - Episode: 1769/2000, Reward: 221.702, Step: 463 +2023-04-02 17:59:36 - r - INFO: - Episode: 1770/2000, Reward: 271.415, Step: 214 +2023-04-02 17:59:40 - r - INFO: - Episode: 1771/2000, Reward: 171.063, Step: 500 +2023-04-02 17:59:41 - r - INFO: - Episode: 1772/2000, Reward: 278.201, Step: 245 +2023-04-02 17:59:42 - r - INFO: - Episode: 1773/2000, Reward: 305.626, Step: 314 +2023-04-02 17:59:42 - r - INFO: - Episode: 1774/2000, Reward: 281.239, Step: 192 +2023-04-02 17:59:44 - r - INFO: - Episode: 1775/2000, Reward: 165.780, Step: 500 +2023-04-02 17:59:48 - r - INFO: - Episode: 1776/2000, Reward: 195.867, Step: 500 +2023-04-02 17:59:49 - r - INFO: - Episode: 1777/2000, Reward: 260.582, Step: 330 +2023-04-02 17:59:51 - r - INFO: - Episode: 1778/2000, Reward: 39.366, Step: 500 +2023-04-02 17:59:52 - r - INFO: - Episode: 1779/2000, Reward: 267.159, Step: 317 +2023-04-02 17:59:53 - r - INFO: - Episode: 1780/2000, Reward: 145.306, Step: 500 +2023-04-02 17:59:57 - r - INFO: - Episode: 1781/2000, Reward: 133.769, Step: 500 +2023-04-02 17:59:59 - r - INFO: - Episode: 1782/2000, Reward: 172.413, Step: 500 +2023-04-02 18:00:01 - r - INFO: - Episode: 1783/2000, Reward: 144.480, Step: 500 +2023-04-02 18:00:02 - r - INFO: - Episode: 1784/2000, Reward: 262.289, Step: 304 +2023-04-02 18:00:03 - r - INFO: - Episode: 1785/2000, Reward: 245.866, Step: 394 +2023-04-02 18:00:05 - r - INFO: - Episode: 1786/2000, Reward: 52.726, Step: 223 +2023-04-02 18:00:07 - r - INFO: - Episode: 1787/2000, Reward: 47.312, Step: 500 +2023-04-02 18:00:08 - r - INFO: - Episode: 1788/2000, Reward: 299.968, Step: 333 +2023-04-02 18:00:09 - r - INFO: - Episode: 1789/2000, Reward: 262.772, Step: 222 +2023-04-02 18:00:11 - r - INFO: - Episode: 1790/2000, Reward: 154.149, Step: 500 +2023-04-02 18:00:14 - r - INFO: - Episode: 1791/2000, Reward: 247.692, Step: 342 +2023-04-02 18:00:15 - r - INFO: - Episode: 1792/2000, Reward: 155.588, Step: 500 +2023-04-02 18:00:16 - r - INFO: - Episode: 1793/2000, Reward: 23.128, Step: 113 +2023-04-02 18:00:17 - r - INFO: - Episode: 1794/2000, Reward: 128.727, Step: 500 +2023-04-02 18:00:18 - r - INFO: - Episode: 1795/2000, Reward: 260.910, Step: 230 +2023-04-02 18:00:21 - r - INFO: - Episode: 1796/2000, Reward: 31.967, Step: 500 +2023-04-02 18:00:23 - r - INFO: - Episode: 1797/2000, Reward: 152.373, Step: 500 +2023-04-02 18:00:24 - r - INFO: - Episode: 1798/2000, Reward: 173.497, Step: 500 +2023-04-02 18:00:25 - r - INFO: - Episode: 1799/2000, Reward: 254.568, Step: 226 +2023-04-02 18:00:26 - r - INFO: - Episode: 1800/2000, Reward: 294.226, Step: 289 +2023-04-02 18:00:29 - r - INFO: - Episode: 1801/2000, Reward: 111.001, Step: 500 +2023-04-02 18:00:30 - r - INFO: - Episode: 1802/2000, Reward: 73.261, Step: 500 +2023-04-02 18:00:31 - r - INFO: - Episode: 1803/2000, Reward: 251.583, Step: 351 +2023-04-02 18:00:33 - r - INFO: - Episode: 1804/2000, Reward: 178.228, Step: 500 +2023-04-02 18:00:33 - r - INFO: - Episode: 1805/2000, Reward: 266.977, Step: 156 +2023-04-02 18:00:37 - r - INFO: - Episode: 1806/2000, Reward: 278.554, Step: 473 +2023-04-02 18:00:38 - r - INFO: - Episode: 1807/2000, Reward: 184.732, Step: 500 +2023-04-02 18:00:40 - r - INFO: - Episode: 1808/2000, Reward: 119.262, Step: 500 +2023-04-02 18:00:41 - r - INFO: - Episode: 1809/2000, Reward: 240.136, Step: 345 +2023-04-02 18:00:43 - r - INFO: - Episode: 1810/2000, Reward: 130.424, Step: 500 +2023-04-02 18:00:47 - r - INFO: - Episode: 1811/2000, Reward: 174.589, Step: 500 +2023-04-02 18:00:48 - r - INFO: - Episode: 1812/2000, Reward: 299.176, Step: 333 +2023-04-02 18:00:48 - r - INFO: - Episode: 1813/2000, Reward: 252.097, Step: 173 +2023-04-02 18:00:49 - r - INFO: - Episode: 1814/2000, Reward: 251.129, Step: 337 +2023-04-02 18:00:50 - r - INFO: - Episode: 1815/2000, Reward: 274.986, Step: 299 +2023-04-02 18:00:53 - r - INFO: - Episode: 1816/2000, Reward: 158.640, Step: 500 +2023-04-02 18:00:55 - r - INFO: - Episode: 1817/2000, Reward: 155.573, Step: 500 +2023-04-02 18:00:55 - r - INFO: - Episode: 1818/2000, Reward: 268.837, Step: 164 +2023-04-02 18:00:57 - r - INFO: - Episode: 1819/2000, Reward: 65.657, Step: 500 +2023-04-02 18:00:58 - r - INFO: - Episode: 1820/2000, Reward: 264.277, Step: 286 +2023-04-02 18:01:02 - r - INFO: - Episode: 1821/2000, Reward: 41.618, Step: 500 +2023-04-02 18:01:04 - r - INFO: - Episode: 1822/2000, Reward: 307.210, Step: 420 +2023-04-02 18:01:05 - r - INFO: - Episode: 1823/2000, Reward: 139.944, Step: 500 +2023-04-02 18:01:07 - r - INFO: - Episode: 1824/2000, Reward: 152.827, Step: 500 +2023-04-02 18:01:08 - r - INFO: - Episode: 1825/2000, Reward: 248.082, Step: 321 +2023-04-02 18:01:11 - r - INFO: - Episode: 1826/2000, Reward: 237.279, Step: 286 +2023-04-02 18:01:13 - r - INFO: - Episode: 1827/2000, Reward: 226.097, Step: 396 +2023-04-02 18:01:14 - r - INFO: - Episode: 1828/2000, Reward: 277.687, Step: 409 +2023-04-02 18:01:15 - r - INFO: - Episode: 1829/2000, Reward: 283.772, Step: 338 +2023-04-02 18:01:17 - r - INFO: - Episode: 1830/2000, Reward: 152.478, Step: 500 +2023-04-02 18:01:19 - r - INFO: - Episode: 1831/2000, Reward: 250.872, Step: 382 +2023-04-02 18:01:20 - r - INFO: - Episode: 1832/2000, Reward: 242.849, Step: 418 +2023-04-02 18:01:21 - r - INFO: - Episode: 1833/2000, Reward: 257.841, Step: 302 +2023-04-02 18:01:23 - r - INFO: - Episode: 1834/2000, Reward: 306.834, Step: 410 +2023-04-02 18:01:24 - r - INFO: - Episode: 1835/2000, Reward: 183.545, Step: 500 +2023-04-02 18:01:28 - r - INFO: - Episode: 1836/2000, Reward: 27.122, Step: 500 +2023-04-02 18:01:29 - r - INFO: - Episode: 1837/2000, Reward: 285.215, Step: 236 +2023-04-02 18:01:29 - r - INFO: - Episode: 1838/2000, Reward: 283.179, Step: 193 +2023-04-02 18:01:31 - r - INFO: - Episode: 1839/2000, Reward: 267.184, Step: 376 +2023-04-02 18:01:32 - r - INFO: - Episode: 1840/2000, Reward: 94.194, Step: 500 +2023-04-02 18:01:35 - r - INFO: - Episode: 1841/2000, Reward: 6.378, Step: 454 +2023-04-02 18:01:36 - r - INFO: - Episode: 1842/2000, Reward: 245.832, Step: 264 +2023-04-02 18:01:37 - r - INFO: - Episode: 1843/2000, Reward: 219.537, Step: 315 +2023-04-02 18:01:39 - r - INFO: - Episode: 1844/2000, Reward: 165.178, Step: 500 +2023-04-02 18:01:40 - r - INFO: - Episode: 1845/2000, Reward: 157.542, Step: 500 +2023-04-02 18:01:43 - r - INFO: - Episode: 1846/2000, Reward: 278.667, Step: 436 +2023-04-02 18:01:45 - r - INFO: - Episode: 1847/2000, Reward: 160.064, Step: 500 +2023-04-02 18:01:46 - r - INFO: - Episode: 1848/2000, Reward: 244.960, Step: 212 +2023-04-02 18:01:47 - r - INFO: - Episode: 1849/2000, Reward: 185.158, Step: 500 +2023-04-02 18:01:49 - r - INFO: - Episode: 1850/2000, Reward: 167.495, Step: 500 +2023-04-02 18:01:52 - r - INFO: - Episode: 1851/2000, Reward: 159.935, Step: 500 +2023-04-02 18:01:53 - r - INFO: - Episode: 1852/2000, Reward: 64.494, Step: 500 +2023-04-02 18:01:55 - r - INFO: - Episode: 1853/2000, Reward: 173.954, Step: 500 +2023-04-02 18:01:57 - r - INFO: - Episode: 1854/2000, Reward: 197.607, Step: 500 +2023-04-02 18:01:58 - r - INFO: - Episode: 1855/2000, Reward: 200.149, Step: 500 +2023-04-02 18:02:01 - r - INFO: - Episode: 1856/2000, Reward: 174.886, Step: 500 +2023-04-02 18:02:02 - r - INFO: - Episode: 1857/2000, Reward: 34.251, Step: 144 +2023-04-02 18:02:03 - r - INFO: - Episode: 1858/2000, Reward: 254.590, Step: 485 +2023-04-02 18:02:04 - r - INFO: - Episode: 1859/2000, Reward: 308.105, Step: 387 +2023-04-02 18:02:05 - r - INFO: - Episode: 1860/2000, Reward: 232.213, Step: 298 +2023-04-02 18:02:09 - r - INFO: - Episode: 1861/2000, Reward: 142.826, Step: 500 +2023-04-02 18:02:11 - r - INFO: - Episode: 1862/2000, Reward: 167.472, Step: 500 +2023-04-02 18:02:13 - r - INFO: - Episode: 1863/2000, Reward: 180.559, Step: 500 +2023-04-02 18:02:13 - r - INFO: - Episode: 1864/2000, Reward: 18.889, Step: 120 +2023-04-02 18:02:15 - r - INFO: - Episode: 1865/2000, Reward: 139.363, Step: 500 +2023-04-02 18:02:19 - r - INFO: - Episode: 1866/2000, Reward: 166.354, Step: 500 +2023-04-02 18:02:20 - r - INFO: - Episode: 1867/2000, Reward: 246.586, Step: 452 +2023-04-02 18:02:21 - r - INFO: - Episode: 1868/2000, Reward: 255.579, Step: 296 +2023-04-02 18:02:23 - r - INFO: - Episode: 1869/2000, Reward: 250.928, Step: 441 +2023-04-02 18:02:24 - r - INFO: - Episode: 1870/2000, Reward: 71.694, Step: 500 +2023-04-02 18:02:30 - r - INFO: - Episode: 1871/2000, Reward: 271.883, Step: 318 +2023-04-02 18:02:31 - r - INFO: - Episode: 1872/2000, Reward: 242.700, Step: 372 +2023-04-02 18:02:32 - r - INFO: - Episode: 1873/2000, Reward: 250.758, Step: 419 +2023-04-02 18:02:34 - r - INFO: - Episode: 1874/2000, Reward: 269.582, Step: 464 +2023-04-02 18:02:34 - r - INFO: - Episode: 1875/2000, Reward: 33.291, Step: 129 +2023-04-02 18:02:39 - r - INFO: - Episode: 1876/2000, Reward: 169.600, Step: 500 +2023-04-02 18:02:40 - r - INFO: - Episode: 1877/2000, Reward: 252.774, Step: 373 +2023-04-02 18:02:41 - r - INFO: - Episode: 1878/2000, Reward: 176.335, Step: 500 +2023-04-02 18:02:43 - r - INFO: - Episode: 1879/2000, Reward: 279.543, Step: 454 +2023-04-02 18:02:44 - r - INFO: - Episode: 1880/2000, Reward: 171.110, Step: 500 +2023-04-02 18:02:49 - r - INFO: - Episode: 1881/2000, Reward: 259.414, Step: 265 +2023-04-02 18:02:50 - r - INFO: - Episode: 1882/2000, Reward: 270.826, Step: 174 +2023-04-02 18:02:51 - r - INFO: - Episode: 1883/2000, Reward: 284.103, Step: 324 +2023-04-02 18:02:52 - r - INFO: - Episode: 1884/2000, Reward: 169.414, Step: 500 +2023-04-02 18:02:54 - r - INFO: - Episode: 1885/2000, Reward: 160.272, Step: 500 +2023-04-02 18:02:58 - r - INFO: - Episode: 1886/2000, Reward: 261.887, Step: 485 +2023-04-02 18:03:00 - r - INFO: - Episode: 1887/2000, Reward: 161.170, Step: 500 +2023-04-02 18:03:02 - r - INFO: - Episode: 1888/2000, Reward: 157.926, Step: 500 +2023-04-02 18:03:03 - r - INFO: - Episode: 1889/2000, Reward: 166.584, Step: 500 +2023-04-02 18:03:05 - r - INFO: - Episode: 1890/2000, Reward: 61.192, Step: 500 +2023-04-02 18:03:10 - r - INFO: - Episode: 1891/2000, Reward: 150.086, Step: 500 +2023-04-02 18:03:12 - r - INFO: - Episode: 1892/2000, Reward: 154.557, Step: 500 +2023-04-02 18:03:14 - r - INFO: - Episode: 1893/2000, Reward: 144.972, Step: 500 +2023-04-02 18:03:15 - r - INFO: - Episode: 1894/2000, Reward: 191.090, Step: 500 +2023-04-02 18:03:16 - r - INFO: - Episode: 1895/2000, Reward: 242.871, Step: 253 +2023-04-02 18:03:19 - r - INFO: - Episode: 1896/2000, Reward: 157.684, Step: 500 +2023-04-02 18:03:20 - r - INFO: - Episode: 1897/2000, Reward: 232.767, Step: 267 +2023-04-02 18:03:22 - r - INFO: - Episode: 1898/2000, Reward: 172.542, Step: 500 +2023-04-02 18:03:24 - r - INFO: - Episode: 1899/2000, Reward: 171.528, Step: 500 +2023-04-02 18:03:25 - r - INFO: - Episode: 1900/2000, Reward: 140.027, Step: 500 +2023-04-02 18:03:29 - r - INFO: - Episode: 1901/2000, Reward: 221.234, Step: 487 +2023-04-02 18:03:31 - r - INFO: - Episode: 1902/2000, Reward: 281.384, Step: 459 +2023-04-02 18:03:31 - r - INFO: - Episode: 1903/2000, Reward: 258.707, Step: 198 +2023-04-02 18:03:33 - r - INFO: - Episode: 1904/2000, Reward: 81.834, Step: 500 +2023-04-02 18:03:35 - r - INFO: - Episode: 1905/2000, Reward: 143.057, Step: 500 +2023-04-02 18:03:38 - r - INFO: - Episode: 1906/2000, Reward: 150.466, Step: 500 +2023-04-02 18:03:40 - r - INFO: - Episode: 1907/2000, Reward: 154.971, Step: 500 +2023-04-02 18:03:42 - r - INFO: - Episode: 1908/2000, Reward: 149.656, Step: 500 +2023-04-02 18:03:43 - r - INFO: - Episode: 1909/2000, Reward: 161.332, Step: 500 +2023-04-02 18:03:45 - r - INFO: - Episode: 1910/2000, Reward: 152.214, Step: 500 +2023-04-02 18:03:51 - r - INFO: - Episode: 1911/2000, Reward: 129.857, Step: 500 +2023-04-02 18:03:53 - r - INFO: - Episode: 1912/2000, Reward: 98.902, Step: 500 +2023-04-02 18:03:55 - r - INFO: - Episode: 1913/2000, Reward: 106.177, Step: 500 +2023-04-02 18:03:56 - r - INFO: - Episode: 1914/2000, Reward: 166.530, Step: 500 +2023-04-02 18:03:58 - r - INFO: - Episode: 1915/2000, Reward: 168.225, Step: 500 +2023-04-02 18:04:03 - r - INFO: - Episode: 1916/2000, Reward: 166.954, Step: 500 +2023-04-02 18:04:04 - r - INFO: - Episode: 1917/2000, Reward: 123.901, Step: 500 +2023-04-02 18:04:06 - r - INFO: - Episode: 1918/2000, Reward: 34.804, Step: 500 +2023-04-02 18:04:08 - r - INFO: - Episode: 1919/2000, Reward: 155.512, Step: 500 +2023-04-02 18:04:09 - r - INFO: - Episode: 1920/2000, Reward: 57.197, Step: 500 +2023-04-02 18:04:14 - r - INFO: - Episode: 1921/2000, Reward: 144.786, Step: 500 +2023-04-02 18:04:16 - r - INFO: - Episode: 1922/2000, Reward: 53.037, Step: 500 +2023-04-02 18:04:17 - r - INFO: - Episode: 1923/2000, Reward: 295.615, Step: 325 +2023-04-02 18:04:18 - r - INFO: - Episode: 1924/2000, Reward: 291.124, Step: 362 +2023-04-02 18:04:20 - r - INFO: - Episode: 1925/2000, Reward: 166.901, Step: 500 +2023-04-02 18:04:26 - r - INFO: - Episode: 1926/2000, Reward: 72.967, Step: 500 +2023-04-02 18:04:28 - r - INFO: - Episode: 1927/2000, Reward: 98.462, Step: 500 +2023-04-02 18:04:29 - r - INFO: - Episode: 1928/2000, Reward: 50.546, Step: 500 +2023-04-02 18:04:31 - r - INFO: - Episode: 1929/2000, Reward: 51.811, Step: 500 +2023-04-02 18:04:32 - r - INFO: - Episode: 1930/2000, Reward: 277.554, Step: 225 +2023-04-02 18:04:34 - r - INFO: - Episode: 1931/2000, Reward: 285.939, Step: 298 +2023-04-02 18:04:36 - r - INFO: - Episode: 1932/2000, Reward: 75.865, Step: 500 +2023-04-02 18:04:38 - r - INFO: - Episode: 1933/2000, Reward: 133.413, Step: 500 +2023-04-02 18:04:40 - r - INFO: - Episode: 1934/2000, Reward: 127.376, Step: 500 +2023-04-02 18:04:41 - r - INFO: - Episode: 1935/2000, Reward: 295.954, Step: 296 +2023-04-02 18:04:46 - r - INFO: - Episode: 1936/2000, Reward: 158.266, Step: 500 +2023-04-02 18:04:48 - r - INFO: - Episode: 1937/2000, Reward: 248.513, Step: 434 +2023-04-02 18:04:49 - r - INFO: - Episode: 1938/2000, Reward: 269.941, Step: 264 +2023-04-02 18:04:50 - r - INFO: - Episode: 1939/2000, Reward: 281.748, Step: 299 +2023-04-02 18:04:51 - r - INFO: - Episode: 1940/2000, Reward: 155.329, Step: 500 +2023-04-02 18:04:54 - r - INFO: - Episode: 1941/2000, Reward: 273.567, Step: 245 +2023-04-02 18:04:56 - r - INFO: - Episode: 1942/2000, Reward: 163.705, Step: 500 +2023-04-02 18:04:57 - r - INFO: - Episode: 1943/2000, Reward: 168.201, Step: 500 +2023-04-02 18:04:59 - r - INFO: - Episode: 1944/2000, Reward: 147.139, Step: 500 +2023-04-02 18:05:01 - r - INFO: - Episode: 1945/2000, Reward: 109.509, Step: 500 +2023-04-02 18:05:04 - r - INFO: - Episode: 1946/2000, Reward: 160.796, Step: 500 +2023-04-02 18:05:05 - r - INFO: - Episode: 1947/2000, Reward: 253.808, Step: 233 +2023-04-02 18:05:07 - r - INFO: - Episode: 1948/2000, Reward: 54.663, Step: 500 +2023-04-02 18:05:07 - r - INFO: - Episode: 1949/2000, Reward: 251.154, Step: 193 +2023-04-02 18:05:09 - r - INFO: - Episode: 1950/2000, Reward: 156.109, Step: 500 +2023-04-02 18:05:11 - r - INFO: - Episode: 1951/2000, Reward: 264.787, Step: 173 +2023-04-02 18:05:13 - r - INFO: - Episode: 1952/2000, Reward: 157.017, Step: 500 +2023-04-02 18:05:14 - r - INFO: - Episode: 1953/2000, Reward: 289.508, Step: 258 +2023-04-02 18:05:15 - r - INFO: - Episode: 1954/2000, Reward: 155.806, Step: 500 +2023-04-02 18:05:17 - r - INFO: - Episode: 1955/2000, Reward: 152.578, Step: 500 +2023-04-02 18:05:20 - r - INFO: - Episode: 1956/2000, Reward: 155.342, Step: 500 +2023-04-02 18:05:21 - r - INFO: - Episode: 1957/2000, Reward: 268.070, Step: 446 +2023-04-02 18:05:22 - r - INFO: - Episode: 1958/2000, Reward: 260.359, Step: 157 +2023-04-02 18:05:23 - r - INFO: - Episode: 1959/2000, Reward: 253.341, Step: 479 +2023-04-02 18:05:25 - r - INFO: - Episode: 1960/2000, Reward: 183.579, Step: 500 +2023-04-02 18:05:28 - r - INFO: - Episode: 1961/2000, Reward: 94.436, Step: 500 +2023-04-02 18:05:29 - r - INFO: - Episode: 1962/2000, Reward: 171.812, Step: 500 +2023-04-02 18:05:31 - r - INFO: - Episode: 1963/2000, Reward: 281.312, Step: 324 +2023-04-02 18:05:32 - r - INFO: - Episode: 1964/2000, Reward: 115.498, Step: 500 +2023-04-02 18:05:33 - r - INFO: - Episode: 1965/2000, Reward: 240.134, Step: 359 +2023-04-02 18:05:37 - r - INFO: - Episode: 1966/2000, Reward: 163.695, Step: 500 +2023-04-02 18:05:38 - r - INFO: - Episode: 1967/2000, Reward: 115.325, Step: 500 +2023-04-02 18:05:40 - r - INFO: - Episode: 1968/2000, Reward: 176.853, Step: 500 +2023-04-02 18:05:41 - r - INFO: - Episode: 1969/2000, Reward: 253.236, Step: 354 +2023-04-02 18:05:42 - r - INFO: - Episode: 1970/2000, Reward: 288.765, Step: 306 +2023-04-02 18:05:45 - r - INFO: - Episode: 1971/2000, Reward: 151.404, Step: 500 +2023-04-02 18:05:46 - r - INFO: - Episode: 1972/2000, Reward: 177.078, Step: 500 +2023-04-02 18:05:48 - r - INFO: - Episode: 1973/2000, Reward: 27.162, Step: 500 +2023-04-02 18:05:48 - r - INFO: - Episode: 1974/2000, Reward: -1.368, Step: 106 +2023-04-02 18:05:50 - r - INFO: - Episode: 1975/2000, Reward: 195.549, Step: 500 +2023-04-02 18:05:53 - r - INFO: - Episode: 1976/2000, Reward: 194.752, Step: 500 +2023-04-02 18:05:54 - r - INFO: - Episode: 1977/2000, Reward: 298.403, Step: 362 +2023-04-02 18:05:56 - r - INFO: - Episode: 1978/2000, Reward: 129.611, Step: 500 +2023-04-02 18:05:57 - r - INFO: - Episode: 1979/2000, Reward: 199.882, Step: 500 +2023-04-02 18:05:59 - r - INFO: - Episode: 1980/2000, Reward: 145.560, Step: 500 +2023-04-02 18:06:01 - r - INFO: - Episode: 1981/2000, Reward: 275.932, Step: 382 +2023-04-02 18:06:03 - r - INFO: - Episode: 1982/2000, Reward: 86.926, Step: 500 +2023-04-02 18:06:05 - r - INFO: - Episode: 1983/2000, Reward: 178.372, Step: 500 +2023-04-02 18:06:06 - r - INFO: - Episode: 1984/2000, Reward: 152.649, Step: 500 +2023-04-02 18:06:07 - r - INFO: - Episode: 1985/2000, Reward: 50.765, Step: 90 +2023-04-02 18:06:09 - r - INFO: - Episode: 1986/2000, Reward: 176.951, Step: 500 +2023-04-02 18:06:11 - r - INFO: - Episode: 1987/2000, Reward: 151.665, Step: 500 +2023-04-02 18:06:13 - r - INFO: - Episode: 1988/2000, Reward: 177.136, Step: 500 +2023-04-02 18:06:14 - r - INFO: - Episode: 1989/2000, Reward: 140.775, Step: 500 +2023-04-02 18:06:16 - r - INFO: - Episode: 1990/2000, Reward: 186.385, Step: 500 +2023-04-02 18:06:19 - r - INFO: - Episode: 1991/2000, Reward: 175.794, Step: 500 +2023-04-02 18:06:21 - r - INFO: - Episode: 1992/2000, Reward: 164.626, Step: 500 +2023-04-02 18:06:22 - r - INFO: - Episode: 1993/2000, Reward: 174.408, Step: 500 +2023-04-02 18:06:24 - r - INFO: - Episode: 1994/2000, Reward: 167.655, Step: 500 +2023-04-02 18:06:25 - r - INFO: - Episode: 1995/2000, Reward: 286.461, Step: 449 +2023-04-02 18:06:29 - r - INFO: - Episode: 1996/2000, Reward: 177.011, Step: 500 +2023-04-02 18:06:30 - r - INFO: - Episode: 1997/2000, Reward: 260.270, Step: 301 +2023-04-02 18:06:31 - r - INFO: - Episode: 1998/2000, Reward: 175.627, Step: 500 +2023-04-02 18:06:33 - r - INFO: - Episode: 1999/2000, Reward: 165.808, Step: 500 +2023-04-02 18:06:34 - r - INFO: - Episode: 2000/2000, Reward: 277.259, Step: 330 +2023-04-02 18:06:36 - r - INFO: - Finish training! diff --git a/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/models/checkpoint.pt b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..b93823d1377848f70a1b2954db2ba20b2f7c906e --- /dev/null +++ b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3053c611ca1584643dbde084d5d528b98089a4388b3a5ee7b39032f06780030 +size 3054989 diff --git a/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/results/learning_curve.png b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..be8ba1ac8bd7995e99f4730dc90fedef2066a6aa Binary files /dev/null and b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/results/learning_curve.png differ diff --git a/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/results/res.csv b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..9cd1507e402c7145caafd6488c5d1949cbafa861 --- /dev/null +++ b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/results/res.csv @@ -0,0 +1,2001 @@ +episodes,rewards,steps +0,-335.1857237321518,123 +1,-146.33197658450902,117 +2,-49.34662352014156,86 +3,-7.131513518140579,134 +4,-136.54569543088468,173 +5,-271.14124724561316,84 +6,-389.1027512007529,97 +7,-148.46609098313064,91 +8,-333.8466937356742,101 +9,-362.73459927942156,121 +10,-375.2073484643999,128 +11,-109.79113648394251,122 +12,-87.35590491372409,94 +13,-81.82018774450765,105 +14,-202.1453406903699,211 +15,-188.19920697681704,90 +16,-467.21533101568673,111 +17,-219.56892007221305,144 +18,-91.69033101399833,76 +19,-138.18277818453473,112 +20,-397.2172456745353,96 +21,-84.06699026442976,95 +22,-92.9977477912214,119 +23,-71.45394351861083,118 +24,-311.5511689939282,94 +25,-70.25986763070776,66 +26,-282.4437384700192,115 +27,-250.23951083194677,96 +28,-125.75926668820719,67 +29,-102.4509584045276,95 +30,-147.8423093636055,118 +31,-93.58585344373658,98 +32,-137.0249693016,143 +33,-128.48167725148818,82 +34,-109.85956454018849,90 +35,-227.67941266355348,179 +36,-106.31367520480454,122 +37,-69.04618116889586,76 +38,-290.3488193632976,147 +39,-264.1109914845483,99 +40,-327.21157425193593,83 +41,-71.32403704005799,74 +42,-102.27250248243992,94 +43,-78.85486884117992,98 +44,-339.1993619086739,115 +45,-131.5239565033496,144 +46,-252.21884151025685,98 +47,-211.6918769493817,172 +48,-56.505545199580794,163 +49,-398.42392793663004,100 +50,-224.2031903689568,104 +51,-130.31584312667513,120 +52,-339.99545330456135,89 +53,-83.40752569664765,89 +54,-199.25921307183324,151 +55,-109.9427072823704,70 +56,-401.64376363553424,152 +57,-364.89366317964914,137 +58,-138.21169179076463,79 +59,-64.85772869366487,105 +60,-375.9892723800548,88 +61,-92.47870489718572,98 +62,-134.20893943452086,144 +63,-431.6850352240026,112 +64,-109.81680907305143,117 +65,-8.705964580656648,500 +66,-231.90608408278825,177 +67,-105.43713690191188,73 +68,-168.65022695331163,112 +69,-330.78645854953186,121 +70,-290.08286866342326,146 +71,-160.0084561693073,83 +72,-181.91014076921095,101 +73,-351.9798889772708,99 +74,-251.5051673313791,115 +75,-213.52085161900794,109 +76,32.14682519596204,80 +77,-63.694424048843416,68 +78,-94.62913271203752,81 +79,-323.12281333870703,95 +80,-185.84651067245275,86 +81,17.53424129011175,97 +82,-352.61097485443264,124 +83,-144.59061143949063,125 +84,-81.37713345535936,127 +85,-449.57919909423777,137 +86,-296.89207995610286,96 +87,216.32816624890418,416 +88,-128.509638133711,429 +89,-22.118249546947524,170 +90,-118.99165252645804,363 +91,-222.07473578736966,162 +92,67.7338135866681,500 +93,-76.77015675157416,135 +94,7.709576332445521,500 +95,-111.43742964449461,123 +96,-207.81130491928442,185 +97,26.64182328958913,500 +98,-187.69657589345007,415 +99,-153.78344336806452,478 +100,-146.70202775773367,500 +101,-44.651143970963254,500 +102,93.26690853626701,500 +103,-124.24545612600204,500 +104,-157.67112938188922,436 +105,-132.9457561135667,408 +106,67.28609187719101,500 +107,25.65666443831373,500 +108,139.0366704993004,500 +109,-120.71239153340133,320 +110,-119.31564433364915,243 +111,5.9717388628370225,500 +112,0.5488200835310977,263 +113,-0.512426186582071,500 +114,-6.916296600345954,500 +115,43.148118557172005,500 +116,58.622854767147096,500 +117,-59.77931938872634,367 +118,27.161840088256614,500 +119,-331.2975330433161,288 +120,66.0161408232541,500 +121,98.26298537465819,500 +122,85.69814341532671,500 +123,52.12075149713028,500 +124,44.96468140935435,500 +125,67.25221567489307,500 +126,44.393050481332914,500 +127,48.0013999892917,500 +128,60.07846779252989,500 +129,83.93519122248271,500 +130,4.327933091083679,500 +131,107.03750231068621,500 +132,125.0208747718782,500 +133,40.74699615326696,500 +134,50.92068614886296,500 +135,68.34090555506361,500 +136,-25.493755699381623,296 +137,79.88048979561655,500 +138,85.21628006961815,500 +139,64.89338430994542,500 +140,52.418979089514366,500 +141,65.88823854772062,500 +142,74.97706571883673,500 +143,81.17295021796656,500 +144,72.12077891165993,500 +145,68.5669605476233,500 +146,70.24390259524368,500 +147,80.28093286276747,500 +148,-53.25021574497138,193 +149,59.54912212471368,500 +150,62.83892911716514,500 +151,68.57341806025333,500 +152,57.28438835797539,500 +153,22.236732788372244,500 +154,49.39899664919875,500 +155,99.90522842550978,500 +156,48.11618435820098,500 +157,47.77115001447048,500 +158,62.65002286678233,500 +159,42.42726109115208,500 +160,40.484664728665216,500 +161,67.3840578273796,500 +162,72.45854953840278,500 +163,54.667291852928436,500 +164,41.188634317420394,500 +165,59.1221755362032,500 +166,73.41364053656093,500 +167,30.79965284806958,500 +168,49.29586567198497,500 +169,22.6892828429908,500 +170,84.0116135315893,500 +171,47.43035188018893,500 +172,50.82369435459311,500 +173,0.8432996053229578,500 +174,83.85615792138361,500 +175,72.400518606228,500 +176,59.342362950949514,500 +177,55.131063707041534,500 +178,5.335386646271553,500 +179,13.7181772577359,500 +180,-6.817143799330129,500 +181,65.0568301861886,500 +182,27.471730164162647,500 +183,52.486964358497694,500 +184,61.2309463755856,500 +185,37.87996264210685,500 +186,-1.0112310576177272,500 +187,63.35167395546793,500 +188,63.43294371459393,500 +189,46.32752949378909,500 +190,63.837618930039326,500 +191,73.87583434943112,500 +192,89.81736194335608,500 +193,74.41570150818515,500 +194,220.8071282272394,392 +195,96.70551535423107,500 +196,66.967862266038,500 +197,100.05568630601617,500 +198,62.5182288441437,500 +199,85.49806237013074,500 +200,80.49349436463254,500 +201,75.36471963753257,500 +202,241.56370692412304,391 +203,66.48074701345269,500 +204,47.22092565911042,500 +205,67.23158954466489,500 +206,21.19576507114141,500 +207,29.92176308134135,500 +208,-219.8084395803172,500 +209,28.14807109323035,500 +210,119.30555778670272,500 +211,41.217793710432,500 +212,48.15722441775016,500 +213,40.82473016098593,500 +214,36.23659759579064,500 +215,55.477377324180274,500 +216,91.11406857134116,500 +217,49.976906950476234,500 +218,50.97128604745577,500 +219,11.90426978445014,500 +220,40.02978054832668,500 +221,48.165706489140305,500 +222,65.46421455394334,500 +223,59.39609229198244,500 +224,52.24783968353568,500 +225,49.614731744256154,500 +226,81.17322213733931,500 +227,87.67952315806134,500 +228,113.53913442818097,500 +229,96.73596027736887,500 +230,55.29066860630504,500 +231,69.91443915913658,500 +232,62.675647357472336,500 +233,95.23628859404269,500 +234,75.09645534373746,500 +235,54.479311771984776,500 +236,96.79567554964825,500 +237,61.501471542667524,500 +238,108.8692093899993,500 +239,132.81333657867134,500 +240,168.31008995726754,500 +241,207.37024043736483,382 +242,87.30391172293862,500 +243,76.85948241378361,500 +244,261.0959525644049,245 +245,107.39893286640934,500 +246,53.49261661352761,500 +247,103.0345571719846,500 +248,244.0694898779832,283 +249,70.0826767617161,500 +250,245.5964704235691,421 +251,51.37168075518861,500 +252,102.81711048998011,500 +253,65.06123168423136,500 +254,68.4202421315476,500 +255,241.9079415311934,272 +256,111.01183435722018,500 +257,240.69420671634526,247 +258,84.15243749879451,500 +259,238.46935150414382,444 +260,118.71611981863363,500 +261,78.81787636346675,500 +262,89.85766895614972,500 +263,87.14225891162812,500 +264,115.95484327262929,500 +265,48.37642566140835,500 +266,236.2710183255975,218 +267,86.92673042324833,500 +268,75.4596754746324,500 +269,55.03064751774899,500 +270,104.02477299653474,500 +271,221.70306734280936,314 +272,77.34167997050528,500 +273,108.01026274676198,500 +274,50.223888278796906,500 +275,82.22231225936736,500 +276,151.52402362538243,500 +277,74.85713075575639,500 +278,49.07694122682003,500 +279,275.2577043256988,375 +280,230.59519654346292,437 +281,67.87160032229701,500 +282,19.102422355524205,500 +283,224.03592508132547,319 +284,221.91160151223633,466 +285,61.45443105753413,500 +286,249.15841702100082,223 +287,149.40733065319455,500 +288,80.13437351956419,500 +289,274.0784269429413,425 +290,232.33590613339948,367 +291,239.7399774796342,481 +292,248.37902130461495,460 +293,252.62977283392956,329 +294,123.9375993882737,500 +295,283.00203632104393,424 +296,242.52944970941417,416 +297,284.93158353836577,499 +298,231.7520056670157,318 +299,185.98190554920183,324 +300,122.7007287019295,500 +301,233.97662808634962,404 +302,275.7416143220524,444 +303,266.1087902839235,367 +304,219.9374059618578,466 +305,5.922675576757601,500 +306,235.9176436786007,355 +307,203.81783278755069,432 +308,57.37724700603364,500 +309,247.66485888640716,346 +310,273.2248457306265,290 +311,132.60692341251635,500 +312,49.315888905973104,500 +313,220.8772738969443,443 +314,-70.14844214645348,500 +315,60.214748908953894,500 +316,94.78731171174132,500 +317,66.4323655361939,500 +318,231.03623661419508,361 +319,70.2843455465967,500 +320,142.7743834106451,500 +321,238.74221553998592,245 +322,68.93970451577712,500 +323,41.17428800403241,500 +324,229.24713957254735,460 +325,243.42267360236625,495 +326,176.54928916850486,500 +327,83.89421080782193,500 +328,270.52491361255846,326 +329,210.7206161365197,423 +330,90.85295251384353,500 +331,238.08782732630914,409 +332,19.870534988589682,268 +333,78.11206673286654,500 +334,257.91880941676004,463 +335,255.65592180699852,222 +336,71.9701209622765,500 +337,249.30947838670883,411 +338,261.7806437003163,318 +339,74.79281968029154,500 +340,47.84831374903184,500 +341,164.86914183876388,500 +342,166.29326675017646,500 +343,299.4485671891721,320 +344,107.06074808101823,500 +345,69.10989766880935,500 +346,250.03182249958235,314 +347,114.9377741130492,500 +348,91.27017007003414,500 +349,61.31484568773065,500 +350,237.91860833430678,278 +351,-192.67366628553503,454 +352,235.42391959030766,405 +353,56.25272840793466,500 +354,110.4345046775505,500 +355,300.8612387253638,231 +356,291.9869730366015,425 +357,275.44269588804286,216 +358,-2.7694305930015872,500 +359,93.31509752174148,500 +360,63.733609528285356,500 +361,251.98890851858008,201 +362,102.28334741255935,500 +363,316.78805750660285,263 +364,292.4437058412635,175 +365,278.6723267475123,186 +366,218.51131110766136,496 +367,234.83072156437342,229 +368,275.8472061467229,393 +369,142.1429173332867,500 +370,196.2207543662306,425 +371,234.2685309860888,310 +372,248.37514217417115,225 +373,68.94506267022768,500 +374,70.3771331480246,500 +375,98.78547267561505,500 +376,154.4273933694426,500 +377,272.53878961566056,253 +378,91.0711055419984,500 +379,290.92557987104976,240 +380,262.2922302502951,245 +381,25.823123581699974,146 +382,194.54827348810332,500 +383,269.67228684257105,346 +384,134.07557998792885,500 +385,173.04512295225925,500 +386,224.58873362675095,446 +387,240.8117070768953,255 +388,173.89783132058565,500 +389,108.27972267203674,500 +390,284.38206922706524,404 +391,136.21592962076872,500 +392,235.23332077489826,255 +393,147.37995243500373,500 +394,80.42696037144,500 +395,192.6833307467172,500 +396,158.31334124222036,500 +397,95.54223569755669,500 +398,288.4520258895242,336 +399,59.04610733164444,500 +400,74.89686042680277,500 +401,163.00924066340238,500 +402,256.0886145688653,350 +403,279.46188262636167,302 +404,125.19424059454354,500 +405,103.79010775977231,500 +406,227.76902710158936,240 +407,197.26147831317377,472 +408,228.68495851969308,245 +409,257.5389126965403,244 +410,255.27168872716487,176 +411,150.73141480849236,500 +412,131.95857373671998,500 +413,115.70618897028822,500 +414,154.2951410568268,500 +415,126.8883871423331,500 +416,240.24989462204633,369 +417,109.04031508811192,500 +418,168.3473334512263,500 +419,222.25981438931285,370 +420,258.3993629468536,439 +421,129.85379281061938,500 +422,94.45765952938244,500 +423,113.40338498872613,500 +424,136.92519188538492,500 +425,-94.72583488915583,469 +426,115.16238965318378,500 +427,82.10636812189806,500 +428,81.88513771593955,500 +429,60.60319375112212,500 +430,-143.09984735187354,412 +431,230.56774465224092,478 +432,120.33753366842049,500 +433,44.07868298231253,500 +434,152.7590439662164,500 +435,144.57507730925514,500 +436,145.32176709570533,500 +437,79.26562628763243,500 +438,94.46677317906698,500 +439,60.34303479440039,500 +440,104.83421707145058,500 +441,124.20630701747396,500 +442,268.52085378580273,324 +443,131.4129673725378,500 +444,144.07233370221365,500 +445,171.705162511729,500 +446,165.0300882985389,500 +447,163.7112682989296,500 +448,148.21805456509628,500 +449,148.25652740370407,500 +450,90.48402967168373,500 +451,135.40428328321062,500 +452,144.5109246560961,500 +453,234.63103678491098,352 +454,88.20936699872583,500 +455,111.25444545466891,500 +456,199.10191875529992,444 +457,68.83943681552667,500 +458,141.11201761563456,500 +459,173.35318164261687,500 +460,95.63740963645691,500 +461,121.91825692994547,500 +462,193.35957337983845,500 +463,152.9851838889674,500 +464,119.45756448259614,500 +465,99.69766312355573,500 +466,80.06693874499106,500 +467,41.52516781189282,500 +468,62.50114345606357,500 +469,179.43942838297002,500 +470,131.54874903632637,500 +471,194.72217487132318,500 +472,105.24507243906157,500 +473,44.40048251013194,500 +474,113.21654370245638,500 +475,227.56725462473838,469 +476,139.58899783024373,500 +477,157.5287075326,500 +478,179.5482436831508,500 +479,189.18958797261308,500 +480,174.70571294659078,500 +481,80.29898476157332,500 +482,97.30017560381924,500 +483,120.44521353957491,500 +484,39.95600904168039,500 +485,124.27397518058447,500 +486,173.57198979695053,500 +487,156.924756282762,500 +488,136.89706639938723,500 +489,125.31594835974983,500 +490,60.583294774987344,500 +491,42.15069439853899,500 +492,36.822231400751335,500 +493,201.2346402905321,500 +494,214.00923284805748,404 +495,116.1902911145804,500 +496,178.8085041429473,500 +497,147.58542597720697,500 +498,140.59938901566537,500 +499,147.71428887997953,500 +500,165.8169283350966,500 +501,139.850529549145,500 +502,127.9528741206982,500 +503,153.3466701484894,500 +504,48.806386441251135,127 +505,138.31407091735514,500 +506,124.92147918272447,500 +507,140.21776058065146,500 +508,119.5593990721228,500 +509,154.3498683803798,500 +510,143.24169503858946,500 +511,167.69406319363873,500 +512,169.05228448658133,500 +513,168.8362846779819,500 +514,179.313595694161,500 +515,166.2072128899399,500 +516,144.85358379302386,500 +517,253.81747802629747,267 +518,242.51909196480435,284 +519,170.78038161996,500 +520,231.0576097797035,294 +521,113.63773600493401,500 +522,154.74573223020982,500 +523,199.39219370496033,500 +524,125.14588664528151,500 +525,176.61852996423212,500 +526,128.6804114203174,500 +527,163.28899456258458,500 +528,186.39776920896173,500 +529,73.89230506240847,500 +530,244.72475834075235,497 +531,253.0706721713969,448 +532,55.16252528128193,500 +533,145.11719037831523,500 +534,145.65032520073333,500 +535,78.81679418740306,500 +536,169.90495543997392,500 +537,140.6776203709727,500 +538,161.70876437787734,500 +539,143.7623919501416,500 +540,137.74030743235838,500 +541,159.34034706444785,500 +542,155.03541987422454,500 +543,206.49019485313474,500 +544,118.11648545392362,500 +545,163.79025414442532,500 +546,45.737604162862574,500 +547,173.18514310471122,500 +548,152.52483542336614,500 +549,45.6106749804352,500 +550,161.8602340692988,500 +551,143.8370503179388,500 +552,160.3172200828268,500 +553,140.1314336697382,500 +554,144.39464891965864,500 +555,174.22970794942364,500 +556,243.93536653308496,437 +557,106.56515430131904,500 +558,131.28149288619383,500 +559,147.74854191011738,500 +560,136.36727423425066,500 +561,135.08412808799164,500 +562,141.1375930997944,500 +563,191.16297194542946,500 +564,145.3748051053705,500 +565,154.4342027386667,500 +566,140.4942628886445,500 +567,161.7359798137648,500 +568,146.85896523904432,500 +569,130.82773764517762,500 +570,135.7982030655493,500 +571,236.41548242794937,202 +572,171.20905540207843,500 +573,152.71400904618096,500 +574,96.33285599102271,500 +575,195.21977364484033,500 +576,156.1344454460501,500 +577,85.45547442171548,500 +578,165.53856852542,500 +579,104.33852327521684,500 +580,168.18367887122574,500 +581,70.33787283750686,500 +582,157.88048871383634,500 +583,270.934920173459,333 +584,147.9258823338298,500 +585,143.24580121089218,500 +586,157.73720159658944,500 +587,169.82952492163983,500 +588,152.45314801729666,500 +589,175.0065163902205,500 +590,159.31620789708703,500 +591,288.78389568929515,371 +592,272.62216142385216,476 +593,159.22527851651176,500 +594,147.64207339887167,500 +595,173.86104344153975,500 +596,191.68684733953594,500 +597,160.5183627784056,500 +598,184.45428874973362,500 +599,159.43089120919043,500 +600,171.57106280938095,500 +601,130.02202611689611,500 +602,171.69733239565494,500 +603,167.20625013445445,500 +604,115.09834156231474,500 +605,147.045732902395,500 +606,162.68804664833095,500 +607,162.24880085757368,500 +608,147.70981952278657,500 +609,243.1505681970364,288 +610,177.25341777437427,500 +611,165.7002661989493,500 +612,148.88885501506036,500 +613,226.87880715778016,200 +614,154.3647789687756,500 +615,173.45449529259767,500 +616,175.1112331666955,500 +617,179.4684384395566,500 +618,141.7919273787867,500 +619,166.79827038702115,500 +620,160.329375665067,500 +621,60.5139102083594,500 +622,142.30162806776292,500 +623,159.23383199431046,500 +624,148.04862297275577,500 +625,147.04975652240597,500 +626,160.0852954764089,500 +627,246.93954035337714,237 +628,140.5692400406028,500 +629,220.17852732627375,254 +630,146.77219559132195,500 +631,160.99047305768795,500 +632,45.44279525321255,500 +633,157.21860179920444,500 +634,170.00204879644156,500 +635,138.1593011287825,500 +636,141.84243301373738,500 +637,68.44755783138469,500 +638,172.11776976922854,500 +639,210.8997758741869,500 +640,190.503504166276,500 +641,138.35982564779488,500 +642,146.21981341975103,500 +643,166.86846806916213,500 +644,168.48495385598056,500 +645,187.70534278401948,500 +646,268.5884000583329,477 +647,194.175944964931,500 +648,177.34256486729794,500 +649,158.41924691127983,500 +650,83.66726343547968,500 +651,194.8844151478728,500 +652,170.18858342288235,500 +653,210.27734015534148,268 +654,257.6306958120763,300 +655,172.85147580636303,500 +656,79.61786952325093,500 +657,174.7059227926003,500 +658,155.6517400106963,500 +659,171.95146094409478,500 +660,64.39253963159769,500 +661,143.9652396257662,500 +662,120.03758929555498,500 +663,159.38817115701673,500 +664,171.71332516640612,500 +665,146.70720130870964,500 +666,200.71384199039719,500 +667,136.02605321819092,500 +668,150.45182598136023,500 +669,72.89743911236707,500 +670,170.2432359743839,500 +671,114.41004210051027,500 +672,167.82218867536545,500 +673,116.93592487890989,500 +674,173.13568080401197,500 +675,253.53065292360398,401 +676,152.84561310046365,500 +677,58.61499866753663,500 +678,132.04867426334957,500 +679,172.06219619683026,500 +680,154.86218773782883,500 +681,161.9475483316215,500 +682,152.77260739151274,500 +683,162.97721685856027,500 +684,189.37593417926126,500 +685,161.61399527169667,500 +686,149.13435657729113,500 +687,66.38609352204554,500 +688,192.7340031905377,500 +689,165.88565852625828,500 +690,114.34275006878698,500 +691,149.71717797517005,500 +692,54.85094713385511,500 +693,264.54248582560297,484 +694,233.19400637577496,197 +695,127.61610039585953,500 +696,157.02699257754858,500 +697,129.939944245375,500 +698,145.462709125277,500 +699,253.0069188688872,345 +700,102.38926878121552,500 +701,156.28453350185455,500 +702,145.21559064277878,500 +703,154.97579211083894,500 +704,270.9571077664908,324 +705,161.41567374975227,500 +706,140.49601126633107,500 +707,147.49572778766412,500 +708,164.89144776201024,500 +709,191.65953564048647,500 +710,178.54862625039456,500 +711,254.5181067574079,275 +712,150.21853969368038,500 +713,139.65328194926678,500 +714,161.0009597789229,500 +715,158.85547144863597,500 +716,150.7509779119654,500 +717,142.3572444497239,500 +718,180.24618406511124,500 +719,172.5910664114578,500 +720,161.3194568090556,500 +721,276.8634086475175,466 +722,170.25139417277768,500 +723,226.63142544154778,350 +724,161.94851737048396,500 +725,256.41031356556573,371 +726,131.85285376624302,500 +727,159.93431428960298,500 +728,192.14939471904165,500 +729,162.63029456709515,500 +730,178.45560413762635,500 +731,240.37898545650324,322 +732,145.99771884653077,500 +733,199.95756684003808,500 +734,255.74705356578752,392 +735,187.91863633338224,500 +736,255.44901355573177,155 +737,187.97286319611848,500 +738,166.17454403943952,500 +739,267.8256571809973,174 +740,254.5865451909628,409 +741,149.95468292188804,500 +742,266.43008907743797,364 +743,143.21864867270244,500 +744,140.02454808618992,500 +745,137.1002797983269,500 +746,146.91977405177673,500 +747,131.32553220306744,500 +748,153.06690433578123,500 +749,146.85406481188602,500 +750,156.109871998767,500 +751,144.64862992809833,500 +752,143.31638492286805,500 +753,165.2870914518598,500 +754,194.33235204621374,500 +755,180.0811805942356,500 +756,170.3982013570247,500 +757,148.45345115127347,500 +758,274.9432412840355,287 +759,188.3477672262737,500 +760,136.49420239667614,500 +761,173.84484161349175,500 +762,166.22890920197065,500 +763,167.80841642653638,500 +764,163.15945788928835,500 +765,127.00192740582303,500 +766,157.31430204324627,500 +767,130.2073297041003,500 +768,131.4342150814597,500 +769,170.44534747193816,500 +770,178.41803778232534,500 +771,128.24650466880414,500 +772,143.57964601414628,500 +773,190.5790869600398,500 +774,303.2117529657021,339 +775,259.5822953748246,327 +776,199.34581940830097,500 +777,176.06120959013242,500 +778,110.99324037827743,500 +779,141.70508978037148,500 +780,150.00657820991174,500 +781,61.28609353124958,234 +782,224.27810078073395,356 +783,178.35945864987198,500 +784,259.0067645898756,349 +785,229.02904456313112,425 +786,148.31927011925256,500 +787,133.49412245166798,500 +788,294.8497797019389,469 +789,133.06538497485624,500 +790,162.02572972045274,500 +791,286.28552579074125,425 +792,121.96187066460585,500 +793,128.71754122566935,500 +794,155.28222380716625,500 +795,145.6819934669396,500 +796,154.7587086240456,500 +797,190.05063730361678,500 +798,149.31453588102133,500 +799,175.0070032386778,500 +800,169.72020902337658,500 +801,203.45863517350887,500 +802,188.19671214953297,500 +803,183.19108448060288,500 +804,275.05059265826094,190 +805,267.07468204909765,342 +806,279.02199215767655,249 +807,181.3660819129512,500 +808,259.60336272250566,292 +809,156.13431924292593,500 +810,264.25126741240683,308 +811,147.98489947374748,500 +812,188.48480981748457,500 +813,162.78394374436286,500 +814,46.947078080979225,101 +815,127.79665015158318,500 +816,134.6180232500629,500 +817,265.64739646590135,426 +818,279.4299004677526,410 +819,266.51642184066276,218 +820,266.21235886250804,375 +821,267.09200661130785,391 +822,274.1190513272284,348 +823,264.5459497713381,314 +824,283.04374889311634,177 +825,166.4267556530884,500 +826,172.52112102685663,500 +827,86.29096624441499,500 +828,173.8114356024157,500 +829,285.6390141979237,295 +830,285.83644276505527,223 +831,239.11454052328185,414 +832,165.27737066231367,500 +833,172.9140703557666,500 +834,152.4868113731186,500 +835,173.2396972832624,500 +836,201.65369721878994,500 +837,143.44666382768057,500 +838,183.32724218011663,500 +839,192.25724985132015,500 +840,160.9022991359687,500 +841,106.15778563352558,500 +842,203.4465131519525,500 +843,183.2818475502408,500 +844,151.22105773569956,500 +845,121.93677748639381,500 +846,183.4546724078063,500 +847,274.00054924829044,404 +848,151.77009167318056,500 +849,67.19193338281006,500 +850,157.2588267329082,500 +851,171.40851918230462,500 +852,174.1635398056641,500 +853,267.0831639196201,293 +854,178.86557745738742,500 +855,122.24327049744217,500 +856,157.17196817031603,500 +857,240.8325045381683,285 +858,191.62411502299685,500 +859,173.35705929527606,500 +860,133.48182914419408,500 +861,260.1509954169067,333 +862,179.18579003571028,500 +863,285.0750241220918,347 +864,260.4438857234407,328 +865,133.92607360430497,500 +866,181.06070227341553,500 +867,192.24266909356112,500 +868,186.07585028146406,500 +869,186.25096400567313,500 +870,196.65788592370384,500 +871,117.93523119369769,500 +872,113.04796072754087,500 +873,239.121409290357,248 +874,166.0998813261377,500 +875,150.88603763809073,500 +876,150.9575984811946,500 +877,140.9597159399786,500 +878,157.25079070584295,500 +879,163.32677573044998,500 +880,149.28320754952878,500 +881,124.9718782684048,500 +882,140.22620422354606,500 +883,170.30635343784593,500 +884,154.92948436929896,500 +885,171.04287912649662,500 +886,184.29909453507375,500 +887,144.33064406933318,500 +888,186.99822039495507,500 +889,142.25612041489973,500 +890,69.77966439845659,500 +891,143.7243785882498,500 +892,157.74304505066152,500 +893,124.00893211680375,500 +894,174.47666435362194,500 +895,181.6636720690271,500 +896,142.5032836911709,500 +897,266.1995971797988,276 +898,162.97331444560018,500 +899,245.0626359281626,383 +900,158.16106793044324,500 +901,220.95475233552617,422 +902,169.3923794538564,500 +903,168.98959347147718,500 +904,277.37719901333776,230 +905,75.52941036602212,500 +906,46.473702552841374,500 +907,174.56766874746222,500 +908,62.74565348758623,500 +909,71.02194821175667,500 +910,175.02174215834816,500 +911,154.7873034236328,500 +912,160.73174570910015,500 +913,42.5838904844551,500 +914,55.69103810921671,500 +915,169.24696987441794,500 +916,255.88830368661323,366 +917,172.84365228040335,500 +918,183.66305057565944,500 +919,163.10619676706213,500 +920,53.33041163102527,500 +921,155.91291791460947,500 +922,264.50627771599807,244 +923,281.856572964004,348 +924,240.650286239672,464 +925,105.51678717659222,500 +926,115.79735474878623,500 +927,174.73235822214446,500 +928,33.51287650064708,121 +929,167.90509055813172,500 +930,90.08480005125348,500 +931,156.06167915387417,500 +932,150.20632888317013,500 +933,255.02559330806096,390 +934,141.9974974634276,500 +935,178.90626093183005,500 +936,191.18418176536315,500 +937,272.60386361885764,223 +938,285.0872176872522,211 +939,305.65982968219953,475 +940,192.6482978969607,500 +941,162.22026167184163,500 +942,181.10441908027767,500 +943,178.52130629359422,500 +944,184.12815024737083,500 +945,266.84178278764307,234 +946,289.7561759792629,388 +947,178.63253237917607,500 +948,249.79181132780943,242 +949,184.05589297117882,500 +950,238.34519193818863,286 +951,297.86681154454345,319 +952,283.5014252933963,220 +953,149.8954589606982,500 +954,168.5523295216601,500 +955,183.18496276917654,500 +956,277.09523392596736,488 +957,186.47961720295655,500 +958,209.1140598725672,500 +959,156.28879063943495,500 +960,178.34466750713236,500 +961,159.12490781368874,500 +962,273.3016095617544,444 +963,161.62945521070284,500 +964,181.9793797423348,500 +965,154.72128833662438,500 +966,175.98765614139205,500 +967,140.37505273373398,500 +968,179.0805851701676,500 +969,179.0206670830182,500 +970,165.7410257416635,500 +971,147.48299131913578,500 +972,161.14338047229418,500 +973,175.40313632722922,500 +974,149.6499566450647,500 +975,154.56823511552358,500 +976,193.00133224569961,500 +977,145.85314831261138,500 +978,150.48725707508888,500 +979,105.90341392221245,500 +980,167.52199845520923,500 +981,174.3524132389625,500 +982,158.82737387965108,500 +983,159.92412131473844,500 +984,177.72619621519252,500 +985,176.1526654320127,500 +986,275.4849177359491,224 +987,263.4779755681653,351 +988,166.66074388935564,500 +989,183.75029259620743,500 +990,148.42559325324146,500 +991,259.9012531226783,499 +992,186.97923931915167,500 +993,305.41323234833976,304 +994,180.92790942136136,500 +995,208.0263643366691,500 +996,-30.511206278948535,122 +997,-50.063104825771106,146 +998,164.2578720550988,500 +999,281.22341863162444,376 +1000,-30.391876055290126,76 +1001,139.69974398729235,500 +1002,-99.94198135074194,106 +1003,-75.55873881924622,99 +1004,199.9676381346198,500 +1005,163.0499124685519,500 +1006,-121.82194186573683,120 +1007,-76.3821194782238,102 +1008,-4.492951680355816,112 +1009,152.3459331464222,500 +1010,0.07114553528543865,124 +1011,-48.28781167056889,116 +1012,257.53753066407586,276 +1013,290.49018037973303,487 +1014,-88.32964241983595,127 +1015,-34.102845104250704,114 +1016,304.45773798181426,397 +1017,-113.0672413392867,155 +1018,135.40199423992323,500 +1019,149.66625088386124,500 +1020,313.44731660172533,247 +1021,296.5692103449752,238 +1022,-118.76937584438183,117 +1023,-75.86893761906798,166 +1024,132.88783022349423,500 +1025,185.75658911154898,500 +1026,181.19441214874402,500 +1027,-75.12433943808972,139 +1028,129.97389929893495,500 +1029,-124.81756073038345,328 +1030,-122.13888891129018,321 +1031,-87.38068959590534,142 +1032,145.16681904400056,500 +1033,-78.50761330896792,154 +1034,256.1981267044188,375 +1035,252.59665676979705,302 +1036,279.28617980221543,416 +1037,172.6297818424115,500 +1038,234.56589987725584,289 +1039,151.1475320717125,500 +1040,186.23436893584534,500 +1041,-75.09978850724558,178 +1042,180.63010739937832,500 +1043,132.60467947991947,500 +1044,246.56579156755296,249 +1045,-76.38111265186492,190 +1046,167.6714745102107,500 +1047,124.88576411582807,500 +1048,232.96856478244936,476 +1049,-98.09351991092059,195 +1050,155.04466933355195,500 +1051,-115.99453434569654,189 +1052,-121.24983600314766,227 +1053,273.594565298425,451 +1054,145.70085341031438,500 +1055,142.44432856113391,500 +1056,150.8425393494675,500 +1057,172.86832784671526,500 +1058,18.65715150219525,500 +1059,115.20634256658104,500 +1060,173.1034151591153,500 +1061,96.78999746311591,500 +1062,287.1093570895912,298 +1063,272.0548757149836,434 +1064,267.30921667840744,279 +1065,177.60792443086336,500 +1066,298.9817723784304,259 +1067,183.4255900834333,500 +1068,166.5628546849047,500 +1069,235.37004796899643,254 +1070,119.21214424396479,500 +1071,182.63878199446964,500 +1072,187.75190404920804,500 +1073,172.06885807836244,500 +1074,167.8275847096383,500 +1075,218.5869974124321,414 +1076,264.73980398373493,402 +1077,262.2051943743451,185 +1078,299.7985814980477,253 +1079,269.09440186423694,358 +1080,145.1614555662815,500 +1081,304.74276901797765,368 +1082,267.4626824539595,348 +1083,258.77909775614904,361 +1084,139.6987245289529,500 +1085,154.75328460458556,500 +1086,271.4971910260082,393 +1087,171.77616815242294,500 +1088,177.54927558879356,500 +1089,150.0110176883084,500 +1090,125.96669591283792,500 +1091,143.69746887755315,500 +1092,246.97940742142762,208 +1093,151.97519768713687,500 +1094,133.59924228044022,500 +1095,126.4232857431029,500 +1096,298.1637640362233,353 +1097,134.7194584723097,500 +1098,253.4015851587179,392 +1099,244.18995152932416,420 +1100,276.0289911934275,310 +1101,251.95913552294863,388 +1102,173.28858804204665,500 +1103,235.75758460440827,401 +1104,266.51367268024023,366 +1105,141.77395564464942,500 +1106,155.95172158746442,500 +1107,141.46205395461152,500 +1108,188.19768522006703,500 +1109,141.09633481012563,500 +1110,131.2098040118143,500 +1111,171.60470854573214,500 +1112,194.10886072667137,500 +1113,149.13144327801132,500 +1114,183.0338910233726,500 +1115,230.02343979149185,434 +1116,182.47781111555665,500 +1117,122.20557629835136,500 +1118,135.09078191524887,500 +1119,159.1380581976107,500 +1120,14.307404027253872,500 +1121,267.83703825245664,364 +1122,167.06290879497396,500 +1123,154.83985605979487,500 +1124,259.39482939134484,288 +1125,175.43820309655223,500 +1126,300.8663063348442,481 +1127,177.35278909579677,500 +1128,188.25529845589,500 +1129,147.82155721538183,500 +1130,157.93626603902405,500 +1131,285.71686109123243,416 +1132,41.54160712016447,500 +1133,279.474116199247,484 +1134,278.44464197880774,433 +1135,143.95128094310198,500 +1136,240.3785586632003,340 +1137,117.68747949173252,500 +1138,128.30181341121764,500 +1139,97.75842624096462,500 +1140,182.2889817539894,500 +1141,155.89685897616994,500 +1142,-25.210367075276103,500 +1143,147.9283883545498,500 +1144,38.863972378534285,500 +1145,188.41758624537462,500 +1146,112.53376423147857,500 +1147,131.9168653961583,500 +1148,158.12865642426218,500 +1149,115.47544100806722,500 +1150,167.5726555693826,500 +1151,171.88136468236533,500 +1152,193.25333323273188,500 +1153,161.4944053900394,500 +1154,104.52757320825548,500 +1155,223.04887024375148,464 +1156,184.17341257271607,500 +1157,179.39564042067224,500 +1158,185.30358106598572,500 +1159,166.37315215731513,500 +1160,169.66238366751386,500 +1161,49.669513039163895,500 +1162,49.80148252052084,500 +1163,166.1302145016106,500 +1164,60.24734538672413,500 +1165,120.50479433142785,500 +1166,130.13006249294918,500 +1167,162.64494335920108,500 +1168,142.34059409218645,500 +1169,185.42687931267946,500 +1170,167.6601863375647,500 +1171,164.4123140554695,500 +1172,142.00668428186682,500 +1173,198.54839173323836,500 +1174,227.54143894242446,306 +1175,165.052207071657,500 +1176,150.32662475624682,500 +1177,147.3617191616431,500 +1178,146.72936381520879,500 +1179,152.6309632956753,500 +1180,160.19681945236027,500 +1181,169.7332014335313,500 +1182,150.6273133448891,500 +1183,159.9771056652576,500 +1184,150.26385557010568,500 +1185,184.85386886330593,500 +1186,181.853711017474,500 +1187,41.08480378594769,500 +1188,164.38082958575433,500 +1189,186.0243882869355,500 +1190,113.0667649933614,500 +1191,173.95291842724305,500 +1192,158.3782272551618,500 +1193,155.99811622665425,500 +1194,163.77041897834275,500 +1195,152.68969049056153,500 +1196,70.5670679238144,500 +1197,178.0962486093856,500 +1198,183.25907130581885,500 +1199,138.45886742874893,500 +1200,127.47877067210415,500 +1201,184.17213484190077,500 +1202,90.24109716953326,500 +1203,180.4155926402578,500 +1204,145.8552653307774,500 +1205,235.45352011831721,370 +1206,156.5126693559129,500 +1207,122.44421918348398,500 +1208,159.14210931101036,500 +1209,167.3628388746988,500 +1210,170.8744199632308,500 +1211,117.08735533392931,500 +1212,145.85709236223656,500 +1213,149.0827755563572,500 +1214,54.685712671012595,97 +1215,156.11744232423956,500 +1216,188.93547164381593,500 +1217,98.86548749999432,500 +1218,94.26935353113674,500 +1219,94.90518874462838,500 +1220,72.68668661871696,500 +1221,85.98815353926705,500 +1222,81.26302402282668,500 +1223,112.45252795721372,500 +1224,208.57669278352495,339 +1225,71.9082985764093,500 +1226,77.99458774655476,500 +1227,132.13079784369137,500 +1228,37.141809463973125,500 +1229,149.74853707821202,500 +1230,240.92317005715546,245 +1231,268.59638924093025,383 +1232,250.73824854704554,263 +1233,141.81001684099797,500 +1234,141.22484249189452,500 +1235,85.07442658107567,500 +1236,109.80967259246813,500 +1237,188.89745035655173,500 +1238,259.4212175488593,486 +1239,95.78927988887546,500 +1240,269.0535176849072,402 +1241,243.10641006448364,410 +1242,140.12764210251848,500 +1243,134.95810676129744,500 +1244,105.35540527840077,500 +1245,77.88034166240406,500 +1246,165.12526941333044,500 +1247,206.35029671691882,500 +1248,168.42905539620926,500 +1249,84.37444597006628,500 +1250,263.25252058952253,362 +1251,124.81926630284043,500 +1252,150.8989366270141,500 +1253,162.64392925637998,500 +1254,162.67614484460364,500 +1255,167.0829779953099,500 +1256,149.3575049120233,500 +1257,158.81720066248823,500 +1258,152.7408418542654,500 +1259,101.43216306856176,500 +1260,87.12960957648954,500 +1261,138.95920652670344,500 +1262,152.79352712467596,500 +1263,139.9678698589959,500 +1264,97.06067668602715,500 +1265,151.44738359797435,500 +1266,142.34437581372998,500 +1267,165.63956554098675,500 +1268,169.71249687523346,500 +1269,173.09844895591064,500 +1270,167.29350394883338,500 +1271,235.78829127969863,437 +1272,153.69305578669503,500 +1273,0.6899703760858245,119 +1274,183.3295482432536,500 +1275,157.1304871725429,500 +1276,203.67770812915995,500 +1277,160.70741054073198,500 +1278,167.32481273333798,500 +1279,170.34436604065252,500 +1280,196.58839088879319,500 +1281,138.21364471765898,500 +1282,135.38661384671371,500 +1283,99.43096909811497,500 +1284,175.20532691168623,500 +1285,178.0797549719674,500 +1286,144.1153096441811,500 +1287,160.715970010058,500 +1288,154.90605902830822,500 +1289,160.29021213122942,500 +1290,62.90264954530241,185 +1291,191.6150414314482,500 +1292,185.42868659115518,500 +1293,148.85039786785893,500 +1294,153.10145016273145,500 +1295,166.47131921834387,500 +1296,113.92512747451075,500 +1297,142.4321431194974,500 +1298,189.82247139754963,500 +1299,177.8573652204783,500 +1300,177.27085594744514,500 +1301,176.16805216283848,500 +1302,156.80463800055747,500 +1303,152.0770611094366,500 +1304,176.28712629734324,500 +1305,147.63706696251623,500 +1306,173.57440757363713,500 +1307,161.94506358623372,500 +1308,183.70650033521792,500 +1309,290.1320241039332,208 +1310,191.50131956008372,500 +1311,164.63505929928743,500 +1312,193.18467718964177,500 +1313,177.10507800238972,500 +1314,136.98957041009535,500 +1315,151.8401035402973,500 +1316,171.02197400139178,500 +1317,179.9469480755513,500 +1318,182.34209778131813,500 +1319,175.89744774919296,500 +1320,174.35241343126623,500 +1321,193.2027680298207,500 +1322,167.97318314075417,500 +1323,72.73350968700888,105 +1324,182.58394015100032,500 +1325,34.199069956236826,101 +1326,206.36113435022384,500 +1327,166.39854404400785,500 +1328,17.40861620772847,120 +1329,195.6596519552323,500 +1330,165.26283403235593,500 +1331,184.22454744650295,500 +1332,159.05570167937975,500 +1333,185.41876710355297,500 +1334,160.0038351212168,500 +1335,162.42307840452426,500 +1336,142.44207866000096,500 +1337,158.43865201883898,500 +1338,175.0647212478618,500 +1339,209.0947776825944,500 +1340,186.04576057648447,500 +1341,68.52010244406151,89 +1342,187.93532077392675,500 +1343,29.42063152751635,112 +1344,269.82367095975394,421 +1345,166.66151078548967,500 +1346,201.28252372823644,500 +1347,256.84608571617525,475 +1348,145.37791120230736,500 +1349,39.482192766647046,168 +1350,288.25140247502316,286 +1351,170.46355487050096,500 +1352,172.21570347719359,500 +1353,170.64176362396216,500 +1354,262.93461530499377,390 +1355,158.68022259806196,500 +1356,159.87261027541228,500 +1357,157.1794363116416,500 +1358,160.1950354189548,500 +1359,149.0222801166495,500 +1360,159.18591181783384,500 +1361,287.86301155059255,317 +1362,180.5630368546528,500 +1363,109.40374257306146,500 +1364,265.5651403691759,400 +1365,244.58768194389404,472 +1366,134.94715631750623,500 +1367,142.87796395810616,500 +1368,248.5025355889588,477 +1369,148.0963643584679,500 +1370,246.35296536950318,282 +1371,188.4054799241219,500 +1372,167.21064457661413,500 +1373,138.96633366577944,500 +1374,115.42838089949815,500 +1375,225.60458717428637,333 +1376,75.0995570313525,500 +1377,171.6306276450823,500 +1378,277.7805128211662,359 +1379,281.21082585598424,230 +1380,159.9571608716223,500 +1381,77.42019410837615,500 +1382,263.979010137404,237 +1383,235.84745500155995,464 +1384,239.4537497349118,340 +1385,160.00947945995924,500 +1386,163.48971402143866,500 +1387,270.8772862459951,419 +1388,-169.01900129603834,389 +1389,-149.65754908251932,419 +1390,203.29217729220298,500 +1391,152.40473118831338,500 +1392,28.148751859031535,120 +1393,179.67665823327465,500 +1394,158.9147977635007,500 +1395,172.77872064139302,500 +1396,176.1540238425264,500 +1397,170.59505773460444,500 +1398,122.23510841646105,500 +1399,172.67109226735505,500 +1400,147.41232954520126,500 +1401,239.0502163074453,272 +1402,241.07158087300343,371 +1403,138.6451415563053,500 +1404,242.96707721465472,458 +1405,176.48698578199918,500 +1406,163.5411500494525,500 +1407,148.69594461924123,500 +1408,163.79393718835195,500 +1409,174.0722764043939,500 +1410,199.27111504862646,500 +1411,175.11537267114005,500 +1412,173.6921201031736,500 +1413,272.25216430143894,315 +1414,153.0874040809182,500 +1415,172.0078562690015,500 +1416,81.99360079040929,500 +1417,175.50060715399874,500 +1418,178.41246631573708,500 +1419,193.3651464255431,500 +1420,264.56457847693537,237 +1421,163.52633818497577,500 +1422,177.58398761094907,500 +1423,264.54392946179405,287 +1424,302.51028485207706,439 +1425,148.90300354696814,500 +1426,162.5556926649182,500 +1427,265.08765923392593,448 +1428,260.6619572868598,427 +1429,171.94568838111027,500 +1430,111.22632814873398,500 +1431,228.72609061087496,471 +1432,163.916242086193,500 +1433,147.29276947037263,500 +1434,141.0174892198229,500 +1435,100.28544353694299,500 +1436,125.92251387334228,500 +1437,252.56315716269324,397 +1438,92.73182639680499,500 +1439,144.8101099855162,500 +1440,173.47247258239418,500 +1441,148.27873942786906,500 +1442,115.53844459343865,500 +1443,181.985163978229,500 +1444,113.00134488918926,500 +1445,141.3568206582538,500 +1446,239.63181613696193,500 +1447,155.7044901042616,500 +1448,153.9108845290257,500 +1449,145.37645934359168,500 +1450,142.36377583126512,500 +1451,167.89123640852202,500 +1452,143.954662717843,500 +1453,180.37938945832997,500 +1454,130.87383881200398,500 +1455,123.11973299435189,500 +1456,164.36864910997417,500 +1457,233.70316988850368,322 +1458,143.10703165248356,500 +1459,160.40558736698927,500 +1460,87.88838700564874,500 +1461,180.02583719438599,500 +1462,127.50126788111609,500 +1463,165.28104956182787,500 +1464,152.74389928881368,500 +1465,99.81252457604825,500 +1466,62.346418741661864,500 +1467,180.24032856053122,500 +1468,154.07986342466154,500 +1469,168.43262423246156,500 +1470,140.5215954402793,500 +1471,179.11419509592997,500 +1472,160.0969409572327,500 +1473,181.55708057386553,500 +1474,139.52782333261817,500 +1475,147.357715263127,500 +1476,170.89828161550804,500 +1477,141.25673425605967,500 +1478,271.1053306584651,324 +1479,194.65282506326633,500 +1480,197.72847145118493,500 +1481,154.36298846742665,500 +1482,108.32593371871855,500 +1483,184.29329818273936,500 +1484,198.8347025438432,500 +1485,247.36605079253695,418 +1486,191.85971956731677,500 +1487,137.71147239443002,500 +1488,145.69941378836413,500 +1489,146.4788074562568,500 +1490,161.9820281446436,500 +1491,143.8840218738323,500 +1492,183.70885530957875,500 +1493,269.52894261021356,491 +1494,160.46124427668934,500 +1495,164.51518500862875,500 +1496,159.0152095607036,500 +1497,157.1298315101697,500 +1498,149.8302340126811,500 +1499,256.63427854927875,418 +1500,297.8301554474489,473 +1501,134.63546686405755,500 +1502,287.94646061118266,445 +1503,257.3491741941353,442 +1504,136.06995292732452,500 +1505,165.5819741948222,500 +1506,141.82970733378167,500 +1507,297.61244886579533,425 +1508,260.6747275113645,403 +1509,167.44958988401777,500 +1510,308.9618510277177,257 +1511,260.13630875550075,173 +1512,174.98024803674502,500 +1513,226.25811038880636,482 +1514,231.53325392783415,404 +1515,251.23578932491168,441 +1516,125.37525207994764,500 +1517,187.71858707326655,500 +1518,254.03714339777972,411 +1519,154.29112306571534,500 +1520,255.91709982602364,301 +1521,160.47207522062018,500 +1522,126.54002952433952,500 +1523,270.49720550042406,390 +1524,248.31012809538544,296 +1525,195.44622371738598,500 +1526,245.96153071220158,474 +1527,254.26179681715035,455 +1528,241.75819601713522,245 +1529,274.8204453549657,348 +1530,269.07017047048754,266 +1531,259.1105992668045,252 +1532,268.2004605978253,251 +1533,277.4132288383143,209 +1534,304.46404619262785,261 +1535,280.5442232256507,200 +1536,264.82733952676926,212 +1537,295.93140609900865,382 +1538,275.130122238966,218 +1539,280.08888169552534,337 +1540,215.62109892964702,443 +1541,135.7376676505715,500 +1542,256.1965598317555,433 +1543,269.53972772552663,286 +1544,290.9367312230013,273 +1545,268.15650935140786,269 +1546,258.26987871728045,194 +1547,177.2798184500653,500 +1548,177.02519430533175,500 +1549,134.9990105715062,500 +1550,132.79177257261364,500 +1551,253.54882607336094,250 +1552,69.10788359146419,500 +1553,246.1831341851267,344 +1554,297.57185606778984,213 +1555,257.4021042580648,296 +1556,276.01093762815975,278 +1557,199.5025549830316,500 +1558,249.8085028896224,339 +1559,279.44674528186033,316 +1560,277.0811576012035,392 +1561,180.2148625669948,500 +1562,157.54502591806917,500 +1563,304.3171688039,375 +1564,126.02167468861347,500 +1565,166.36011587427683,500 +1566,279.41469005424017,441 +1567,256.60078489621685,249 +1568,293.09180351768475,365 +1569,162.79706346972768,500 +1570,290.3903254339166,341 +1571,188.52321383333486,500 +1572,161.84729971752463,500 +1573,179.73153589671128,500 +1574,277.79787403158946,332 +1575,170.94574185441945,500 +1576,273.1021940788951,464 +1577,267.92791469903733,297 +1578,166.09860889211046,500 +1579,162.87344778113172,500 +1580,184.0859949773231,500 +1581,185.08657345656624,500 +1582,262.4558555592147,281 +1583,159.68354215522723,500 +1584,182.60808091708768,500 +1585,162.4004571138879,500 +1586,190.6359379995434,500 +1587,301.8195815949847,246 +1588,137.5487057794984,500 +1589,169.9453997814478,500 +1590,154.0690502368745,500 +1591,175.8877139892446,500 +1592,159.32714345079498,500 +1593,169.4985161423462,500 +1594,175.83742619565945,500 +1595,193.75369457632559,500 +1596,292.9029473358955,493 +1597,153.6276103244635,500 +1598,287.23020971787645,272 +1599,177.52519302192297,500 +1600,158.69310029508208,500 +1601,163.45603497503186,500 +1602,187.72415251237538,500 +1603,161.32536677369882,500 +1604,159.5514997809911,500 +1605,164.1706783538047,500 +1606,321.02975429133085,446 +1607,185.27777837878764,500 +1608,169.57012203185582,500 +1609,171.50144846499,500 +1610,264.62592780596356,402 +1611,305.29426618156765,405 +1612,31.805186197778788,500 +1613,-138.7786059220438,475 +1614,269.34219771639937,471 +1615,159.8310642440357,500 +1616,257.11759998490726,337 +1617,270.6626396227839,328 +1618,166.3015195018077,500 +1619,167.35682217929687,500 +1620,255.99211228356035,414 +1621,267.2398665620386,208 +1622,249.12167213058342,286 +1623,209.9058605825952,500 +1624,148.31148474758214,500 +1625,177.46399698575038,500 +1626,252.6185679632112,337 +1627,245.12390108648995,475 +1628,140.02092730370148,500 +1629,293.6495630498656,461 +1630,248.07893822850582,236 +1631,195.72703440881392,500 +1632,227.63537549727798,208 +1633,162.57969736094356,500 +1634,238.88605737169718,452 +1635,176.2628679468286,500 +1636,158.47537735109267,500 +1637,164.8079917495705,500 +1638,162.81129609855884,500 +1639,158.95568182331007,500 +1640,171.17593077092,500 +1641,241.91576872429968,291 +1642,189.9696753714451,500 +1643,269.1225676974592,272 +1644,153.22200120147372,500 +1645,268.09661720483757,426 +1646,192.73047258659582,500 +1647,179.73369246048767,500 +1648,151.81625788613653,500 +1649,267.5193707012853,459 +1650,227.43988917553781,397 +1651,289.0284340506049,325 +1652,161.34109350811863,500 +1653,289.04344522537144,276 +1654,270.71757562806806,361 +1655,287.2220023783037,389 +1656,169.97218938308484,500 +1657,189.02543388702458,500 +1658,253.55797034277848,473 +1659,149.32519259824784,500 +1660,143.25015262799334,500 +1661,264.5524214149401,210 +1662,250.78241623424438,322 +1663,258.5040509935592,251 +1664,164.1672842740288,500 +1665,66.99157673871342,500 +1666,144.65668096293032,500 +1667,231.81631112627238,359 +1668,234.6550351665292,407 +1669,241.8499196446127,304 +1670,133.87437599483056,500 +1671,290.67065194033773,190 +1672,288.5565072356609,204 +1673,230.4228435615384,256 +1674,164.68683936103793,500 +1675,244.7981442595739,346 +1676,248.81366688127991,291 +1677,154.7444928108678,500 +1678,259.8167002544392,494 +1679,283.4263657664286,314 +1680,261.645244098537,259 +1681,263.0115183845345,184 +1682,131.8735508359715,500 +1683,246.2826027390746,444 +1684,284.50583423396324,220 +1685,-3.4330293068034763,155 +1686,268.4029015344463,393 +1687,263.6578280273419,283 +1688,170.7719019350333,500 +1689,258.2505608054255,399 +1690,152.8227555366325,500 +1691,255.56191034559154,379 +1692,240.8429324931063,486 +1693,125.21424953363677,500 +1694,161.39069224608556,500 +1695,276.1296473523237,493 +1696,110.54706069343216,500 +1697,286.76943095411593,323 +1698,262.7662584311813,465 +1699,154.83777837119527,500 +1700,190.77656701924397,500 +1701,181.0680524283195,500 +1702,234.4915446019391,302 +1703,289.4793837153037,293 +1704,180.89785269860224,500 +1705,158.40921261561064,500 +1706,198.38866291050243,500 +1707,175.6219553538599,500 +1708,254.5735275246547,295 +1709,270.17443831349965,375 +1710,230.63133694035037,454 +1711,252.91011873735133,367 +1712,280.2034024237521,498 +1713,292.60407109338087,340 +1714,225.01348082763923,264 +1715,166.4215437956867,500 +1716,144.08487936337877,500 +1717,157.6878007970581,500 +1718,279.3044393145294,494 +1719,272.74577643146927,218 +1720,183.06245454004934,500 +1721,267.1903627947363,433 +1722,183.1142348269536,500 +1723,170.95636913913506,500 +1724,168.95126990805355,500 +1725,148.86022102639535,500 +1726,278.87719946666203,214 +1727,250.93919083984628,306 +1728,273.9822971141052,159 +1729,284.72640623962315,190 +1730,287.7131780450663,289 +1731,281.7064153294699,412 +1732,264.3774814117037,432 +1733,274.6689780844587,236 +1734,265.4167026208347,259 +1735,289.19316529426214,202 +1736,274.6471713757392,294 +1737,115.5696680199556,500 +1738,255.91571112190994,200 +1739,68.57140620918476,500 +1740,244.07691464034636,237 +1741,248.02287349335285,325 +1742,285.8053543468766,232 +1743,197.2250277323277,500 +1744,242.9577607027271,212 +1745,256.6594760661792,186 +1746,56.26690518989957,500 +1747,262.7505398239705,449 +1748,140.38433510881606,500 +1749,47.95107326954544,500 +1750,189.2103888593619,500 +1751,40.116671712110524,500 +1752,267.6898751168179,212 +1753,274.26192784090057,215 +1754,271.9293786035767,188 +1755,145.65333182737953,500 +1756,144.71983281064615,500 +1757,248.14191966399815,333 +1758,238.23200681258336,357 +1759,37.04100021918786,500 +1760,246.44169633101305,283 +1761,270.6974031439338,340 +1762,46.91515236489587,500 +1763,-124.16971448980054,500 +1764,264.98500207994255,176 +1765,272.4318103619396,335 +1766,184.67994420870363,500 +1767,169.2494785380969,500 +1768,221.70219885237168,463 +1769,271.41464634638123,214 +1770,171.06291573629466,500 +1771,278.2012381862014,245 +1772,305.6262532143434,314 +1773,281.239026128085,192 +1774,165.78034767589546,500 +1775,195.86691438407502,500 +1776,260.5819427516773,330 +1777,39.365728023368845,500 +1778,267.15912128221987,317 +1779,145.30602115549152,500 +1780,133.76896560365503,500 +1781,172.41276688552435,500 +1782,144.47990661558447,500 +1783,262.2892120512618,304 +1784,245.86629586333103,394 +1785,52.72589995066599,223 +1786,47.31184400053695,500 +1787,299.96771149335046,333 +1788,262.7717209852532,222 +1789,154.14931496151954,500 +1790,247.69184428182567,342 +1791,155.58783073619773,500 +1792,23.12795164839757,113 +1793,128.72699374648423,500 +1794,260.9102625009707,230 +1795,31.966692485950215,500 +1796,152.37339085529894,500 +1797,173.49730573454627,500 +1798,254.56777681348248,226 +1799,294.2264317809148,289 +1800,111.00137019447702,500 +1801,73.26087381817537,500 +1802,251.58334726359718,351 +1803,178.22774890067316,500 +1804,266.976505850396,156 +1805,278.55392338742706,473 +1806,184.7317144921546,500 +1807,119.26182312968591,500 +1808,240.13563264022298,345 +1809,130.4240668847301,500 +1810,174.5890787005682,500 +1811,299.17607145228584,333 +1812,252.09669025111296,173 +1813,251.12880178943877,337 +1814,274.98648780531596,299 +1815,158.64012192309767,500 +1816,155.57253010608383,500 +1817,268.83749511589065,164 +1818,65.65721791257907,500 +1819,264.2765794939315,286 +1820,41.61811924581658,500 +1821,307.2095348129088,420 +1822,139.9444327529991,500 +1823,152.827324781369,500 +1824,248.0823366771297,321 +1825,237.27927498386236,286 +1826,226.0969629720782,396 +1827,277.6872388939346,409 +1828,283.7715091266298,338 +1829,152.47780747726813,500 +1830,250.87234195911597,382 +1831,242.84879599172936,418 +1832,257.84104634346903,302 +1833,306.83368393880994,410 +1834,183.54466949948548,500 +1835,27.122231914608385,500 +1836,285.2153107805,236 +1837,283.1790670683889,193 +1838,267.1842885229891,376 +1839,94.19420509261195,500 +1840,6.378101290720608,454 +1841,245.83151033938867,264 +1842,219.53735723250566,315 +1843,165.17817977465143,500 +1844,157.54183325739677,500 +1845,278.6668804645586,436 +1846,160.0638868021003,500 +1847,244.96031333418324,212 +1848,185.15763213306946,500 +1849,167.4945236708156,500 +1850,159.93450493030474,500 +1851,64.49391515479762,500 +1852,173.95357643506645,500 +1853,197.60710800692914,500 +1854,200.14853276170385,500 +1855,174.88584678867414,500 +1856,34.25057609113966,144 +1857,254.58967119662825,485 +1858,308.1049870656548,387 +1859,232.21297680983596,298 +1860,142.82610582165103,500 +1861,167.4723504480381,500 +1862,180.55877632845744,500 +1863,18.88914795091644,120 +1864,139.36342110399076,500 +1865,166.35444612576327,500 +1866,246.58626240052482,452 +1867,255.578806219165,296 +1868,250.92806068418747,441 +1869,71.69384465669599,500 +1870,271.8827095426893,318 +1871,242.6999000090626,372 +1872,250.75849680841193,419 +1873,269.58213014053746,464 +1874,33.29095175882679,129 +1875,169.60013015969977,500 +1876,252.77381718056512,373 +1877,176.33515119437504,500 +1878,279.5430733432613,454 +1879,171.11013194322575,500 +1880,259.41393889168586,265 +1881,270.8256532587688,174 +1882,284.1026263519225,324 +1883,169.41404732467967,500 +1884,160.2724130043987,500 +1885,261.8874119165903,485 +1886,161.16965332591914,500 +1887,157.9262890703557,500 +1888,166.58427273084467,500 +1889,61.19164143878467,500 +1890,150.08602663501347,500 +1891,154.5569247413144,500 +1892,144.97230034496403,500 +1893,191.08984101190896,500 +1894,242.87081272688908,253 +1895,157.68364960297748,500 +1896,232.76664113779503,267 +1897,172.54158839325547,500 +1898,171.5275864558997,500 +1899,140.0268434853445,500 +1900,221.23354485345263,487 +1901,281.3840538653195,459 +1902,258.70696774557973,198 +1903,81.83437123081902,500 +1904,143.05679562528778,500 +1905,150.46574361420647,500 +1906,154.9712342467976,500 +1907,149.65589751004367,500 +1908,161.3319326704685,500 +1909,152.21397979700959,500 +1910,129.85744848350407,500 +1911,98.90167826761638,500 +1912,106.177410990882,500 +1913,166.53041300365007,500 +1914,168.22510105476286,500 +1915,166.95392249641836,500 +1916,123.90148574274632,500 +1917,34.80396843147126,500 +1918,155.5122153100952,500 +1919,57.196894575907976,500 +1920,144.78619644787327,500 +1921,53.03701299447987,500 +1922,295.6148512900687,325 +1923,291.12421667941476,362 +1924,166.90127757366133,500 +1925,72.9666371107036,500 +1926,98.46174168112073,500 +1927,50.54613666824542,500 +1928,51.81099796302183,500 +1929,277.55442597005765,225 +1930,285.9390080844404,298 +1931,75.86549987985441,500 +1932,133.4126928512472,500 +1933,127.3759140393114,500 +1934,295.95410782993156,296 +1935,158.26567884616827,500 +1936,248.51280858880907,434 +1937,269.9406653073189,264 +1938,281.7484768871346,299 +1939,155.32937355395967,500 +1940,273.5667091955313,245 +1941,163.70484904400115,500 +1942,168.2012878302778,500 +1943,147.13913560653864,500 +1944,109.50850633575666,500 +1945,160.7955652956411,500 +1946,253.80843192282222,233 +1947,54.66260766289343,500 +1948,251.15418976659942,193 +1949,156.10881065362906,500 +1950,264.7870590640953,173 +1951,157.0169774144052,500 +1952,289.5075809760132,258 +1953,155.80565612139458,500 +1954,152.57775300537952,500 +1955,155.34164076344223,500 +1956,268.0699847903705,446 +1957,260.35893156702787,157 +1958,253.34131270359885,479 +1959,183.57862219233118,500 +1960,94.43611157586167,500 +1961,171.81210735240685,500 +1962,281.31191031269105,324 +1963,115.49799887832445,500 +1964,240.1338470425453,359 +1965,163.69524895835397,500 +1966,115.32531086162935,500 +1967,176.8530785043737,500 +1968,253.23624197186595,354 +1969,288.7648810216808,306 +1970,151.4044583783478,500 +1971,177.07802295312172,500 +1972,27.161531382132807,500 +1973,-1.3677508827598501,106 +1974,195.54874548478176,500 +1975,194.7515272302086,500 +1976,298.4030511747202,362 +1977,129.61125705397433,500 +1978,199.88212357517085,500 +1979,145.56009558434405,500 +1980,275.932135351486,382 +1981,86.9263139871182,500 +1982,178.3720428086926,500 +1983,152.64923990745163,500 +1984,50.76545124815678,90 +1985,176.9511310565773,500 +1986,151.66511287416682,500 +1987,177.1364133864843,500 +1988,140.7754183778112,500 +1989,186.38476289873856,500 +1990,175.79427035539155,500 +1991,164.62575487551763,500 +1992,174.40765486502164,500 +1993,167.6549446240523,500 +1994,286.46064362885954,449 +1995,177.01142354657676,500 +1996,260.26971602229446,301 +1997,175.62726495966803,500 +1998,165.80760378430293,500 +1999,277.25918068690123,330 diff --git a/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/tb_logs/events.out.tfevents.1680426118.mekeypan-MS-7E07.240232.0 b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/tb_logs/events.out.tfevents.1680426118.mekeypan-MS-7E07.240232.0 new file mode 100644 index 0000000000000000000000000000000000000000..73fb2a713eaffca41fab62dbaa4309c96a6a68c4 --- /dev/null +++ b/LunarLanderContinuous-v2/Train_LunarLanderContinuous-v2_SAC_20230402-170158/tb_logs/events.out.tfevents.1680426118.mekeypan-MS-7E07.240232.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14cf09865cfff9c6fb7d5ea57121d65fe6a3ec5cdd53a9c39b2478f900a99c29 +size 40 diff --git a/Mario/Test_Mario_DQN_CNN_20221207-193032/config.yaml b/Mario/Test_Mario_DQN_CNN_20221207-193032/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4afe784419976a93489639c8bfc193310aabac7c --- /dev/null +++ b/Mario/Test_Mario_DQN_CNN_20221207-193032/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: DQN_CNN + device: cuda + env_name: Mario + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_Mario_DQN_CNN_20221207-155552 + max_steps: 1000 + mode: test + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 20 + wrapper: envs.wrappers.MarioWrappers +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 diff --git a/Mario/Test_Mario_DQN_CNN_20221207-193032/logs/log.txt b/Mario/Test_Mario_DQN_CNN_20221207-193032/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..150b1e6090f6b8ac6795fd6f40d07485a1b4b673 --- /dev/null +++ b/Mario/Test_Mario_DQN_CNN_20221207-193032/logs/log.txt @@ -0,0 +1,43 @@ +2022-12-07 19:30:32 - r - INFO: - Hyperparameters: +2022-12-07 19:30:32 - r - INFO: - ================================================================================ +2022-12-07 19:30:32 - r - INFO: - Name Value Type +2022-12-07 19:30:32 - r - INFO: - env_name Mario +2022-12-07 19:30:32 - r - INFO: - new_step_api 1 +2022-12-07 19:30:32 - r - INFO: - wrapper envs.wrappers.MarioWrappers +2022-12-07 19:30:32 - r - INFO: - render 0 +2022-12-07 19:30:32 - r - INFO: - algo_name DQN_CNN +2022-12-07 19:30:32 - r - INFO: - mode test +2022-12-07 19:30:32 - r - INFO: - seed 1 +2022-12-07 19:30:32 - r - INFO: - device cuda +2022-12-07 19:30:32 - r - INFO: - train_eps 20 +2022-12-07 19:30:32 - r - INFO: - test_eps 10 +2022-12-07 19:30:32 - r - INFO: - eval_eps 10 +2022-12-07 19:30:32 - r - INFO: - eval_per_episode 5 +2022-12-07 19:30:32 - r - INFO: - max_steps 1000 +2022-12-07 19:30:32 - r - INFO: - load_checkpoint 1 +2022-12-07 19:30:32 - r - INFO: - load_path Train_Mario_DQN_CNN_20221207-155552 +2022-12-07 19:30:32 - r - INFO: - show_fig 0 +2022-12-07 19:30:32 - r - INFO: - save_fig 1 +2022-12-07 19:30:32 - r - INFO: - epsilon_start 0.95 +2022-12-07 19:30:32 - r - INFO: - epsilon_end 0.01 +2022-12-07 19:30:32 - r - INFO: - epsilon_decay 500 +2022-12-07 19:30:32 - r - INFO: - gamma 0.99 +2022-12-07 19:30:32 - r - INFO: - lr 0.0001 +2022-12-07 19:30:32 - r - INFO: - buffer_size 100000 +2022-12-07 19:30:32 - r - INFO: - batch_size 64 +2022-12-07 19:30:32 - r - INFO: - target_update 4 +2022-12-07 19:30:32 - r - INFO: - ================================================================================ +2022-12-07 19:30:33 - r - INFO: - n_states: 4, n_actions: 256 +2022-12-07 19:30:36 - r - INFO: - Start testing! +2022-12-07 19:30:36 - r - INFO: - Env: Mario, Algorithm: DQN_CNN, Device: cuda +2022-12-07 19:30:37 - r - INFO: - Episode: 1/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:38 - r - INFO: - Episode: 2/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:39 - r - INFO: - Episode: 3/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:40 - r - INFO: - Episode: 4/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:41 - r - INFO: - Episode: 5/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:42 - r - INFO: - Episode: 6/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:43 - r - INFO: - Episode: 7/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:43 - r - INFO: - Episode: 8/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:44 - r - INFO: - Episode: 9/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:45 - r - INFO: - Episode: 10/10, Reward: 737.000, Step: 91 +2022-12-07 19:30:45 - r - INFO: - Finish testing! diff --git a/Mario/Test_Mario_DQN_CNN_20221207-193032/models/checkpoint.pt b/Mario/Test_Mario_DQN_CNN_20221207-193032/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..b36f6a06b7ab606b4bfefd988b20242c62cabfb2 --- /dev/null +++ b/Mario/Test_Mario_DQN_CNN_20221207-193032/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a7fa90cd9ae8dd2e3a2ede80a92a2fbd636c3762b2ea1ec0ad3e02f364d0d4 +size 7264839 diff --git a/Mario/Test_Mario_DQN_CNN_20221207-193032/results/learning_curve.png b/Mario/Test_Mario_DQN_CNN_20221207-193032/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..e29be2d4fe9b10b8a577542138282d55945eda22 Binary files /dev/null and b/Mario/Test_Mario_DQN_CNN_20221207-193032/results/learning_curve.png differ diff --git a/Mario/Test_Mario_DQN_CNN_20221207-193032/results/res.csv b/Mario/Test_Mario_DQN_CNN_20221207-193032/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..689f64bf824906cf910f10deb0856dcecba4fda0 --- /dev/null +++ b/Mario/Test_Mario_DQN_CNN_20221207-193032/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,737.0,91 +1,737.0,91 +2,737.0,91 +3,737.0,91 +4,737.0,91 +5,737.0,91 +6,737.0,91 +7,737.0,91 +8,737.0,91 +9,737.0,91 diff --git a/Mario/Train_Mario_DQN_CNN_20221207-155552/config.yaml b/Mario/Train_Mario_DQN_CNN_20221207-155552/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..050a6e2bc4e3fafb042085233ff2002c7adf34ec --- /dev/null +++ b/Mario/Train_Mario_DQN_CNN_20221207-155552/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: DQN_CNN + device: cuda + env_name: Mario + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_Mario_DQN_CNN_20221205-205203 + max_steps: 1000 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: envs.wrappers.MarioWrappers +algo_cfg: + batch_size: 64 + buffer_size: 100000 + epsilon_decay: 500 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.99 + lr: 0.0001 + target_update: 4 diff --git a/Mario/Train_Mario_DQN_CNN_20221207-155552/logs/log.txt b/Mario/Train_Mario_DQN_CNN_20221207-155552/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..07dcdfd1778c18015054615cd805d5d07a16d44b --- /dev/null +++ b/Mario/Train_Mario_DQN_CNN_20221207-155552/logs/log.txt @@ -0,0 +1,238 @@ +2022-12-07 15:55:52 - r - INFO: - Hyperparameters: +2022-12-07 15:55:52 - r - INFO: - ================================================================================ +2022-12-07 15:55:52 - r - INFO: - Name Value Type +2022-12-07 15:55:52 - r - INFO: - env_name Mario +2022-12-07 15:55:52 - r - INFO: - new_step_api 1 +2022-12-07 15:55:52 - r - INFO: - wrapper envs.wrappers.MarioWrappers +2022-12-07 15:55:52 - r - INFO: - render 0 +2022-12-07 15:55:52 - r - INFO: - algo_name DQN_CNN +2022-12-07 15:55:52 - r - INFO: - mode train +2022-12-07 15:55:52 - r - INFO: - seed 1 +2022-12-07 15:55:52 - r - INFO: - device cuda +2022-12-07 15:55:52 - r - INFO: - train_eps 200 +2022-12-07 15:55:52 - r - INFO: - test_eps 10 +2022-12-07 15:55:52 - r - INFO: - eval_eps 10 +2022-12-07 15:55:52 - r - INFO: - eval_per_episode 5 +2022-12-07 15:55:52 - r - INFO: - max_steps 1000 +2022-12-07 15:55:52 - r - INFO: - load_checkpoint 0 +2022-12-07 15:55:52 - r - INFO: - load_path Train_Mario_DQN_CNN_20221205-205203 +2022-12-07 15:55:52 - r - INFO: - show_fig 0 +2022-12-07 15:55:52 - r - INFO: - save_fig 1 +2022-12-07 15:55:52 - r - INFO: - epsilon_start 0.95 +2022-12-07 15:55:52 - r - INFO: - epsilon_end 0.01 +2022-12-07 15:55:52 - r - INFO: - epsilon_decay 500 +2022-12-07 15:55:52 - r - INFO: - gamma 0.99 +2022-12-07 15:55:52 - r - INFO: - lr 0.0001 +2022-12-07 15:55:52 - r - INFO: - buffer_size 100000 +2022-12-07 15:55:52 - r - INFO: - batch_size 64 +2022-12-07 15:55:52 - r - INFO: - target_update 4 +2022-12-07 15:55:52 - r - INFO: - ================================================================================ +2022-12-07 15:55:53 - r - INFO: - n_states: 4, n_actions: 256 +2022-12-07 15:55:56 - r - INFO: - Start training! +2022-12-07 15:55:56 - r - INFO: - Env: Mario, Algorithm: DQN_CNN, Device: cuda +2022-12-07 15:56:31 - r - INFO: - Episode: 1/200, Reward: 697.000, Step: 1000 +2022-12-07 15:57:10 - r - INFO: - Episode: 2/200, Reward: 330.000, Step: 1000 +2022-12-07 15:57:48 - r - INFO: - Episode: 3/200, Reward: 164.000, Step: 1000 +2022-12-07 15:58:26 - r - INFO: - Episode: 4/200, Reward: 939.000, Step: 1000 +2022-12-07 15:59:05 - r - INFO: - Episode: 5/200, Reward: 757.000, Step: 1000 +2022-12-07 16:00:35 - r - INFO: - Current episode 5 has the best eval reward: -240.000 +2022-12-07 16:01:13 - r - INFO: - Episode: 6/200, Reward: 347.000, Step: 1000 +2022-12-07 16:01:52 - r - INFO: - Episode: 7/200, Reward: 727.000, Step: 1000 +2022-12-07 16:02:17 - r - INFO: - Episode: 8/200, Reward: 430.000, Step: 626 +2022-12-07 16:02:55 - r - INFO: - Episode: 9/200, Reward: 911.000, Step: 1000 +2022-12-07 16:03:26 - r - INFO: - Episode: 10/200, Reward: 350.000, Step: 796 +2022-12-07 16:04:56 - r - INFO: - Current episode 10 has the best eval reward: -200.000 +2022-12-07 16:05:34 - r - INFO: - Episode: 11/200, Reward: 273.000, Step: 1000 +2022-12-07 16:06:13 - r - INFO: - Episode: 12/200, Reward: 531.000, Step: 1000 +2022-12-07 16:06:52 - r - INFO: - Episode: 13/200, Reward: 131.000, Step: 1000 +2022-12-07 16:07:31 - r - INFO: - Episode: 14/200, Reward: 650.000, Step: 1000 +2022-12-07 16:08:10 - r - INFO: - Episode: 15/200, Reward: 670.000, Step: 1000 +2022-12-07 16:08:22 - r - INFO: - Current episode 15 has the best eval reward: 680.000 +2022-12-07 16:09:01 - r - INFO: - Episode: 16/200, Reward: 277.000, Step: 1000 +2022-12-07 16:09:39 - r - INFO: - Episode: 17/200, Reward: 555.000, Step: 1000 +2022-12-07 16:10:13 - r - INFO: - Episode: 18/200, Reward: 610.000, Step: 879 +2022-12-07 16:10:52 - r - INFO: - Episode: 19/200, Reward: 161.000, Step: 1000 +2022-12-07 16:11:31 - r - INFO: - Episode: 20/200, Reward: 442.000, Step: 1000 +2022-12-07 16:13:29 - r - INFO: - Episode: 21/200, Reward: 326.000, Step: 742 +2022-12-07 16:14:07 - r - INFO: - Episode: 22/200, Reward: 431.000, Step: 1000 +2022-12-07 16:14:46 - r - INFO: - Episode: 23/200, Reward: 1110.000, Step: 1000 +2022-12-07 16:15:25 - r - INFO: - Episode: 24/200, Reward: 339.000, Step: 1000 +2022-12-07 16:16:04 - r - INFO: - Episode: 25/200, Reward: 316.000, Step: 1000 +2022-12-07 16:17:34 - r - INFO: - Episode: 26/200, Reward: 493.000, Step: 575 +2022-12-07 16:18:13 - r - INFO: - Episode: 27/200, Reward: 755.000, Step: 1000 +2022-12-07 16:18:53 - r - INFO: - Episode: 28/200, Reward: 589.000, Step: 1000 +2022-12-07 16:19:26 - r - INFO: - Episode: 29/200, Reward: 518.000, Step: 840 +2022-12-07 16:20:01 - r - INFO: - Episode: 30/200, Reward: 372.000, Step: 910 +2022-12-07 16:22:04 - r - INFO: - Episode: 31/200, Reward: 398.000, Step: 659 +2022-12-07 16:22:42 - r - INFO: - Episode: 32/200, Reward: 207.000, Step: 1000 +2022-12-07 16:23:15 - r - INFO: - Episode: 33/200, Reward: 594.000, Step: 822 +2022-12-07 16:23:36 - r - INFO: - Episode: 34/200, Reward: 587.000, Step: 555 +2022-12-07 16:23:47 - r - INFO: - Episode: 35/200, Reward: 646.000, Step: 264 +2022-12-07 16:25:56 - r - INFO: - Episode: 36/200, Reward: 579.000, Step: 1000 +2022-12-07 16:26:35 - r - INFO: - Episode: 37/200, Reward: 116.000, Step: 1000 +2022-12-07 16:27:14 - r - INFO: - Episode: 38/200, Reward: 416.000, Step: 1000 +2022-12-07 16:27:54 - r - INFO: - Episode: 39/200, Reward: 636.000, Step: 1000 +2022-12-07 16:28:33 - r - INFO: - Episode: 40/200, Reward: 235.000, Step: 1000 +2022-12-07 16:30:41 - r - INFO: - Episode: 41/200, Reward: 381.000, Step: 1000 +2022-12-07 16:31:20 - r - INFO: - Episode: 42/200, Reward: 655.000, Step: 1000 +2022-12-07 16:31:47 - r - INFO: - Episode: 43/200, Reward: 319.000, Step: 684 +2022-12-07 16:32:14 - r - INFO: - Episode: 44/200, Reward: 487.000, Step: 716 +2022-12-07 16:32:43 - r - INFO: - Episode: 45/200, Reward: 568.000, Step: 741 +2022-12-07 16:34:52 - r - INFO: - Episode: 46/200, Reward: 1336.000, Step: 1000 +2022-12-07 16:35:31 - r - INFO: - Episode: 47/200, Reward: 416.000, Step: 1000 +2022-12-07 16:35:50 - r - INFO: - Episode: 48/200, Reward: 497.000, Step: 506 +2022-12-07 16:36:29 - r - INFO: - Episode: 49/200, Reward: 730.000, Step: 1000 +2022-12-07 16:37:08 - r - INFO: - Episode: 50/200, Reward: 508.000, Step: 1000 +2022-12-07 16:39:17 - r - INFO: - Episode: 51/200, Reward: 310.000, Step: 1000 +2022-12-07 16:39:56 - r - INFO: - Episode: 52/200, Reward: 438.000, Step: 1000 +2022-12-07 16:40:34 - r - INFO: - Episode: 53/200, Reward: 346.000, Step: 1000 +2022-12-07 16:41:13 - r - INFO: - Episode: 54/200, Reward: 386.000, Step: 1000 +2022-12-07 16:41:49 - r - INFO: - Episode: 55/200, Reward: 474.000, Step: 913 +2022-12-07 16:43:57 - r - INFO: - Episode: 56/200, Reward: 718.000, Step: 1000 +2022-12-07 16:44:36 - r - INFO: - Episode: 57/200, Reward: 592.000, Step: 1000 +2022-12-07 16:44:55 - r - INFO: - Episode: 58/200, Reward: 463.000, Step: 493 +2022-12-07 16:45:35 - r - INFO: - Episode: 59/200, Reward: 871.000, Step: 1000 +2022-12-07 16:46:14 - r - INFO: - Episode: 60/200, Reward: 815.000, Step: 1000 +2022-12-07 16:48:24 - r - INFO: - Episode: 61/200, Reward: 532.000, Step: 860 +2022-12-07 16:48:51 - r - INFO: - Episode: 62/200, Reward: 318.000, Step: 705 +2022-12-07 16:49:31 - r - INFO: - Episode: 63/200, Reward: 177.000, Step: 1000 +2022-12-07 16:50:09 - r - INFO: - Episode: 64/200, Reward: 352.000, Step: 1000 +2022-12-07 16:50:48 - r - INFO: - Episode: 65/200, Reward: 632.000, Step: 1000 +2022-12-07 16:52:58 - r - INFO: - Episode: 66/200, Reward: 435.000, Step: 1000 +2022-12-07 16:53:37 - r - INFO: - Episode: 67/200, Reward: 829.000, Step: 1000 +2022-12-07 16:54:16 - r - INFO: - Episode: 68/200, Reward: 389.000, Step: 1000 +2022-12-07 16:54:55 - r - INFO: - Episode: 69/200, Reward: 410.000, Step: 1000 +2022-12-07 16:55:34 - r - INFO: - Episode: 70/200, Reward: 439.000, Step: 1000 +2022-12-07 16:57:30 - r - INFO: - Episode: 71/200, Reward: 501.000, Step: 726 +2022-12-07 16:57:59 - r - INFO: - Episode: 72/200, Reward: 373.000, Step: 737 +2022-12-07 16:58:32 - r - INFO: - Episode: 73/200, Reward: 408.000, Step: 857 +2022-12-07 16:59:11 - r - INFO: - Episode: 74/200, Reward: 731.000, Step: 1000 +2022-12-07 16:59:34 - r - INFO: - Episode: 75/200, Reward: 548.000, Step: 579 +2022-12-07 17:01:42 - r - INFO: - Episode: 76/200, Reward: 794.000, Step: 1000 +2022-12-07 17:02:22 - r - INFO: - Episode: 77/200, Reward: 571.000, Step: 1000 +2022-12-07 17:02:47 - r - INFO: - Episode: 78/200, Reward: 532.000, Step: 648 +2022-12-07 17:03:26 - r - INFO: - Episode: 79/200, Reward: 613.000, Step: 1000 +2022-12-07 17:03:49 - r - INFO: - Episode: 80/200, Reward: 528.000, Step: 625 +2022-12-07 17:05:55 - r - INFO: - Episode: 81/200, Reward: 394.000, Step: 1000 +2022-12-07 17:06:34 - r - INFO: - Episode: 82/200, Reward: 11.000, Step: 1000 +2022-12-07 17:07:14 - r - INFO: - Episode: 83/200, Reward: 480.000, Step: 1000 +2022-12-07 17:07:52 - r - INFO: - Episode: 84/200, Reward: 829.000, Step: 1000 +2022-12-07 17:08:32 - r - INFO: - Episode: 85/200, Reward: 557.000, Step: 1000 +2022-12-07 17:10:39 - r - INFO: - Episode: 86/200, Reward: 642.000, Step: 1000 +2022-12-07 17:11:17 - r - INFO: - Episode: 87/200, Reward: 810.000, Step: 1000 +2022-12-07 17:11:36 - r - INFO: - Episode: 88/200, Reward: 633.000, Step: 473 +2022-12-07 17:12:15 - r - INFO: - Episode: 89/200, Reward: 862.000, Step: 1000 +2022-12-07 17:12:54 - r - INFO: - Episode: 90/200, Reward: 639.000, Step: 1000 +2022-12-07 17:15:01 - r - INFO: - Episode: 91/200, Reward: 582.000, Step: 1000 +2022-12-07 17:15:40 - r - INFO: - Episode: 92/200, Reward: 455.000, Step: 1000 +2022-12-07 17:16:19 - r - INFO: - Episode: 93/200, Reward: 859.000, Step: 1000 +2022-12-07 17:16:58 - r - INFO: - Episode: 94/200, Reward: 481.000, Step: 1000 +2022-12-07 17:17:37 - r - INFO: - Episode: 95/200, Reward: 267.000, Step: 1000 +2022-12-07 17:19:44 - r - INFO: - Episode: 96/200, Reward: 776.000, Step: 1000 +2022-12-07 17:20:14 - r - INFO: - Episode: 97/200, Reward: 513.000, Step: 783 +2022-12-07 17:20:47 - r - INFO: - Episode: 98/200, Reward: 194.000, Step: 856 +2022-12-07 17:21:26 - r - INFO: - Episode: 99/200, Reward: 656.000, Step: 1000 +2022-12-07 17:22:03 - r - INFO: - Episode: 100/200, Reward: 937.000, Step: 941 +2022-12-07 17:24:11 - r - INFO: - Episode: 101/200, Reward: 897.000, Step: 1000 +2022-12-07 17:24:28 - r - INFO: - Episode: 102/200, Reward: 599.000, Step: 437 +2022-12-07 17:24:54 - r - INFO: - Episode: 103/200, Reward: 1188.000, Step: 669 +2022-12-07 17:25:18 - r - INFO: - Episode: 104/200, Reward: 1011.000, Step: 620 +2022-12-07 17:25:57 - r - INFO: - Episode: 105/200, Reward: 570.000, Step: 1000 +2022-12-07 17:28:02 - r - INFO: - Episode: 106/200, Reward: 1064.000, Step: 922 +2022-12-07 17:28:40 - r - INFO: - Episode: 107/200, Reward: 466.000, Step: 1000 +2022-12-07 17:29:19 - r - INFO: - Episode: 108/200, Reward: 34.000, Step: 1000 +2022-12-07 17:29:47 - r - INFO: - Episode: 109/200, Reward: 543.000, Step: 732 +2022-12-07 17:30:05 - r - INFO: - Episode: 110/200, Reward: 480.000, Step: 477 +2022-12-07 17:32:14 - r - INFO: - Episode: 111/200, Reward: 832.000, Step: 1000 +2022-12-07 17:32:53 - r - INFO: - Episode: 112/200, Reward: 911.000, Step: 1000 +2022-12-07 17:33:02 - r - INFO: - Episode: 113/200, Reward: 712.000, Step: 241 +2022-12-07 17:33:09 - r - INFO: - Episode: 114/200, Reward: 609.000, Step: 198 +2022-12-07 17:33:33 - r - INFO: - Episode: 115/200, Reward: 1059.000, Step: 610 +2022-12-07 17:35:36 - r - INFO: - Episode: 116/200, Reward: 1422.000, Step: 870 +2022-12-07 17:36:15 - r - INFO: - Episode: 117/200, Reward: 1119.000, Step: 1000 +2022-12-07 17:36:32 - r - INFO: - Episode: 118/200, Reward: 1040.000, Step: 452 +2022-12-07 17:37:05 - r - INFO: - Episode: 119/200, Reward: 1486.000, Step: 833 +2022-12-07 17:37:28 - r - INFO: - Episode: 120/200, Reward: 1047.000, Step: 601 +2022-12-07 17:39:40 - r - INFO: - Episode: 121/200, Reward: 1867.000, Step: 1000 +2022-12-07 17:39:48 - r - INFO: - Episode: 122/200, Reward: 656.000, Step: 209 +2022-12-07 17:40:11 - r - INFO: - Episode: 123/200, Reward: 1407.000, Step: 572 +2022-12-07 17:40:24 - r - INFO: - Episode: 124/200, Reward: 627.000, Step: 334 +2022-12-07 17:40:44 - r - INFO: - Episode: 125/200, Reward: 980.000, Step: 521 +2022-12-07 17:42:53 - r - INFO: - Episode: 126/200, Reward: 1444.000, Step: 1000 +2022-12-07 17:43:19 - r - INFO: - Episode: 127/200, Reward: 1411.000, Step: 672 +2022-12-07 17:43:58 - r - INFO: - Episode: 128/200, Reward: 1343.000, Step: 1000 +2022-12-07 17:44:33 - r - INFO: - Episode: 129/200, Reward: 1552.000, Step: 891 +2022-12-07 17:44:53 - r - INFO: - Episode: 130/200, Reward: 582.000, Step: 531 +2022-12-07 17:46:59 - r - INFO: - Episode: 131/200, Reward: 1768.000, Step: 1000 +2022-12-07 17:47:39 - r - INFO: - Episode: 132/200, Reward: 1337.000, Step: 1000 +2022-12-07 17:47:59 - r - INFO: - Episode: 133/200, Reward: 1090.000, Step: 520 +2022-12-07 17:48:06 - r - INFO: - Episode: 134/200, Reward: 709.000, Step: 186 +2022-12-07 17:48:32 - r - INFO: - Episode: 135/200, Reward: 1481.000, Step: 650 +2022-12-07 17:50:10 - r - INFO: - Episode: 136/200, Reward: 1453.000, Step: 776 +2022-12-07 17:50:18 - r - INFO: - Episode: 137/200, Reward: 655.000, Step: 191 +2022-12-07 17:50:35 - r - INFO: - Episode: 138/200, Reward: 1454.000, Step: 437 +2022-12-07 17:51:09 - r - INFO: - Episode: 139/200, Reward: 1342.000, Step: 858 +2022-12-07 17:51:34 - r - INFO: - Episode: 140/200, Reward: 1036.000, Step: 638 +2022-12-07 17:51:45 - r - INFO: - Current episode 140 has the best eval reward: 680.000 +2022-12-07 17:52:24 - r - INFO: - Episode: 141/200, Reward: 1744.000, Step: 1000 +2022-12-07 17:52:42 - r - INFO: - Episode: 142/200, Reward: 1072.000, Step: 445 +2022-12-07 17:53:21 - r - INFO: - Episode: 143/200, Reward: 1863.000, Step: 1000 +2022-12-07 17:53:43 - r - INFO: - Episode: 144/200, Reward: 986.000, Step: 555 +2022-12-07 17:53:57 - r - INFO: - Episode: 145/200, Reward: 681.000, Step: 363 +2022-12-07 17:54:06 - r - INFO: - Current episode 145 has the best eval reward: 737.000 +2022-12-07 17:54:42 - r - INFO: - Episode: 146/200, Reward: 1168.000, Step: 940 +2022-12-07 17:55:15 - r - INFO: - Episode: 147/200, Reward: 1033.000, Step: 840 +2022-12-07 17:55:31 - r - INFO: - Episode: 148/200, Reward: 500.000, Step: 387 +2022-12-07 17:56:10 - r - INFO: - Episode: 149/200, Reward: 1030.000, Step: 1000 +2022-12-07 17:56:45 - r - INFO: - Episode: 150/200, Reward: 1420.000, Step: 898 +2022-12-07 17:58:53 - r - INFO: - Episode: 151/200, Reward: 670.000, Step: 1000 +2022-12-07 17:59:25 - r - INFO: - Episode: 152/200, Reward: 939.000, Step: 824 +2022-12-07 17:59:41 - r - INFO: - Episode: 153/200, Reward: 1095.000, Step: 427 +2022-12-07 17:59:49 - r - INFO: - Episode: 154/200, Reward: 633.000, Step: 208 +2022-12-07 18:00:29 - r - INFO: - Episode: 155/200, Reward: 1058.000, Step: 1000 +2022-12-07 18:02:21 - r - INFO: - Episode: 156/200, Reward: 1055.000, Step: 629 +2022-12-07 18:02:47 - r - INFO: - Episode: 157/200, Reward: 1022.000, Step: 664 +2022-12-07 18:02:57 - r - INFO: - Episode: 158/200, Reward: 648.000, Step: 258 +2022-12-07 18:03:15 - r - INFO: - Episode: 159/200, Reward: 512.000, Step: 465 +2022-12-07 18:03:54 - r - INFO: - Episode: 160/200, Reward: 728.000, Step: 1000 +2022-12-07 18:05:48 - r - INFO: - Episode: 161/200, Reward: 1141.000, Step: 654 +2022-12-07 18:06:21 - r - INFO: - Episode: 162/200, Reward: 1192.000, Step: 832 +2022-12-07 18:06:29 - r - INFO: - Episode: 163/200, Reward: 653.000, Step: 232 +2022-12-07 18:07:10 - r - INFO: - Episode: 164/200, Reward: 1021.000, Step: 1000 +2022-12-07 18:07:51 - r - INFO: - Episode: 165/200, Reward: 1202.000, Step: 1000 +2022-12-07 18:10:00 - r - INFO: - Episode: 166/200, Reward: 1169.000, Step: 1000 +2022-12-07 18:10:17 - r - INFO: - Episode: 167/200, Reward: 512.000, Step: 413 +2022-12-07 18:10:38 - r - INFO: - Episode: 168/200, Reward: 445.000, Step: 529 +2022-12-07 18:11:18 - r - INFO: - Episode: 169/200, Reward: 656.000, Step: 1000 +2022-12-07 18:11:58 - r - INFO: - Episode: 170/200, Reward: 637.000, Step: 1000 +2022-12-07 18:14:06 - r - INFO: - Episode: 171/200, Reward: 596.000, Step: 1000 +2022-12-07 18:14:47 - r - INFO: - Episode: 172/200, Reward: 578.000, Step: 1000 +2022-12-07 18:15:27 - r - INFO: - Episode: 173/200, Reward: 662.000, Step: 1000 +2022-12-07 18:16:08 - r - INFO: - Episode: 174/200, Reward: 822.000, Step: 1000 +2022-12-07 18:16:25 - r - INFO: - Episode: 175/200, Reward: 513.000, Step: 416 +2022-12-07 18:18:35 - r - INFO: - Episode: 176/200, Reward: 978.000, Step: 1000 +2022-12-07 18:19:16 - r - INFO: - Episode: 177/200, Reward: 758.000, Step: 1000 +2022-12-07 18:19:56 - r - INFO: - Episode: 178/200, Reward: 815.000, Step: 1000 +2022-12-07 18:20:37 - r - INFO: - Episode: 179/200, Reward: 1084.000, Step: 1000 +2022-12-07 18:21:18 - r - INFO: - Episode: 180/200, Reward: 997.000, Step: 1000 +2022-12-07 18:23:27 - r - INFO: - Episode: 181/200, Reward: 669.000, Step: 1000 +2022-12-07 18:24:08 - r - INFO: - Episode: 182/200, Reward: 782.000, Step: 1000 +2022-12-07 18:24:48 - r - INFO: - Episode: 183/200, Reward: 753.000, Step: 1000 +2022-12-07 18:25:30 - r - INFO: - Episode: 184/200, Reward: 848.000, Step: 1000 +2022-12-07 18:25:59 - r - INFO: - Episode: 185/200, Reward: 558.000, Step: 727 +2022-12-07 18:28:10 - r - INFO: - Episode: 186/200, Reward: 836.000, Step: 1000 +2022-12-07 18:28:51 - r - INFO: - Episode: 187/200, Reward: 820.000, Step: 1000 +2022-12-07 18:29:08 - r - INFO: - Episode: 188/200, Reward: 580.000, Step: 427 +2022-12-07 18:29:28 - r - INFO: - Episode: 189/200, Reward: 1089.000, Step: 480 +2022-12-07 18:30:08 - r - INFO: - Episode: 190/200, Reward: 1092.000, Step: 1000 +2022-12-07 18:32:02 - r - INFO: - Episode: 191/200, Reward: 675.000, Step: 635 +2022-12-07 18:32:27 - r - INFO: - Episode: 192/200, Reward: 986.000, Step: 642 +2022-12-07 18:33:06 - r - INFO: - Episode: 193/200, Reward: 993.000, Step: 986 +2022-12-07 18:33:31 - r - INFO: - Episode: 194/200, Reward: 894.000, Step: 666 +2022-12-07 18:33:48 - r - INFO: - Episode: 195/200, Reward: 635.000, Step: 428 +2022-12-07 18:35:55 - r - INFO: - Episode: 196/200, Reward: 953.000, Step: 1000 +2022-12-07 18:36:27 - r - INFO: - Episode: 197/200, Reward: 1583.000, Step: 818 +2022-12-07 18:37:06 - r - INFO: - Episode: 198/200, Reward: 501.000, Step: 1000 +2022-12-07 18:37:46 - r - INFO: - Episode: 199/200, Reward: 1045.000, Step: 1000 +2022-12-07 18:38:24 - r - INFO: - Episode: 200/200, Reward: 749.000, Step: 969 +2022-12-07 18:39:54 - r - INFO: - Finish training! diff --git a/Mario/Train_Mario_DQN_CNN_20221207-155552/models/checkpoint.pt b/Mario/Train_Mario_DQN_CNN_20221207-155552/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..b36f6a06b7ab606b4bfefd988b20242c62cabfb2 --- /dev/null +++ b/Mario/Train_Mario_DQN_CNN_20221207-155552/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a7fa90cd9ae8dd2e3a2ede80a92a2fbd636c3762b2ea1ec0ad3e02f364d0d4 +size 7264839 diff --git a/Mario/Train_Mario_DQN_CNN_20221207-155552/results/learning_curve.png b/Mario/Train_Mario_DQN_CNN_20221207-155552/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..f8b259bd0515e0ccbd38afe4a83c4c165bb6c9a7 Binary files /dev/null and b/Mario/Train_Mario_DQN_CNN_20221207-155552/results/learning_curve.png differ diff --git a/Mario/Train_Mario_DQN_CNN_20221207-155552/results/res.csv b/Mario/Train_Mario_DQN_CNN_20221207-155552/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..f3a601420577a1a042f142d5da8c17fe8c3e4dfc --- /dev/null +++ b/Mario/Train_Mario_DQN_CNN_20221207-155552/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,697.0,1000 +1,330.0,1000 +2,164.0,1000 +3,939.0,1000 +4,757.0,1000 +5,347.0,1000 +6,727.0,1000 +7,430.0,626 +8,911.0,1000 +9,350.0,796 +10,273.0,1000 +11,531.0,1000 +12,131.0,1000 +13,650.0,1000 +14,670.0,1000 +15,277.0,1000 +16,555.0,1000 +17,610.0,879 +18,161.0,1000 +19,442.0,1000 +20,326.0,742 +21,431.0,1000 +22,1110.0,1000 +23,339.0,1000 +24,316.0,1000 +25,493.0,575 +26,755.0,1000 +27,589.0,1000 +28,518.0,840 +29,372.0,910 +30,398.0,659 +31,207.0,1000 +32,594.0,822 +33,587.0,555 +34,646.0,264 +35,579.0,1000 +36,116.0,1000 +37,416.0,1000 +38,636.0,1000 +39,235.0,1000 +40,381.0,1000 +41,655.0,1000 +42,319.0,684 +43,487.0,716 +44,568.0,741 +45,1336.0,1000 +46,416.0,1000 +47,497.0,506 +48,730.0,1000 +49,508.0,1000 +50,310.0,1000 +51,438.0,1000 +52,346.0,1000 +53,386.0,1000 +54,474.0,913 +55,718.0,1000 +56,592.0,1000 +57,463.0,493 +58,871.0,1000 +59,815.0,1000 +60,532.0,860 +61,318.0,705 +62,177.0,1000 +63,352.0,1000 +64,632.0,1000 +65,435.0,1000 +66,829.0,1000 +67,389.0,1000 +68,410.0,1000 +69,439.0,1000 +70,501.0,726 +71,373.0,737 +72,408.0,857 +73,731.0,1000 +74,548.0,579 +75,794.0,1000 +76,571.0,1000 +77,532.0,648 +78,613.0,1000 +79,528.0,625 +80,394.0,1000 +81,11.0,1000 +82,480.0,1000 +83,829.0,1000 +84,557.0,1000 +85,642.0,1000 +86,810.0,1000 +87,633.0,473 +88,862.0,1000 +89,639.0,1000 +90,582.0,1000 +91,455.0,1000 +92,859.0,1000 +93,481.0,1000 +94,267.0,1000 +95,776.0,1000 +96,513.0,783 +97,194.0,856 +98,656.0,1000 +99,937.0,941 +100,897.0,1000 +101,599.0,437 +102,1188.0,669 +103,1011.0,620 +104,570.0,1000 +105,1064.0,922 +106,466.0,1000 +107,34.0,1000 +108,543.0,732 +109,480.0,477 +110,832.0,1000 +111,911.0,1000 +112,712.0,241 +113,609.0,198 +114,1059.0,610 +115,1422.0,870 +116,1119.0,1000 +117,1040.0,452 +118,1486.0,833 +119,1047.0,601 +120,1867.0,1000 +121,656.0,209 +122,1407.0,572 +123,627.0,334 +124,980.0,521 +125,1444.0,1000 +126,1411.0,672 +127,1343.0,1000 +128,1552.0,891 +129,582.0,531 +130,1768.0,1000 +131,1337.0,1000 +132,1090.0,520 +133,709.0,186 +134,1481.0,650 +135,1453.0,776 +136,655.0,191 +137,1454.0,437 +138,1342.0,858 +139,1036.0,638 +140,1744.0,1000 +141,1072.0,445 +142,1863.0,1000 +143,986.0,555 +144,681.0,363 +145,1168.0,940 +146,1033.0,840 +147,500.0,387 +148,1030.0,1000 +149,1420.0,898 +150,670.0,1000 +151,939.0,824 +152,1095.0,427 +153,633.0,208 +154,1058.0,1000 +155,1055.0,629 +156,1022.0,664 +157,648.0,258 +158,512.0,465 +159,728.0,1000 +160,1141.0,654 +161,1192.0,832 +162,653.0,232 +163,1021.0,1000 +164,1202.0,1000 +165,1169.0,1000 +166,512.0,413 +167,445.0,529 +168,656.0,1000 +169,637.0,1000 +170,596.0,1000 +171,578.0,1000 +172,662.0,1000 +173,822.0,1000 +174,513.0,416 +175,978.0,1000 +176,758.0,1000 +177,815.0,1000 +178,1084.0,1000 +179,997.0,1000 +180,669.0,1000 +181,782.0,1000 +182,753.0,1000 +183,848.0,1000 +184,558.0,727 +185,836.0,1000 +186,820.0,1000 +187,580.0,427 +188,1089.0,480 +189,1092.0,1000 +190,675.0,635 +191,986.0,642 +192,993.0,986 +193,894.0,666 +194,635.0,428 +195,953.0,1000 +196,1583.0,818 +197,501.0,1000 +198,1045.0,1000 +199,749.0,969 diff --git a/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/config.yaml b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08ced0a2566ab1d8928d8fdb8c55175fc184b955 --- /dev/null +++ b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/config.yaml @@ -0,0 +1,45 @@ +general_cfg: + algo_name: DQN + device: cpu + env_name: MountainCar-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_MountainCar-v0_DQN_20230404-211546 + max_steps: 200 + mode: test + new_step_api: true + render: true + save_fig: true + seed: 3 + show_fig: false + test_eps: 20 + train_eps: 200 + wrapper: null + render_mode: rgb_array + n_workers: 1 +algo_cfg: + batch_size: 64 + buffer_size: 40000 + epsilon_decay: 800 + epsilon_end: 0.01 + epsilon_start: 0.99 + gamma: 0.99 + lr: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/logs/log.txt b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..8e683c571f62c92e3eeb463c9e467bcdb20d156f --- /dev/null +++ b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/logs/log.txt @@ -0,0 +1,59 @@ +2023-04-04 21:24:50 - r - INFO: - Hyperparameters: +2023-04-04 21:24:50 - r - INFO: - ================================================================================ +2023-04-04 21:24:50 - r - INFO: - Name Value Type +2023-04-04 21:24:50 - r - INFO: - env_name MountainCar-v0 +2023-04-04 21:24:50 - r - INFO: - new_step_api 1 +2023-04-04 21:24:50 - r - INFO: - wrapper None +2023-04-04 21:24:50 - r - INFO: - render 1 +2023-04-04 21:24:50 - r - INFO: - algo_name DQN +2023-04-04 21:24:50 - r - INFO: - mode test +2023-04-04 21:24:50 - r - INFO: - seed 3 +2023-04-04 21:24:50 - r - INFO: - device cpu +2023-04-04 21:24:50 - r - INFO: - train_eps 200 +2023-04-04 21:24:50 - r - INFO: - test_eps 20 +2023-04-04 21:24:50 - r - INFO: - eval_eps 10 +2023-04-04 21:24:50 - r - INFO: - eval_per_episode 5 +2023-04-04 21:24:50 - r - INFO: - max_steps 200 +2023-04-04 21:24:50 - r - INFO: - load_checkpoint 1 +2023-04-04 21:24:50 - r - INFO: - load_path Train_MountainCar-v0_DQN_20230404-211546 +2023-04-04 21:24:50 - r - INFO: - show_fig 0 +2023-04-04 21:24:50 - r - INFO: - save_fig 1 +2023-04-04 21:24:50 - r - INFO: - epsilon_start 0.99 +2023-04-04 21:24:50 - r - INFO: - epsilon_end 0.01 +2023-04-04 21:24:50 - r - INFO: - epsilon_decay 800 +2023-04-04 21:24:50 - r - INFO: - gamma 0.99 +2023-04-04 21:24:50 - r - INFO: - lr 0.01 +2023-04-04 21:24:50 - r - INFO: - buffer_size 40000 +2023-04-04 21:24:50 - r - INFO: - batch_size 64 +2023-04-04 21:24:50 - r - INFO: - target_update 4 +2023-04-04 21:24:50 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-04 21:24:50 - r - INFO: - task_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Test_MountainCar-v0_DQN_20230404-212450 +2023-04-04 21:24:50 - r - INFO: - model_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Test_MountainCar-v0_DQN_20230404-212450/models +2023-04-04 21:24:50 - r - INFO: - res_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Test_MountainCar-v0_DQN_20230404-212450/results +2023-04-04 21:24:50 - r - INFO: - log_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Test_MountainCar-v0_DQN_20230404-212450/logs +2023-04-04 21:24:50 - r - INFO: - traj_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Test_MountainCar-v0_DQN_20230404-212450/traj +2023-04-04 21:24:50 - r - INFO: - ================================================================================ +2023-04-04 21:24:50 - r - INFO: - n_states: 2, n_actions: 3 +2023-04-04 21:24:50 - r - INFO: - Start testing! +2023-04-04 21:24:50 - r - INFO: - Env: MountainCar-v0, Algorithm: DQN, Device: cpu +2023-04-04 21:24:54 - r - INFO: - Episode: 1/20, Reward: -109.000, Step: 109 +2023-04-04 21:24:58 - r - INFO: - Episode: 2/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:01 - r - INFO: - Episode: 3/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:05 - r - INFO: - Episode: 4/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:09 - r - INFO: - Episode: 5/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:12 - r - INFO: - Episode: 6/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:16 - r - INFO: - Episode: 7/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:19 - r - INFO: - Episode: 8/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:23 - r - INFO: - Episode: 9/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:27 - r - INFO: - Episode: 10/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:30 - r - INFO: - Episode: 11/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:34 - r - INFO: - Episode: 12/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:38 - r - INFO: - Episode: 13/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:41 - r - INFO: - Episode: 14/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:45 - r - INFO: - Episode: 15/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:49 - r - INFO: - Episode: 16/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:52 - r - INFO: - Episode: 17/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:56 - r - INFO: - Episode: 18/20, Reward: -109.000, Step: 109 +2023-04-04 21:25:59 - r - INFO: - Episode: 19/20, Reward: -109.000, Step: 109 +2023-04-04 21:26:03 - r - INFO: - Episode: 20/20, Reward: -109.000, Step: 109 +2023-04-04 21:26:03 - r - INFO: - Finish testing! diff --git a/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/models/checkpoint.pt b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a06a50488ece05e6d94c8ac69b4a85c989451e9 --- /dev/null +++ b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79dc9f4203f492acea03e1b9dfda4bcc05b1be2f85e26536088a58e9b130adc3 +size 271383 diff --git a/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/results/learning_curve.png b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..8eacea9b3773cc4ec0aa6ac11531b1e542cb31c9 Binary files /dev/null and b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/results/learning_curve.png differ diff --git a/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/results/res.csv b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..d84c3a428347c789b3815f27daf3d3f730d12c8a --- /dev/null +++ b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,-109.0,109 +1,-109.0,109 +2,-109.0,109 +3,-109.0,109 +4,-109.0,109 +5,-109.0,109 +6,-109.0,109 +7,-109.0,109 +8,-109.0,109 +9,-109.0,109 +10,-109.0,109 +11,-109.0,109 +12,-109.0,109 +13,-109.0,109 +14,-109.0,109 +15,-109.0,109 +16,-109.0,109 +17,-109.0,109 +18,-109.0,109 +19,-109.0,109 diff --git a/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/videos/video.gif b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/videos/video.gif new file mode 100644 index 0000000000000000000000000000000000000000..0fd39f9aedee1e9ee46832cca970331857731574 --- /dev/null +++ b/MountainCar-v0/Test_MountainCar-v0_DQN_20230404-212450/videos/video.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc8d8eaec6a12a28adb8a607309716d622c5c3193cad9250583a796ab2559e2e +size 341470 diff --git a/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/config.yaml b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d9a7c85333ccc19e8fb6cf221fa0f3f40c0e18d --- /dev/null +++ b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/config.yaml @@ -0,0 +1,43 @@ +general_cfg: + algo_name: DQN + device: cpu + env_name: MountainCar-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: tasks + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 3 + show_fig: false + test_eps: 20 + train_eps: 200 + wrapper: null +algo_cfg: + batch_size: 64 + buffer_size: 40000 + epsilon_decay: 800 + epsilon_end: 0.01 + epsilon_start: 0.99 + gamma: 0.99 + lr: 0.01 + target_update: 4 + value_layers: + - activation: relu + layer_dim: + - n_states + - 256 + layer_type: linear + - activation: relu + layer_dim: + - 256 + - 256 + layer_type: linear + - activation: none + layer_dim: + - 256 + - n_actions + layer_type: linear diff --git a/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/logs/log.txt b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6a6e799352a3d5cb7665afdabb6cb0672827782a --- /dev/null +++ b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/logs/log.txt @@ -0,0 +1,266 @@ +2023-04-04 21:15:46 - r - INFO: - Hyperparameters: +2023-04-04 21:15:46 - r - INFO: - ================================================================================ +2023-04-04 21:15:46 - r - INFO: - Name Value Type +2023-04-04 21:15:46 - r - INFO: - env_name MountainCar-v0 +2023-04-04 21:15:46 - r - INFO: - new_step_api 1 +2023-04-04 21:15:46 - r - INFO: - wrapper None +2023-04-04 21:15:46 - r - INFO: - render 0 +2023-04-04 21:15:46 - r - INFO: - algo_name DQN +2023-04-04 21:15:46 - r - INFO: - mode train +2023-04-04 21:15:46 - r - INFO: - seed 3 +2023-04-04 21:15:46 - r - INFO: - device cpu +2023-04-04 21:15:46 - r - INFO: - train_eps 200 +2023-04-04 21:15:46 - r - INFO: - test_eps 20 +2023-04-04 21:15:46 - r - INFO: - eval_eps 10 +2023-04-04 21:15:46 - r - INFO: - eval_per_episode 5 +2023-04-04 21:15:46 - r - INFO: - max_steps 200 +2023-04-04 21:15:46 - r - INFO: - load_checkpoint 0 +2023-04-04 21:15:46 - r - INFO: - load_path tasks +2023-04-04 21:15:46 - r - INFO: - show_fig 0 +2023-04-04 21:15:46 - r - INFO: - save_fig 1 +2023-04-04 21:15:46 - r - INFO: - epsilon_start 0.99 +2023-04-04 21:15:46 - r - INFO: - epsilon_end 0.01 +2023-04-04 21:15:46 - r - INFO: - epsilon_decay 800 +2023-04-04 21:15:46 - r - INFO: - gamma 0.99 +2023-04-04 21:15:46 - r - INFO: - lr 0.01 +2023-04-04 21:15:46 - r - INFO: - buffer_size 40000 +2023-04-04 21:15:46 - r - INFO: - batch_size 64 +2023-04-04 21:15:46 - r - INFO: - target_update 4 +2023-04-04 21:15:46 - r - INFO: - value_layers [{'layer_type': 'linear', 'layer_dim': ['n_states', 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 256], 'activation': 'relu'}, {'layer_type': 'linear', 'layer_dim': [256, 'n_actions'], 'activation': 'none'}] +2023-04-04 21:15:46 - r - INFO: - task_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Train_MountainCar-v0_DQN_20230404-211546 +2023-04-04 21:15:46 - r - INFO: - model_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Train_MountainCar-v0_DQN_20230404-211546/models +2023-04-04 21:15:46 - r - INFO: - res_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Train_MountainCar-v0_DQN_20230404-211546/results +2023-04-04 21:15:46 - r - INFO: - log_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Train_MountainCar-v0_DQN_20230404-211546/logs +2023-04-04 21:15:46 - r - INFO: - traj_dir /home/PJLAB/geyuhong/rl-tutorials/joyrl/tasks/Train_MountainCar-v0_DQN_20230404-211546/traj +2023-04-04 21:15:46 - r - INFO: - ================================================================================ +2023-04-04 21:15:46 - r - INFO: - n_states: 2, n_actions: 3 +2023-04-04 21:15:46 - r - INFO: - Start training! +2023-04-04 21:15:46 - r - INFO: - Env: MountainCar-v0, Algorithm: DQN, Device: cpu +2023-04-04 21:15:47 - r - INFO: - Episode: 1/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:47 - r - INFO: - Episode: 2/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:47 - r - INFO: - Episode: 3/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:47 - r - INFO: - Episode: 4/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:48 - r - INFO: - Episode: 5/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:48 - r - INFO: - Current episode 5 has the best eval reward: -200.000 +2023-04-04 21:15:48 - r - INFO: - Episode: 6/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:48 - r - INFO: - Episode: 7/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:49 - r - INFO: - Episode: 8/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:49 - r - INFO: - Episode: 9/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:49 - r - INFO: - Episode: 10/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:50 - r - INFO: - Current episode 10 has the best eval reward: -200.000 +2023-04-04 21:15:50 - r - INFO: - Episode: 11/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:50 - r - INFO: - Episode: 12/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:51 - r - INFO: - Episode: 13/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:51 - r - INFO: - Episode: 14/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:51 - r - INFO: - Episode: 15/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:52 - r - INFO: - Current episode 15 has the best eval reward: -200.000 +2023-04-04 21:15:52 - r - INFO: - Episode: 16/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:52 - r - INFO: - Episode: 17/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:53 - r - INFO: - Episode: 18/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:53 - r - INFO: - Episode: 19/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:53 - r - INFO: - Episode: 20/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:54 - r - INFO: - Current episode 20 has the best eval reward: -200.000 +2023-04-04 21:15:54 - r - INFO: - Episode: 21/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:54 - r - INFO: - Episode: 22/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:55 - r - INFO: - Episode: 23/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:55 - r - INFO: - Episode: 24/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:55 - r - INFO: - Episode: 25/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:56 - r - INFO: - Current episode 25 has the best eval reward: -200.000 +2023-04-04 21:15:56 - r - INFO: - Episode: 26/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:56 - r - INFO: - Episode: 27/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:57 - r - INFO: - Episode: 28/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:57 - r - INFO: - Episode: 29/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:57 - r - INFO: - Episode: 30/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:58 - r - INFO: - Current episode 30 has the best eval reward: -200.000 +2023-04-04 21:15:58 - r - INFO: - Episode: 31/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:58 - r - INFO: - Episode: 32/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:59 - r - INFO: - Episode: 33/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:59 - r - INFO: - Episode: 34/200, Reward: -200.000, Step: 200 +2023-04-04 21:15:59 - r - INFO: - Episode: 35/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:00 - r - INFO: - Current episode 35 has the best eval reward: -200.000 +2023-04-04 21:16:00 - r - INFO: - Episode: 36/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:00 - r - INFO: - Episode: 37/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:01 - r - INFO: - Episode: 38/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:01 - r - INFO: - Episode: 39/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:02 - r - INFO: - Episode: 40/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:02 - r - INFO: - Current episode 40 has the best eval reward: -200.000 +2023-04-04 21:16:02 - r - INFO: - Episode: 41/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:03 - r - INFO: - Episode: 42/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:03 - r - INFO: - Episode: 43/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:03 - r - INFO: - Episode: 44/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:04 - r - INFO: - Episode: 45/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:04 - r - INFO: - Current episode 45 has the best eval reward: -200.000 +2023-04-04 21:16:04 - r - INFO: - Episode: 46/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:05 - r - INFO: - Episode: 47/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:05 - r - INFO: - Episode: 48/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:06 - r - INFO: - Episode: 49/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:06 - r - INFO: - Episode: 50/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:06 - r - INFO: - Current episode 50 has the best eval reward: -200.000 +2023-04-04 21:16:07 - r - INFO: - Episode: 51/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:07 - r - INFO: - Episode: 52/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:07 - r - INFO: - Episode: 53/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:08 - r - INFO: - Episode: 54/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:08 - r - INFO: - Episode: 55/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:08 - r - INFO: - Current episode 55 has the best eval reward: -200.000 +2023-04-04 21:16:09 - r - INFO: - Episode: 56/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:09 - r - INFO: - Episode: 57/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:09 - r - INFO: - Episode: 58/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:10 - r - INFO: - Episode: 59/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:10 - r - INFO: - Episode: 60/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:10 - r - INFO: - Current episode 60 has the best eval reward: -200.000 +2023-04-04 21:16:11 - r - INFO: - Episode: 61/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:11 - r - INFO: - Episode: 62/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:11 - r - INFO: - Episode: 63/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:12 - r - INFO: - Episode: 64/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:12 - r - INFO: - Episode: 65/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:12 - r - INFO: - Current episode 65 has the best eval reward: -200.000 +2023-04-04 21:16:13 - r - INFO: - Episode: 66/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:13 - r - INFO: - Episode: 67/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:14 - r - INFO: - Episode: 68/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:14 - r - INFO: - Episode: 69/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:15 - r - INFO: - Episode: 70/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:15 - r - INFO: - Current episode 70 has the best eval reward: -200.000 +2023-04-04 21:16:15 - r - INFO: - Episode: 71/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:16 - r - INFO: - Episode: 72/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:16 - r - INFO: - Episode: 73/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:16 - r - INFO: - Episode: 74/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:17 - r - INFO: - Episode: 75/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:17 - r - INFO: - Current episode 75 has the best eval reward: -200.000 +2023-04-04 21:16:17 - r - INFO: - Episode: 76/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:18 - r - INFO: - Episode: 77/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:18 - r - INFO: - Episode: 78/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:18 - r - INFO: - Episode: 79/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:19 - r - INFO: - Episode: 80/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:19 - r - INFO: - Current episode 80 has the best eval reward: -200.000 +2023-04-04 21:16:19 - r - INFO: - Episode: 81/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:20 - r - INFO: - Episode: 82/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:20 - r - INFO: - Episode: 83/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:20 - r - INFO: - Episode: 84/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:21 - r - INFO: - Episode: 85/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:21 - r - INFO: - Current episode 85 has the best eval reward: -200.000 +2023-04-04 21:16:21 - r - INFO: - Episode: 86/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:22 - r - INFO: - Episode: 87/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:22 - r - INFO: - Episode: 88/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:22 - r - INFO: - Episode: 89/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:23 - r - INFO: - Episode: 90/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:23 - r - INFO: - Current episode 90 has the best eval reward: -200.000 +2023-04-04 21:16:23 - r - INFO: - Episode: 91/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:24 - r - INFO: - Episode: 92/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:24 - r - INFO: - Episode: 93/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:24 - r - INFO: - Episode: 94/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:25 - r - INFO: - Episode: 95/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:25 - r - INFO: - Current episode 95 has the best eval reward: -200.000 +2023-04-04 21:16:26 - r - INFO: - Episode: 96/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:26 - r - INFO: - Episode: 97/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:26 - r - INFO: - Episode: 98/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:27 - r - INFO: - Episode: 99/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:27 - r - INFO: - Episode: 100/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:27 - r - INFO: - Current episode 100 has the best eval reward: -200.000 +2023-04-04 21:16:28 - r - INFO: - Episode: 101/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:28 - r - INFO: - Episode: 102/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:28 - r - INFO: - Episode: 103/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:29 - r - INFO: - Episode: 104/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:29 - r - INFO: - Episode: 105/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:29 - r - INFO: - Current episode 105 has the best eval reward: -200.000 +2023-04-04 21:16:30 - r - INFO: - Episode: 106/200, Reward: -190.000, Step: 190 +2023-04-04 21:16:30 - r - INFO: - Episode: 107/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:30 - r - INFO: - Episode: 108/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:31 - r - INFO: - Episode: 109/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:31 - r - INFO: - Episode: 110/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:31 - r - INFO: - Current episode 110 has the best eval reward: -200.000 +2023-04-04 21:16:32 - r - INFO: - Episode: 111/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:32 - r - INFO: - Episode: 112/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:32 - r - INFO: - Episode: 113/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:33 - r - INFO: - Episode: 114/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:33 - r - INFO: - Episode: 115/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:33 - r - INFO: - Current episode 115 has the best eval reward: -200.000 +2023-04-04 21:16:34 - r - INFO: - Episode: 116/200, Reward: -149.000, Step: 149 +2023-04-04 21:16:34 - r - INFO: - Episode: 117/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:34 - r - INFO: - Episode: 118/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:35 - r - INFO: - Episode: 119/200, Reward: -105.000, Step: 105 +2023-04-04 21:16:35 - r - INFO: - Episode: 120/200, Reward: -162.000, Step: 162 +2023-04-04 21:16:35 - r - INFO: - Current episode 120 has the best eval reward: -148.000 +2023-04-04 21:16:36 - r - INFO: - Episode: 121/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:36 - r - INFO: - Episode: 122/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:36 - r - INFO: - Episode: 123/200, Reward: -195.000, Step: 195 +2023-04-04 21:16:37 - r - INFO: - Episode: 124/200, Reward: -173.000, Step: 173 +2023-04-04 21:16:37 - r - INFO: - Episode: 125/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:38 - r - INFO: - Episode: 126/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:38 - r - INFO: - Episode: 127/200, Reward: -149.000, Step: 149 +2023-04-04 21:16:38 - r - INFO: - Episode: 128/200, Reward: -127.000, Step: 127 +2023-04-04 21:16:38 - r - INFO: - Episode: 129/200, Reward: -132.000, Step: 132 +2023-04-04 21:16:39 - r - INFO: - Episode: 130/200, Reward: -145.000, Step: 145 +2023-04-04 21:16:39 - r - INFO: - Episode: 131/200, Reward: -125.000, Step: 125 +2023-04-04 21:16:39 - r - INFO: - Episode: 132/200, Reward: -127.000, Step: 127 +2023-04-04 21:16:40 - r - INFO: - Episode: 133/200, Reward: -126.000, Step: 126 +2023-04-04 21:16:40 - r - INFO: - Episode: 134/200, Reward: -115.000, Step: 115 +2023-04-04 21:16:40 - r - INFO: - Episode: 135/200, Reward: -168.000, Step: 168 +2023-04-04 21:16:41 - r - INFO: - Episode: 136/200, Reward: -174.000, Step: 174 +2023-04-04 21:16:41 - r - INFO: - Episode: 137/200, Reward: -118.000, Step: 118 +2023-04-04 21:16:41 - r - INFO: - Episode: 138/200, Reward: -153.000, Step: 153 +2023-04-04 21:16:42 - r - INFO: - Episode: 139/200, Reward: -118.000, Step: 118 +2023-04-04 21:16:42 - r - INFO: - Episode: 140/200, Reward: -117.000, Step: 117 +2023-04-04 21:16:42 - r - INFO: - Current episode 140 has the best eval reward: -112.000 +2023-04-04 21:16:42 - r - INFO: - Episode: 141/200, Reward: -113.000, Step: 113 +2023-04-04 21:16:42 - r - INFO: - Episode: 142/200, Reward: -114.000, Step: 114 +2023-04-04 21:16:43 - r - INFO: - Episode: 143/200, Reward: -116.000, Step: 116 +2023-04-04 21:16:43 - r - INFO: - Episode: 144/200, Reward: -117.000, Step: 117 +2023-04-04 21:16:43 - r - INFO: - Episode: 145/200, Reward: -117.000, Step: 117 +2023-04-04 21:16:43 - r - INFO: - Episode: 146/200, Reward: -114.000, Step: 114 +2023-04-04 21:16:44 - r - INFO: - Episode: 147/200, Reward: -132.000, Step: 132 +2023-04-04 21:16:44 - r - INFO: - Episode: 148/200, Reward: -124.000, Step: 124 +2023-04-04 21:16:44 - r - INFO: - Episode: 149/200, Reward: -118.000, Step: 118 +2023-04-04 21:16:44 - r - INFO: - Episode: 150/200, Reward: -125.000, Step: 125 +2023-04-04 21:16:45 - r - INFO: - Episode: 151/200, Reward: -111.000, Step: 111 +2023-04-04 21:16:45 - r - INFO: - Episode: 152/200, Reward: -131.000, Step: 131 +2023-04-04 21:16:45 - r - INFO: - Episode: 153/200, Reward: -114.000, Step: 114 +2023-04-04 21:16:45 - r - INFO: - Episode: 154/200, Reward: -125.000, Step: 125 +2023-04-04 21:16:46 - r - INFO: - Episode: 155/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:46 - r - INFO: - Episode: 156/200, Reward: -107.000, Step: 107 +2023-04-04 21:16:46 - r - INFO: - Episode: 157/200, Reward: -127.000, Step: 127 +2023-04-04 21:16:47 - r - INFO: - Episode: 158/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:47 - r - INFO: - Episode: 159/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:47 - r - INFO: - Episode: 160/200, Reward: -113.000, Step: 113 +2023-04-04 21:16:48 - r - INFO: - Episode: 161/200, Reward: -166.000, Step: 166 +2023-04-04 21:16:48 - r - INFO: - Episode: 162/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:49 - r - INFO: - Episode: 163/200, Reward: -171.000, Step: 171 +2023-04-04 21:16:49 - r - INFO: - Episode: 164/200, Reward: -156.000, Step: 156 +2023-04-04 21:16:49 - r - INFO: - Episode: 165/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:50 - r - INFO: - Episode: 166/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:50 - r - INFO: - Episode: 167/200, Reward: -128.000, Step: 128 +2023-04-04 21:16:50 - r - INFO: - Episode: 168/200, Reward: -171.000, Step: 171 +2023-04-04 21:16:51 - r - INFO: - Episode: 169/200, Reward: -123.000, Step: 123 +2023-04-04 21:16:51 - r - INFO: - Episode: 170/200, Reward: -173.000, Step: 173 +2023-04-04 21:16:51 - r - INFO: - Current episode 170 has the best eval reward: -111.000 +2023-04-04 21:16:51 - r - INFO: - Episode: 171/200, Reward: -157.000, Step: 157 +2023-04-04 21:16:52 - r - INFO: - Episode: 172/200, Reward: -122.000, Step: 122 +2023-04-04 21:16:52 - r - INFO: - Episode: 173/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:52 - r - INFO: - Episode: 174/200, Reward: -123.000, Step: 123 +2023-04-04 21:16:52 - r - INFO: - Episode: 175/200, Reward: -152.000, Step: 152 +2023-04-04 21:16:53 - r - INFO: - Episode: 176/200, Reward: -116.000, Step: 116 +2023-04-04 21:16:53 - r - INFO: - Episode: 177/200, Reward: -141.000, Step: 141 +2023-04-04 21:16:53 - r - INFO: - Episode: 178/200, Reward: -158.000, Step: 158 +2023-04-04 21:16:54 - r - INFO: - Episode: 179/200, Reward: -153.000, Step: 153 +2023-04-04 21:16:54 - r - INFO: - Episode: 180/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:55 - r - INFO: - Episode: 181/200, Reward: -199.000, Step: 199 +2023-04-04 21:16:55 - r - INFO: - Episode: 182/200, Reward: -194.000, Step: 194 +2023-04-04 21:16:55 - r - INFO: - Episode: 183/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:56 - r - INFO: - Episode: 184/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:56 - r - INFO: - Episode: 185/200, Reward: -149.000, Step: 149 +2023-04-04 21:16:57 - r - INFO: - Episode: 186/200, Reward: -162.000, Step: 162 +2023-04-04 21:16:57 - r - INFO: - Episode: 187/200, Reward: -191.000, Step: 191 +2023-04-04 21:16:57 - r - INFO: - Episode: 188/200, Reward: -196.000, Step: 196 +2023-04-04 21:16:58 - r - INFO: - Episode: 189/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:58 - r - INFO: - Episode: 190/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:59 - r - INFO: - Episode: 191/200, Reward: -200.000, Step: 200 +2023-04-04 21:16:59 - r - INFO: - Episode: 192/200, Reward: -191.000, Step: 191 +2023-04-04 21:16:59 - r - INFO: - Episode: 193/200, Reward: -200.000, Step: 200 +2023-04-04 21:17:00 - r - INFO: - Episode: 194/200, Reward: -198.000, Step: 198 +2023-04-04 21:17:00 - r - INFO: - Episode: 195/200, Reward: -200.000, Step: 200 +2023-04-04 21:17:00 - r - INFO: - Current episode 195 has the best eval reward: -109.000 +2023-04-04 21:17:01 - r - INFO: - Episode: 196/200, Reward: -181.000, Step: 181 +2023-04-04 21:17:01 - r - INFO: - Episode: 197/200, Reward: -181.000, Step: 181 +2023-04-04 21:17:01 - r - INFO: - Episode: 198/200, Reward: -187.000, Step: 187 +2023-04-04 21:17:02 - r - INFO: - Episode: 199/200, Reward: -200.000, Step: 200 +2023-04-04 21:17:02 - r - INFO: - Episode: 200/200, Reward: -200.000, Step: 200 +2023-04-04 21:17:02 - r - INFO: - Finish training! diff --git a/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/models/checkpoint.pt b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a06a50488ece05e6d94c8ac69b4a85c989451e9 --- /dev/null +++ b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79dc9f4203f492acea03e1b9dfda4bcc05b1be2f85e26536088a58e9b130adc3 +size 271383 diff --git a/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/results/learning_curve.png b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..3e247050a975c17d47a7fc9c3834521c6f1a596e Binary files /dev/null and b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/results/learning_curve.png differ diff --git a/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/results/res.csv b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..9cf68a4965d1dad01053d18767462775ccf98dac --- /dev/null +++ b/MountainCar-v0/Train_MountainCar-v0_DQN_20230404-211546/results/res.csv @@ -0,0 +1,201 @@ +episodes,rewards,steps +0,-200.0,200 +1,-200.0,200 +2,-200.0,200 +3,-200.0,200 +4,-200.0,200 +5,-200.0,200 +6,-200.0,200 +7,-200.0,200 +8,-200.0,200 +9,-200.0,200 +10,-200.0,200 +11,-200.0,200 +12,-200.0,200 +13,-200.0,200 +14,-200.0,200 +15,-200.0,200 +16,-200.0,200 +17,-200.0,200 +18,-200.0,200 +19,-200.0,200 +20,-200.0,200 +21,-200.0,200 +22,-200.0,200 +23,-200.0,200 +24,-200.0,200 +25,-200.0,200 +26,-200.0,200 +27,-200.0,200 +28,-200.0,200 +29,-200.0,200 +30,-200.0,200 +31,-200.0,200 +32,-200.0,200 +33,-200.0,200 +34,-200.0,200 +35,-200.0,200 +36,-200.0,200 +37,-200.0,200 +38,-200.0,200 +39,-200.0,200 +40,-200.0,200 +41,-200.0,200 +42,-200.0,200 +43,-200.0,200 +44,-200.0,200 +45,-200.0,200 +46,-200.0,200 +47,-200.0,200 +48,-200.0,200 +49,-200.0,200 +50,-200.0,200 +51,-200.0,200 +52,-200.0,200 +53,-200.0,200 +54,-200.0,200 +55,-200.0,200 +56,-200.0,200 +57,-200.0,200 +58,-200.0,200 +59,-200.0,200 +60,-200.0,200 +61,-200.0,200 +62,-200.0,200 +63,-200.0,200 +64,-200.0,200 +65,-200.0,200 +66,-200.0,200 +67,-200.0,200 +68,-200.0,200 +69,-200.0,200 +70,-200.0,200 +71,-200.0,200 +72,-200.0,200 +73,-200.0,200 +74,-200.0,200 +75,-200.0,200 +76,-200.0,200 +77,-200.0,200 +78,-200.0,200 +79,-200.0,200 +80,-200.0,200 +81,-200.0,200 +82,-200.0,200 +83,-200.0,200 +84,-200.0,200 +85,-200.0,200 +86,-200.0,200 +87,-200.0,200 +88,-200.0,200 +89,-200.0,200 +90,-200.0,200 +91,-200.0,200 +92,-200.0,200 +93,-200.0,200 +94,-200.0,200 +95,-200.0,200 +96,-200.0,200 +97,-200.0,200 +98,-200.0,200 +99,-200.0,200 +100,-200.0,200 +101,-200.0,200 +102,-200.0,200 +103,-200.0,200 +104,-200.0,200 +105,-190.0,190 +106,-200.0,200 +107,-200.0,200 +108,-200.0,200 +109,-200.0,200 +110,-200.0,200 +111,-200.0,200 +112,-200.0,200 +113,-200.0,200 +114,-200.0,200 +115,-149.0,149 +116,-200.0,200 +117,-200.0,200 +118,-105.0,105 +119,-162.0,162 +120,-200.0,200 +121,-200.0,200 +122,-195.0,195 +123,-173.0,173 +124,-200.0,200 +125,-200.0,200 +126,-149.0,149 +127,-127.0,127 +128,-132.0,132 +129,-145.0,145 +130,-125.0,125 +131,-127.0,127 +132,-126.0,126 +133,-115.0,115 +134,-168.0,168 +135,-174.0,174 +136,-118.0,118 +137,-153.0,153 +138,-118.0,118 +139,-117.0,117 +140,-113.0,113 +141,-114.0,114 +142,-116.0,116 +143,-117.0,117 +144,-117.0,117 +145,-114.0,114 +146,-132.0,132 +147,-124.0,124 +148,-118.0,118 +149,-125.0,125 +150,-111.0,111 +151,-131.0,131 +152,-114.0,114 +153,-125.0,125 +154,-200.0,200 +155,-107.0,107 +156,-127.0,127 +157,-200.0,200 +158,-200.0,200 +159,-113.0,113 +160,-166.0,166 +161,-200.0,200 +162,-171.0,171 +163,-156.0,156 +164,-200.0,200 +165,-200.0,200 +166,-128.0,128 +167,-171.0,171 +168,-123.0,123 +169,-173.0,173 +170,-157.0,157 +171,-122.0,122 +172,-200.0,200 +173,-123.0,123 +174,-152.0,152 +175,-116.0,116 +176,-141.0,141 +177,-158.0,158 +178,-153.0,153 +179,-200.0,200 +180,-199.0,199 +181,-194.0,194 +182,-200.0,200 +183,-200.0,200 +184,-149.0,149 +185,-162.0,162 +186,-191.0,191 +187,-196.0,196 +188,-200.0,200 +189,-200.0,200 +190,-200.0,200 +191,-191.0,191 +192,-200.0,200 +193,-198.0,198 +194,-200.0,200 +195,-181.0,181 +196,-181.0,181 +197,-187.0,187 +198,-200.0,200 +199,-200.0,200 diff --git a/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/config.yaml b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6dad4b42fb703ba7ed81df02ae82f70178146d3 --- /dev/null +++ b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/config.yaml @@ -0,0 +1,31 @@ +general_cfg: + algo_name: DDPG_HER + device: cpu + env_name: Pendulum-v1 + eval_eps: 10 + eval_per_episode: 5 + her_sample_num: 4 + load_checkpoint: true + load_path: tasks + max_steps: 200 + mode: test + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 0 + show_fig: false + test_eps: 20 + train_eps: 150 + update_every: 100 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0001 + batch_size: 128 + buffer_size: 8000 + critic_hidden_dim: 256 + critic_lr: 0.001 + gamma: 0.99 + tau: 0.001 diff --git a/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/logs/log.txt b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a5ed918cc1982988243922ce52ca82ec28d785b6 --- /dev/null +++ b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/logs/log.txt @@ -0,0 +1,63 @@ +2023-04-14 15:16:11 - r - INFO: - Hyperparameters: +2023-04-14 15:16:11 - r - INFO: - ================================================================================ +2023-04-14 15:16:11 - r - INFO: - Name Value Type +2023-04-14 15:16:11 - r - INFO: - env_name Pendulum-v1 +2023-04-14 15:16:11 - r - INFO: - new_step_api 1 +2023-04-14 15:16:11 - r - INFO: - wrapper None +2023-04-14 15:16:11 - r - INFO: - render 0 +2023-04-14 15:16:11 - r - INFO: - render_mode human +2023-04-14 15:16:11 - r - INFO: - algo_name DDPG_HER +2023-04-14 15:16:11 - r - INFO: - mode test +2023-04-14 15:16:11 - r - INFO: - mp_backend mp +2023-04-14 15:16:11 - r - INFO: - seed 0 +2023-04-14 15:16:11 - r - INFO: - device cpu +2023-04-14 15:16:11 - r - INFO: - her_sample_num 4 +2023-04-14 15:16:11 - r - INFO: - update_every 100 +2023-04-14 15:16:11 - r - INFO: - train_eps 150 +2023-04-14 15:16:11 - r - INFO: - test_eps 20 +2023-04-14 15:16:11 - r - INFO: - eval_eps 10 +2023-04-14 15:16:11 - r - INFO: - eval_per_episode 5 +2023-04-14 15:16:11 - r - INFO: - max_steps 200 +2023-04-14 15:16:11 - r - INFO: - load_checkpoint 1 +2023-04-14 15:16:11 - r - INFO: - load_path tasks +2023-04-14 15:16:11 - r - INFO: - show_fig 0 +2023-04-14 15:16:11 - r - INFO: - save_fig 1 +2023-04-14 15:16:11 - r - INFO: - gamma 0.99 +2023-04-14 15:16:11 - r - INFO: - critic_lr 0.001 +2023-04-14 15:16:11 - r - INFO: - actor_lr 0.0001 +2023-04-14 15:16:11 - r - INFO: - buffer_size 8000 +2023-04-14 15:16:11 - r - INFO: - batch_size 128 +2023-04-14 15:16:11 - r - INFO: - tau 0.001 +2023-04-14 15:16:11 - r - INFO: - critic_hidden_dim 256 +2023-04-14 15:16:11 - r - INFO: - actor_hidden_dim 256 +2023-04-14 15:16:11 - r - INFO: - task_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_Pendulum-v1_DDPG_HER_20230414-151611 +2023-04-14 15:16:11 - r - INFO: - res_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_Pendulum-v1_DDPG_HER_20230414-151611/results +2023-04-14 15:16:11 - r - INFO: - log_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_Pendulum-v1_DDPG_HER_20230414-151611/logs +2023-04-14 15:16:11 - r - INFO: - traj_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_Pendulum-v1_DDPG_HER_20230414-151611/traj +2023-04-14 15:16:11 - r - INFO: - video_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Test_Pendulum-v1_DDPG_HER_20230414-151611/videos +2023-04-14 15:16:11 - r - INFO: - ================================================================================ +2023-04-14 15:16:11 - r - INFO: - action_bound: 2.0 +2023-04-14 15:16:11 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-14 15:16:11 - r - INFO: - Start testing! +2023-04-14 15:16:11 - r - INFO: - Env: Pendulum-v1, Algorithm: DDPG_HER, Device: cpu +2023-04-14 15:16:11 - r - INFO: - Episode: 1/20, Reward: -119.619, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 2/20, Reward: -125.829, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 3/20, Reward: -311.731, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 4/20, Reward: -122.448, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 5/20, Reward: -123.622, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 6/20, Reward: -255.851, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 7/20, Reward: -135.118, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 8/20, Reward: -131.147, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 9/20, Reward: -128.532, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 10/20, Reward: -237.524, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 11/20, Reward: -127.244, Step: 200 +2023-04-14 15:16:11 - r - INFO: - Episode: 12/20, Reward: -124.922, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 13/20, Reward: -115.935, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 14/20, Reward: -129.591, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 15/20, Reward: -124.751, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 16/20, Reward: -132.203, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 17/20, Reward: -2.005, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 18/20, Reward: -129.287, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 19/20, Reward: -129.070, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Episode: 20/20, Reward: -123.223, Step: 200 +2023-04-14 15:16:12 - r - INFO: - Finish testing! diff --git a/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/models/actor_checkpoint.pt b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..72a6a9340e781ac7c817baeee4f45827cd1a80f4 --- /dev/null +++ b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f7a7922068cee2ed0b4813e1699b0876afc982bb3ff1322a2561ef580a2657 +size 270167 diff --git a/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/results/learning_curve.png b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..2264ea50ef8b7a199b4ba062f4149dba367bf17c Binary files /dev/null and b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/results/learning_curve.png differ diff --git a/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/results/res.csv b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..b00558524f34bce96142c66030eb90d6724e0c47 --- /dev/null +++ b/Pendulum-v1/Test_Pendulum-v1_DDPG_HER_20230414-151611/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,-119.61920876540275,200 +1,-125.82893782683291,200 +2,-311.73095108760737,200 +3,-122.44831167856046,200 +4,-123.6224323006854,200 +5,-255.85127591923364,200 +6,-135.11788384957563,200 +7,-131.14691370142617,200 +8,-128.53218824691203,200 +9,-237.52396602186374,200 +10,-127.24392722065872,200 +11,-124.92183808056497,200 +12,-115.93462539523514,200 +13,-129.59096000796794,200 +14,-124.750557710735,200 +15,-132.20308579173368,200 +16,-2.0046774824491123,200 +17,-129.2870748287406,200 +18,-129.07048695344062,200 +19,-123.22322820920151,200 diff --git a/Pendulum-v1/Test_gym_TD3_20230416-113300/config.yaml b/Pendulum-v1/Test_gym_TD3_20230416-113300/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..779a71314eb23dddfa7ad8633d6af21c6f87bcca --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_20230416-113300/config.yaml @@ -0,0 +1,38 @@ +general_cfg: + algo_name: TD3 + device: cuda + env_name: gym + eval_eps: 5 + eval_per_episode: 10 + load_checkpoint: true + load_path: Train_gym_TD3_20230416-110359 + max_steps: 200 + mode: test + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.001 + batch_size: 100 + buffer_size: 1000000 + critic_hidden_dim: 256 + critic_lr: 0.001 + expl_noise: 0.1 + explore_steps: 1000 + gamma: 0.99 + noise_clip: 0.5 + policy_freq: 2 + policy_noise: 0.2 + tau: 0.005 +env_cfg: + id: Pendulum-v1 + new_step_api: true + render_mode: null diff --git a/Pendulum-v1/Test_gym_TD3_20230416-113300/logs/log.txt b/Pendulum-v1/Test_gym_TD3_20230416-113300/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..cea71cb65a883bb6fc1bdff563c2900a65e43d49 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_20230416-113300/logs/log.txt @@ -0,0 +1,57 @@ +2023-04-16 11:33:00 - r - INFO: - Hyperparameters: +2023-04-16 11:33:00 - r - INFO: - ================================================================================ +2023-04-16 11:33:00 - r - INFO: - Name Value Type +2023-04-16 11:33:00 - r - INFO: - env_name gym +2023-04-16 11:33:00 - r - INFO: - new_step_api 1 +2023-04-16 11:33:00 - r - INFO: - wrapper None +2023-04-16 11:33:00 - r - INFO: - render 0 +2023-04-16 11:33:00 - r - INFO: - render_mode None +2023-04-16 11:33:00 - r - INFO: - algo_name TD3 +2023-04-16 11:33:00 - r - INFO: - mode test +2023-04-16 11:33:00 - r - INFO: - mp_backend mp +2023-04-16 11:33:00 - r - INFO: - seed 1 +2023-04-16 11:33:00 - r - INFO: - device cuda +2023-04-16 11:33:00 - r - INFO: - train_eps 100 +2023-04-16 11:33:00 - r - INFO: - test_eps 10 +2023-04-16 11:33:00 - r - INFO: - eval_eps 5 +2023-04-16 11:33:00 - r - INFO: - eval_per_episode 10 +2023-04-16 11:33:00 - r - INFO: - max_steps 200 +2023-04-16 11:33:00 - r - INFO: - load_checkpoint 1 +2023-04-16 11:33:00 - r - INFO: - load_path Train_gym_TD3_20230416-110359 +2023-04-16 11:33:00 - r - INFO: - show_fig 0 +2023-04-16 11:33:00 - r - INFO: - save_fig 1 +2023-04-16 11:33:00 - r - INFO: - explore_steps 1000 +2023-04-16 11:33:00 - r - INFO: - policy_freq 2 +2023-04-16 11:33:00 - r - INFO: - actor_lr 0.001 +2023-04-16 11:33:00 - r - INFO: - critic_lr 0.001 +2023-04-16 11:33:00 - r - INFO: - actor_hidden_dim 256 +2023-04-16 11:33:00 - r - INFO: - critic_hidden_dim 256 +2023-04-16 11:33:00 - r - INFO: - gamma 0.99 +2023-04-16 11:33:00 - r - INFO: - tau 0.005 +2023-04-16 11:33:00 - r - INFO: - policy_noise 0.2 +2023-04-16 11:33:00 - r - INFO: - expl_noise 0.1 +2023-04-16 11:33:00 - r - INFO: - noise_clip 0.5 +2023-04-16 11:33:00 - r - INFO: - batch_size 100 +2023-04-16 11:33:00 - r - INFO: - buffer_size 1000000 +2023-04-16 11:33:00 - r - INFO: - id Pendulum-v1 +2023-04-16 11:33:00 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-113300 +2023-04-16 11:33:00 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-113300/results +2023-04-16 11:33:00 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-113300/logs +2023-04-16 11:33:00 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-113300/traj +2023-04-16 11:33:00 - r - INFO: - video_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-113300/videos +2023-04-16 11:33:00 - r - INFO: - ================================================================================ +2023-04-16 11:33:00 - r - INFO: - action_bound: 2.0 +2023-04-16 11:33:00 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-16 11:33:06 - r - INFO: - Start testing! +2023-04-16 11:33:06 - r - INFO: - Env: gym, Algorithm: TD3, Device: cuda +2023-04-16 11:33:06 - r - INFO: - Episode: 1/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:07 - r - INFO: - Episode: 2/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:08 - r - INFO: - Episode: 3/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:08 - r - INFO: - Episode: 4/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:09 - r - INFO: - Episode: 5/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:09 - r - INFO: - Episode: 6/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:10 - r - INFO: - Episode: 7/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:11 - r - INFO: - Episode: 8/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:11 - r - INFO: - Episode: 9/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:12 - r - INFO: - Episode: 10/10, Reward: -0.514, Step: 200 +2023-04-16 11:33:12 - r - INFO: - Finish testing! diff --git a/Pendulum-v1/Test_gym_TD3_20230416-113300/models/actor.pth b/Pendulum-v1/Test_gym_TD3_20230416-113300/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..c17690ddbd28cdbb98cedb83dc0a4361c3a87922 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_20230416-113300/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:214724c4b215e10e761cc6d58e6297c203462f3f06fadae121fcf7292ec326d0 +size 270167 diff --git a/Pendulum-v1/Test_gym_TD3_20230416-113300/models/critic_1.pth b/Pendulum-v1/Test_gym_TD3_20230416-113300/models/critic_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..a0f0b09e903928cdeb9a0c6bf547ed141a2fa528 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_20230416-113300/models/critic_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5170fbe3f10fd4f32c1cdd0635c4d70411fb765915ea58aafcf8e7f2fb921687 +size 271191 diff --git a/Pendulum-v1/Test_gym_TD3_20230416-113300/models/critic_2.pth b/Pendulum-v1/Test_gym_TD3_20230416-113300/models/critic_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..113773da9b235c99c3dfe129c6a72d570497bedc --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_20230416-113300/models/critic_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1647dea7ffb4bd6c42f116180b782c7fc8737774c7db2d15765968441ceb6e5b +size 271191 diff --git a/Pendulum-v1/Test_gym_TD3_20230416-113300/results/learning_curve.png b/Pendulum-v1/Test_gym_TD3_20230416-113300/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..160cc5abb9b3705be8059c3cb8e06f461ce92f8d Binary files /dev/null and b/Pendulum-v1/Test_gym_TD3_20230416-113300/results/learning_curve.png differ diff --git a/Pendulum-v1/Test_gym_TD3_20230416-113300/results/res.csv b/Pendulum-v1/Test_gym_TD3_20230416-113300/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e38efdfd816af7e7350e20060ab367447217c37 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_20230416-113300/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,-0.5136373548809647,200 +1,-0.5136373548809647,200 +2,-0.5136373548809647,200 +3,-0.5136373548809647,200 +4,-0.5136373548809647,200 +5,-0.5136373548809647,200 +6,-0.5136373548809647,200 +7,-0.5136373548809647,200 +8,-0.5136373548809647,200 +9,-0.5136373548809647,200 diff --git a/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/config.yaml b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fbc223a0bf760257ccbd8e77066d98aceef8093 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/config.yaml @@ -0,0 +1,43 @@ +general_cfg: + algo_name: TD3_BC + device: cuda + env_name: gym + eval_eps: 5 + eval_per_episode: 10 + load_checkpoint: true + load_path: Train_gym_TD3_BC_20230416-111154 + max_steps: 200 + mode: test + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 1 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + alpha: 5 + batch_size: 100 + buffer_size: 1000000 + critic_hidden_dim: 256 + critic_lr: 0.0003 + expert_path: tasks/Collect_gym_TD3_20230416-111040/traj/traj.pkl + expl_noise: 0.1 + explore_steps: 1000 + gamma: 0.99 + lmbda: 1 + noise_clip: 0.5 + normalize: false + policy_freq: 2 + policy_noise: 0.2 + tau: 0.005 + train_iterations: 2000 +env_cfg: + id: Pendulum-v1 + new_step_api: true + render_mode: null diff --git a/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/logs/log.txt b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..ece7d02c0d4c2f9f144ce3c77e34600ed1313d03 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/logs/log.txt @@ -0,0 +1,62 @@ +2023-04-16 11:31:55 - r - INFO: - Hyperparameters: +2023-04-16 11:31:55 - r - INFO: - ================================================================================ +2023-04-16 11:31:55 - r - INFO: - Name Value Type +2023-04-16 11:31:55 - r - INFO: - env_name gym +2023-04-16 11:31:55 - r - INFO: - new_step_api 1 +2023-04-16 11:31:55 - r - INFO: - wrapper None +2023-04-16 11:31:55 - r - INFO: - render 0 +2023-04-16 11:31:55 - r - INFO: - render_mode None +2023-04-16 11:31:55 - r - INFO: - algo_name TD3_BC +2023-04-16 11:31:55 - r - INFO: - mode test +2023-04-16 11:31:55 - r - INFO: - mp_backend mp +2023-04-16 11:31:55 - r - INFO: - seed 1 +2023-04-16 11:31:55 - r - INFO: - device cuda +2023-04-16 11:31:55 - r - INFO: - train_eps 1 +2023-04-16 11:31:55 - r - INFO: - test_eps 10 +2023-04-16 11:31:55 - r - INFO: - eval_eps 5 +2023-04-16 11:31:55 - r - INFO: - eval_per_episode 10 +2023-04-16 11:31:55 - r - INFO: - max_steps 200 +2023-04-16 11:31:55 - r - INFO: - load_checkpoint 1 +2023-04-16 11:31:55 - r - INFO: - load_path Train_gym_TD3_BC_20230416-111154 +2023-04-16 11:31:55 - r - INFO: - show_fig 0 +2023-04-16 11:31:55 - r - INFO: - save_fig 1 +2023-04-16 11:31:55 - r - INFO: - explore_steps 1000 +2023-04-16 11:31:55 - r - INFO: - policy_freq 2 +2023-04-16 11:31:55 - r - INFO: - actor_lr 0.0003 +2023-04-16 11:31:55 - r - INFO: - critic_lr 0.0003 +2023-04-16 11:31:55 - r - INFO: - actor_hidden_dim 256 +2023-04-16 11:31:55 - r - INFO: - critic_hidden_dim 256 +2023-04-16 11:31:55 - r - INFO: - gamma 0.99 +2023-04-16 11:31:55 - r - INFO: - tau 0.005 +2023-04-16 11:31:55 - r - INFO: - policy_noise 0.2 +2023-04-16 11:31:55 - r - INFO: - expl_noise 0.1 +2023-04-16 11:31:55 - r - INFO: - noise_clip 0.5 +2023-04-16 11:31:55 - r - INFO: - batch_size 100 +2023-04-16 11:31:55 - r - INFO: - buffer_size 1000000 +2023-04-16 11:31:55 - r - INFO: - alpha 5 +2023-04-16 11:31:55 - r - INFO: - lmbda 1 +2023-04-16 11:31:55 - r - INFO: - train_iterations 2000 +2023-04-16 11:31:55 - r - INFO: - normalize 0 +2023-04-16 11:31:55 - r - INFO: - expert_path tasks/Collect_gym_TD3_20230416-111040/traj/traj.pkl +2023-04-16 11:31:55 - r - INFO: - id Pendulum-v1 +2023-04-16 11:31:55 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_BC_20230416-113155 +2023-04-16 11:31:55 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_BC_20230416-113155/results +2023-04-16 11:31:55 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_BC_20230416-113155/logs +2023-04-16 11:31:55 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_BC_20230416-113155/traj +2023-04-16 11:31:55 - r - INFO: - video_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_BC_20230416-113155/videos +2023-04-16 11:31:55 - r - INFO: - ================================================================================ +2023-04-16 11:31:55 - r - INFO: - action_bound: 2.0 +2023-04-16 11:31:55 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-16 11:32:00 - r - INFO: - Start testing! +2023-04-16 11:32:00 - r - INFO: - Env: gym, Algorithm: TD3_BC, Device: cuda +2023-04-16 11:32:01 - r - INFO: - Episode: 1/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:02 - r - INFO: - Episode: 2/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:02 - r - INFO: - Episode: 3/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:03 - r - INFO: - Episode: 4/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:03 - r - INFO: - Episode: 5/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:04 - r - INFO: - Episode: 6/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:05 - r - INFO: - Episode: 7/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:05 - r - INFO: - Episode: 8/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:06 - r - INFO: - Episode: 9/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:06 - r - INFO: - Episode: 10/10, Reward: -0.477, Step: 200 +2023-04-16 11:32:06 - r - INFO: - Finish testing! diff --git a/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/actor.pth b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..d01f0446b73b2992fa96229d97626fda87daf727 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b872a4af1157564eebb890940a46d0f1df2d2c1fc369f98cfe0d6424fc43a301 +size 270167 diff --git a/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/critic_1.pth b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/critic_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..b4410954afd07ce999fbdaf1dc4ecd8c785cddda --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/critic_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0427328c4fbb4d5cf265d77ac129550e7c291cfd776d483b85f38b6b743f91d +size 271191 diff --git a/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/critic_2.pth b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/critic_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..629a8140417ea7113f3248704110b8bbd3ece107 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/models/critic_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31df1a65697a1573a5def537c431e5f284bd32ea3c05d5071d9e06b13a176b38 +size 271191 diff --git a/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/results/learning_curve.png b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..a3993f7e14157d87a0a0084054b86fa95209fdf0 Binary files /dev/null and b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/results/learning_curve.png differ diff --git a/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/results/res.csv b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e52acb9672e7827e9907a8aed2bb1ee6d5763b2 --- /dev/null +++ b/Pendulum-v1/Test_gym_TD3_BC_20230416-113155/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,-0.4769480296830484,200 +1,-0.4769480296830484,200 +2,-0.4769480296830484,200 +3,-0.4769480296830484,200 +4,-0.4769480296830484,200 +5,-0.4769480296830484,200 +6,-0.4769480296830484,200 +7,-0.4769480296830484,200 +8,-0.4769480296830484,200 +9,-0.4769480296830484,200 diff --git a/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/config.yaml b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79e2efb7b4485d29460dbc37310c0e7b642a726f --- /dev/null +++ b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/config.yaml @@ -0,0 +1,40 @@ +general_cfg: + algo_name: TD3 + device: cpu + env_name: gym + eval_eps: 5 + eval_per_episode: 10 + load_checkpoint: true + load_path: Train_gym_TD3_20230416-214019 + max_steps: 200 + mode: test + mp_backend: mp + n_workers: 1 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.001 + batch_size: 100 + buffer_size: 1000000 + critic_hidden_dim: 256 + critic_lr: 0.001 + expl_noise: 0.1 + explore_steps: 1000 + gamma: 0.99 + global_best_reward: -1800 + noise_clip: 0.5 + policy_freq: 2 + policy_noise: 0.2 + tau: 0.005 +env_cfg: + id: Pendulum-v1 + new_step_api: true + render_mode: null diff --git a/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/logs/log.txt b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..58a7d49edd9547e18445f1dd543bf57122ce9be1 --- /dev/null +++ b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/logs/log.txt @@ -0,0 +1,59 @@ +2023-04-16 22:14:28 - r - INFO: - Hyperparameters: +2023-04-16 22:14:28 - r - INFO: - ================================================================================ +2023-04-16 22:14:28 - r - INFO: - Name Value Type +2023-04-16 22:14:28 - r - INFO: - env_name gym +2023-04-16 22:14:28 - r - INFO: - new_step_api 1 +2023-04-16 22:14:28 - r - INFO: - wrapper None +2023-04-16 22:14:28 - r - INFO: - render 0 +2023-04-16 22:14:28 - r - INFO: - render_mode None +2023-04-16 22:14:28 - r - INFO: - algo_name TD3 +2023-04-16 22:14:28 - r - INFO: - mode test +2023-04-16 22:14:28 - r - INFO: - mp_backend mp +2023-04-16 22:14:28 - r - INFO: - seed 1 +2023-04-16 22:14:28 - r - INFO: - device cpu +2023-04-16 22:14:28 - r - INFO: - train_eps 200 +2023-04-16 22:14:28 - r - INFO: - test_eps 10 +2023-04-16 22:14:28 - r - INFO: - eval_eps 5 +2023-04-16 22:14:28 - r - INFO: - eval_per_episode 10 +2023-04-16 22:14:28 - r - INFO: - max_steps 200 +2023-04-16 22:14:28 - r - INFO: - load_checkpoint 1 +2023-04-16 22:14:28 - r - INFO: - load_path Train_gym_TD3_20230416-214019 +2023-04-16 22:14:28 - r - INFO: - show_fig 0 +2023-04-16 22:14:28 - r - INFO: - save_fig 1 +2023-04-16 22:14:28 - r - INFO: - n_workers 1 +2023-04-16 22:14:28 - r - INFO: - explore_steps 1000 +2023-04-16 22:14:28 - r - INFO: - policy_freq 2 +2023-04-16 22:14:28 - r - INFO: - actor_lr 0.001 +2023-04-16 22:14:28 - r - INFO: - critic_lr 0.001 +2023-04-16 22:14:28 - r - INFO: - actor_hidden_dim 256 +2023-04-16 22:14:28 - r - INFO: - critic_hidden_dim 256 +2023-04-16 22:14:28 - r - INFO: - gamma 0.99 +2023-04-16 22:14:28 - r - INFO: - tau 0.005 +2023-04-16 22:14:28 - r - INFO: - policy_noise 0.2 +2023-04-16 22:14:28 - r - INFO: - expl_noise 0.1 +2023-04-16 22:14:28 - r - INFO: - noise_clip 0.5 +2023-04-16 22:14:28 - r - INFO: - batch_size 100 +2023-04-16 22:14:28 - r - INFO: - buffer_size 1000000 +2023-04-16 22:14:28 - r - INFO: - global_best_reward -1800 +2023-04-16 22:14:28 - r - INFO: - id Pendulum-v1 +2023-04-16 22:14:28 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-221428 +2023-04-16 22:14:28 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-221428/results +2023-04-16 22:14:28 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-221428/logs +2023-04-16 22:14:28 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-221428/traj +2023-04-16 22:14:28 - r - INFO: - video_dir /home/dingli/joyrl_offline/tasks/Test_gym_TD3_20230416-221428/videos +2023-04-16 22:14:28 - r - INFO: - ================================================================================ +2023-04-16 22:14:28 - r - INFO: - action_bound: 2.0 +2023-04-16 22:14:28 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-16 22:14:28 - r - INFO: - Start testing! +2023-04-16 22:14:28 - r - INFO: - Env: gym, Algorithm: TD3, Device: cpu +2023-04-16 22:14:28 - r - INFO: - Episode: 1/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 2/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 3/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 4/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 5/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 6/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 7/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 8/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 9/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Episode: 10/10, Reward: -0.421, Step: 200 +2023-04-16 22:14:28 - r - INFO: - Finish testing! diff --git a/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/actor.pth b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..69cd81f9ba4724e9f8555c6dedc4bab222f9c5d4 --- /dev/null +++ b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c980b556a5a7de268589d9cb3f0d6753c443ebcf13f413245b3868298b706bd +size 270103 diff --git a/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/critic_1.pth b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/critic_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..dde1a62afe4109ff2066c1ffded39944a2241c9f --- /dev/null +++ b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/critic_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f23d87610b7fa55de3c0021d57f5cee965b2ddeffbfd5faafba5bc580eff9a15 +size 271127 diff --git a/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/critic_2.pth b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/critic_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..fef1f280978433a251648779a34d74f837e6b015 --- /dev/null +++ b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/models/critic_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49eac81686ca615c879ff219962324a4ced1994172109f54d9a7fd0eadd0f20f +size 271127 diff --git a/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/results/learning_curve.png b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..14d9c2fb1cf7724cc7004c89fb330c4c735ffe37 Binary files /dev/null and b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/results/learning_curve.png differ diff --git a/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/results/res.csv b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..584ab531998fd9c2ede8a2d3d6649df8645e5258 --- /dev/null +++ b/Pendulum-v1/Test_gym_mp_TD3_20230416-221428/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,-0.4213114874583766,200 +1,-0.4213114874583766,200 +2,-0.4213114874583766,200 +3,-0.4213114874583766,200 +4,-0.4213114874583766,200 +5,-0.4213114874583766,200 +6,-0.4213114874583766,200 +7,-0.4213114874583766,200 +8,-0.4213114874583766,200 +9,-0.4213114874583766,200 diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/config.yaml b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e27b3dca8fe4c2a3174684a23aa950b22540d994 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/config.yaml @@ -0,0 +1,27 @@ +general_cfg: + algo_name: DDPG + device: cuda + env_name: Pendulum-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 10 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0001 + batch_size: 128 + buffer_size: 8000 + critic_hidden_dim: 256 + critic_lr: 0.001 + gamma: 0.99 + tau: 0.001 diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/logs/log.txt b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..34d6b2285ac685ecbdb243cb427b4c5592942b59 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/logs/log.txt @@ -0,0 +1,454 @@ +2022-12-01 11:47:04 - r - INFO: - Hyperparameters: +2022-12-01 11:47:04 - r - INFO: - ================================================================================ +2022-12-01 11:47:04 - r - INFO: - Name Value Type +2022-12-01 11:47:04 - r - INFO: - env_name Pendulum-v1 +2022-12-01 11:47:04 - r - INFO: - new_step_api 1 +2022-12-01 11:47:04 - r - INFO: - wrapper None +2022-12-01 11:47:04 - r - INFO: - render 0 +2022-12-01 11:47:04 - r - INFO: - algo_name DDPG +2022-12-01 11:47:04 - r - INFO: - mode train +2022-12-01 11:47:04 - r - INFO: - seed 10 +2022-12-01 11:47:04 - r - INFO: - device cuda +2022-12-01 11:47:04 - r - INFO: - train_eps 400 +2022-12-01 11:47:04 - r - INFO: - test_eps 10 +2022-12-01 11:47:04 - r - INFO: - eval_eps 10 +2022-12-01 11:47:04 - r - INFO: - eval_per_episode 5 +2022-12-01 11:47:04 - r - INFO: - max_steps 200 +2022-12-01 11:47:04 - r - INFO: - load_checkpoint 0 +2022-12-01 11:47:04 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2022-12-01 11:47:04 - r - INFO: - show_fig 0 +2022-12-01 11:47:04 - r - INFO: - save_fig 1 +2022-12-01 11:47:04 - r - INFO: - gamma 0.99 +2022-12-01 11:47:04 - r - INFO: - critic_lr 0.001 +2022-12-01 11:47:04 - r - INFO: - actor_lr 0.0001 +2022-12-01 11:47:04 - r - INFO: - buffer_size 8000 +2022-12-01 11:47:04 - r - INFO: - batch_size 128 +2022-12-01 11:47:04 - r - INFO: - tau 0.001 +2022-12-01 11:47:04 - r - INFO: - critic_hidden_dim 256 +2022-12-01 11:47:04 - r - INFO: - actor_hidden_dim 256 +2022-12-01 11:47:04 - r - INFO: - ================================================================================ +2022-12-01 11:47:04 - r - INFO: - action_bound: 2.0 +2022-12-01 11:47:04 - r - INFO: - n_states: 3, n_actions: 1 +2022-12-01 11:47:07 - r - INFO: - Start training! +2022-12-01 11:47:07 - r - INFO: - Env: Pendulum-v1, Algorithm: DDPG, Device: cuda +2022-12-01 11:47:08 - r - INFO: - Episode: 1/400, Reward: -1212.568, Step: 200 +2022-12-01 11:47:08 - r - INFO: - Episode: 2/400, Reward: -1623.901, Step: 200 +2022-12-01 11:47:09 - r - INFO: - Episode: 3/400, Reward: -1487.254, Step: 200 +2022-12-01 11:47:10 - r - INFO: - Episode: 4/400, Reward: -1453.605, Step: 200 +2022-12-01 11:47:10 - r - INFO: - Episode: 5/400, Reward: -1673.476, Step: 200 +2022-12-01 11:47:11 - r - INFO: - Current episode 5 has the best eval reward: -1636.256 +2022-12-01 11:47:12 - r - INFO: - Episode: 6/400, Reward: -1414.224, Step: 200 +2022-12-01 11:47:12 - r - INFO: - Episode: 7/400, Reward: -1510.705, Step: 200 +2022-12-01 11:47:13 - r - INFO: - Episode: 8/400, Reward: -1501.576, Step: 200 +2022-12-01 11:47:14 - r - INFO: - Episode: 9/400, Reward: -1512.594, Step: 200 +2022-12-01 11:47:14 - r - INFO: - Episode: 10/400, Reward: -1524.774, Step: 200 +2022-12-01 11:47:15 - r - INFO: - Current episode 10 has the best eval reward: -1574.442 +2022-12-01 11:47:16 - r - INFO: - Episode: 11/400, Reward: -1354.692, Step: 200 +2022-12-01 11:47:16 - r - INFO: - Episode: 12/400, Reward: -1554.375, Step: 200 +2022-12-01 11:47:17 - r - INFO: - Episode: 13/400, Reward: -1569.261, Step: 200 +2022-12-01 11:47:18 - r - INFO: - Episode: 14/400, Reward: -1497.939, Step: 200 +2022-12-01 11:47:18 - r - INFO: - Episode: 15/400, Reward: -1569.101, Step: 200 +2022-12-01 11:47:20 - r - INFO: - Episode: 16/400, Reward: -1253.003, Step: 200 +2022-12-01 11:47:20 - r - INFO: - Episode: 17/400, Reward: -1545.806, Step: 200 +2022-12-01 11:47:21 - r - INFO: - Episode: 18/400, Reward: -1265.231, Step: 200 +2022-12-01 11:47:22 - r - INFO: - Episode: 19/400, Reward: -305.823, Step: 200 +2022-12-01 11:47:22 - r - INFO: - Episode: 20/400, Reward: -1357.265, Step: 200 +2022-12-01 11:47:23 - r - INFO: - Current episode 20 has the best eval reward: -1434.902 +2022-12-01 11:47:24 - r - INFO: - Episode: 21/400, Reward: -1296.912, Step: 200 +2022-12-01 11:47:25 - r - INFO: - Episode: 22/400, Reward: -1314.462, Step: 200 +2022-12-01 11:47:25 - r - INFO: - Episode: 23/400, Reward: -1249.900, Step: 200 +2022-12-01 11:47:26 - r - INFO: - Episode: 24/400, Reward: -1339.911, Step: 200 +2022-12-01 11:47:27 - r - INFO: - Episode: 25/400, Reward: -1366.518, Step: 200 +2022-12-01 11:47:27 - r - INFO: - Current episode 25 has the best eval reward: -1369.715 +2022-12-01 11:47:28 - r - INFO: - Episode: 26/400, Reward: -1260.492, Step: 200 +2022-12-01 11:47:29 - r - INFO: - Episode: 27/400, Reward: -1194.335, Step: 200 +2022-12-01 11:47:29 - r - INFO: - Episode: 28/400, Reward: -1153.404, Step: 200 +2022-12-01 11:47:30 - r - INFO: - Episode: 29/400, Reward: -1082.368, Step: 200 +2022-12-01 11:47:31 - r - INFO: - Episode: 30/400, Reward: -1171.469, Step: 200 +2022-12-01 11:47:31 - r - INFO: - Current episode 30 has the best eval reward: -1277.884 +2022-12-01 11:47:32 - r - INFO: - Episode: 31/400, Reward: -1200.771, Step: 200 +2022-12-01 11:47:33 - r - INFO: - Episode: 32/400, Reward: -1032.577, Step: 200 +2022-12-01 11:47:33 - r - INFO: - Episode: 33/400, Reward: -1118.504, Step: 200 +2022-12-01 11:47:34 - r - INFO: - Episode: 34/400, Reward: -812.889, Step: 200 +2022-12-01 11:47:35 - r - INFO: - Episode: 35/400, Reward: -1346.486, Step: 200 +2022-12-01 11:47:35 - r - INFO: - Current episode 35 has the best eval reward: -1259.006 +2022-12-01 11:47:36 - r - INFO: - Episode: 36/400, Reward: -1150.214, Step: 200 +2022-12-01 11:47:37 - r - INFO: - Episode: 37/400, Reward: -1270.615, Step: 200 +2022-12-01 11:47:38 - r - INFO: - Episode: 38/400, Reward: -1016.817, Step: 200 +2022-12-01 11:47:38 - r - INFO: - Episode: 39/400, Reward: -663.595, Step: 200 +2022-12-01 11:47:39 - r - INFO: - Episode: 40/400, Reward: -138.012, Step: 200 +2022-12-01 11:47:40 - r - INFO: - Current episode 40 has the best eval reward: -1166.679 +2022-12-01 11:47:41 - r - INFO: - Episode: 41/400, Reward: -934.253, Step: 200 +2022-12-01 11:47:41 - r - INFO: - Episode: 42/400, Reward: -260.424, Step: 200 +2022-12-01 11:47:42 - r - INFO: - Episode: 43/400, Reward: -326.425, Step: 200 +2022-12-01 11:47:43 - r - INFO: - Episode: 44/400, Reward: -133.962, Step: 200 +2022-12-01 11:47:43 - r - INFO: - Episode: 45/400, Reward: -264.038, Step: 200 +2022-12-01 11:47:44 - r - INFO: - Current episode 45 has the best eval reward: -917.545 +2022-12-01 11:47:45 - r - INFO: - Episode: 46/400, Reward: -1091.756, Step: 200 +2022-12-01 11:47:45 - r - INFO: - Episode: 47/400, Reward: -0.407, Step: 200 +2022-12-01 11:47:46 - r - INFO: - Episode: 48/400, Reward: -131.840, Step: 200 +2022-12-01 11:47:47 - r - INFO: - Episode: 49/400, Reward: -129.614, Step: 200 +2022-12-01 11:47:47 - r - INFO: - Episode: 50/400, Reward: -128.357, Step: 200 +2022-12-01 11:47:48 - r - INFO: - Current episode 50 has the best eval reward: -781.569 +2022-12-01 11:47:49 - r - INFO: - Episode: 51/400, Reward: -407.419, Step: 200 +2022-12-01 11:47:49 - r - INFO: - Episode: 52/400, Reward: -402.595, Step: 200 +2022-12-01 11:47:50 - r - INFO: - Episode: 53/400, Reward: -0.787, Step: 200 +2022-12-01 11:47:51 - r - INFO: - Episode: 54/400, Reward: -442.559, Step: 200 +2022-12-01 11:47:52 - r - INFO: - Episode: 55/400, Reward: -244.390, Step: 200 +2022-12-01 11:47:52 - r - INFO: - Current episode 55 has the best eval reward: -613.927 +2022-12-01 11:47:53 - r - INFO: - Episode: 56/400, Reward: -281.037, Step: 200 +2022-12-01 11:47:54 - r - INFO: - Episode: 57/400, Reward: -131.178, Step: 200 +2022-12-01 11:47:54 - r - INFO: - Episode: 58/400, Reward: -1.174, Step: 200 +2022-12-01 11:47:55 - r - INFO: - Episode: 59/400, Reward: -260.898, Step: 200 +2022-12-01 11:47:56 - r - INFO: - Episode: 60/400, Reward: -389.764, Step: 200 +2022-12-01 11:47:56 - r - INFO: - Current episode 60 has the best eval reward: -581.082 +2022-12-01 11:47:57 - r - INFO: - Episode: 61/400, Reward: -263.920, Step: 200 +2022-12-01 11:47:58 - r - INFO: - Episode: 62/400, Reward: -1.243, Step: 200 +2022-12-01 11:47:58 - r - INFO: - Episode: 63/400, Reward: -1256.841, Step: 200 +2022-12-01 11:47:59 - r - INFO: - Episode: 64/400, Reward: -132.499, Step: 200 +2022-12-01 11:48:00 - r - INFO: - Episode: 65/400, Reward: -397.011, Step: 200 +2022-12-01 11:48:00 - r - INFO: - Current episode 65 has the best eval reward: -390.975 +2022-12-01 11:48:01 - r - INFO: - Episode: 66/400, Reward: -124.641, Step: 200 +2022-12-01 11:48:02 - r - INFO: - Episode: 67/400, Reward: -263.838, Step: 200 +2022-12-01 11:48:02 - r - INFO: - Episode: 68/400, Reward: -384.373, Step: 200 +2022-12-01 11:48:03 - r - INFO: - Episode: 69/400, Reward: -123.299, Step: 200 +2022-12-01 11:48:04 - r - INFO: - Episode: 70/400, Reward: -1502.847, Step: 200 +2022-12-01 11:48:05 - r - INFO: - Episode: 71/400, Reward: -354.070, Step: 200 +2022-12-01 11:48:06 - r - INFO: - Episode: 72/400, Reward: -1039.914, Step: 200 +2022-12-01 11:48:06 - r - INFO: - Episode: 73/400, Reward: -242.399, Step: 200 +2022-12-01 11:48:07 - r - INFO: - Episode: 74/400, Reward: -122.664, Step: 200 +2022-12-01 11:48:08 - r - INFO: - Episode: 75/400, Reward: -125.647, Step: 200 +2022-12-01 11:48:09 - r - INFO: - Episode: 76/400, Reward: -785.472, Step: 200 +2022-12-01 11:48:10 - r - INFO: - Episode: 77/400, Reward: -239.213, Step: 200 +2022-12-01 11:48:10 - r - INFO: - Episode: 78/400, Reward: -237.476, Step: 200 +2022-12-01 11:48:11 - r - INFO: - Episode: 79/400, Reward: -240.762, Step: 200 +2022-12-01 11:48:12 - r - INFO: - Episode: 80/400, Reward: -497.246, Step: 200 +2022-12-01 11:48:13 - r - INFO: - Episode: 81/400, Reward: -389.137, Step: 200 +2022-12-01 11:48:14 - r - INFO: - Episode: 82/400, Reward: -0.271, Step: 200 +2022-12-01 11:48:15 - r - INFO: - Episode: 83/400, Reward: -0.309, Step: 200 +2022-12-01 11:48:16 - r - INFO: - Episode: 84/400, Reward: -243.787, Step: 200 +2022-12-01 11:48:17 - r - INFO: - Episode: 85/400, Reward: -0.285, Step: 200 +2022-12-01 11:48:18 - r - INFO: - Episode: 86/400, Reward: -129.952, Step: 200 +2022-12-01 11:48:19 - r - INFO: - Episode: 87/400, Reward: -128.368, Step: 200 +2022-12-01 11:48:19 - r - INFO: - Episode: 88/400, Reward: -0.698, Step: 200 +2022-12-01 11:48:20 - r - INFO: - Episode: 89/400, Reward: -120.313, Step: 200 +2022-12-01 11:48:21 - r - INFO: - Episode: 90/400, Reward: -127.307, Step: 200 +2022-12-01 11:48:22 - r - INFO: - Current episode 90 has the best eval reward: -359.406 +2022-12-01 11:48:22 - r - INFO: - Episode: 91/400, Reward: -126.289, Step: 200 +2022-12-01 11:48:23 - r - INFO: - Episode: 92/400, Reward: -129.531, Step: 200 +2022-12-01 11:48:24 - r - INFO: - Episode: 93/400, Reward: -1.561, Step: 200 +2022-12-01 11:48:25 - r - INFO: - Episode: 94/400, Reward: -130.035, Step: 200 +2022-12-01 11:48:26 - r - INFO: - Episode: 95/400, Reward: -241.012, Step: 200 +2022-12-01 11:48:26 - r - INFO: - Current episode 95 has the best eval reward: -306.512 +2022-12-01 11:48:27 - r - INFO: - Episode: 96/400, Reward: -118.295, Step: 200 +2022-12-01 11:48:28 - r - INFO: - Episode: 97/400, Reward: -245.900, Step: 200 +2022-12-01 11:48:29 - r - INFO: - Episode: 98/400, Reward: -1.181, Step: 200 +2022-12-01 11:48:29 - r - INFO: - Episode: 99/400, Reward: -126.194, Step: 200 +2022-12-01 11:48:30 - r - INFO: - Episode: 100/400, Reward: -127.803, Step: 200 +2022-12-01 11:48:31 - r - INFO: - Episode: 101/400, Reward: -117.819, Step: 200 +2022-12-01 11:48:32 - r - INFO: - Episode: 102/400, Reward: -119.513, Step: 200 +2022-12-01 11:48:33 - r - INFO: - Episode: 103/400, Reward: -221.549, Step: 200 +2022-12-01 11:48:34 - r - INFO: - Episode: 104/400, Reward: -244.916, Step: 200 +2022-12-01 11:48:35 - r - INFO: - Episode: 105/400, Reward: -126.866, Step: 200 +2022-12-01 11:48:36 - r - INFO: - Episode: 106/400, Reward: -358.937, Step: 200 +2022-12-01 11:48:37 - r - INFO: - Episode: 107/400, Reward: -125.261, Step: 200 +2022-12-01 11:48:38 - r - INFO: - Episode: 108/400, Reward: -127.865, Step: 200 +2022-12-01 11:48:39 - r - INFO: - Episode: 109/400, Reward: -357.778, Step: 200 +2022-12-01 11:48:40 - r - INFO: - Episode: 110/400, Reward: -124.891, Step: 200 +2022-12-01 11:48:40 - r - INFO: - Current episode 110 has the best eval reward: -303.949 +2022-12-01 11:48:41 - r - INFO: - Episode: 111/400, Reward: -383.057, Step: 200 +2022-12-01 11:48:42 - r - INFO: - Episode: 112/400, Reward: -3.618, Step: 200 +2022-12-01 11:48:43 - r - INFO: - Episode: 113/400, Reward: -248.158, Step: 200 +2022-12-01 11:48:44 - r - INFO: - Episode: 114/400, Reward: -349.098, Step: 200 +2022-12-01 11:48:44 - r - INFO: - Episode: 115/400, Reward: -126.353, Step: 200 +2022-12-01 11:48:45 - r - INFO: - Current episode 115 has the best eval reward: -201.906 +2022-12-01 11:48:46 - r - INFO: - Episode: 116/400, Reward: -237.635, Step: 200 +2022-12-01 11:48:47 - r - INFO: - Episode: 117/400, Reward: -118.228, Step: 200 +2022-12-01 11:48:48 - r - INFO: - Episode: 118/400, Reward: -119.511, Step: 200 +2022-12-01 11:48:48 - r - INFO: - Episode: 119/400, Reward: -120.025, Step: 200 +2022-12-01 11:48:49 - r - INFO: - Episode: 120/400, Reward: -377.781, Step: 200 +2022-12-01 11:48:51 - r - INFO: - Episode: 121/400, Reward: -129.686, Step: 200 +2022-12-01 11:48:51 - r - INFO: - Episode: 122/400, Reward: -248.889, Step: 200 +2022-12-01 11:48:52 - r - INFO: - Episode: 123/400, Reward: -128.352, Step: 200 +2022-12-01 11:48:53 - r - INFO: - Episode: 124/400, Reward: -242.038, Step: 200 +2022-12-01 11:48:54 - r - INFO: - Episode: 125/400, Reward: -116.703, Step: 200 +2022-12-01 11:48:55 - r - INFO: - Episode: 126/400, Reward: -364.826, Step: 200 +2022-12-01 11:48:56 - r - INFO: - Episode: 127/400, Reward: -1.162, Step: 200 +2022-12-01 11:48:57 - r - INFO: - Episode: 128/400, Reward: -126.903, Step: 200 +2022-12-01 11:48:58 - r - INFO: - Episode: 129/400, Reward: -249.294, Step: 200 +2022-12-01 11:48:59 - r - INFO: - Episode: 130/400, Reward: -1.437, Step: 200 +2022-12-01 11:49:00 - r - INFO: - Episode: 131/400, Reward: -1.518, Step: 200 +2022-12-01 11:49:01 - r - INFO: - Episode: 132/400, Reward: -1.269, Step: 200 +2022-12-01 11:49:02 - r - INFO: - Episode: 133/400, Reward: -242.660, Step: 200 +2022-12-01 11:49:03 - r - INFO: - Episode: 134/400, Reward: -122.925, Step: 200 +2022-12-01 11:49:03 - r - INFO: - Episode: 135/400, Reward: -121.282, Step: 200 +2022-12-01 11:49:05 - r - INFO: - Episode: 136/400, Reward: -122.574, Step: 200 +2022-12-01 11:49:06 - r - INFO: - Episode: 137/400, Reward: -121.215, Step: 200 +2022-12-01 11:49:06 - r - INFO: - Episode: 138/400, Reward: -250.123, Step: 200 +2022-12-01 11:49:07 - r - INFO: - Episode: 139/400, Reward: -327.829, Step: 200 +2022-12-01 11:49:08 - r - INFO: - Episode: 140/400, Reward: -125.407, Step: 200 +2022-12-01 11:49:10 - r - INFO: - Episode: 141/400, Reward: -116.500, Step: 200 +2022-12-01 11:49:10 - r - INFO: - Episode: 142/400, Reward: -336.258, Step: 200 +2022-12-01 11:49:11 - r - INFO: - Episode: 143/400, Reward: -230.379, Step: 200 +2022-12-01 11:49:12 - r - INFO: - Episode: 144/400, Reward: -127.153, Step: 200 +2022-12-01 11:49:13 - r - INFO: - Episode: 145/400, Reward: -244.796, Step: 200 +2022-12-01 11:49:14 - r - INFO: - Episode: 146/400, Reward: -325.592, Step: 200 +2022-12-01 11:49:15 - r - INFO: - Episode: 147/400, Reward: -1.020, Step: 200 +2022-12-01 11:49:16 - r - INFO: - Episode: 148/400, Reward: -123.722, Step: 200 +2022-12-01 11:49:17 - r - INFO: - Episode: 149/400, Reward: -123.822, Step: 200 +2022-12-01 11:49:18 - r - INFO: - Episode: 150/400, Reward: -128.729, Step: 200 +2022-12-01 11:49:19 - r - INFO: - Episode: 151/400, Reward: -119.312, Step: 200 +2022-12-01 11:49:20 - r - INFO: - Episode: 152/400, Reward: -269.220, Step: 200 +2022-12-01 11:49:21 - r - INFO: - Episode: 153/400, Reward: -364.687, Step: 200 +2022-12-01 11:49:22 - r - INFO: - Episode: 154/400, Reward: -224.135, Step: 200 +2022-12-01 11:49:22 - r - INFO: - Episode: 155/400, Reward: -125.227, Step: 200 +2022-12-01 11:49:23 - r - INFO: - Current episode 155 has the best eval reward: -194.534 +2022-12-01 11:49:24 - r - INFO: - Episode: 156/400, Reward: -127.929, Step: 200 +2022-12-01 11:49:25 - r - INFO: - Episode: 157/400, Reward: -129.891, Step: 200 +2022-12-01 11:49:25 - r - INFO: - Episode: 158/400, Reward: -126.725, Step: 200 +2022-12-01 11:49:26 - r - INFO: - Episode: 159/400, Reward: -119.785, Step: 200 +2022-12-01 11:49:27 - r - INFO: - Episode: 160/400, Reward: -346.210, Step: 200 +2022-12-01 11:49:28 - r - INFO: - Episode: 161/400, Reward: -368.187, Step: 200 +2022-12-01 11:49:29 - r - INFO: - Episode: 162/400, Reward: -246.304, Step: 200 +2022-12-01 11:49:30 - r - INFO: - Episode: 163/400, Reward: -1.275, Step: 200 +2022-12-01 11:49:31 - r - INFO: - Episode: 164/400, Reward: -117.674, Step: 200 +2022-12-01 11:49:32 - r - INFO: - Episode: 165/400, Reward: -336.285, Step: 200 +2022-12-01 11:49:33 - r - INFO: - Episode: 166/400, Reward: -1.146, Step: 200 +2022-12-01 11:49:34 - r - INFO: - Episode: 167/400, Reward: -1.339, Step: 200 +2022-12-01 11:49:35 - r - INFO: - Episode: 168/400, Reward: -118.357, Step: 200 +2022-12-01 11:49:36 - r - INFO: - Episode: 169/400, Reward: -242.254, Step: 200 +2022-12-01 11:49:37 - r - INFO: - Episode: 170/400, Reward: -123.858, Step: 200 +2022-12-01 11:49:38 - r - INFO: - Episode: 171/400, Reward: -117.109, Step: 200 +2022-12-01 11:49:39 - r - INFO: - Episode: 172/400, Reward: -123.739, Step: 200 +2022-12-01 11:49:40 - r - INFO: - Episode: 173/400, Reward: -115.141, Step: 200 +2022-12-01 11:49:41 - r - INFO: - Episode: 174/400, Reward: -119.847, Step: 200 +2022-12-01 11:49:42 - r - INFO: - Episode: 175/400, Reward: -120.644, Step: 200 +2022-12-01 11:49:43 - r - INFO: - Episode: 176/400, Reward: -116.678, Step: 200 +2022-12-01 11:49:44 - r - INFO: - Episode: 177/400, Reward: -117.131, Step: 200 +2022-12-01 11:49:45 - r - INFO: - Episode: 178/400, Reward: -122.806, Step: 200 +2022-12-01 11:49:46 - r - INFO: - Episode: 179/400, Reward: -127.810, Step: 200 +2022-12-01 11:49:47 - r - INFO: - Episode: 180/400, Reward: -239.768, Step: 200 +2022-12-01 11:49:48 - r - INFO: - Episode: 181/400, Reward: -221.090, Step: 200 +2022-12-01 11:49:49 - r - INFO: - Episode: 182/400, Reward: -121.223, Step: 200 +2022-12-01 11:49:50 - r - INFO: - Episode: 183/400, Reward: -227.651, Step: 200 +2022-12-01 11:49:51 - r - INFO: - Episode: 184/400, Reward: -322.285, Step: 200 +2022-12-01 11:49:52 - r - INFO: - Episode: 185/400, Reward: -123.356, Step: 200 +2022-12-01 11:49:52 - r - INFO: - Current episode 185 has the best eval reward: -169.112 +2022-12-01 11:49:53 - r - INFO: - Episode: 186/400, Reward: -119.883, Step: 200 +2022-12-01 11:49:54 - r - INFO: - Episode: 187/400, Reward: -121.378, Step: 200 +2022-12-01 11:49:55 - r - INFO: - Episode: 188/400, Reward: -0.875, Step: 200 +2022-12-01 11:49:56 - r - INFO: - Episode: 189/400, Reward: -358.294, Step: 200 +2022-12-01 11:49:57 - r - INFO: - Episode: 190/400, Reward: -119.036, Step: 200 +2022-12-01 11:49:58 - r - INFO: - Episode: 191/400, Reward: -1.170, Step: 200 +2022-12-01 11:49:59 - r - INFO: - Episode: 192/400, Reward: -113.781, Step: 200 +2022-12-01 11:50:00 - r - INFO: - Episode: 193/400, Reward: -119.419, Step: 200 +2022-12-01 11:50:01 - r - INFO: - Episode: 194/400, Reward: -121.292, Step: 200 +2022-12-01 11:50:02 - r - INFO: - Episode: 195/400, Reward: -330.155, Step: 200 +2022-12-01 11:50:03 - r - INFO: - Episode: 196/400, Reward: -119.069, Step: 200 +2022-12-01 11:50:04 - r - INFO: - Episode: 197/400, Reward: -123.463, Step: 200 +2022-12-01 11:50:05 - r - INFO: - Episode: 198/400, Reward: -126.123, Step: 200 +2022-12-01 11:50:06 - r - INFO: - Episode: 199/400, Reward: -0.956, Step: 200 +2022-12-01 11:50:07 - r - INFO: - Episode: 200/400, Reward: -115.547, Step: 200 +2022-12-01 11:50:09 - r - INFO: - Episode: 201/400, Reward: -345.353, Step: 200 +2022-12-01 11:50:09 - r - INFO: - Episode: 202/400, Reward: -122.789, Step: 200 +2022-12-01 11:50:10 - r - INFO: - Episode: 203/400, Reward: -129.462, Step: 200 +2022-12-01 11:50:11 - r - INFO: - Episode: 204/400, Reward: -126.246, Step: 200 +2022-12-01 11:50:12 - r - INFO: - Episode: 205/400, Reward: -128.631, Step: 200 +2022-12-01 11:50:14 - r - INFO: - Episode: 206/400, Reward: -354.559, Step: 200 +2022-12-01 11:50:15 - r - INFO: - Episode: 207/400, Reward: -1.449, Step: 200 +2022-12-01 11:50:16 - r - INFO: - Episode: 208/400, Reward: -121.555, Step: 200 +2022-12-01 11:50:17 - r - INFO: - Episode: 209/400, Reward: -118.753, Step: 200 +2022-12-01 11:50:18 - r - INFO: - Episode: 210/400, Reward: -361.083, Step: 200 +2022-12-01 11:50:19 - r - INFO: - Episode: 211/400, Reward: -227.722, Step: 200 +2022-12-01 11:50:20 - r - INFO: - Episode: 212/400, Reward: -256.454, Step: 200 +2022-12-01 11:50:21 - r - INFO: - Episode: 213/400, Reward: -123.373, Step: 200 +2022-12-01 11:50:22 - r - INFO: - Episode: 214/400, Reward: -114.304, Step: 200 +2022-12-01 11:50:23 - r - INFO: - Episode: 215/400, Reward: -1.312, Step: 200 +2022-12-01 11:50:24 - r - INFO: - Episode: 216/400, Reward: -239.796, Step: 200 +2022-12-01 11:50:25 - r - INFO: - Episode: 217/400, Reward: -126.548, Step: 200 +2022-12-01 11:50:26 - r - INFO: - Episode: 218/400, Reward: -126.639, Step: 200 +2022-12-01 11:50:27 - r - INFO: - Episode: 219/400, Reward: -125.775, Step: 200 +2022-12-01 11:50:28 - r - INFO: - Episode: 220/400, Reward: -1.940, Step: 200 +2022-12-01 11:50:29 - r - INFO: - Episode: 221/400, Reward: -124.998, Step: 200 +2022-12-01 11:50:30 - r - INFO: - Episode: 222/400, Reward: -353.236, Step: 200 +2022-12-01 11:50:31 - r - INFO: - Episode: 223/400, Reward: -357.056, Step: 200 +2022-12-01 11:50:31 - r - INFO: - Episode: 224/400, Reward: -114.204, Step: 200 +2022-12-01 11:50:32 - r - INFO: - Episode: 225/400, Reward: -125.223, Step: 200 +2022-12-01 11:50:34 - r - INFO: - Episode: 226/400, Reward: -227.426, Step: 200 +2022-12-01 11:50:35 - r - INFO: - Episode: 227/400, Reward: -3.664, Step: 200 +2022-12-01 11:50:35 - r - INFO: - Episode: 228/400, Reward: -123.205, Step: 200 +2022-12-01 11:50:36 - r - INFO: - Episode: 229/400, Reward: -124.514, Step: 200 +2022-12-01 11:50:37 - r - INFO: - Episode: 230/400, Reward: -118.301, Step: 200 +2022-12-01 11:50:38 - r - INFO: - Episode: 231/400, Reward: -243.096, Step: 200 +2022-12-01 11:50:39 - r - INFO: - Episode: 232/400, Reward: -126.697, Step: 200 +2022-12-01 11:50:40 - r - INFO: - Episode: 233/400, Reward: -1.867, Step: 200 +2022-12-01 11:50:41 - r - INFO: - Episode: 234/400, Reward: -119.831, Step: 200 +2022-12-01 11:50:42 - r - INFO: - Episode: 235/400, Reward: -117.760, Step: 200 +2022-12-01 11:50:43 - r - INFO: - Episode: 236/400, Reward: -123.939, Step: 200 +2022-12-01 11:50:44 - r - INFO: - Episode: 237/400, Reward: -1.717, Step: 200 +2022-12-01 11:50:45 - r - INFO: - Episode: 238/400, Reward: -128.808, Step: 200 +2022-12-01 11:50:46 - r - INFO: - Episode: 239/400, Reward: -131.194, Step: 200 +2022-12-01 11:50:46 - r - INFO: - Episode: 240/400, Reward: -352.905, Step: 200 +2022-12-01 11:50:48 - r - INFO: - Episode: 241/400, Reward: -233.712, Step: 200 +2022-12-01 11:50:49 - r - INFO: - Episode: 242/400, Reward: -262.387, Step: 200 +2022-12-01 11:50:49 - r - INFO: - Episode: 243/400, Reward: -3.707, Step: 200 +2022-12-01 11:50:50 - r - INFO: - Episode: 244/400, Reward: -126.616, Step: 200 +2022-12-01 11:50:51 - r - INFO: - Episode: 245/400, Reward: -230.692, Step: 200 +2022-12-01 11:50:52 - r - INFO: - Episode: 246/400, Reward: -1.252, Step: 200 +2022-12-01 11:50:53 - r - INFO: - Episode: 247/400, Reward: -118.523, Step: 200 +2022-12-01 11:50:53 - r - INFO: - Episode: 248/400, Reward: -125.407, Step: 200 +2022-12-01 11:50:54 - r - INFO: - Episode: 249/400, Reward: -117.500, Step: 200 +2022-12-01 11:50:55 - r - INFO: - Episode: 250/400, Reward: -393.070, Step: 200 +2022-12-01 11:50:56 - r - INFO: - Episode: 251/400, Reward: -118.367, Step: 200 +2022-12-01 11:50:57 - r - INFO: - Episode: 252/400, Reward: -128.272, Step: 200 +2022-12-01 11:50:57 - r - INFO: - Episode: 253/400, Reward: -245.614, Step: 200 +2022-12-01 11:50:58 - r - INFO: - Episode: 254/400, Reward: -122.880, Step: 200 +2022-12-01 11:50:59 - r - INFO: - Episode: 255/400, Reward: -239.682, Step: 200 +2022-12-01 11:51:00 - r - INFO: - Episode: 256/400, Reward: -124.537, Step: 200 +2022-12-01 11:51:01 - r - INFO: - Episode: 257/400, Reward: -1.111, Step: 200 +2022-12-01 11:51:01 - r - INFO: - Episode: 258/400, Reward: -117.201, Step: 200 +2022-12-01 11:51:02 - r - INFO: - Episode: 259/400, Reward: -128.891, Step: 200 +2022-12-01 11:51:03 - r - INFO: - Episode: 260/400, Reward: -126.265, Step: 200 +2022-12-01 11:51:04 - r - INFO: - Episode: 261/400, Reward: -231.952, Step: 200 +2022-12-01 11:51:05 - r - INFO: - Episode: 262/400, Reward: -119.314, Step: 200 +2022-12-01 11:51:06 - r - INFO: - Episode: 263/400, Reward: -125.479, Step: 200 +2022-12-01 11:51:07 - r - INFO: - Episode: 264/400, Reward: -3.259, Step: 200 +2022-12-01 11:51:07 - r - INFO: - Episode: 265/400, Reward: -127.222, Step: 200 +2022-12-01 11:51:09 - r - INFO: - Episode: 266/400, Reward: -117.150, Step: 200 +2022-12-01 11:51:10 - r - INFO: - Episode: 267/400, Reward: -126.775, Step: 200 +2022-12-01 11:51:10 - r - INFO: - Episode: 268/400, Reward: -286.837, Step: 200 +2022-12-01 11:51:11 - r - INFO: - Episode: 269/400, Reward: -119.631, Step: 200 +2022-12-01 11:51:12 - r - INFO: - Episode: 270/400, Reward: -117.221, Step: 200 +2022-12-01 11:51:13 - r - INFO: - Episode: 271/400, Reward: -127.008, Step: 200 +2022-12-01 11:51:14 - r - INFO: - Episode: 272/400, Reward: -125.160, Step: 200 +2022-12-01 11:51:15 - r - INFO: - Episode: 273/400, Reward: -119.650, Step: 200 +2022-12-01 11:51:16 - r - INFO: - Episode: 274/400, Reward: -126.034, Step: 200 +2022-12-01 11:51:17 - r - INFO: - Episode: 275/400, Reward: -117.445, Step: 200 +2022-12-01 11:51:17 - r - INFO: - Current episode 275 has the best eval reward: -168.495 +2022-12-01 11:51:18 - r - INFO: - Episode: 276/400, Reward: -353.132, Step: 200 +2022-12-01 11:51:19 - r - INFO: - Episode: 277/400, Reward: -123.565, Step: 200 +2022-12-01 11:51:20 - r - INFO: - Episode: 278/400, Reward: -226.007, Step: 200 +2022-12-01 11:51:21 - r - INFO: - Episode: 279/400, Reward: -0.719, Step: 200 +2022-12-01 11:51:21 - r - INFO: - Episode: 280/400, Reward: -119.678, Step: 200 +2022-12-01 11:51:23 - r - INFO: - Episode: 281/400, Reward: -124.168, Step: 200 +2022-12-01 11:51:24 - r - INFO: - Episode: 282/400, Reward: -126.560, Step: 200 +2022-12-01 11:51:25 - r - INFO: - Episode: 283/400, Reward: -300.111, Step: 200 +2022-12-01 11:51:25 - r - INFO: - Episode: 284/400, Reward: -360.951, Step: 200 +2022-12-01 11:51:26 - r - INFO: - Episode: 285/400, Reward: -125.889, Step: 200 +2022-12-01 11:51:28 - r - INFO: - Episode: 286/400, Reward: -247.302, Step: 200 +2022-12-01 11:51:28 - r - INFO: - Episode: 287/400, Reward: -131.289, Step: 200 +2022-12-01 11:51:29 - r - INFO: - Episode: 288/400, Reward: -240.206, Step: 200 +2022-12-01 11:51:30 - r - INFO: - Episode: 289/400, Reward: -287.705, Step: 200 +2022-12-01 11:51:31 - r - INFO: - Episode: 290/400, Reward: -227.766, Step: 200 +2022-12-01 11:51:32 - r - INFO: - Episode: 291/400, Reward: -123.641, Step: 200 +2022-12-01 11:51:33 - r - INFO: - Episode: 292/400, Reward: -1.027, Step: 200 +2022-12-01 11:51:34 - r - INFO: - Episode: 293/400, Reward: -251.142, Step: 200 +2022-12-01 11:51:35 - r - INFO: - Episode: 294/400, Reward: -124.858, Step: 200 +2022-12-01 11:51:36 - r - INFO: - Episode: 295/400, Reward: -120.365, Step: 200 +2022-12-01 11:51:37 - r - INFO: - Episode: 296/400, Reward: -116.219, Step: 200 +2022-12-01 11:51:38 - r - INFO: - Episode: 297/400, Reward: -126.511, Step: 200 +2022-12-01 11:51:39 - r - INFO: - Episode: 298/400, Reward: -245.523, Step: 200 +2022-12-01 11:51:39 - r - INFO: - Episode: 299/400, Reward: -234.450, Step: 200 +2022-12-01 11:51:40 - r - INFO: - Episode: 300/400, Reward: -119.541, Step: 200 +2022-12-01 11:51:41 - r - INFO: - Episode: 301/400, Reward: -378.112, Step: 200 +2022-12-01 11:51:42 - r - INFO: - Episode: 302/400, Reward: -235.862, Step: 200 +2022-12-01 11:51:43 - r - INFO: - Episode: 303/400, Reward: -231.766, Step: 200 +2022-12-01 11:51:44 - r - INFO: - Episode: 304/400, Reward: -114.889, Step: 200 +2022-12-01 11:51:45 - r - INFO: - Episode: 305/400, Reward: -124.536, Step: 200 +2022-12-01 11:51:46 - r - INFO: - Episode: 306/400, Reward: -127.555, Step: 200 +2022-12-01 11:51:47 - r - INFO: - Episode: 307/400, Reward: -242.659, Step: 200 +2022-12-01 11:51:48 - r - INFO: - Episode: 308/400, Reward: -121.678, Step: 200 +2022-12-01 11:51:48 - r - INFO: - Episode: 309/400, Reward: -233.681, Step: 200 +2022-12-01 11:51:49 - r - INFO: - Episode: 310/400, Reward: -125.020, Step: 200 +2022-12-01 11:51:51 - r - INFO: - Episode: 311/400, Reward: -121.913, Step: 200 +2022-12-01 11:51:51 - r - INFO: - Episode: 312/400, Reward: -123.341, Step: 200 +2022-12-01 11:51:52 - r - INFO: - Episode: 313/400, Reward: -251.091, Step: 200 +2022-12-01 11:51:53 - r - INFO: - Episode: 314/400, Reward: -119.395, Step: 200 +2022-12-01 11:51:54 - r - INFO: - Episode: 315/400, Reward: -233.410, Step: 200 +2022-12-01 11:51:55 - r - INFO: - Episode: 316/400, Reward: -124.669, Step: 200 +2022-12-01 11:51:56 - r - INFO: - Episode: 317/400, Reward: -240.765, Step: 200 +2022-12-01 11:51:57 - r - INFO: - Episode: 318/400, Reward: -116.002, Step: 200 +2022-12-01 11:51:58 - r - INFO: - Episode: 319/400, Reward: -233.707, Step: 200 +2022-12-01 11:51:59 - r - INFO: - Episode: 320/400, Reward: -126.942, Step: 200 +2022-12-01 11:52:00 - r - INFO: - Episode: 321/400, Reward: -120.706, Step: 200 +2022-12-01 11:52:01 - r - INFO: - Episode: 322/400, Reward: -118.342, Step: 200 +2022-12-01 11:52:02 - r - INFO: - Episode: 323/400, Reward: -230.450, Step: 200 +2022-12-01 11:52:03 - r - INFO: - Episode: 324/400, Reward: -0.747, Step: 200 +2022-12-01 11:52:04 - r - INFO: - Episode: 325/400, Reward: -123.279, Step: 200 +2022-12-01 11:52:04 - r - INFO: - Current episode 325 has the best eval reward: -159.430 +2022-12-01 11:52:05 - r - INFO: - Episode: 326/400, Reward: -227.798, Step: 200 +2022-12-01 11:52:06 - r - INFO: - Episode: 327/400, Reward: -244.460, Step: 200 +2022-12-01 11:52:07 - r - INFO: - Episode: 328/400, Reward: -118.017, Step: 200 +2022-12-01 11:52:08 - r - INFO: - Episode: 329/400, Reward: -0.858, Step: 200 +2022-12-01 11:52:08 - r - INFO: - Episode: 330/400, Reward: -342.191, Step: 200 +2022-12-01 11:52:10 - r - INFO: - Episode: 331/400, Reward: -113.292, Step: 200 +2022-12-01 11:52:11 - r - INFO: - Episode: 332/400, Reward: -125.138, Step: 200 +2022-12-01 11:52:12 - r - INFO: - Episode: 333/400, Reward: -236.427, Step: 200 +2022-12-01 11:52:12 - r - INFO: - Episode: 334/400, Reward: -353.551, Step: 200 +2022-12-01 11:52:13 - r - INFO: - Episode: 335/400, Reward: -126.451, Step: 200 +2022-12-01 11:52:15 - r - INFO: - Episode: 336/400, Reward: -123.233, Step: 200 +2022-12-01 11:52:15 - r - INFO: - Episode: 337/400, Reward: -242.726, Step: 200 +2022-12-01 11:52:16 - r - INFO: - Episode: 338/400, Reward: -118.301, Step: 200 +2022-12-01 11:52:17 - r - INFO: - Episode: 339/400, Reward: -238.164, Step: 200 +2022-12-01 11:52:18 - r - INFO: - Episode: 340/400, Reward: -393.018, Step: 200 +2022-12-01 11:52:19 - r - INFO: - Episode: 341/400, Reward: -4.361, Step: 200 +2022-12-01 11:52:20 - r - INFO: - Episode: 342/400, Reward: -118.507, Step: 200 +2022-12-01 11:52:21 - r - INFO: - Episode: 343/400, Reward: -122.249, Step: 200 +2022-12-01 11:52:22 - r - INFO: - Episode: 344/400, Reward: -253.743, Step: 200 +2022-12-01 11:52:23 - r - INFO: - Episode: 345/400, Reward: -339.840, Step: 200 +2022-12-01 11:52:24 - r - INFO: - Episode: 346/400, Reward: -124.519, Step: 200 +2022-12-01 11:52:25 - r - INFO: - Episode: 347/400, Reward: -125.941, Step: 200 +2022-12-01 11:52:26 - r - INFO: - Episode: 348/400, Reward: -119.085, Step: 200 +2022-12-01 11:52:27 - r - INFO: - Episode: 349/400, Reward: -123.989, Step: 200 +2022-12-01 11:52:28 - r - INFO: - Episode: 350/400, Reward: -242.460, Step: 200 +2022-12-01 11:52:29 - r - INFO: - Episode: 351/400, Reward: -236.760, Step: 200 +2022-12-01 11:52:30 - r - INFO: - Episode: 352/400, Reward: -1.233, Step: 200 +2022-12-01 11:52:30 - r - INFO: - Episode: 353/400, Reward: -118.372, Step: 200 +2022-12-01 11:52:31 - r - INFO: - Episode: 354/400, Reward: -355.525, Step: 200 +2022-12-01 11:52:32 - r - INFO: - Episode: 355/400, Reward: -357.921, Step: 200 +2022-12-01 11:52:34 - r - INFO: - Episode: 356/400, Reward: -238.332, Step: 200 +2022-12-01 11:52:34 - r - INFO: - Episode: 357/400, Reward: -130.658, Step: 200 +2022-12-01 11:52:35 - r - INFO: - Episode: 358/400, Reward: -126.854, Step: 200 +2022-12-01 11:52:36 - r - INFO: - Episode: 359/400, Reward: -116.509, Step: 200 +2022-12-01 11:52:37 - r - INFO: - Episode: 360/400, Reward: -123.745, Step: 200 +2022-12-01 11:52:38 - r - INFO: - Episode: 361/400, Reward: -368.148, Step: 200 +2022-12-01 11:52:39 - r - INFO: - Episode: 362/400, Reward: -126.382, Step: 200 +2022-12-01 11:52:40 - r - INFO: - Episode: 363/400, Reward: -236.524, Step: 200 +2022-12-01 11:52:41 - r - INFO: - Episode: 364/400, Reward: -240.952, Step: 200 +2022-12-01 11:52:42 - r - INFO: - Episode: 365/400, Reward: -117.773, Step: 200 +2022-12-01 11:52:43 - r - INFO: - Episode: 366/400, Reward: -247.744, Step: 200 +2022-12-01 11:52:44 - r - INFO: - Episode: 367/400, Reward: -124.078, Step: 200 +2022-12-01 11:52:45 - r - INFO: - Episode: 368/400, Reward: -126.035, Step: 200 +2022-12-01 11:52:46 - r - INFO: - Episode: 369/400, Reward: -244.663, Step: 200 +2022-12-01 11:52:47 - r - INFO: - Episode: 370/400, Reward: -250.769, Step: 200 +2022-12-01 11:52:48 - r - INFO: - Episode: 371/400, Reward: -2.284, Step: 200 +2022-12-01 11:52:49 - r - INFO: - Episode: 372/400, Reward: -124.358, Step: 200 +2022-12-01 11:52:50 - r - INFO: - Episode: 373/400, Reward: -124.564, Step: 200 +2022-12-01 11:52:51 - r - INFO: - Episode: 374/400, Reward: -352.073, Step: 200 +2022-12-01 11:52:51 - r - INFO: - Episode: 375/400, Reward: -1.262, Step: 200 +2022-12-01 11:52:53 - r - INFO: - Episode: 376/400, Reward: -241.049, Step: 200 +2022-12-01 11:52:54 - r - INFO: - Episode: 377/400, Reward: -125.765, Step: 200 +2022-12-01 11:52:54 - r - INFO: - Episode: 378/400, Reward: -118.270, Step: 200 +2022-12-01 11:52:55 - r - INFO: - Episode: 379/400, Reward: -1.971, Step: 200 +2022-12-01 11:52:56 - r - INFO: - Episode: 380/400, Reward: -234.409, Step: 200 +2022-12-01 11:52:57 - r - INFO: - Episode: 381/400, Reward: -347.749, Step: 200 +2022-12-01 11:52:58 - r - INFO: - Episode: 382/400, Reward: -116.178, Step: 200 +2022-12-01 11:52:59 - r - INFO: - Episode: 383/400, Reward: -231.128, Step: 200 +2022-12-01 11:53:00 - r - INFO: - Episode: 384/400, Reward: -254.934, Step: 200 +2022-12-01 11:53:01 - r - INFO: - Episode: 385/400, Reward: -236.741, Step: 200 +2022-12-01 11:53:02 - r - INFO: - Episode: 386/400, Reward: -121.548, Step: 200 +2022-12-01 11:53:03 - r - INFO: - Episode: 387/400, Reward: -235.556, Step: 200 +2022-12-01 11:53:04 - r - INFO: - Episode: 388/400, Reward: -364.689, Step: 200 +2022-12-01 11:53:05 - r - INFO: - Episode: 389/400, Reward: -234.825, Step: 200 +2022-12-01 11:53:06 - r - INFO: - Episode: 390/400, Reward: -122.418, Step: 200 +2022-12-01 11:53:07 - r - INFO: - Episode: 391/400, Reward: -114.974, Step: 200 +2022-12-01 11:53:08 - r - INFO: - Episode: 392/400, Reward: -126.728, Step: 200 +2022-12-01 11:53:09 - r - INFO: - Episode: 393/400, Reward: -0.804, Step: 200 +2022-12-01 11:53:09 - r - INFO: - Episode: 394/400, Reward: -122.792, Step: 200 +2022-12-01 11:53:10 - r - INFO: - Episode: 395/400, Reward: -122.466, Step: 200 +2022-12-01 11:53:12 - r - INFO: - Episode: 396/400, Reward: -368.470, Step: 200 +2022-12-01 11:53:12 - r - INFO: - Episode: 397/400, Reward: -122.965, Step: 200 +2022-12-01 11:53:13 - r - INFO: - Episode: 398/400, Reward: -228.855, Step: 200 +2022-12-01 11:53:14 - r - INFO: - Episode: 399/400, Reward: -233.288, Step: 200 +2022-12-01 11:53:15 - r - INFO: - Episode: 400/400, Reward: -124.666, Step: 200 +2022-12-01 11:53:15 - r - INFO: - Finish training! diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/models/actor_checkpoint.pt b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..da3a5d2ee860077b7903a5a01180b99cd52bba1c --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e597766d4cbb7f45b4829f08fb296e02f880c06a5f1c7e8914ae3edfffd5b27 +size 270231 diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/results/learning_curve.png b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..639711d84263c662ab96c5cf47c8407ddf991c9f Binary files /dev/null and b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/results/learning_curve.png differ diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/results/res.csv b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..850114376768eb0712bfd7e776daa0b4617abd7f --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_20221201-114704/results/res.csv @@ -0,0 +1,401 @@ +episodes,rewards,steps +0,-1212.567790509667,200 +1,-1623.90050898633,200 +2,-1487.2540570510778,200 +3,-1453.6050208134404,200 +4,-1673.4762545138892,200 +5,-1414.2243811733215,200 +6,-1510.7052200180894,200 +7,-1501.575665761993,200 +8,-1512.5944531574316,200 +9,-1524.7744177918894,200 +10,-1354.692085392766,200 +11,-1554.3746703287636,200 +12,-1569.2612397054797,200 +13,-1497.9386660259095,200 +14,-1569.1011909824151,200 +15,-1253.0033379748509,200 +16,-1545.8060145381855,200 +17,-1265.2311705490215,200 +18,-305.82276280873407,200 +19,-1357.2647833743379,200 +20,-1296.9116241813176,200 +21,-1314.4621209531194,200 +22,-1249.9004458012248,200 +23,-1339.9108332691255,200 +24,-1366.5184464046633,200 +25,-1260.4917067598642,200 +26,-1194.3352827285053,200 +27,-1153.4043499440857,200 +28,-1082.3681931274152,200 +29,-1171.4685875728028,200 +30,-1200.7706507466314,200 +31,-1032.5769585027751,200 +32,-1118.503759148228,200 +33,-812.889261032345,200 +34,-1346.485781190277,200 +35,-1150.2140348230303,200 +36,-1270.614689092599,200 +37,-1016.8168272147682,200 +38,-663.5954619737352,200 +39,-138.01214689008125,200 +40,-934.2527412964954,200 +41,-260.4236560115692,200 +42,-326.42485912322536,200 +43,-133.96159257769474,200 +44,-264.0377370283736,200 +45,-1091.7560930763104,200 +46,-0.4068611028228804,200 +47,-131.84009403179755,200 +48,-129.61386019610077,200 +49,-128.35718529417167,200 +50,-407.41856177348257,200 +51,-402.5949012730138,200 +52,-0.7871212444572683,200 +53,-442.5590939848539,200 +54,-244.39037528009618,200 +55,-281.0373284085536,200 +56,-131.17759345587862,200 +57,-1.1744234280309984,200 +58,-260.8982646552037,200 +59,-389.7637508199495,200 +60,-263.9201033034666,200 +61,-1.242859023783189,200 +62,-1256.8407202519154,200 +63,-132.49857524719138,200 +64,-397.0109133415189,200 +65,-124.64071860239733,200 +66,-263.83847787328176,200 +67,-384.3725051863925,200 +68,-123.29930919574642,200 +69,-1502.8471749197145,200 +70,-354.07000984950906,200 +71,-1039.9138673842492,200 +72,-242.39933426350547,200 +73,-122.6644138115817,200 +74,-125.64699662000082,200 +75,-785.4715017999798,200 +76,-239.2134448316515,200 +77,-237.47579487864564,200 +78,-240.7615577353036,200 +79,-497.2460269290864,200 +80,-389.137335314073,200 +81,-0.27112735186688924,200 +82,-0.3086774869748233,200 +83,-243.78677419611907,200 +84,-0.2849053573155253,200 +85,-129.95249846428172,200 +86,-128.36770876164985,200 +87,-0.6982052614212529,200 +88,-120.31324877134472,200 +89,-127.30701950283134,200 +90,-126.28860194823548,200 +91,-129.5309528856722,200 +92,-1.5610275380910392,200 +93,-130.0354013298848,200 +94,-241.0117247029227,200 +95,-118.29451152645855,200 +96,-245.90024271631833,200 +97,-1.1813720795550908,200 +98,-126.19409898092339,200 +99,-127.8032220593728,200 +100,-117.81936279930302,200 +101,-119.51315376165093,200 +102,-221.54855948109116,200 +103,-244.9161107009129,200 +104,-126.86578869001644,200 +105,-358.9366505581329,200 +106,-125.26087242054723,200 +107,-127.86492374431437,200 +108,-357.7782206605127,200 +109,-124.89145418273924,200 +110,-383.05708561048414,200 +111,-3.6178136145142172,200 +112,-248.15786033784366,200 +113,-349.09786434723947,200 +114,-126.35251122697167,200 +115,-237.63522300410204,200 +116,-118.22756919715725,200 +117,-119.5107377539306,200 +118,-120.02511282093975,200 +119,-377.78066789153604,200 +120,-129.68571250046196,200 +121,-248.8885368760876,200 +122,-128.35179793591774,200 +123,-242.03786282740847,200 +124,-116.70300814145506,200 +125,-364.82645829556594,200 +126,-1.1622811754968918,200 +127,-126.90344264006548,200 +128,-249.2942262287832,200 +129,-1.4371265779706741,200 +130,-1.5182421074658987,200 +131,-1.2689996792696698,200 +132,-242.66029747420913,200 +133,-122.9252356038692,200 +134,-121.28216804628428,200 +135,-122.57424341607928,200 +136,-121.21456479996444,200 +137,-250.12296949439366,200 +138,-327.82923872836636,200 +139,-125.40699584708844,200 +140,-116.5002658634688,200 +141,-336.25842699755134,200 +142,-230.37864406642584,200 +143,-127.15311834771302,200 +144,-244.79627483953058,200 +145,-325.5923495673209,200 +146,-1.0204374551306619,200 +147,-123.72243711538715,200 +148,-123.82166233262521,200 +149,-128.7294156369154,200 +150,-119.31209044975706,200 +151,-269.22027954032365,200 +152,-364.6870939981257,200 +153,-224.13468420403024,200 +154,-125.22730098623303,200 +155,-127.92895036690234,200 +156,-129.89097457605777,200 +157,-126.72531700681934,200 +158,-119.7850032893606,200 +159,-346.2098208053867,200 +160,-368.1867319651628,200 +161,-246.30367497072865,200 +162,-1.275297347971024,200 +163,-117.67350478628695,200 +164,-336.28518788986725,200 +165,-1.1462767825080664,200 +166,-1.3387542122282372,200 +167,-118.35652425364356,200 +168,-242.2539368184398,200 +169,-123.85838031428564,200 +170,-117.10879553128855,200 +171,-123.73929437000524,200 +172,-115.14099346906151,200 +173,-119.84652598178756,200 +174,-120.6439962098665,200 +175,-116.67769122806183,200 +176,-117.13116926949454,200 +177,-122.80600166027432,200 +178,-127.8101911457023,200 +179,-239.76783023149943,200 +180,-221.0901580192667,200 +181,-121.22306496994257,200 +182,-227.6510141301169,200 +183,-322.2846238986938,200 +184,-123.35597643103705,200 +185,-119.88288970007227,200 +186,-121.37823501360883,200 +187,-0.8747798993382542,200 +188,-358.29401380245605,200 +189,-119.03620186476329,200 +190,-1.170334861365693,200 +191,-113.78093682410294,200 +192,-119.41905481185384,200 +193,-121.29170944293014,200 +194,-330.15508271137554,200 +195,-119.06855822794175,200 +196,-123.46342605614734,200 +197,-126.12295482657818,200 +198,-0.955593205782399,200 +199,-115.54698221933135,200 +200,-345.35321174334746,200 +201,-122.7889263059338,200 +202,-129.46245591059238,200 +203,-126.24627838077089,200 +204,-128.6314658171212,200 +205,-354.558816181193,200 +206,-1.4488718026188616,200 +207,-121.55541576988286,200 +208,-118.75282946826736,200 +209,-361.0832345263783,200 +210,-227.72158476257837,200 +211,-256.45444130035406,200 +212,-123.37300964883983,200 +213,-114.30419425385213,200 +214,-1.3116631626515545,200 +215,-239.79627193349634,200 +216,-126.54756458516205,200 +217,-126.63939754488048,200 +218,-125.77463478109455,200 +219,-1.9399912963507502,200 +220,-124.99821624523541,200 +221,-353.2355290431278,200 +222,-357.0561822988093,200 +223,-114.20404230922577,200 +224,-125.2228216148647,200 +225,-227.42607572499656,200 +226,-3.66384583645844,200 +227,-123.20482956417794,200 +228,-124.5137660280843,200 +229,-118.301096765625,200 +230,-243.09590482871403,200 +231,-126.69723215895019,200 +232,-1.8669578864483127,200 +233,-119.8310527422825,200 +234,-117.76012570850753,200 +235,-123.93871282364343,200 +236,-1.7168797790561883,200 +237,-128.80820035416008,200 +238,-131.19391957459277,200 +239,-352.9046065501171,200 +240,-233.71249387417777,200 +241,-262.38672404161645,200 +242,-3.7069104404948297,200 +243,-126.6161194862307,200 +244,-230.69221157544033,200 +245,-1.2520344535505652,200 +246,-118.52263815670227,200 +247,-125.4068792476078,200 +248,-117.50017500673853,200 +249,-393.0699409612341,200 +250,-118.36656247252151,200 +251,-128.272032138697,200 +252,-245.6135875368202,200 +253,-122.88025794109298,200 +254,-239.6816900060018,200 +255,-124.53719203788097,200 +256,-1.1110322441728246,200 +257,-117.20115177155951,200 +258,-128.8907428246222,200 +259,-126.26518534927192,200 +260,-231.9515597068849,200 +261,-119.31353102451676,200 +262,-125.47907416411866,200 +263,-3.2585418147910055,200 +264,-127.22228346638764,200 +265,-117.15003991870962,200 +266,-126.77484246849679,200 +267,-286.8372750075032,200 +268,-119.63108646198032,200 +269,-117.220849256608,200 +270,-127.00824262198856,200 +271,-125.15956992929272,200 +272,-119.6498891175291,200 +273,-126.03370110237906,200 +274,-117.44506818609379,200 +275,-353.1321992630821,200 +276,-123.565078688204,200 +277,-226.00677123406874,200 +278,-0.7194826217112447,200 +279,-119.67832887804394,200 +280,-124.16811583775699,200 +281,-126.55993876412806,200 +282,-300.110749858184,200 +283,-360.95109190345244,200 +284,-125.88939413633598,200 +285,-247.30239878930985,200 +286,-131.289320755518,200 +287,-240.20601948049932,200 +288,-287.7052948722776,200 +289,-227.7658190183464,200 +290,-123.64110999426921,200 +291,-1.0271528840463287,200 +292,-251.14233972806213,200 +293,-124.85806098908452,200 +294,-120.36459740707978,200 +295,-116.21907154645311,200 +296,-126.51066109761726,200 +297,-245.52260164448222,200 +298,-234.4504681050107,200 +299,-119.54139726250581,200 +300,-378.11171464427173,200 +301,-235.86244730196216,200 +302,-231.76624873360237,200 +303,-114.88863260226945,200 +304,-124.5356898260653,200 +305,-127.55482707894846,200 +306,-242.65874525848005,200 +307,-121.6784359890847,200 +308,-233.6810974953129,200 +309,-125.02006124532345,200 +310,-121.91305443133672,200 +311,-123.34111389595492,200 +312,-251.0910698407238,200 +313,-119.39482989042334,200 +314,-233.40984081275317,200 +315,-124.66862728081996,200 +316,-240.7649857554929,200 +317,-116.00214081668108,200 +318,-233.7073249326516,200 +319,-126.94237135466841,200 +320,-120.70553886869959,200 +321,-118.34220148948498,200 +322,-230.44955958950447,200 +323,-0.7470952202906396,200 +324,-123.27938468334166,200 +325,-227.79804661024497,200 +326,-244.46032058496195,200 +327,-118.01693537671495,200 +328,-0.8577964297833797,200 +329,-342.1907112204193,200 +330,-113.29167706692641,200 +331,-125.13843346097002,200 +332,-236.42717212029493,200 +333,-353.5506948429681,200 +334,-126.4512296912333,200 +335,-123.23258988570352,200 +336,-242.72594865912376,200 +337,-118.30122180805739,200 +338,-238.16429341163462,200 +339,-393.01794480450354,200 +340,-4.361374395916245,200 +341,-118.50650965284454,200 +342,-122.2488070906401,200 +343,-253.74250826722954,200 +344,-339.8398544082274,200 +345,-124.51878691095172,200 +346,-125.94094059928794,200 +347,-119.08482859899843,200 +348,-123.98865217909652,200 +349,-242.46016054535525,200 +350,-236.76009860870204,200 +351,-1.2333958948568258,200 +352,-118.37166997000097,200 +353,-355.52450936234607,200 +354,-357.92143547778466,200 +355,-238.33173281615038,200 +356,-130.65792019641003,200 +357,-126.8544313444508,200 +358,-116.50940457252848,200 +359,-123.74509048396844,200 +360,-368.14830160700626,200 +361,-126.38208229049773,200 +362,-236.523521747964,200 +363,-240.95223515670057,200 +364,-117.77302567733554,200 +365,-247.7438030784602,200 +366,-124.07828253073873,200 +367,-126.03525592405485,200 +368,-244.66335723451874,200 +369,-250.76922693289163,200 +370,-2.2836473101239783,200 +371,-124.35833030447026,200 +372,-124.56362051691382,200 +373,-352.07315122967464,200 +374,-1.261706373658124,200 +375,-241.04932240175708,200 +376,-125.76478713066989,200 +377,-118.26990865551241,200 +378,-1.971486854330923,200 +379,-234.40885710658657,200 +380,-347.7489301944504,200 +381,-116.17842157604629,200 +382,-231.12814154316845,200 +383,-254.93448140941447,200 +384,-236.74147909076814,200 +385,-121.5477601851976,200 +386,-235.55563255180687,200 +387,-364.6886726900155,200 +388,-234.8246762310216,200 +389,-122.41849376434521,200 +390,-114.97416181206125,200 +391,-126.7282667590554,200 +392,-0.8040550197181987,200 +393,-122.79156418968286,200 +394,-122.46642961981983,200 +395,-368.4695035569944,200 +396,-122.96452798166366,200 +397,-228.85482334086666,200 +398,-233.28839642937024,200 +399,-124.66577454411869,200 diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/config.yaml b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f9be7b945a858b86bbdc4e1e916cd9c34f92286 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/config.yaml @@ -0,0 +1,31 @@ +general_cfg: + algo_name: DDPG_HER + device: cpu + env_name: Pendulum-v1 + eval_eps: 10 + eval_per_episode: 5 + her_sample_num: 4 + load_checkpoint: false + load_path: tasks + max_steps: 200 + mode: train + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 0 + show_fig: false + test_eps: 20 + train_eps: 150 + update_every: 100 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0001 + batch_size: 128 + buffer_size: 8000 + critic_hidden_dim: 256 + critic_lr: 0.001 + gamma: 0.99 + tau: 0.001 diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/logs/log.txt b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9bb3468b5f6ca8683657f3759c54ba6982f113e --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/logs/log.txt @@ -0,0 +1,206 @@ +2023-04-14 15:02:20 - r - INFO: - Hyperparameters: +2023-04-14 15:02:20 - r - INFO: - ================================================================================ +2023-04-14 15:02:20 - r - INFO: - Name Value Type +2023-04-14 15:02:20 - r - INFO: - env_name Pendulum-v1 +2023-04-14 15:02:20 - r - INFO: - new_step_api 1 +2023-04-14 15:02:20 - r - INFO: - wrapper None +2023-04-14 15:02:20 - r - INFO: - render 0 +2023-04-14 15:02:20 - r - INFO: - render_mode human +2023-04-14 15:02:20 - r - INFO: - algo_name DDPG_HER +2023-04-14 15:02:20 - r - INFO: - mode train +2023-04-14 15:02:20 - r - INFO: - mp_backend mp +2023-04-14 15:02:20 - r - INFO: - seed 0 +2023-04-14 15:02:20 - r - INFO: - device cpu +2023-04-14 15:02:20 - r - INFO: - her_sample_num 4 +2023-04-14 15:02:20 - r - INFO: - update_every 100 +2023-04-14 15:02:20 - r - INFO: - train_eps 150 +2023-04-14 15:02:20 - r - INFO: - test_eps 20 +2023-04-14 15:02:20 - r - INFO: - eval_eps 10 +2023-04-14 15:02:20 - r - INFO: - eval_per_episode 5 +2023-04-14 15:02:20 - r - INFO: - max_steps 200 +2023-04-14 15:02:20 - r - INFO: - load_checkpoint 0 +2023-04-14 15:02:20 - r - INFO: - load_path tasks +2023-04-14 15:02:20 - r - INFO: - show_fig 0 +2023-04-14 15:02:20 - r - INFO: - save_fig 1 +2023-04-14 15:02:20 - r - INFO: - gamma 0.99 +2023-04-14 15:02:20 - r - INFO: - critic_lr 0.001 +2023-04-14 15:02:20 - r - INFO: - actor_lr 0.0001 +2023-04-14 15:02:20 - r - INFO: - buffer_size 8000 +2023-04-14 15:02:20 - r - INFO: - batch_size 128 +2023-04-14 15:02:20 - r - INFO: - tau 0.001 +2023-04-14 15:02:20 - r - INFO: - critic_hidden_dim 256 +2023-04-14 15:02:20 - r - INFO: - actor_hidden_dim 256 +2023-04-14 15:02:20 - r - INFO: - task_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Train_Pendulum-v1_DDPG_HER_20230414-150220 +2023-04-14 15:02:20 - r - INFO: - res_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Train_Pendulum-v1_DDPG_HER_20230414-150220/results +2023-04-14 15:02:20 - r - INFO: - log_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Train_Pendulum-v1_DDPG_HER_20230414-150220/logs +2023-04-14 15:02:20 - r - INFO: - traj_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Train_Pendulum-v1_DDPG_HER_20230414-150220/traj +2023-04-14 15:02:20 - r - INFO: - video_dir D:\MyDoc\joyrl\joyrl-offline/tasks/Train_Pendulum-v1_DDPG_HER_20230414-150220/videos +2023-04-14 15:02:20 - r - INFO: - ================================================================================ +2023-04-14 15:02:20 - r - INFO: - action_bound: 2.0 +2023-04-14 15:02:20 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-14 15:02:20 - r - INFO: - Start training! +2023-04-14 15:02:20 - r - INFO: - Env: Pendulum-v1, Algorithm: DDPG_HER, Device: cpu +2023-04-14 15:02:21 - r - INFO: - Episode: 1/150, Reward: -1430.631, Step: 200 +2023-04-14 15:02:22 - r - INFO: - Episode: 2/150, Reward: -1190.537, Step: 200 +2023-04-14 15:02:24 - r - INFO: - Episode: 3/150, Reward: -1079.347, Step: 200 +2023-04-14 15:02:26 - r - INFO: - Episode: 4/150, Reward: -1579.293, Step: 200 +2023-04-14 15:02:28 - r - INFO: - Episode: 5/150, Reward: -1523.855, Step: 200 +2023-04-14 15:02:29 - r - INFO: - Current episode 5 has the best eval reward: -1410.344 +2023-04-14 15:02:31 - r - INFO: - Episode: 6/150, Reward: -1764.677, Step: 200 +2023-04-14 15:02:33 - r - INFO: - Episode: 7/150, Reward: -1739.242, Step: 200 +2023-04-14 15:02:35 - r - INFO: - Episode: 8/150, Reward: -1730.528, Step: 200 +2023-04-14 15:02:37 - r - INFO: - Episode: 9/150, Reward: -1780.244, Step: 200 +2023-04-14 15:02:39 - r - INFO: - Episode: 10/150, Reward: -1712.587, Step: 200 +2023-04-14 15:02:41 - r - INFO: - Episode: 11/150, Reward: -1750.423, Step: 200 +2023-04-14 15:02:43 - r - INFO: - Episode: 12/150, Reward: -1589.055, Step: 200 +2023-04-14 15:02:45 - r - INFO: - Episode: 13/150, Reward: -1601.020, Step: 200 +2023-04-14 15:02:47 - r - INFO: - Episode: 14/150, Reward: -1538.405, Step: 200 +2023-04-14 15:02:49 - r - INFO: - Episode: 15/150, Reward: -1437.579, Step: 200 +2023-04-14 15:02:52 - r - INFO: - Episode: 16/150, Reward: -1479.413, Step: 200 +2023-04-14 15:02:54 - r - INFO: - Episode: 17/150, Reward: -1260.055, Step: 200 +2023-04-14 15:02:56 - r - INFO: - Episode: 18/150, Reward: -1442.884, Step: 200 +2023-04-14 15:02:58 - r - INFO: - Episode: 19/150, Reward: -1143.685, Step: 200 +2023-04-14 15:03:00 - r - INFO: - Episode: 20/150, Reward: -1305.908, Step: 200 +2023-04-14 15:03:00 - r - INFO: - Current episode 20 has the best eval reward: -1222.109 +2023-04-14 15:03:02 - r - INFO: - Episode: 21/150, Reward: -1303.816, Step: 200 +2023-04-14 15:03:04 - r - INFO: - Episode: 22/150, Reward: -1539.392, Step: 200 +2023-04-14 15:03:06 - r - INFO: - Episode: 23/150, Reward: -1265.643, Step: 200 +2023-04-14 15:03:08 - r - INFO: - Episode: 24/150, Reward: -1190.020, Step: 200 +2023-04-14 15:03:10 - r - INFO: - Episode: 25/150, Reward: -1509.776, Step: 200 +2023-04-14 15:03:11 - r - INFO: - Current episode 25 has the best eval reward: -1219.529 +2023-04-14 15:03:13 - r - INFO: - Episode: 26/150, Reward: -1139.533, Step: 200 +2023-04-14 15:03:15 - r - INFO: - Episode: 27/150, Reward: -1356.016, Step: 200 +2023-04-14 15:03:17 - r - INFO: - Episode: 28/150, Reward: -1212.430, Step: 200 +2023-04-14 15:03:19 - r - INFO: - Episode: 29/150, Reward: -1173.039, Step: 200 +2023-04-14 15:03:21 - r - INFO: - Episode: 30/150, Reward: -1134.589, Step: 200 +2023-04-14 15:03:22 - r - INFO: - Current episode 30 has the best eval reward: -1103.859 +2023-04-14 15:03:24 - r - INFO: - Episode: 31/150, Reward: -1539.600, Step: 200 +2023-04-14 15:03:26 - r - INFO: - Episode: 32/150, Reward: -906.160, Step: 200 +2023-04-14 15:03:29 - r - INFO: - Episode: 33/150, Reward: -1051.940, Step: 200 +2023-04-14 15:03:31 - r - INFO: - Episode: 34/150, Reward: -1015.502, Step: 200 +2023-04-14 15:03:33 - r - INFO: - Episode: 35/150, Reward: -1291.428, Step: 200 +2023-04-14 15:03:36 - r - INFO: - Episode: 36/150, Reward: -968.884, Step: 200 +2023-04-14 15:03:38 - r - INFO: - Episode: 37/150, Reward: -1275.947, Step: 200 +2023-04-14 15:03:41 - r - INFO: - Episode: 38/150, Reward: -942.761, Step: 200 +2023-04-14 15:03:43 - r - INFO: - Episode: 39/150, Reward: -825.685, Step: 200 +2023-04-14 15:03:45 - r - INFO: - Episode: 40/150, Reward: -944.424, Step: 200 +2023-04-14 15:03:46 - r - INFO: - Current episode 40 has the best eval reward: -878.044 +2023-04-14 15:03:48 - r - INFO: - Episode: 41/150, Reward: -882.493, Step: 200 +2023-04-14 15:03:50 - r - INFO: - Episode: 42/150, Reward: -927.014, Step: 200 +2023-04-14 15:03:52 - r - INFO: - Episode: 43/150, Reward: -812.261, Step: 200 +2023-04-14 15:03:55 - r - INFO: - Episode: 44/150, Reward: -815.314, Step: 200 +2023-04-14 15:03:57 - r - INFO: - Episode: 45/150, Reward: -661.461, Step: 200 +2023-04-14 15:03:58 - r - INFO: - Current episode 45 has the best eval reward: -548.563 +2023-04-14 15:04:00 - r - INFO: - Episode: 46/150, Reward: -845.423, Step: 200 +2023-04-14 15:04:02 - r - INFO: - Episode: 47/150, Reward: -792.809, Step: 200 +2023-04-14 15:04:05 - r - INFO: - Episode: 48/150, Reward: -990.300, Step: 200 +2023-04-14 15:04:07 - r - INFO: - Episode: 49/150, Reward: -383.060, Step: 200 +2023-04-14 15:04:10 - r - INFO: - Episode: 50/150, Reward: -538.956, Step: 200 +2023-04-14 15:04:11 - r - INFO: - Current episode 50 has the best eval reward: -383.151 +2023-04-14 15:04:13 - r - INFO: - Episode: 51/150, Reward: -130.052, Step: 200 +2023-04-14 15:04:15 - r - INFO: - Episode: 52/150, Reward: -377.034, Step: 200 +2023-04-14 15:04:17 - r - INFO: - Episode: 53/150, Reward: -132.334, Step: 200 +2023-04-14 15:04:19 - r - INFO: - Episode: 54/150, Reward: -354.477, Step: 200 +2023-04-14 15:04:21 - r - INFO: - Episode: 55/150, Reward: -553.161, Step: 200 +2023-04-14 15:04:22 - r - INFO: - Current episode 55 has the best eval reward: -299.486 +2023-04-14 15:04:24 - r - INFO: - Episode: 56/150, Reward: -132.422, Step: 200 +2023-04-14 15:04:26 - r - INFO: - Episode: 57/150, Reward: -386.039, Step: 200 +2023-04-14 15:04:28 - r - INFO: - Episode: 58/150, Reward: -132.089, Step: 200 +2023-04-14 15:04:30 - r - INFO: - Episode: 59/150, Reward: -131.928, Step: 200 +2023-04-14 15:04:32 - r - INFO: - Episode: 60/150, Reward: -402.214, Step: 200 +2023-04-14 15:04:35 - r - INFO: - Episode: 61/150, Reward: -259.670, Step: 200 +2023-04-14 15:04:37 - r - INFO: - Episode: 62/150, Reward: -161.742, Step: 200 +2023-04-14 15:04:39 - r - INFO: - Episode: 63/150, Reward: -136.568, Step: 200 +2023-04-14 15:04:41 - r - INFO: - Episode: 64/150, Reward: -254.675, Step: 200 +2023-04-14 15:04:43 - r - INFO: - Episode: 65/150, Reward: -1490.135, Step: 200 +2023-04-14 15:04:46 - r - INFO: - Episode: 66/150, Reward: -255.318, Step: 200 +2023-04-14 15:04:47 - r - INFO: - Episode: 67/150, Reward: -1386.437, Step: 200 +2023-04-14 15:04:50 - r - INFO: - Episode: 68/150, Reward: -358.886, Step: 200 +2023-04-14 15:04:52 - r - INFO: - Episode: 69/150, Reward: -134.174, Step: 200 +2023-04-14 15:04:54 - r - INFO: - Episode: 70/150, Reward: -121.258, Step: 200 +2023-04-14 15:04:54 - r - INFO: - Current episode 70 has the best eval reward: -220.980 +2023-04-14 15:04:56 - r - INFO: - Episode: 71/150, Reward: -244.039, Step: 200 +2023-04-14 15:04:58 - r - INFO: - Episode: 72/150, Reward: -2.718, Step: 200 +2023-04-14 15:05:00 - r - INFO: - Episode: 73/150, Reward: -127.649, Step: 200 +2023-04-14 15:05:02 - r - INFO: - Episode: 74/150, Reward: -132.686, Step: 200 +2023-04-14 15:05:04 - r - INFO: - Episode: 75/150, Reward: -235.846, Step: 200 +2023-04-14 15:05:05 - r - INFO: - Current episode 75 has the best eval reward: -201.119 +2023-04-14 15:05:07 - r - INFO: - Episode: 76/150, Reward: -396.815, Step: 200 +2023-04-14 15:05:09 - r - INFO: - Episode: 77/150, Reward: -122.913, Step: 200 +2023-04-14 15:05:11 - r - INFO: - Episode: 78/150, Reward: -117.540, Step: 200 +2023-04-14 15:05:13 - r - INFO: - Episode: 79/150, Reward: -127.221, Step: 200 +2023-04-14 15:05:15 - r - INFO: - Episode: 80/150, Reward: -126.340, Step: 200 +2023-04-14 15:05:16 - r - INFO: - Current episode 80 has the best eval reward: -125.201 +2023-04-14 15:05:18 - r - INFO: - Episode: 81/150, Reward: -127.639, Step: 200 +2023-04-14 15:05:20 - r - INFO: - Episode: 82/150, Reward: -128.077, Step: 200 +2023-04-14 15:05:22 - r - INFO: - Episode: 83/150, Reward: -116.016, Step: 200 +2023-04-14 15:05:24 - r - INFO: - Episode: 84/150, Reward: -124.603, Step: 200 +2023-04-14 15:05:26 - r - INFO: - Episode: 85/150, Reward: -126.350, Step: 200 +2023-04-14 15:05:29 - r - INFO: - Episode: 86/150, Reward: -117.398, Step: 200 +2023-04-14 15:05:31 - r - INFO: - Episode: 87/150, Reward: -125.239, Step: 200 +2023-04-14 15:05:33 - r - INFO: - Episode: 88/150, Reward: -1.460, Step: 200 +2023-04-14 15:05:35 - r - INFO: - Episode: 89/150, Reward: -367.643, Step: 200 +2023-04-14 15:05:37 - r - INFO: - Episode: 90/150, Reward: -1.008, Step: 200 +2023-04-14 15:05:39 - r - INFO: - Episode: 91/150, Reward: -126.855, Step: 200 +2023-04-14 15:05:41 - r - INFO: - Episode: 92/150, Reward: -374.736, Step: 200 +2023-04-14 15:05:43 - r - INFO: - Episode: 93/150, Reward: -251.304, Step: 200 +2023-04-14 15:05:45 - r - INFO: - Episode: 94/150, Reward: -238.669, Step: 200 +2023-04-14 15:05:47 - r - INFO: - Episode: 95/150, Reward: -119.575, Step: 200 +2023-04-14 15:05:50 - r - INFO: - Episode: 96/150, Reward: -370.054, Step: 200 +2023-04-14 15:05:52 - r - INFO: - Episode: 97/150, Reward: -392.496, Step: 200 +2023-04-14 15:05:54 - r - INFO: - Episode: 98/150, Reward: -240.950, Step: 200 +2023-04-14 15:05:56 - r - INFO: - Episode: 99/150, Reward: -235.973, Step: 200 +2023-04-14 15:05:58 - r - INFO: - Episode: 100/150, Reward: -126.822, Step: 200 +2023-04-14 15:05:58 - r - INFO: - Current episode 100 has the best eval reward: -111.559 +2023-04-14 15:06:00 - r - INFO: - Episode: 101/150, Reward: -1.238, Step: 200 +2023-04-14 15:06:02 - r - INFO: - Episode: 102/150, Reward: -129.029, Step: 200 +2023-04-14 15:06:04 - r - INFO: - Episode: 103/150, Reward: -126.249, Step: 200 +2023-04-14 15:06:06 - r - INFO: - Episode: 104/150, Reward: -121.460, Step: 200 +2023-04-14 15:06:08 - r - INFO: - Episode: 105/150, Reward: -275.772, Step: 200 +2023-04-14 15:06:11 - r - INFO: - Episode: 106/150, Reward: -127.109, Step: 200 +2023-04-14 15:06:13 - r - INFO: - Episode: 107/150, Reward: -3.014, Step: 200 +2023-04-14 15:06:15 - r - INFO: - Episode: 108/150, Reward: -0.725, Step: 200 +2023-04-14 15:06:17 - r - INFO: - Episode: 109/150, Reward: -260.152, Step: 200 +2023-04-14 15:06:19 - r - INFO: - Episode: 110/150, Reward: -122.403, Step: 200 +2023-04-14 15:06:21 - r - INFO: - Episode: 111/150, Reward: -1.335, Step: 200 +2023-04-14 15:06:23 - r - INFO: - Episode: 112/150, Reward: -368.960, Step: 200 +2023-04-14 15:06:25 - r - INFO: - Episode: 113/150, Reward: -239.405, Step: 200 +2023-04-14 15:06:27 - r - INFO: - Episode: 114/150, Reward: -241.047, Step: 200 +2023-04-14 15:06:29 - r - INFO: - Episode: 115/150, Reward: -124.533, Step: 200 +2023-04-14 15:06:32 - r - INFO: - Episode: 116/150, Reward: -123.599, Step: 200 +2023-04-14 15:06:34 - r - INFO: - Episode: 117/150, Reward: -125.264, Step: 200 +2023-04-14 15:06:37 - r - INFO: - Episode: 118/150, Reward: -123.005, Step: 200 +2023-04-14 15:06:39 - r - INFO: - Episode: 119/150, Reward: -128.860, Step: 200 +2023-04-14 15:06:41 - r - INFO: - Episode: 120/150, Reward: -126.551, Step: 200 +2023-04-14 15:06:43 - r - INFO: - Episode: 121/150, Reward: -117.272, Step: 200 +2023-04-14 15:06:45 - r - INFO: - Episode: 122/150, Reward: -122.532, Step: 200 +2023-04-14 15:06:47 - r - INFO: - Episode: 123/150, Reward: -333.176, Step: 200 +2023-04-14 15:06:49 - r - INFO: - Episode: 124/150, Reward: -128.796, Step: 200 +2023-04-14 15:06:51 - r - INFO: - Episode: 125/150, Reward: -127.390, Step: 200 +2023-04-14 15:06:54 - r - INFO: - Episode: 126/150, Reward: -124.982, Step: 200 +2023-04-14 15:06:56 - r - INFO: - Episode: 127/150, Reward: -376.673, Step: 200 +2023-04-14 15:06:58 - r - INFO: - Episode: 128/150, Reward: -374.774, Step: 200 +2023-04-14 15:07:00 - r - INFO: - Episode: 129/150, Reward: -236.785, Step: 200 +2023-04-14 15:07:02 - r - INFO: - Episode: 130/150, Reward: -122.334, Step: 200 +2023-04-14 15:07:04 - r - INFO: - Episode: 131/150, Reward: -254.140, Step: 200 +2023-04-14 15:07:06 - r - INFO: - Episode: 132/150, Reward: -247.884, Step: 200 +2023-04-14 15:07:08 - r - INFO: - Episode: 133/150, Reward: -129.876, Step: 200 +2023-04-14 15:07:10 - r - INFO: - Episode: 134/150, Reward: -251.932, Step: 200 +2023-04-14 15:07:12 - r - INFO: - Episode: 135/150, Reward: -124.185, Step: 200 +2023-04-14 15:07:15 - r - INFO: - Episode: 136/150, Reward: -244.707, Step: 200 +2023-04-14 15:07:17 - r - INFO: - Episode: 137/150, Reward: -123.827, Step: 200 +2023-04-14 15:07:19 - r - INFO: - Episode: 138/150, Reward: -242.895, Step: 200 +2023-04-14 15:07:21 - r - INFO: - Episode: 139/150, Reward: -125.282, Step: 200 +2023-04-14 15:07:23 - r - INFO: - Episode: 140/150, Reward: -353.050, Step: 200 +2023-04-14 15:07:26 - r - INFO: - Episode: 141/150, Reward: -1.573, Step: 200 +2023-04-14 15:07:27 - r - INFO: - Episode: 142/150, Reward: -364.479, Step: 200 +2023-04-14 15:07:29 - r - INFO: - Episode: 143/150, Reward: -0.872, Step: 200 +2023-04-14 15:07:31 - r - INFO: - Episode: 144/150, Reward: -120.558, Step: 200 +2023-04-14 15:07:33 - r - INFO: - Episode: 145/150, Reward: -246.312, Step: 200 +2023-04-14 15:07:34 - r - INFO: - Current episode 145 has the best eval reward: -99.201 +2023-04-14 15:07:36 - r - INFO: - Episode: 146/150, Reward: -0.580, Step: 200 +2023-04-14 15:07:38 - r - INFO: - Episode: 147/150, Reward: -245.957, Step: 200 +2023-04-14 15:07:40 - r - INFO: - Episode: 148/150, Reward: -120.189, Step: 200 +2023-04-14 15:07:42 - r - INFO: - Episode: 149/150, Reward: -242.257, Step: 200 +2023-04-14 15:07:44 - r - INFO: - Episode: 150/150, Reward: -1.290, Step: 200 +2023-04-14 15:07:45 - r - INFO: - Finish training! diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/models/actor_checkpoint.pt b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/models/actor_checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3db3aaf36ba0cfca1995aa2584471cc3052b8e6 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/models/actor_checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c9bbbc02145eebc39c0f763f2935f7121a73c055a0cdd8ea4e8f4cc49de79b +size 270167 diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/results/learning_curve.png b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..c59db5acc8f43ca739d1fe9ce2d35e8263d5b68c Binary files /dev/null and b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/results/learning_curve.png differ diff --git a/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/results/res.csv b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..70c598d9209375768f8ec9bb5f0341b06ca4ce33 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_DDPG_HER_20230414-150220/results/res.csv @@ -0,0 +1,151 @@ +episodes,rewards,steps +0,-1430.6308492158244,200 +1,-1190.5367970500984,200 +2,-1079.3471795745675,200 +3,-1579.2933332899154,200 +4,-1523.8545135635886,200 +5,-1764.6770197817314,200 +6,-1739.241508091033,200 +7,-1730.5276652666748,200 +8,-1780.244283391058,200 +9,-1712.587380985927,200 +10,-1750.4225143588756,200 +11,-1589.0554765682489,200 +12,-1601.020434474761,200 +13,-1538.4049507680515,200 +14,-1437.5786142193342,200 +15,-1479.4130402789551,200 +16,-1260.0547562430675,200 +17,-1442.883652483688,200 +18,-1143.6853913143643,200 +19,-1305.908168694182,200 +20,-1303.81606982542,200 +21,-1539.3915323539622,200 +22,-1265.642759299553,200 +23,-1190.0198219007082,200 +24,-1509.7764248612998,200 +25,-1139.5331950559082,200 +26,-1356.0163888626528,200 +27,-1212.4303747303516,200 +28,-1173.0390127304488,200 +29,-1134.5893609697564,200 +30,-1539.6001153987495,200 +31,-906.1600599095619,200 +32,-1051.9396183893075,200 +33,-1015.5015598159622,200 +34,-1291.4276117386553,200 +35,-968.8843465646914,200 +36,-1275.9473868828154,200 +37,-942.7614870431684,200 +38,-825.685468725092,200 +39,-944.424091435172,200 +40,-882.493105112489,200 +41,-927.014329528836,200 +42,-812.2613670272619,200 +43,-815.3141232861163,200 +44,-661.4610558213548,200 +45,-845.422675295838,200 +46,-792.8088089116783,200 +47,-990.3000092618887,200 +48,-383.06003537718817,200 +49,-538.95590403877,200 +50,-130.05156550915444,200 +51,-377.0335407075482,200 +52,-132.33429855691475,200 +53,-354.4771547749317,200 +54,-553.1607546422097,200 +55,-132.42213794782236,200 +56,-386.03945856878397,200 +57,-132.08868225309894,200 +58,-131.92803623243867,200 +59,-402.2138929822295,200 +60,-259.66984455829385,200 +61,-161.74193652325351,200 +62,-136.5680714930932,200 +63,-254.67505951708083,200 +64,-1490.1352203495762,200 +65,-255.31764333435822,200 +66,-1386.4370407611004,200 +67,-358.88571014307405,200 +68,-134.17424231435749,200 +69,-121.25794383779807,200 +70,-244.0391358203264,200 +71,-2.718432450469906,200 +72,-127.64864314711062,200 +73,-132.68604195770834,200 +74,-235.8463720517245,200 +75,-396.81501649288043,200 +76,-122.91344508184179,200 +77,-117.54019231950215,200 +78,-127.22060394491794,200 +79,-126.34008688161772,200 +80,-127.63882608404063,200 +81,-128.07712057784045,200 +82,-116.01607161411498,200 +83,-124.6025746919546,200 +84,-126.34953015339337,200 +85,-117.39820228542744,200 +86,-125.23907011924322,200 +87,-1.4603195542576626,200 +88,-367.6433077930441,200 +89,-1.0084386015369422,200 +90,-126.85495552988425,200 +91,-374.7357847208059,200 +92,-251.3038784592841,200 +93,-238.66898646973328,200 +94,-119.57474394122339,200 +95,-370.05436998886864,200 +96,-392.4958453235724,200 +97,-240.94993697137264,200 +98,-235.97258808527206,200 +99,-126.82248929227423,200 +100,-1.2377784113899366,200 +101,-129.0292330503135,200 +102,-126.24876826768202,200 +103,-121.45981534849007,200 +104,-275.7716252159668,200 +105,-127.10858742124495,200 +106,-3.0138686710924225,200 +107,-0.7250940254172238,200 +108,-260.15211505075996,200 +109,-122.402534601475,200 +110,-1.3347805314197814,200 +111,-368.95958219453473,200 +112,-239.40475621478535,200 +113,-241.04667354926255,200 +114,-124.53300054647515,200 +115,-123.59910497666924,200 +116,-125.26449800007374,200 +117,-123.00454770860556,200 +118,-128.86029430629196,200 +119,-126.55100731609157,200 +120,-117.27242967986406,200 +121,-122.53166242635218,200 +122,-333.1757961105794,200 +123,-128.7963799137116,200 +124,-127.39021687415912,200 +125,-124.98241748599165,200 +126,-376.67331746137637,200 +127,-374.7743484635078,200 +128,-236.7849383212303,200 +129,-122.3343081869815,200 +130,-254.1399748208669,200 +131,-247.8838036125661,200 +132,-129.8760483364584,200 +133,-251.93176403276203,200 +134,-124.18468835786423,200 +135,-244.70657952729815,200 +136,-123.82660065271533,200 +137,-242.8954240363246,200 +138,-125.28191955366951,200 +139,-353.0501639674728,200 +140,-1.5732736223053416,200 +141,-364.4789662879637,200 +142,-0.8717135195373841,200 +143,-120.55828747315104,200 +144,-246.3122657224132,200 +145,-0.5797920615695951,200 +146,-245.95705803544502,200 +147,-120.18929825568993,200 +148,-242.25702503052074,200 +149,-1.2900164167859671,200 diff --git a/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/config.yaml b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14d68cf0ac8a6c844228c4e97662f597fca38772 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/config.yaml @@ -0,0 +1,32 @@ +general_cfg: + algo_name: SAC + device: cuda + env_name: Pendulum-v1 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 10 + show_fig: false + test_eps: 10 + train_eps: 400 + wrapper: null +algo_cfg: + alpha: 0.2 + automatic_entropy_tuning: false + batch_size: 64 + buffer_size: 1000000 + continous: false + gamma: 0.99 + hidden_dim: 64 + lr: 0.001 + n_epochs: 1 + policy_type: Gaussian + start_steps: 10000 + target_update_fre: 1 + tau: 0.005 diff --git a/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/logs/log.txt b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..19b0a54dd33177193a9a4077028c38f0f0744261 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/logs/log.txt @@ -0,0 +1,448 @@ +2023-03-05 11:42:17 - r - INFO: - Hyperparameters: +2023-03-05 11:42:17 - r - INFO: - ================================================================================ +2023-03-05 11:42:17 - r - INFO: - Name Value Type +2023-03-05 11:42:17 - r - INFO: - env_name Pendulum-v1 +2023-03-05 11:42:17 - r - INFO: - new_step_api 1 +2023-03-05 11:42:17 - r - INFO: - wrapper None +2023-03-05 11:42:17 - r - INFO: - render 0 +2023-03-05 11:42:17 - r - INFO: - algo_name SAC +2023-03-05 11:42:17 - r - INFO: - mode train +2023-03-05 11:42:17 - r - INFO: - seed 10 +2023-03-05 11:42:17 - r - INFO: - device cuda +2023-03-05 11:42:17 - r - INFO: - train_eps 400 +2023-03-05 11:42:17 - r - INFO: - test_eps 10 +2023-03-05 11:42:17 - r - INFO: - eval_eps 10 +2023-03-05 11:42:17 - r - INFO: - eval_per_episode 5 +2023-03-05 11:42:17 - r - INFO: - max_steps 200 +2023-03-05 11:42:17 - r - INFO: - load_checkpoint 0 +2023-03-05 11:42:17 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-03-05 11:42:17 - r - INFO: - show_fig 0 +2023-03-05 11:42:17 - r - INFO: - save_fig 1 +2023-03-05 11:42:17 - r - INFO: - policy_type Gaussian +2023-03-05 11:42:17 - r - INFO: - lr 0.001 +2023-03-05 11:42:17 - r - INFO: - gamma 0.99 +2023-03-05 11:42:17 - r - INFO: - tau 0.005 +2023-03-05 11:42:17 - r - INFO: - alpha 0.2 +2023-03-05 11:42:17 - r - INFO: - automatic_entropy_tuning 0 +2023-03-05 11:42:17 - r - INFO: - batch_size 64 +2023-03-05 11:42:17 - r - INFO: - hidden_dim 64 +2023-03-05 11:42:17 - r - INFO: - n_epochs 1 +2023-03-05 11:42:17 - r - INFO: - start_steps 10000 +2023-03-05 11:42:17 - r - INFO: - target_update_fre 1 +2023-03-05 11:42:17 - r - INFO: - buffer_size 1000000 +2023-03-05 11:42:17 - r - INFO: - continous 0 +2023-03-05 11:42:17 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Train_Pendulum-v1_SAC_20230305-114217 +2023-03-05 11:42:17 - r - INFO: - model_dir /home/dingli/joyrl_offline/tasks/Train_Pendulum-v1_SAC_20230305-114217/models +2023-03-05 11:42:17 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Train_Pendulum-v1_SAC_20230305-114217/results +2023-03-05 11:42:17 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Train_Pendulum-v1_SAC_20230305-114217/logs +2023-03-05 11:42:17 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Train_Pendulum-v1_SAC_20230305-114217/traj +2023-03-05 11:42:17 - r - INFO: - ================================================================================ +2023-03-05 11:42:17 - r - INFO: - action_bound: 2.0 +2023-03-05 11:42:17 - r - INFO: - n_states: 3, n_actions: 1 +2023-03-05 11:42:19 - r - INFO: - Start training! +2023-03-05 11:42:19 - r - INFO: - Env: Pendulum-v1, Algorithm: SAC, Device: cuda +2023-03-05 11:42:21 - r - INFO: - Episode: 1/400, Reward: -1740.831, Step: 200 +2023-03-05 11:42:23 - r - INFO: - Episode: 2/400, Reward: -961.069, Step: 200 +2023-03-05 11:42:26 - r - INFO: - Episode: 3/400, Reward: -967.685, Step: 200 +2023-03-05 11:42:28 - r - INFO: - Episode: 4/400, Reward: -1261.628, Step: 200 +2023-03-05 11:42:31 - r - INFO: - Episode: 5/400, Reward: -966.460, Step: 200 +2023-03-05 11:42:32 - r - INFO: - Current episode 5 has the best eval reward: -1766.117 +2023-03-05 11:42:34 - r - INFO: - Episode: 6/400, Reward: -886.180, Step: 200 +2023-03-05 11:42:37 - r - INFO: - Episode: 7/400, Reward: -1621.125, Step: 200 +2023-03-05 11:42:40 - r - INFO: - Episode: 8/400, Reward: -886.914, Step: 200 +2023-03-05 11:42:42 - r - INFO: - Episode: 9/400, Reward: -872.599, Step: 200 +2023-03-05 11:42:45 - r - INFO: - Episode: 10/400, Reward: -1263.833, Step: 200 +2023-03-05 11:42:46 - r - INFO: - Current episode 10 has the best eval reward: -875.561 +2023-03-05 11:42:49 - r - INFO: - Episode: 11/400, Reward: -994.983, Step: 200 +2023-03-05 11:42:51 - r - INFO: - Episode: 12/400, Reward: -768.399, Step: 200 +2023-03-05 11:42:54 - r - INFO: - Episode: 13/400, Reward: -1660.733, Step: 200 +2023-03-05 11:42:56 - r - INFO: - Episode: 14/400, Reward: -1576.356, Step: 200 +2023-03-05 11:42:59 - r - INFO: - Episode: 15/400, Reward: -968.902, Step: 200 +2023-03-05 11:43:00 - r - INFO: - Current episode 15 has the best eval reward: -106.637 +2023-03-05 11:43:03 - r - INFO: - Episode: 16/400, Reward: -1071.378, Step: 200 +2023-03-05 11:43:05 - r - INFO: - Episode: 17/400, Reward: -1156.690, Step: 200 +2023-03-05 11:43:08 - r - INFO: - Episode: 18/400, Reward: -1514.475, Step: 200 +2023-03-05 11:43:10 - r - INFO: - Episode: 19/400, Reward: -1093.737, Step: 200 +2023-03-05 11:43:13 - r - INFO: - Episode: 20/400, Reward: -968.878, Step: 200 +2023-03-05 11:43:17 - r - INFO: - Episode: 21/400, Reward: -1203.862, Step: 200 +2023-03-05 11:43:19 - r - INFO: - Episode: 22/400, Reward: -1405.155, Step: 200 +2023-03-05 11:43:22 - r - INFO: - Episode: 23/400, Reward: -917.430, Step: 200 +2023-03-05 11:43:24 - r - INFO: - Episode: 24/400, Reward: -1087.190, Step: 200 +2023-03-05 11:43:27 - r - INFO: - Episode: 25/400, Reward: -1116.189, Step: 200 +2023-03-05 11:43:31 - r - INFO: - Episode: 26/400, Reward: -1823.221, Step: 200 +2023-03-05 11:43:33 - r - INFO: - Episode: 27/400, Reward: -1379.747, Step: 200 +2023-03-05 11:43:36 - r - INFO: - Episode: 28/400, Reward: -905.889, Step: 200 +2023-03-05 11:43:38 - r - INFO: - Episode: 29/400, Reward: -1101.764, Step: 200 +2023-03-05 11:43:41 - r - INFO: - Episode: 30/400, Reward: -1175.637, Step: 200 +2023-03-05 11:43:45 - r - INFO: - Episode: 31/400, Reward: -1825.436, Step: 200 +2023-03-05 11:43:47 - r - INFO: - Episode: 32/400, Reward: -1655.561, Step: 200 +2023-03-05 11:43:50 - r - INFO: - Episode: 33/400, Reward: -1180.226, Step: 200 +2023-03-05 11:43:52 - r - INFO: - Episode: 34/400, Reward: -1063.964, Step: 200 +2023-03-05 11:43:55 - r - INFO: - Episode: 35/400, Reward: -1187.600, Step: 200 +2023-03-05 11:43:59 - r - INFO: - Episode: 36/400, Reward: -960.925, Step: 200 +2023-03-05 11:44:01 - r - INFO: - Episode: 37/400, Reward: -1522.463, Step: 200 +2023-03-05 11:44:04 - r - INFO: - Episode: 38/400, Reward: -1310.517, Step: 200 +2023-03-05 11:44:06 - r - INFO: - Episode: 39/400, Reward: -1008.963, Step: 200 +2023-03-05 11:44:09 - r - INFO: - Episode: 40/400, Reward: -1062.704, Step: 200 +2023-03-05 11:44:13 - r - INFO: - Episode: 41/400, Reward: -1519.136, Step: 200 +2023-03-05 11:44:15 - r - INFO: - Episode: 42/400, Reward: -1652.808, Step: 200 +2023-03-05 11:44:18 - r - INFO: - Episode: 43/400, Reward: -1333.117, Step: 200 +2023-03-05 11:44:20 - r - INFO: - Episode: 44/400, Reward: -964.514, Step: 200 +2023-03-05 11:44:23 - r - INFO: - Episode: 45/400, Reward: -906.041, Step: 200 +2023-03-05 11:44:27 - r - INFO: - Episode: 46/400, Reward: -866.323, Step: 200 +2023-03-05 11:44:29 - r - INFO: - Episode: 47/400, Reward: -1611.942, Step: 200 +2023-03-05 11:44:32 - r - INFO: - Episode: 48/400, Reward: -865.089, Step: 200 +2023-03-05 11:44:34 - r - INFO: - Episode: 49/400, Reward: -1789.956, Step: 200 +2023-03-05 11:44:37 - r - INFO: - Episode: 50/400, Reward: -1261.456, Step: 200 +2023-03-05 11:44:41 - r - INFO: - Episode: 51/400, Reward: -121.079, Step: 200 +2023-03-05 11:44:43 - r - INFO: - Episode: 52/400, Reward: -125.196, Step: 200 +2023-03-05 11:44:46 - r - INFO: - Episode: 53/400, Reward: -236.065, Step: 200 +2023-03-05 11:44:48 - r - INFO: - Episode: 54/400, Reward: -230.632, Step: 200 +2023-03-05 11:44:50 - r - INFO: - Episode: 55/400, Reward: -117.968, Step: 200 +2023-03-05 11:44:55 - r - INFO: - Episode: 56/400, Reward: -359.631, Step: 200 +2023-03-05 11:44:57 - r - INFO: - Episode: 57/400, Reward: -115.585, Step: 200 +2023-03-05 11:44:59 - r - INFO: - Episode: 58/400, Reward: -126.048, Step: 200 +2023-03-05 11:45:02 - r - INFO: - Episode: 59/400, Reward: -124.265, Step: 200 +2023-03-05 11:45:05 - r - INFO: - Episode: 60/400, Reward: -122.481, Step: 200 +2023-03-05 11:45:09 - r - INFO: - Episode: 61/400, Reward: -117.759, Step: 200 +2023-03-05 11:45:11 - r - INFO: - Episode: 62/400, Reward: -0.930, Step: 200 +2023-03-05 11:45:14 - r - INFO: - Episode: 63/400, Reward: -121.123, Step: 200 +2023-03-05 11:45:16 - r - INFO: - Episode: 64/400, Reward: -230.469, Step: 200 +2023-03-05 11:45:19 - r - INFO: - Episode: 65/400, Reward: -121.184, Step: 200 +2023-03-05 11:45:23 - r - INFO: - Episode: 66/400, Reward: -291.294, Step: 200 +2023-03-05 11:45:25 - r - INFO: - Episode: 67/400, Reward: -129.429, Step: 200 +2023-03-05 11:45:28 - r - INFO: - Episode: 68/400, Reward: -123.449, Step: 200 +2023-03-05 11:45:31 - r - INFO: - Episode: 69/400, Reward: -0.570, Step: 200 +2023-03-05 11:45:33 - r - INFO: - Episode: 70/400, Reward: -117.034, Step: 200 +2023-03-05 11:45:37 - r - INFO: - Episode: 71/400, Reward: -118.839, Step: 200 +2023-03-05 11:45:39 - r - INFO: - Episode: 72/400, Reward: -117.737, Step: 200 +2023-03-05 11:45:42 - r - INFO: - Episode: 73/400, Reward: -256.074, Step: 200 +2023-03-05 11:45:45 - r - INFO: - Episode: 74/400, Reward: -230.478, Step: 200 +2023-03-05 11:45:47 - r - INFO: - Episode: 75/400, Reward: -123.289, Step: 200 +2023-03-05 11:45:49 - r - INFO: - Current episode 75 has the best eval reward: -85.034 +2023-03-05 11:45:51 - r - INFO: - Episode: 76/400, Reward: -122.168, Step: 200 +2023-03-05 11:45:54 - r - INFO: - Episode: 77/400, Reward: -230.310, Step: 200 +2023-03-05 11:45:56 - r - INFO: - Episode: 78/400, Reward: -118.475, Step: 200 +2023-03-05 11:45:59 - r - INFO: - Episode: 79/400, Reward: -119.281, Step: 200 +2023-03-05 11:46:01 - r - INFO: - Episode: 80/400, Reward: -124.897, Step: 200 +2023-03-05 11:46:05 - r - INFO: - Episode: 81/400, Reward: -243.982, Step: 200 +2023-03-05 11:46:07 - r - INFO: - Episode: 82/400, Reward: -124.768, Step: 200 +2023-03-05 11:46:10 - r - INFO: - Episode: 83/400, Reward: -227.730, Step: 200 +2023-03-05 11:46:12 - r - INFO: - Episode: 84/400, Reward: -1.010, Step: 200 +2023-03-05 11:46:15 - r - INFO: - Episode: 85/400, Reward: -124.081, Step: 200 +2023-03-05 11:46:19 - r - INFO: - Episode: 86/400, Reward: -1.588, Step: 200 +2023-03-05 11:46:21 - r - INFO: - Episode: 87/400, Reward: -121.960, Step: 200 +2023-03-05 11:46:24 - r - INFO: - Episode: 88/400, Reward: -120.874, Step: 200 +2023-03-05 11:46:26 - r - INFO: - Episode: 89/400, Reward: -0.759, Step: 200 +2023-03-05 11:46:29 - r - INFO: - Episode: 90/400, Reward: -120.736, Step: 200 +2023-03-05 11:46:33 - r - INFO: - Episode: 91/400, Reward: -125.261, Step: 200 +2023-03-05 11:46:35 - r - INFO: - Episode: 92/400, Reward: -2.439, Step: 200 +2023-03-05 11:46:38 - r - INFO: - Episode: 93/400, Reward: -119.618, Step: 200 +2023-03-05 11:46:40 - r - INFO: - Episode: 94/400, Reward: -248.041, Step: 200 +2023-03-05 11:46:43 - r - INFO: - Episode: 95/400, Reward: -117.193, Step: 200 +2023-03-05 11:46:47 - r - INFO: - Episode: 96/400, Reward: -237.092, Step: 200 +2023-03-05 11:46:49 - r - INFO: - Episode: 97/400, Reward: -123.495, Step: 200 +2023-03-05 11:46:52 - r - INFO: - Episode: 98/400, Reward: -1.174, Step: 200 +2023-03-05 11:46:54 - r - INFO: - Episode: 99/400, Reward: -128.283, Step: 200 +2023-03-05 11:46:57 - r - INFO: - Episode: 100/400, Reward: -241.865, Step: 200 +2023-03-05 11:47:01 - r - INFO: - Episode: 101/400, Reward: -237.591, Step: 200 +2023-03-05 11:47:03 - r - INFO: - Episode: 102/400, Reward: -2.152, Step: 200 +2023-03-05 11:47:06 - r - INFO: - Episode: 103/400, Reward: -116.715, Step: 200 +2023-03-05 11:47:09 - r - INFO: - Episode: 104/400, Reward: -234.190, Step: 200 +2023-03-05 11:47:11 - r - INFO: - Episode: 105/400, Reward: -117.637, Step: 200 +2023-03-05 11:47:15 - r - INFO: - Episode: 106/400, Reward: -232.310, Step: 200 +2023-03-05 11:47:18 - r - INFO: - Episode: 107/400, Reward: -121.094, Step: 200 +2023-03-05 11:47:20 - r - INFO: - Episode: 108/400, Reward: -223.660, Step: 200 +2023-03-05 11:47:23 - r - INFO: - Episode: 109/400, Reward: -121.571, Step: 200 +2023-03-05 11:47:25 - r - INFO: - Episode: 110/400, Reward: -1.335, Step: 200 +2023-03-05 11:47:29 - r - INFO: - Episode: 111/400, Reward: -123.449, Step: 200 +2023-03-05 11:47:32 - r - INFO: - Episode: 112/400, Reward: -1.152, Step: 200 +2023-03-05 11:47:35 - r - INFO: - Episode: 113/400, Reward: -236.364, Step: 200 +2023-03-05 11:47:37 - r - INFO: - Episode: 114/400, Reward: -2.099, Step: 200 +2023-03-05 11:47:40 - r - INFO: - Episode: 115/400, Reward: -117.181, Step: 200 +2023-03-05 11:47:44 - r - INFO: - Episode: 116/400, Reward: -2.308, Step: 200 +2023-03-05 11:47:46 - r - INFO: - Episode: 117/400, Reward: -122.470, Step: 200 +2023-03-05 11:47:49 - r - INFO: - Episode: 118/400, Reward: -269.947, Step: 200 +2023-03-05 11:47:52 - r - INFO: - Episode: 119/400, Reward: -1.709, Step: 200 +2023-03-05 11:47:54 - r - INFO: - Episode: 120/400, Reward: -123.374, Step: 200 +2023-03-05 11:47:58 - r - INFO: - Episode: 121/400, Reward: -120.821, Step: 200 +2023-03-05 11:48:01 - r - INFO: - Episode: 122/400, Reward: -1.854, Step: 200 +2023-03-05 11:48:03 - r - INFO: - Episode: 123/400, Reward: -122.845, Step: 200 +2023-03-05 11:48:06 - r - INFO: - Episode: 124/400, Reward: -1.950, Step: 200 +2023-03-05 11:48:08 - r - INFO: - Episode: 125/400, Reward: -125.031, Step: 200 +2023-03-05 11:48:12 - r - INFO: - Episode: 126/400, Reward: -119.509, Step: 200 +2023-03-05 11:48:15 - r - INFO: - Episode: 127/400, Reward: -119.325, Step: 200 +2023-03-05 11:48:17 - r - INFO: - Episode: 128/400, Reward: -119.516, Step: 200 +2023-03-05 11:48:20 - r - INFO: - Episode: 129/400, Reward: -120.932, Step: 200 +2023-03-05 11:48:22 - r - INFO: - Episode: 130/400, Reward: -125.258, Step: 200 +2023-03-05 11:48:26 - r - INFO: - Episode: 131/400, Reward: -230.162, Step: 200 +2023-03-05 11:48:29 - r - INFO: - Episode: 132/400, Reward: -343.086, Step: 200 +2023-03-05 11:48:31 - r - INFO: - Episode: 133/400, Reward: -124.625, Step: 200 +2023-03-05 11:48:34 - r - INFO: - Episode: 134/400, Reward: -117.051, Step: 200 +2023-03-05 11:48:36 - r - INFO: - Episode: 135/400, Reward: -123.435, Step: 200 +2023-03-05 11:48:40 - r - INFO: - Episode: 136/400, Reward: -228.665, Step: 200 +2023-03-05 11:48:43 - r - INFO: - Episode: 137/400, Reward: -120.986, Step: 200 +2023-03-05 11:48:45 - r - INFO: - Episode: 138/400, Reward: -123.391, Step: 200 +2023-03-05 11:48:48 - r - INFO: - Episode: 139/400, Reward: -125.514, Step: 200 +2023-03-05 11:48:50 - r - INFO: - Episode: 140/400, Reward: -123.420, Step: 200 +2023-03-05 11:48:54 - r - INFO: - Episode: 141/400, Reward: -120.776, Step: 200 +2023-03-05 11:48:57 - r - INFO: - Episode: 142/400, Reward: -127.112, Step: 200 +2023-03-05 11:48:59 - r - INFO: - Episode: 143/400, Reward: -123.433, Step: 200 +2023-03-05 11:49:02 - r - INFO: - Episode: 144/400, Reward: -226.254, Step: 200 +2023-03-05 11:49:04 - r - INFO: - Episode: 145/400, Reward: -230.954, Step: 200 +2023-03-05 11:49:08 - r - INFO: - Episode: 146/400, Reward: -128.639, Step: 200 +2023-03-05 11:49:11 - r - INFO: - Episode: 147/400, Reward: -129.375, Step: 200 +2023-03-05 11:49:13 - r - INFO: - Episode: 148/400, Reward: -129.193, Step: 200 +2023-03-05 11:49:16 - r - INFO: - Episode: 149/400, Reward: -238.002, Step: 200 +2023-03-05 11:49:18 - r - INFO: - Episode: 150/400, Reward: -234.694, Step: 200 +2023-03-05 11:49:22 - r - INFO: - Episode: 151/400, Reward: -128.960, Step: 200 +2023-03-05 11:49:25 - r - INFO: - Episode: 152/400, Reward: -322.122, Step: 200 +2023-03-05 11:49:27 - r - INFO: - Episode: 153/400, Reward: -242.238, Step: 200 +2023-03-05 11:49:30 - r - INFO: - Episode: 154/400, Reward: -10.029, Step: 200 +2023-03-05 11:49:32 - r - INFO: - Episode: 155/400, Reward: -366.905, Step: 200 +2023-03-05 11:49:37 - r - INFO: - Episode: 156/400, Reward: -230.781, Step: 200 +2023-03-05 11:49:39 - r - INFO: - Episode: 157/400, Reward: -122.230, Step: 200 +2023-03-05 11:49:41 - r - INFO: - Episode: 158/400, Reward: -121.343, Step: 200 +2023-03-05 11:49:44 - r - INFO: - Episode: 159/400, Reward: -9.152, Step: 200 +2023-03-05 11:49:47 - r - INFO: - Episode: 160/400, Reward: -130.778, Step: 200 +2023-03-05 11:49:51 - r - INFO: - Episode: 161/400, Reward: -128.777, Step: 200 +2023-03-05 11:49:53 - r - INFO: - Episode: 162/400, Reward: -327.974, Step: 200 +2023-03-05 11:49:56 - r - INFO: - Episode: 163/400, Reward: -231.541, Step: 200 +2023-03-05 11:49:58 - r - INFO: - Episode: 164/400, Reward: -118.463, Step: 200 +2023-03-05 11:50:01 - r - INFO: - Episode: 165/400, Reward: -119.919, Step: 200 +2023-03-05 11:50:05 - r - INFO: - Episode: 166/400, Reward: -227.345, Step: 200 +2023-03-05 11:50:07 - r - INFO: - Episode: 167/400, Reward: -347.262, Step: 200 +2023-03-05 11:50:10 - r - INFO: - Episode: 168/400, Reward: -117.324, Step: 200 +2023-03-05 11:50:12 - r - INFO: - Episode: 169/400, Reward: -126.777, Step: 200 +2023-03-05 11:50:15 - r - INFO: - Episode: 170/400, Reward: -222.402, Step: 200 +2023-03-05 11:50:19 - r - INFO: - Episode: 171/400, Reward: -336.967, Step: 200 +2023-03-05 11:50:21 - r - INFO: - Episode: 172/400, Reward: -4.295, Step: 200 +2023-03-05 11:50:24 - r - INFO: - Episode: 173/400, Reward: -127.348, Step: 200 +2023-03-05 11:50:26 - r - INFO: - Episode: 174/400, Reward: -230.956, Step: 200 +2023-03-05 11:50:29 - r - INFO: - Episode: 175/400, Reward: -350.648, Step: 200 +2023-03-05 11:50:33 - r - INFO: - Episode: 176/400, Reward: -129.078, Step: 200 +2023-03-05 11:50:35 - r - INFO: - Episode: 177/400, Reward: -126.609, Step: 200 +2023-03-05 11:50:38 - r - INFO: - Episode: 178/400, Reward: -120.766, Step: 200 +2023-03-05 11:50:40 - r - INFO: - Episode: 179/400, Reward: -121.202, Step: 200 +2023-03-05 11:50:43 - r - INFO: - Episode: 180/400, Reward: -125.929, Step: 200 +2023-03-05 11:50:47 - r - INFO: - Episode: 181/400, Reward: -124.739, Step: 200 +2023-03-05 11:50:49 - r - INFO: - Episode: 182/400, Reward: -245.240, Step: 200 +2023-03-05 11:50:52 - r - INFO: - Episode: 183/400, Reward: -234.015, Step: 200 +2023-03-05 11:50:55 - r - INFO: - Episode: 184/400, Reward: -351.502, Step: 200 +2023-03-05 11:50:57 - r - INFO: - Episode: 185/400, Reward: -126.190, Step: 200 +2023-03-05 11:51:01 - r - INFO: - Episode: 186/400, Reward: -119.752, Step: 200 +2023-03-05 11:51:04 - r - INFO: - Episode: 187/400, Reward: -124.533, Step: 200 +2023-03-05 11:51:06 - r - INFO: - Episode: 188/400, Reward: -121.198, Step: 200 +2023-03-05 11:51:09 - r - INFO: - Episode: 189/400, Reward: -116.399, Step: 200 +2023-03-05 11:51:11 - r - INFO: - Episode: 190/400, Reward: -237.938, Step: 200 +2023-03-05 11:51:15 - r - INFO: - Episode: 191/400, Reward: -342.369, Step: 200 +2023-03-05 11:51:18 - r - INFO: - Episode: 192/400, Reward: -225.576, Step: 200 +2023-03-05 11:51:20 - r - INFO: - Episode: 193/400, Reward: -120.088, Step: 200 +2023-03-05 11:51:23 - r - INFO: - Episode: 194/400, Reward: -126.699, Step: 200 +2023-03-05 11:51:25 - r - INFO: - Episode: 195/400, Reward: -129.906, Step: 200 +2023-03-05 11:51:29 - r - INFO: - Episode: 196/400, Reward: -121.470, Step: 200 +2023-03-05 11:51:32 - r - INFO: - Episode: 197/400, Reward: -2.127, Step: 200 +2023-03-05 11:51:35 - r - INFO: - Episode: 198/400, Reward: -122.947, Step: 200 +2023-03-05 11:51:37 - r - INFO: - Episode: 199/400, Reward: -223.389, Step: 200 +2023-03-05 11:51:40 - r - INFO: - Episode: 200/400, Reward: -131.037, Step: 200 +2023-03-05 11:51:44 - r - INFO: - Episode: 201/400, Reward: -121.685, Step: 200 +2023-03-05 11:51:46 - r - INFO: - Episode: 202/400, Reward: -123.564, Step: 200 +2023-03-05 11:51:49 - r - INFO: - Episode: 203/400, Reward: -234.580, Step: 200 +2023-03-05 11:51:51 - r - INFO: - Episode: 204/400, Reward: -123.657, Step: 200 +2023-03-05 11:51:54 - r - INFO: - Episode: 205/400, Reward: -234.739, Step: 200 +2023-03-05 11:51:58 - r - INFO: - Episode: 206/400, Reward: -119.959, Step: 200 +2023-03-05 11:52:00 - r - INFO: - Episode: 207/400, Reward: -230.505, Step: 200 +2023-03-05 11:52:03 - r - INFO: - Episode: 208/400, Reward: -232.542, Step: 200 +2023-03-05 11:52:05 - r - INFO: - Episode: 209/400, Reward: -124.964, Step: 200 +2023-03-05 11:52:08 - r - INFO: - Episode: 210/400, Reward: -231.458, Step: 200 +2023-03-05 11:52:12 - r - INFO: - Episode: 211/400, Reward: -122.316, Step: 200 +2023-03-05 11:52:14 - r - INFO: - Episode: 212/400, Reward: -2.820, Step: 200 +2023-03-05 11:52:16 - r - INFO: - Episode: 213/400, Reward: -126.555, Step: 200 +2023-03-05 11:52:19 - r - INFO: - Episode: 214/400, Reward: -121.396, Step: 200 +2023-03-05 11:52:22 - r - INFO: - Episode: 215/400, Reward: -126.893, Step: 200 +2023-03-05 11:52:26 - r - INFO: - Episode: 216/400, Reward: -123.645, Step: 200 +2023-03-05 11:52:28 - r - INFO: - Episode: 217/400, Reward: -365.787, Step: 200 +2023-03-05 11:52:31 - r - INFO: - Episode: 218/400, Reward: -125.840, Step: 200 +2023-03-05 11:52:33 - r - INFO: - Episode: 219/400, Reward: -123.968, Step: 200 +2023-03-05 11:52:36 - r - INFO: - Episode: 220/400, Reward: -126.313, Step: 200 +2023-03-05 11:52:40 - r - INFO: - Episode: 221/400, Reward: -121.046, Step: 200 +2023-03-05 11:52:42 - r - INFO: - Episode: 222/400, Reward: -232.492, Step: 200 +2023-03-05 11:52:45 - r - INFO: - Episode: 223/400, Reward: -121.855, Step: 200 +2023-03-05 11:52:47 - r - INFO: - Episode: 224/400, Reward: -1.313, Step: 200 +2023-03-05 11:52:50 - r - INFO: - Episode: 225/400, Reward: -120.833, Step: 200 +2023-03-05 11:52:54 - r - INFO: - Episode: 226/400, Reward: -121.842, Step: 200 +2023-03-05 11:52:56 - r - INFO: - Episode: 227/400, Reward: -2.767, Step: 200 +2023-03-05 11:52:59 - r - INFO: - Episode: 228/400, Reward: -122.450, Step: 200 +2023-03-05 11:53:01 - r - INFO: - Episode: 229/400, Reward: -117.011, Step: 200 +2023-03-05 11:53:04 - r - INFO: - Episode: 230/400, Reward: -121.282, Step: 200 +2023-03-05 11:53:08 - r - INFO: - Episode: 231/400, Reward: -119.668, Step: 200 +2023-03-05 11:53:10 - r - INFO: - Episode: 232/400, Reward: -1.676, Step: 200 +2023-03-05 11:53:13 - r - INFO: - Episode: 233/400, Reward: -325.939, Step: 200 +2023-03-05 11:53:15 - r - INFO: - Episode: 234/400, Reward: -118.521, Step: 200 +2023-03-05 11:53:18 - r - INFO: - Episode: 235/400, Reward: -230.993, Step: 200 +2023-03-05 11:53:22 - r - INFO: - Episode: 236/400, Reward: -126.674, Step: 200 +2023-03-05 11:53:24 - r - INFO: - Episode: 237/400, Reward: -223.550, Step: 200 +2023-03-05 11:53:27 - r - INFO: - Episode: 238/400, Reward: -254.288, Step: 200 +2023-03-05 11:53:30 - r - INFO: - Episode: 239/400, Reward: -238.798, Step: 200 +2023-03-05 11:53:32 - r - INFO: - Episode: 240/400, Reward: -119.634, Step: 200 +2023-03-05 11:53:36 - r - INFO: - Episode: 241/400, Reward: -240.899, Step: 200 +2023-03-05 11:53:39 - r - INFO: - Episode: 242/400, Reward: -231.824, Step: 200 +2023-03-05 11:53:41 - r - INFO: - Episode: 243/400, Reward: -122.801, Step: 200 +2023-03-05 11:53:44 - r - INFO: - Episode: 244/400, Reward: -1.265, Step: 200 +2023-03-05 11:53:46 - r - INFO: - Episode: 245/400, Reward: -242.958, Step: 200 +2023-03-05 11:53:50 - r - INFO: - Episode: 246/400, Reward: -232.867, Step: 200 +2023-03-05 11:53:53 - r - INFO: - Episode: 247/400, Reward: -118.480, Step: 200 +2023-03-05 11:53:55 - r - INFO: - Episode: 248/400, Reward: -114.560, Step: 200 +2023-03-05 11:53:58 - r - INFO: - Episode: 249/400, Reward: -1.909, Step: 200 +2023-03-05 11:54:00 - r - INFO: - Episode: 250/400, Reward: -117.991, Step: 200 +2023-03-05 11:54:04 - r - INFO: - Episode: 251/400, Reward: -227.719, Step: 200 +2023-03-05 11:54:07 - r - INFO: - Episode: 252/400, Reward: -121.283, Step: 200 +2023-03-05 11:54:09 - r - INFO: - Episode: 253/400, Reward: -225.416, Step: 200 +2023-03-05 11:54:12 - r - INFO: - Episode: 254/400, Reward: -124.251, Step: 200 +2023-03-05 11:54:14 - r - INFO: - Episode: 255/400, Reward: -125.539, Step: 200 +2023-03-05 11:54:18 - r - INFO: - Episode: 256/400, Reward: -123.485, Step: 200 +2023-03-05 11:54:21 - r - INFO: - Episode: 257/400, Reward: -122.450, Step: 200 +2023-03-05 11:54:23 - r - INFO: - Episode: 258/400, Reward: -120.159, Step: 200 +2023-03-05 11:54:26 - r - INFO: - Episode: 259/400, Reward: -125.485, Step: 200 +2023-03-05 11:54:28 - r - INFO: - Episode: 260/400, Reward: -122.159, Step: 200 +2023-03-05 11:54:32 - r - INFO: - Episode: 261/400, Reward: -233.975, Step: 200 +2023-03-05 11:54:34 - r - INFO: - Episode: 262/400, Reward: -119.895, Step: 200 +2023-03-05 11:54:37 - r - INFO: - Episode: 263/400, Reward: -124.272, Step: 200 +2023-03-05 11:54:39 - r - INFO: - Episode: 264/400, Reward: -125.546, Step: 200 +2023-03-05 11:54:42 - r - INFO: - Episode: 265/400, Reward: -118.443, Step: 200 +2023-03-05 11:54:46 - r - INFO: - Episode: 266/400, Reward: -119.949, Step: 200 +2023-03-05 11:54:48 - r - INFO: - Episode: 267/400, Reward: -122.551, Step: 200 +2023-03-05 11:54:51 - r - INFO: - Episode: 268/400, Reward: -2.975, Step: 200 +2023-03-05 11:54:53 - r - INFO: - Episode: 269/400, Reward: -123.863, Step: 200 +2023-03-05 11:54:56 - r - INFO: - Episode: 270/400, Reward: -121.591, Step: 200 +2023-03-05 11:55:00 - r - INFO: - Episode: 271/400, Reward: -2.258, Step: 200 +2023-03-05 11:55:02 - r - INFO: - Episode: 272/400, Reward: -363.247, Step: 200 +2023-03-05 11:55:05 - r - INFO: - Episode: 273/400, Reward: -3.673, Step: 200 +2023-03-05 11:55:07 - r - INFO: - Episode: 274/400, Reward: -124.923, Step: 200 +2023-03-05 11:55:10 - r - INFO: - Episode: 275/400, Reward: -127.559, Step: 200 +2023-03-05 11:55:14 - r - INFO: - Episode: 276/400, Reward: -117.881, Step: 200 +2023-03-05 11:55:17 - r - INFO: - Episode: 277/400, Reward: -121.423, Step: 200 +2023-03-05 11:55:19 - r - INFO: - Episode: 278/400, Reward: -126.849, Step: 200 +2023-03-05 11:55:22 - r - INFO: - Episode: 279/400, Reward: -126.276, Step: 200 +2023-03-05 11:55:24 - r - INFO: - Episode: 280/400, Reward: -2.250, Step: 200 +2023-03-05 11:55:28 - r - INFO: - Episode: 281/400, Reward: -121.136, Step: 200 +2023-03-05 11:55:31 - r - INFO: - Episode: 282/400, Reward: -224.689, Step: 200 +2023-03-05 11:55:33 - r - INFO: - Episode: 283/400, Reward: -124.130, Step: 200 +2023-03-05 11:55:35 - r - INFO: - Episode: 284/400, Reward: -120.099, Step: 200 +2023-03-05 11:55:38 - r - INFO: - Episode: 285/400, Reward: -120.851, Step: 200 +2023-03-05 11:55:42 - r - INFO: - Episode: 286/400, Reward: -241.842, Step: 200 +2023-03-05 11:55:44 - r - INFO: - Episode: 287/400, Reward: -1.888, Step: 200 +2023-03-05 11:55:47 - r - INFO: - Episode: 288/400, Reward: -232.046, Step: 200 +2023-03-05 11:55:49 - r - INFO: - Episode: 289/400, Reward: -0.992, Step: 200 +2023-03-05 11:55:52 - r - INFO: - Episode: 290/400, Reward: -242.567, Step: 200 +2023-03-05 11:55:56 - r - INFO: - Episode: 291/400, Reward: -246.054, Step: 200 +2023-03-05 11:55:58 - r - INFO: - Episode: 292/400, Reward: -122.682, Step: 200 +2023-03-05 11:56:01 - r - INFO: - Episode: 293/400, Reward: -241.528, Step: 200 +2023-03-05 11:56:04 - r - INFO: - Episode: 294/400, Reward: -227.902, Step: 200 +2023-03-05 11:56:06 - r - INFO: - Episode: 295/400, Reward: -118.493, Step: 200 +2023-03-05 11:56:10 - r - INFO: - Episode: 296/400, Reward: -122.008, Step: 200 +2023-03-05 11:56:13 - r - INFO: - Episode: 297/400, Reward: -1.455, Step: 200 +2023-03-05 11:56:15 - r - INFO: - Episode: 298/400, Reward: -0.942, Step: 200 +2023-03-05 11:56:18 - r - INFO: - Episode: 299/400, Reward: -232.474, Step: 200 +2023-03-05 11:56:20 - r - INFO: - Episode: 300/400, Reward: -118.554, Step: 200 +2023-03-05 11:56:24 - r - INFO: - Episode: 301/400, Reward: -116.931, Step: 200 +2023-03-05 11:56:27 - r - INFO: - Episode: 302/400, Reward: -233.801, Step: 200 +2023-03-05 11:56:29 - r - INFO: - Episode: 303/400, Reward: -122.254, Step: 200 +2023-03-05 11:56:32 - r - INFO: - Episode: 304/400, Reward: -235.866, Step: 200 +2023-03-05 11:56:34 - r - INFO: - Episode: 305/400, Reward: -1.511, Step: 200 +2023-03-05 11:56:38 - r - INFO: - Episode: 306/400, Reward: -119.010, Step: 200 +2023-03-05 11:56:41 - r - INFO: - Episode: 307/400, Reward: -121.740, Step: 200 +2023-03-05 11:56:43 - r - INFO: - Episode: 308/400, Reward: -122.898, Step: 200 +2023-03-05 11:56:46 - r - INFO: - Episode: 309/400, Reward: -118.441, Step: 200 +2023-03-05 11:56:48 - r - INFO: - Episode: 310/400, Reward: -120.226, Step: 200 +2023-03-05 11:56:52 - r - INFO: - Episode: 311/400, Reward: -118.058, Step: 200 +2023-03-05 11:56:54 - r - INFO: - Episode: 312/400, Reward: -120.073, Step: 200 +2023-03-05 11:56:57 - r - INFO: - Episode: 313/400, Reward: -123.118, Step: 200 +2023-03-05 11:57:00 - r - INFO: - Episode: 314/400, Reward: -307.235, Step: 200 +2023-03-05 11:57:03 - r - INFO: - Episode: 315/400, Reward: -2.070, Step: 200 +2023-03-05 11:57:06 - r - INFO: - Episode: 316/400, Reward: -0.628, Step: 200 +2023-03-05 11:57:09 - r - INFO: - Episode: 317/400, Reward: -230.086, Step: 200 +2023-03-05 11:57:12 - r - INFO: - Episode: 318/400, Reward: -1.442, Step: 200 +2023-03-05 11:57:14 - r - INFO: - Episode: 319/400, Reward: -248.928, Step: 200 +2023-03-05 11:57:17 - r - INFO: - Episode: 320/400, Reward: -246.539, Step: 200 +2023-03-05 11:57:21 - r - INFO: - Episode: 321/400, Reward: -228.111, Step: 200 +2023-03-05 11:57:23 - r - INFO: - Episode: 322/400, Reward: -254.951, Step: 200 +2023-03-05 11:57:26 - r - INFO: - Episode: 323/400, Reward: -120.730, Step: 200 +2023-03-05 11:57:28 - r - INFO: - Episode: 324/400, Reward: -123.069, Step: 200 +2023-03-05 11:57:31 - r - INFO: - Episode: 325/400, Reward: -346.035, Step: 200 +2023-03-05 11:57:35 - r - INFO: - Episode: 326/400, Reward: -118.853, Step: 200 +2023-03-05 11:57:37 - r - INFO: - Episode: 327/400, Reward: -231.077, Step: 200 +2023-03-05 11:57:40 - r - INFO: - Episode: 328/400, Reward: -126.975, Step: 200 +2023-03-05 11:57:42 - r - INFO: - Episode: 329/400, Reward: -121.876, Step: 200 +2023-03-05 11:57:45 - r - INFO: - Episode: 330/400, Reward: -238.283, Step: 200 +2023-03-05 11:57:49 - r - INFO: - Episode: 331/400, Reward: -233.785, Step: 200 +2023-03-05 11:57:52 - r - INFO: - Episode: 332/400, Reward: -239.539, Step: 200 +2023-03-05 11:57:54 - r - INFO: - Episode: 333/400, Reward: -123.722, Step: 200 +2023-03-05 11:57:57 - r - INFO: - Episode: 334/400, Reward: -244.016, Step: 200 +2023-03-05 11:57:59 - r - INFO: - Episode: 335/400, Reward: -122.493, Step: 200 +2023-03-05 11:58:03 - r - INFO: - Episode: 336/400, Reward: -132.925, Step: 200 +2023-03-05 11:58:06 - r - INFO: - Episode: 337/400, Reward: -123.566, Step: 200 +2023-03-05 11:58:09 - r - INFO: - Episode: 338/400, Reward: -228.249, Step: 200 +2023-03-05 11:58:11 - r - INFO: - Episode: 339/400, Reward: -232.270, Step: 200 +2023-03-05 11:58:14 - r - INFO: - Episode: 340/400, Reward: -1.657, Step: 200 +2023-03-05 11:58:18 - r - INFO: - Episode: 341/400, Reward: -3.349, Step: 200 +2023-03-05 11:58:21 - r - INFO: - Episode: 342/400, Reward: -227.909, Step: 200 +2023-03-05 11:58:23 - r - INFO: - Episode: 343/400, Reward: -125.091, Step: 200 +2023-03-05 11:58:26 - r - INFO: - Episode: 344/400, Reward: -119.591, Step: 200 +2023-03-05 11:58:28 - r - INFO: - Episode: 345/400, Reward: -230.799, Step: 200 +2023-03-05 11:58:32 - r - INFO: - Episode: 346/400, Reward: -4.066, Step: 200 +2023-03-05 11:58:35 - r - INFO: - Episode: 347/400, Reward: -118.917, Step: 200 +2023-03-05 11:58:37 - r - INFO: - Episode: 348/400, Reward: -231.116, Step: 200 +2023-03-05 11:58:40 - r - INFO: - Episode: 349/400, Reward: -115.684, Step: 200 +2023-03-05 11:58:42 - r - INFO: - Episode: 350/400, Reward: -123.263, Step: 200 +2023-03-05 11:58:46 - r - INFO: - Episode: 351/400, Reward: -126.788, Step: 200 +2023-03-05 11:58:49 - r - INFO: - Episode: 352/400, Reward: -122.307, Step: 200 +2023-03-05 11:58:51 - r - INFO: - Episode: 353/400, Reward: -123.405, Step: 200 +2023-03-05 11:58:54 - r - INFO: - Episode: 354/400, Reward: -126.424, Step: 200 +2023-03-05 11:58:56 - r - INFO: - Episode: 355/400, Reward: -118.180, Step: 200 +2023-03-05 11:59:00 - r - INFO: - Episode: 356/400, Reward: -125.828, Step: 200 +2023-03-05 11:59:03 - r - INFO: - Episode: 357/400, Reward: -127.383, Step: 200 +2023-03-05 11:59:05 - r - INFO: - Episode: 358/400, Reward: -124.455, Step: 200 +2023-03-05 11:59:08 - r - INFO: - Episode: 359/400, Reward: -233.062, Step: 200 +2023-03-05 11:59:10 - r - INFO: - Episode: 360/400, Reward: -118.345, Step: 200 +2023-03-05 11:59:14 - r - INFO: - Episode: 361/400, Reward: -1.072, Step: 200 +2023-03-05 11:59:17 - r - INFO: - Episode: 362/400, Reward: -127.160, Step: 200 +2023-03-05 11:59:20 - r - INFO: - Episode: 363/400, Reward: -121.396, Step: 200 +2023-03-05 11:59:22 - r - INFO: - Episode: 364/400, Reward: -115.136, Step: 200 +2023-03-05 11:59:25 - r - INFO: - Episode: 365/400, Reward: -233.369, Step: 200 +2023-03-05 11:59:29 - r - INFO: - Episode: 366/400, Reward: -124.808, Step: 200 +2023-03-05 11:59:31 - r - INFO: - Episode: 367/400, Reward: -122.788, Step: 200 +2023-03-05 11:59:34 - r - INFO: - Episode: 368/400, Reward: -123.264, Step: 200 +2023-03-05 11:59:36 - r - INFO: - Episode: 369/400, Reward: -259.776, Step: 200 +2023-03-05 11:59:39 - r - INFO: - Episode: 370/400, Reward: -227.237, Step: 200 +2023-03-05 11:59:43 - r - INFO: - Episode: 371/400, Reward: -1.756, Step: 200 +2023-03-05 11:59:45 - r - INFO: - Episode: 372/400, Reward: -119.952, Step: 200 +2023-03-05 11:59:48 - r - INFO: - Episode: 373/400, Reward: -1.477, Step: 200 +2023-03-05 11:59:50 - r - INFO: - Episode: 374/400, Reward: -128.200, Step: 200 +2023-03-05 11:59:53 - r - INFO: - Episode: 375/400, Reward: -2.361, Step: 200 +2023-03-05 11:59:57 - r - INFO: - Episode: 376/400, Reward: -2.999, Step: 200 +2023-03-05 11:59:59 - r - INFO: - Episode: 377/400, Reward: -119.141, Step: 200 +2023-03-05 12:00:02 - r - INFO: - Episode: 378/400, Reward: -228.394, Step: 200 +2023-03-05 12:00:04 - r - INFO: - Episode: 379/400, Reward: -118.631, Step: 200 +2023-03-05 12:00:07 - r - INFO: - Episode: 380/400, Reward: -118.296, Step: 200 +2023-03-05 12:00:11 - r - INFO: - Episode: 381/400, Reward: -120.981, Step: 200 +2023-03-05 12:00:13 - r - INFO: - Episode: 382/400, Reward: -242.309, Step: 200 +2023-03-05 12:00:16 - r - INFO: - Episode: 383/400, Reward: -128.361, Step: 200 +2023-03-05 12:00:18 - r - INFO: - Episode: 384/400, Reward: -243.632, Step: 200 +2023-03-05 12:00:21 - r - INFO: - Episode: 385/400, Reward: -118.637, Step: 200 +2023-03-05 12:00:25 - r - INFO: - Episode: 386/400, Reward: -330.532, Step: 200 +2023-03-05 12:00:27 - r - INFO: - Episode: 387/400, Reward: -119.244, Step: 200 +2023-03-05 12:00:30 - r - INFO: - Episode: 388/400, Reward: -118.364, Step: 200 +2023-03-05 12:00:32 - r - INFO: - Episode: 389/400, Reward: -118.736, Step: 200 +2023-03-05 12:00:35 - r - INFO: - Episode: 390/400, Reward: -347.443, Step: 200 +2023-03-05 12:00:39 - r - INFO: - Episode: 391/400, Reward: -121.889, Step: 200 +2023-03-05 12:00:42 - r - INFO: - Episode: 392/400, Reward: -123.754, Step: 200 +2023-03-05 12:00:44 - r - INFO: - Episode: 393/400, Reward: -0.573, Step: 200 +2023-03-05 12:00:47 - r - INFO: - Episode: 394/400, Reward: -229.850, Step: 200 +2023-03-05 12:00:49 - r - INFO: - Episode: 395/400, Reward: -343.326, Step: 200 +2023-03-05 12:00:53 - r - INFO: - Episode: 396/400, Reward: -227.782, Step: 200 +2023-03-05 12:00:55 - r - INFO: - Episode: 397/400, Reward: -122.918, Step: 200 +2023-03-05 12:00:58 - r - INFO: - Episode: 398/400, Reward: -119.106, Step: 200 +2023-03-05 12:01:01 - r - INFO: - Episode: 399/400, Reward: -242.566, Step: 200 +2023-03-05 12:01:03 - r - INFO: - Episode: 400/400, Reward: -364.784, Step: 200 +2023-03-05 12:01:05 - r - INFO: - Finish training! diff --git a/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/models/checkpoint.pt b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c33d27b421f22935c546d6d68ac524f703409bc --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffff8b78aa63759f028e5a98d80638825b86be9284bf29df8b68f1bc49121f75 +size 221645 diff --git a/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/results/learning_curve.png b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..310220a359ea0b7cb66a11f652360f12b23f8933 Binary files /dev/null and b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/results/learning_curve.png differ diff --git a/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/results/res.csv b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe2f876cb164b2c560709d2ded9003eb3d6a7401 --- /dev/null +++ b/Pendulum-v1/Train_Pendulum-v1_SAC_20230305-114217/results/res.csv @@ -0,0 +1,401 @@ +episodes,rewards,steps +0,-1740.8312772558186,200 +1,-961.0690893948698,200 +2,-967.6853488323495,200 +3,-1261.6279643072685,200 +4,-966.4601746071279,200 +5,-886.1803839672709,200 +6,-1621.1245482042475,200 +7,-886.9143504872607,200 +8,-872.5987255068338,200 +9,-1263.8330554259642,200 +10,-994.982960821575,200 +11,-768.3987155877307,200 +12,-1660.7334686454524,200 +13,-1576.3557924883983,200 +14,-968.9016073281383,200 +15,-1071.378497320579,200 +16,-1156.6902310747068,200 +17,-1514.4748782964762,200 +18,-1093.7368495153485,200 +19,-968.8778188772778,200 +20,-1203.861725552502,200 +21,-1405.15466697909,200 +22,-917.4296510824465,200 +23,-1087.1896572729286,200 +24,-1116.1893909525902,200 +25,-1823.2207410666947,200 +26,-1379.7470314835666,200 +27,-905.8885528276571,200 +28,-1101.7635146470782,200 +29,-1175.6367430654018,200 +30,-1825.4356886123153,200 +31,-1655.5608517267472,200 +32,-1180.2261216486522,200 +33,-1063.9636213060498,200 +34,-1187.600253309378,200 +35,-960.9250385656012,200 +36,-1522.4630191941196,200 +37,-1310.5170842837936,200 +38,-1008.9631406614152,200 +39,-1062.7039017538477,200 +40,-1519.1360880462714,200 +41,-1652.8081237864747,200 +42,-1333.116727206263,200 +43,-964.51356804181,200 +44,-906.0413801462098,200 +45,-866.3232312673045,200 +46,-1611.9418251117272,200 +47,-865.0887025740575,200 +48,-1789.9561516360952,200 +49,-1261.4560715438718,200 +50,-121.07934062811904,200 +51,-125.1960212161913,200 +52,-236.06450323239793,200 +53,-230.6322919049866,200 +54,-117.96831090432273,200 +55,-359.6309928463407,200 +56,-115.58493007454831,200 +57,-126.04822383879512,200 +58,-124.26477057069509,200 +59,-122.48148244622793,200 +60,-117.75869946092378,200 +61,-0.9297141105385545,200 +62,-121.12289898144112,200 +63,-230.4694966607722,200 +64,-121.18396562482455,200 +65,-291.2939983734708,200 +66,-129.42913657775128,200 +67,-123.44870091280983,200 +68,-0.5702222402873633,200 +69,-117.03381306473737,200 +70,-118.83917224037314,200 +71,-117.73673163261721,200 +72,-256.07355727161575,200 +73,-230.47812672926202,200 +74,-123.28914677369383,200 +75,-122.16823530890674,200 +76,-230.31038295889172,200 +77,-118.47510581957737,200 +78,-119.28089209365994,200 +79,-124.89670812926282,200 +80,-243.9822256108303,200 +81,-124.76784274560563,200 +82,-227.7302022483137,200 +83,-1.0098728749010784,200 +84,-124.08146167145551,200 +85,-1.58809700164591,200 +86,-121.9598303018825,200 +87,-120.87393017867159,200 +88,-0.7587089331107321,200 +89,-120.73595475839734,200 +90,-125.26085820104748,200 +91,-2.438540490909063,200 +92,-119.6175756142322,200 +93,-248.04103101439702,200 +94,-117.19339729306179,200 +95,-237.09195109857026,200 +96,-123.49524967742742,200 +97,-1.1743222437365484,200 +98,-128.28261986979481,200 +99,-241.86489285665772,200 +100,-237.5906369699415,200 +101,-2.151643811552402,200 +102,-116.71526039907259,200 +103,-234.19044621770556,200 +104,-117.63659770515251,200 +105,-232.3097125046084,200 +106,-121.09393575608233,200 +107,-223.66004837398958,200 +108,-121.57131299529145,200 +109,-1.3352310548355606,200 +110,-123.44859316248971,200 +111,-1.1518580377434746,200 +112,-236.3641822041889,200 +113,-2.0988195214441827,200 +114,-117.18057226047364,200 +115,-2.3080552666423992,200 +116,-122.47020772161908,200 +117,-269.94745108241426,200 +118,-1.7087282056121973,200 +119,-123.37369123422117,200 +120,-120.82073391813404,200 +121,-1.8541652321127935,200 +122,-122.84450527641008,200 +123,-1.9499530102196443,200 +124,-125.03077436890683,200 +125,-119.5094754503306,200 +126,-119.32488684967407,200 +127,-119.51626311007207,200 +128,-120.93249108521475,200 +129,-125.25818615068965,200 +130,-230.16160161144745,200 +131,-343.08588053030263,200 +132,-124.62517525091478,200 +133,-117.05060538898005,200 +134,-123.43518201685897,200 +135,-228.66542284667202,200 +136,-120.98607124779437,200 +137,-123.39098530619893,200 +138,-125.51407912854508,200 +139,-123.42031142167917,200 +140,-120.7761178730733,200 +141,-127.11234031663895,200 +142,-123.43285739763634,200 +143,-226.2535750748725,200 +144,-230.95437772725356,200 +145,-128.63944404137808,200 +146,-129.37521020862408,200 +147,-129.19314823289275,200 +148,-238.0015799217973,200 +149,-234.69401702713026,200 +150,-128.95982052949427,200 +151,-322.1218859673061,200 +152,-242.23840400031824,200 +153,-10.02853073310278,200 +154,-366.90519058872934,200 +155,-230.78056017638878,200 +156,-122.22975968098518,200 +157,-121.34343730542,200 +158,-9.15188002998091,200 +159,-130.77809449118513,200 +160,-128.77684354451992,200 +161,-327.97423690890395,200 +162,-231.5407281796477,200 +163,-118.46297982978449,200 +164,-119.91854590608006,200 +165,-227.3448546236956,200 +166,-347.2616076559741,200 +167,-117.3244667866222,200 +168,-126.77709215700749,200 +169,-222.40238847487285,200 +170,-336.9669608843524,200 +171,-4.294707063159415,200 +172,-127.34760961599655,200 +173,-230.95574505557954,200 +174,-350.6481982351329,200 +175,-129.07791802627088,200 +176,-126.60937609159515,200 +177,-120.7663195932923,200 +178,-121.20236098985659,200 +179,-125.9287913022166,200 +180,-124.73861263769442,200 +181,-245.2401869780623,200 +182,-234.01516167674274,200 +183,-351.50229303371,200 +184,-126.18991522845755,200 +185,-119.75215447906655,200 +186,-124.53315138827575,200 +187,-121.19764166298557,200 +188,-116.39883522426905,200 +189,-237.93834867748092,200 +190,-342.3694558181359,200 +191,-225.5759341035521,200 +192,-120.08844995618301,200 +193,-126.69906339419737,200 +194,-129.90625113245318,200 +195,-121.46996034800452,200 +196,-2.127165755778542,200 +197,-122.94681373454888,200 +198,-223.3886863362706,200 +199,-131.0372991467051,200 +200,-121.68500639847933,200 +201,-123.56416981790649,200 +202,-234.57983857744085,200 +203,-123.65661559695077,200 +204,-234.73937310754982,200 +205,-119.95888058903344,200 +206,-230.504727169853,200 +207,-232.54203966937783,200 +208,-124.96416874986504,200 +209,-231.45774223495462,200 +210,-122.31561230172528,200 +211,-2.820291135879688,200 +212,-126.55491590164168,200 +213,-121.39604475613211,200 +214,-126.89327435566437,200 +215,-123.6450604779974,200 +216,-365.78650501074486,200 +217,-125.83981652777061,200 +218,-123.96838218310343,200 +219,-126.3129033507389,200 +220,-121.04623423168414,200 +221,-232.49168744930205,200 +222,-121.85526030086369,200 +223,-1.313210740704022,200 +224,-120.83261911197191,200 +225,-121.84162997734299,200 +226,-2.7667948417862736,200 +227,-122.44962738751354,200 +228,-117.01088893136885,200 +229,-121.2815699162756,200 +230,-119.66849082987936,200 +231,-1.6758218734063999,200 +232,-325.93912641326716,200 +233,-118.52089210951341,200 +234,-230.9929395265159,200 +235,-126.67408839962316,200 +236,-223.5497194486765,200 +237,-254.28769063949682,200 +238,-238.7984222657351,200 +239,-119.63424714669672,200 +240,-240.89936916039284,200 +241,-231.82428563255928,200 +242,-122.80092927826453,200 +243,-1.265453328942679,200 +244,-242.95778554241548,200 +245,-232.8665905579679,200 +246,-118.48020374530488,200 +247,-114.56027815967052,200 +248,-1.9090177083831832,200 +249,-117.99083726209076,200 +250,-227.71915314748455,200 +251,-121.28286242549773,200 +252,-225.41556757806265,200 +253,-124.25089292320273,200 +254,-125.53904545493685,200 +255,-123.48478273896977,200 +256,-122.44990658394663,200 +257,-120.15880652489344,200 +258,-125.4853240536495,200 +259,-122.15923728217913,200 +260,-233.97512448515997,200 +261,-119.8951262043544,200 +262,-124.27174247359389,200 +263,-125.54636309582547,200 +264,-118.44343999976657,200 +265,-119.94896323337845,200 +266,-122.55141634657419,200 +267,-2.9750754356445435,200 +268,-123.86260727849641,200 +269,-121.59088209114171,200 +270,-2.2583701180363867,200 +271,-363.2474812469634,200 +272,-3.673207705969357,200 +273,-124.92345795162294,200 +274,-127.559217332195,200 +275,-117.88131900282809,200 +276,-121.42310339405101,200 +277,-126.84863571709897,200 +278,-126.27614484313939,200 +279,-2.2499873901771696,200 +280,-121.1359979689505,200 +281,-224.68867571332422,200 +282,-124.13041871109385,200 +283,-120.09865197143162,200 +284,-120.85127574233252,200 +285,-241.84157390995722,200 +286,-1.8883316696747514,200 +287,-232.0455668009921,200 +288,-0.9920058093707583,200 +289,-242.56687007115084,200 +290,-246.0539206872767,200 +291,-122.6822691203317,200 +292,-241.52827604924695,200 +293,-227.90186982517383,200 +294,-118.49340628002831,200 +295,-122.0084620908193,200 +296,-1.4554235742568797,200 +297,-0.9417011427405071,200 +298,-232.4735236406664,200 +299,-118.55436299798939,200 +300,-116.93127237236583,200 +301,-233.80149983991703,200 +302,-122.2538058428176,200 +303,-235.8664311974204,200 +304,-1.5112389751013449,200 +305,-119.00953651098925,200 +306,-121.73998679563122,200 +307,-122.89776765845241,200 +308,-118.44132367961865,200 +309,-120.22553736883394,200 +310,-118.05797471030625,200 +311,-120.0733432725947,200 +312,-123.11826114622873,200 +313,-307.23514008371313,200 +314,-2.070075229383631,200 +315,-0.6282053166490618,200 +316,-230.08608216026315,200 +317,-1.4416004137314185,200 +318,-248.92818650945117,200 +319,-246.53862945635166,200 +320,-228.1109197984334,200 +321,-254.95131054177898,200 +322,-120.73034864243611,200 +323,-123.06856974642963,200 +324,-346.03513645400915,200 +325,-118.85325482549345,200 +326,-231.0774178583102,200 +327,-126.97509347790334,200 +328,-121.87634083452872,200 +329,-238.2826937310235,200 +330,-233.78488285983647,200 +331,-239.539213825356,200 +332,-123.72224429749,200 +333,-244.0158868120912,200 +334,-122.4928704493939,200 +335,-132.92513774222564,200 +336,-123.56616866551367,200 +337,-228.24929548641907,200 +338,-232.26960787739284,200 +339,-1.6572037841965508,200 +340,-3.3487306606282905,200 +341,-227.90865233503712,200 +342,-125.09120659907836,200 +343,-119.59121383740974,200 +344,-230.79924812675944,200 +345,-4.065969674328051,200 +346,-118.91746185303818,200 +347,-231.11597418294494,200 +348,-115.68443060416796,200 +349,-123.26318387998084,200 +350,-126.78843932807875,200 +351,-122.30715088305702,200 +352,-123.40518083599186,200 +353,-126.42422180940258,200 +354,-118.17989763679792,200 +355,-125.82786942639923,200 +356,-127.38265773490905,200 +357,-124.45495192756128,200 +358,-233.06163995956527,200 +359,-118.34522621894905,200 +360,-1.072039326115114,200 +361,-127.16044343250464,200 +362,-121.39564580970485,200 +363,-115.13559969500137,200 +364,-233.3686646894422,200 +365,-124.8075052080167,200 +366,-122.78840990635287,200 +367,-123.26365199230351,200 +368,-259.77634647150876,200 +369,-227.2369659169205,200 +370,-1.7555078565729523,200 +371,-119.95172400535714,200 +372,-1.4767166394979265,200 +373,-128.20045257314416,200 +374,-2.360923243668752,200 +375,-2.9993463909267115,200 +376,-119.14118754997314,200 +377,-228.39358255590324,200 +378,-118.63136700320051,200 +379,-118.29607371634711,200 +380,-120.9807536933188,200 +381,-242.30891318513025,200 +382,-128.36067478869478,200 +383,-243.6324284520479,200 +384,-118.6373893662789,200 +385,-330.53162154316414,200 +386,-119.24357296845005,200 +387,-118.36420136290116,200 +388,-118.7361938600814,200 +389,-347.4426681400597,200 +390,-121.88892046535962,200 +391,-123.75394440952341,200 +392,-0.572942503984737,200 +393,-229.84981727676404,200 +394,-343.32564410001936,200 +395,-227.781564121981,200 +396,-122.91777865380416,200 +397,-119.10553245031416,200 +398,-242.56617208309072,200 +399,-364.78388845746565,200 diff --git a/Pendulum-v1/Train_gym_TD3_20230416-110359/config.yaml b/Pendulum-v1/Train_gym_TD3_20230416-110359/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86f8db10925894fa589ce8e4c2dbd0503ceef810 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_20230416-110359/config.yaml @@ -0,0 +1,38 @@ +general_cfg: + algo_name: TD3 + device: cuda + env_name: gym + eval_eps: 5 + eval_per_episode: 10 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.001 + batch_size: 100 + buffer_size: 1000000 + critic_hidden_dim: 256 + critic_lr: 0.001 + expl_noise: 0.1 + explore_steps: 1000 + gamma: 0.99 + noise_clip: 0.5 + policy_freq: 2 + policy_noise: 0.2 + tau: 0.005 +env_cfg: + id: Pendulum-v1 + new_step_api: true + render_mode: null diff --git a/Pendulum-v1/Train_gym_TD3_20230416-110359/logs/log.txt b/Pendulum-v1/Train_gym_TD3_20230416-110359/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..47c5d95e5c41fb037dc4206be69b9e0c6f1c1043 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_20230416-110359/logs/log.txt @@ -0,0 +1,148 @@ +2023-04-16 11:03:59 - r - INFO: - Hyperparameters: +2023-04-16 11:03:59 - r - INFO: - ================================================================================ +2023-04-16 11:03:59 - r - INFO: - Name Value Type +2023-04-16 11:03:59 - r - INFO: - env_name gym +2023-04-16 11:03:59 - r - INFO: - new_step_api 1 +2023-04-16 11:03:59 - r - INFO: - wrapper None +2023-04-16 11:03:59 - r - INFO: - render 0 +2023-04-16 11:03:59 - r - INFO: - render_mode None +2023-04-16 11:03:59 - r - INFO: - algo_name TD3 +2023-04-16 11:03:59 - r - INFO: - mode train +2023-04-16 11:03:59 - r - INFO: - mp_backend mp +2023-04-16 11:03:59 - r - INFO: - seed 1 +2023-04-16 11:03:59 - r - INFO: - device cuda +2023-04-16 11:03:59 - r - INFO: - train_eps 100 +2023-04-16 11:03:59 - r - INFO: - test_eps 10 +2023-04-16 11:03:59 - r - INFO: - eval_eps 5 +2023-04-16 11:03:59 - r - INFO: - eval_per_episode 10 +2023-04-16 11:03:59 - r - INFO: - max_steps 200 +2023-04-16 11:03:59 - r - INFO: - load_checkpoint 0 +2023-04-16 11:03:59 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-16 11:03:59 - r - INFO: - show_fig 0 +2023-04-16 11:03:59 - r - INFO: - save_fig 1 +2023-04-16 11:03:59 - r - INFO: - explore_steps 1000 +2023-04-16 11:03:59 - r - INFO: - policy_freq 2 +2023-04-16 11:03:59 - r - INFO: - actor_lr 0.001 +2023-04-16 11:03:59 - r - INFO: - critic_lr 0.001 +2023-04-16 11:03:59 - r - INFO: - actor_hidden_dim 256 +2023-04-16 11:03:59 - r - INFO: - critic_hidden_dim 256 +2023-04-16 11:03:59 - r - INFO: - gamma 0.99 +2023-04-16 11:03:59 - r - INFO: - tau 0.005 +2023-04-16 11:03:59 - r - INFO: - policy_noise 0.2 +2023-04-16 11:03:59 - r - INFO: - expl_noise 0.1 +2023-04-16 11:03:59 - r - INFO: - noise_clip 0.5 +2023-04-16 11:03:59 - r - INFO: - batch_size 100 +2023-04-16 11:03:59 - r - INFO: - buffer_size 1000000 +2023-04-16 11:03:59 - r - INFO: - id Pendulum-v1 +2023-04-16 11:03:59 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-110359 +2023-04-16 11:03:59 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-110359/results +2023-04-16 11:03:59 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-110359/logs +2023-04-16 11:03:59 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-110359/traj +2023-04-16 11:03:59 - r - INFO: - video_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-110359/videos +2023-04-16 11:03:59 - r - INFO: - ================================================================================ +2023-04-16 11:03:59 - r - INFO: - action_bound: 2.0 +2023-04-16 11:03:59 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-16 11:04:05 - r - INFO: - Start training! +2023-04-16 11:04:05 - r - INFO: - Env: gym, Algorithm: TD3, Device: cuda +2023-04-16 11:04:05 - r - INFO: - Episode: 1/100, Reward: -832.441, Step: 200 +2023-04-16 11:04:05 - r - INFO: - Episode: 2/100, Reward: -968.404, Step: 200 +2023-04-16 11:04:05 - r - INFO: - Episode: 3/100, Reward: -874.586, Step: 200 +2023-04-16 11:04:05 - r - INFO: - Episode: 4/100, Reward: -875.727, Step: 200 +2023-04-16 11:04:05 - r - INFO: - Episode: 5/100, Reward: -897.681, Step: 200 +2023-04-16 11:04:09 - r - INFO: - Episode: 6/100, Reward: -1568.319, Step: 200 +2023-04-16 11:04:13 - r - INFO: - Episode: 7/100, Reward: -1459.121, Step: 200 +2023-04-16 11:04:16 - r - INFO: - Episode: 8/100, Reward: -1240.331, Step: 200 +2023-04-16 11:04:20 - r - INFO: - Episode: 9/100, Reward: -1134.842, Step: 200 +2023-04-16 11:04:24 - r - INFO: - Episode: 10/100, Reward: -5.367, Step: 200 +2023-04-16 11:04:27 - r - INFO: - Current episode 10 has the best eval reward: -0.514 +2023-04-16 11:04:30 - r - INFO: - Episode: 11/100, Reward: -650.500, Step: 200 +2023-04-16 11:04:34 - r - INFO: - Episode: 12/100, Reward: -8.378, Step: 200 +2023-04-16 11:04:38 - r - INFO: - Episode: 13/100, Reward: -4.576, Step: 200 +2023-04-16 11:04:42 - r - INFO: - Episode: 14/100, Reward: -3.628, Step: 200 +2023-04-16 11:04:45 - r - INFO: - Episode: 15/100, Reward: -3.894, Step: 200 +2023-04-16 11:04:49 - r - INFO: - Episode: 16/100, Reward: -3.819, Step: 200 +2023-04-16 11:04:53 - r - INFO: - Episode: 17/100, Reward: -3.415, Step: 200 +2023-04-16 11:04:57 - r - INFO: - Episode: 18/100, Reward: -2.764, Step: 200 +2023-04-16 11:05:00 - r - INFO: - Episode: 19/100, Reward: -2.402, Step: 200 +2023-04-16 11:05:04 - r - INFO: - Episode: 20/100, Reward: -3.090, Step: 200 +2023-04-16 11:05:11 - r - INFO: - Episode: 21/100, Reward: -1.481, Step: 200 +2023-04-16 11:05:15 - r - INFO: - Episode: 22/100, Reward: -1.263, Step: 200 +2023-04-16 11:05:19 - r - INFO: - Episode: 23/100, Reward: -1.318, Step: 200 +2023-04-16 11:05:22 - r - INFO: - Episode: 24/100, Reward: -1.565, Step: 200 +2023-04-16 11:05:26 - r - INFO: - Episode: 25/100, Reward: -1.522, Step: 200 +2023-04-16 11:05:30 - r - INFO: - Episode: 26/100, Reward: -1.708, Step: 200 +2023-04-16 11:05:34 - r - INFO: - Episode: 27/100, Reward: -1.423, Step: 200 +2023-04-16 11:05:37 - r - INFO: - Episode: 28/100, Reward: -1.537, Step: 200 +2023-04-16 11:05:41 - r - INFO: - Episode: 29/100, Reward: -1.524, Step: 200 +2023-04-16 11:05:45 - r - INFO: - Episode: 30/100, Reward: -1.459, Step: 200 +2023-04-16 11:05:52 - r - INFO: - Episode: 31/100, Reward: -1.736, Step: 200 +2023-04-16 11:05:56 - r - INFO: - Episode: 32/100, Reward: -2.068, Step: 200 +2023-04-16 11:05:59 - r - INFO: - Episode: 33/100, Reward: -1.741, Step: 200 +2023-04-16 11:06:03 - r - INFO: - Episode: 34/100, Reward: -1.849, Step: 200 +2023-04-16 11:06:07 - r - INFO: - Episode: 35/100, Reward: -1.399, Step: 200 +2023-04-16 11:06:11 - r - INFO: - Episode: 36/100, Reward: -1.471, Step: 200 +2023-04-16 11:06:15 - r - INFO: - Episode: 37/100, Reward: -1.480, Step: 200 +2023-04-16 11:06:18 - r - INFO: - Episode: 38/100, Reward: -1.453, Step: 200 +2023-04-16 11:06:22 - r - INFO: - Episode: 39/100, Reward: -1.410, Step: 200 +2023-04-16 11:06:26 - r - INFO: - Episode: 40/100, Reward: -1.362, Step: 200 +2023-04-16 11:06:33 - r - INFO: - Episode: 41/100, Reward: -1.323, Step: 200 +2023-04-16 11:06:36 - r - INFO: - Episode: 42/100, Reward: -1.332, Step: 200 +2023-04-16 11:06:40 - r - INFO: - Episode: 43/100, Reward: -1.447, Step: 200 +2023-04-16 11:06:44 - r - INFO: - Episode: 44/100, Reward: -1.354, Step: 200 +2023-04-16 11:06:47 - r - INFO: - Episode: 45/100, Reward: -1.310, Step: 200 +2023-04-16 11:06:51 - r - INFO: - Episode: 46/100, Reward: -1.432, Step: 200 +2023-04-16 11:06:55 - r - INFO: - Episode: 47/100, Reward: -1.422, Step: 200 +2023-04-16 11:06:59 - r - INFO: - Episode: 48/100, Reward: -1.380, Step: 200 +2023-04-16 11:07:02 - r - INFO: - Episode: 49/100, Reward: -1.384, Step: 200 +2023-04-16 11:07:06 - r - INFO: - Episode: 50/100, Reward: -1.286, Step: 200 +2023-04-16 11:07:13 - r - INFO: - Episode: 51/100, Reward: -1.287, Step: 200 +2023-04-16 11:07:17 - r - INFO: - Episode: 52/100, Reward: -1.343, Step: 200 +2023-04-16 11:07:20 - r - INFO: - Episode: 53/100, Reward: -1.389, Step: 200 +2023-04-16 11:07:24 - r - INFO: - Episode: 54/100, Reward: -1.476, Step: 200 +2023-04-16 11:07:28 - r - INFO: - Episode: 55/100, Reward: -1.626, Step: 200 +2023-04-16 11:07:31 - r - INFO: - Episode: 56/100, Reward: -1.645, Step: 200 +2023-04-16 11:07:35 - r - INFO: - Episode: 57/100, Reward: -1.664, Step: 200 +2023-04-16 11:07:39 - r - INFO: - Episode: 58/100, Reward: -1.740, Step: 200 +2023-04-16 11:07:43 - r - INFO: - Episode: 59/100, Reward: -1.680, Step: 200 +2023-04-16 11:07:46 - r - INFO: - Episode: 60/100, Reward: -1.795, Step: 200 +2023-04-16 11:07:53 - r - INFO: - Episode: 61/100, Reward: -1.789, Step: 200 +2023-04-16 11:07:57 - r - INFO: - Episode: 62/100, Reward: -1.763, Step: 200 +2023-04-16 11:08:01 - r - INFO: - Episode: 63/100, Reward: -1.821, Step: 200 +2023-04-16 11:08:05 - r - INFO: - Episode: 64/100, Reward: -1.844, Step: 200 +2023-04-16 11:08:09 - r - INFO: - Episode: 65/100, Reward: -1.918, Step: 200 +2023-04-16 11:08:13 - r - INFO: - Episode: 66/100, Reward: -1.805, Step: 200 +2023-04-16 11:08:16 - r - INFO: - Episode: 67/100, Reward: -1.785, Step: 200 +2023-04-16 11:08:20 - r - INFO: - Episode: 68/100, Reward: -1.861, Step: 200 +2023-04-16 11:08:24 - r - INFO: - Episode: 69/100, Reward: -1.835, Step: 200 +2023-04-16 11:08:27 - r - INFO: - Episode: 70/100, Reward: -1.856, Step: 200 +2023-04-16 11:08:35 - r - INFO: - Episode: 71/100, Reward: -1.933, Step: 200 +2023-04-16 11:08:38 - r - INFO: - Episode: 72/100, Reward: -1.982, Step: 200 +2023-04-16 11:08:42 - r - INFO: - Episode: 73/100, Reward: -1.895, Step: 200 +2023-04-16 11:08:45 - r - INFO: - Episode: 74/100, Reward: -1.956, Step: 200 +2023-04-16 11:08:48 - r - INFO: - Episode: 75/100, Reward: -1.884, Step: 200 +2023-04-16 11:08:51 - r - INFO: - Episode: 76/100, Reward: -1.884, Step: 200 +2023-04-16 11:08:55 - r - INFO: - Episode: 77/100, Reward: -1.821, Step: 200 +2023-04-16 11:08:59 - r - INFO: - Episode: 78/100, Reward: -1.874, Step: 200 +2023-04-16 11:09:02 - r - INFO: - Episode: 79/100, Reward: -1.897, Step: 200 +2023-04-16 11:09:06 - r - INFO: - Episode: 80/100, Reward: -1.519, Step: 200 +2023-04-16 11:09:13 - r - INFO: - Episode: 81/100, Reward: -0.946, Step: 200 +2023-04-16 11:09:17 - r - INFO: - Episode: 82/100, Reward: -0.804, Step: 200 +2023-04-16 11:09:21 - r - INFO: - Episode: 83/100, Reward: -1.212, Step: 200 +2023-04-16 11:09:24 - r - INFO: - Episode: 84/100, Reward: -1.332, Step: 200 +2023-04-16 11:09:28 - r - INFO: - Episode: 85/100, Reward: -1.605, Step: 200 +2023-04-16 11:09:32 - r - INFO: - Episode: 86/100, Reward: -1.488, Step: 200 +2023-04-16 11:09:36 - r - INFO: - Episode: 87/100, Reward: -1.515, Step: 200 +2023-04-16 11:09:40 - r - INFO: - Episode: 88/100, Reward: -1.570, Step: 200 +2023-04-16 11:09:43 - r - INFO: - Episode: 89/100, Reward: -1.669, Step: 200 +2023-04-16 11:09:47 - r - INFO: - Episode: 90/100, Reward: -1.649, Step: 200 +2023-04-16 11:09:54 - r - INFO: - Episode: 91/100, Reward: -1.573, Step: 200 +2023-04-16 11:09:58 - r - INFO: - Episode: 92/100, Reward: -1.586, Step: 200 +2023-04-16 11:10:02 - r - INFO: - Episode: 93/100, Reward: -1.620, Step: 200 +2023-04-16 11:10:06 - r - INFO: - Episode: 94/100, Reward: -1.777, Step: 200 +2023-04-16 11:10:10 - r - INFO: - Episode: 95/100, Reward: -1.823, Step: 200 +2023-04-16 11:10:13 - r - INFO: - Episode: 96/100, Reward: -1.755, Step: 200 +2023-04-16 11:10:17 - r - INFO: - Episode: 97/100, Reward: -1.776, Step: 200 +2023-04-16 11:10:21 - r - INFO: - Episode: 98/100, Reward: -1.732, Step: 200 +2023-04-16 11:10:25 - r - INFO: - Episode: 99/100, Reward: -1.771, Step: 200 +2023-04-16 11:10:28 - r - INFO: - Episode: 100/100, Reward: -1.783, Step: 200 +2023-04-16 11:10:32 - r - INFO: - Finish training! diff --git a/Pendulum-v1/Train_gym_TD3_20230416-110359/models/actor.pth b/Pendulum-v1/Train_gym_TD3_20230416-110359/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..c17690ddbd28cdbb98cedb83dc0a4361c3a87922 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_20230416-110359/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:214724c4b215e10e761cc6d58e6297c203462f3f06fadae121fcf7292ec326d0 +size 270167 diff --git a/Pendulum-v1/Train_gym_TD3_20230416-110359/models/critic_1.pth b/Pendulum-v1/Train_gym_TD3_20230416-110359/models/critic_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..a0f0b09e903928cdeb9a0c6bf547ed141a2fa528 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_20230416-110359/models/critic_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5170fbe3f10fd4f32c1cdd0635c4d70411fb765915ea58aafcf8e7f2fb921687 +size 271191 diff --git a/Pendulum-v1/Train_gym_TD3_20230416-110359/models/critic_2.pth b/Pendulum-v1/Train_gym_TD3_20230416-110359/models/critic_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..113773da9b235c99c3dfe129c6a72d570497bedc --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_20230416-110359/models/critic_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1647dea7ffb4bd6c42f116180b782c7fc8737774c7db2d15765968441ceb6e5b +size 271191 diff --git a/Pendulum-v1/Train_gym_TD3_20230416-110359/results/learning_curve.png b/Pendulum-v1/Train_gym_TD3_20230416-110359/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..331eeaccd97cc8f56e82b07b634a37fa6f52ba9a Binary files /dev/null and b/Pendulum-v1/Train_gym_TD3_20230416-110359/results/learning_curve.png differ diff --git a/Pendulum-v1/Train_gym_TD3_20230416-110359/results/res.csv b/Pendulum-v1/Train_gym_TD3_20230416-110359/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..aadba45e32993fa44dcc3ca6ddfdd72276c365df --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_20230416-110359/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,-832.4413162158323,200 +1,-968.4036079738249,200 +2,-874.5862306381961,200 +3,-875.7265412473744,200 +4,-897.6813364592543,200 +5,-1568.3188263407867,200 +6,-1459.1205728839695,200 +7,-1240.3308986778293,200 +8,-1134.8423085380664,200 +9,-5.36664590710244,200 +10,-650.5003287883866,200 +11,-8.377575660447649,200 +12,-4.5756982919706335,200 +13,-3.627742564378688,200 +14,-3.893889252491577,200 +15,-3.8192645919291257,200 +16,-3.4146845438017186,200 +17,-2.7635981143628925,200 +18,-2.4017956104121017,200 +19,-3.090478935757363,200 +20,-1.4809018908948575,200 +21,-1.2630981739795628,200 +22,-1.3180949949636327,200 +23,-1.564526506341134,200 +24,-1.5223962556642976,200 +25,-1.707998195828492,200 +26,-1.4226643582643066,200 +27,-1.5365465560984268,200 +28,-1.5239148208423863,200 +29,-1.4588095000150678,200 +30,-1.7359744252202507,200 +31,-2.0682890284663578,200 +32,-1.741025565051244,200 +33,-1.8488817690283263,200 +34,-1.39893985171838,200 +35,-1.4707914551719694,200 +36,-1.4797493231032233,200 +37,-1.4525725387039807,200 +38,-1.4103713564691112,200 +39,-1.3618191088451532,200 +40,-1.322506170922359,200 +41,-1.3324320670299856,200 +42,-1.4468348751323956,200 +43,-1.3540026843790918,200 +44,-1.3097180192945221,200 +45,-1.4315286227920205,200 +46,-1.4223011898198696,200 +47,-1.3803119102174926,200 +48,-1.383840087083155,200 +49,-1.285964624772213,200 +50,-1.2869926211551335,200 +51,-1.3431550684354865,200 +52,-1.3886544335397744,200 +53,-1.4759717484102066,200 +54,-1.626338794891603,200 +55,-1.644649473947096,200 +56,-1.6638041634800558,200 +57,-1.7398234937547132,200 +58,-1.680024377278263,200 +59,-1.7947581457511599,200 +60,-1.7891190829763475,200 +61,-1.7632050654476048,200 +62,-1.8211882825454626,200 +63,-1.8442752105447826,200 +64,-1.9176680463940405,200 +65,-1.8047871753057574,200 +66,-1.7846120425902607,200 +67,-1.8608097138458495,200 +68,-1.8351909400796054,200 +69,-1.8555094919167812,200 +70,-1.9327542950993424,200 +71,-1.9822565698772594,200 +72,-1.8954672018586112,200 +73,-1.9557719773884903,200 +74,-1.8844351899739862,200 +75,-1.883524519131489,200 +76,-1.8207078088705415,200 +77,-1.8741586443072578,200 +78,-1.897463137234909,200 +79,-1.5191278207011145,200 +80,-0.9455138132785595,200 +81,-0.8036250047483352,200 +82,-1.2124938658523443,200 +83,-1.3321687125553676,200 +84,-1.605031143258114,200 +85,-1.488155427061823,200 +86,-1.5151577723764103,200 +87,-1.5704170742235621,200 +88,-1.6685888937499647,200 +89,-1.6493756229468635,200 +90,-1.5729133065433396,200 +91,-1.5856557451539273,200 +92,-1.6196725146806101,200 +93,-1.7767693701781986,200 +94,-1.8225698294766743,200 +95,-1.7551168262928307,200 +96,-1.7761588517008813,200 +97,-1.7324583634362947,200 +98,-1.7706494685091234,200 +99,-1.7833268395875401,200 diff --git a/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/config.yaml b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8265cbc71525c09a02d8f26292ed62154a21a622 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/config.yaml @@ -0,0 +1,43 @@ +general_cfg: + algo_name: TD3_BC + device: cuda + env_name: gym + eval_eps: 5 + eval_per_episode: 1 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + mp_backend: mp + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 1 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.0003 + alpha: 5 + batch_size: 100 + buffer_size: 1000000 + critic_hidden_dim: 256 + critic_lr: 0.0003 + expert_path: tasks/Collect_gym_TD3_20230416-111040/traj/traj.pkl + expl_noise: 0.1 + explore_steps: 1000 + gamma: 0.99 + lmbda: 1 + noise_clip: 0.5 + normalize: false + policy_freq: 2 + policy_noise: 0.2 + tau: 0.005 + train_iterations: 1500 +env_cfg: + id: Pendulum-v1 + new_step_api: true + render_mode: null diff --git a/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/logs/log.txt b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..f7ecb7d3b7417c61f26104fca5276411eb3db0b7 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/logs/log.txt @@ -0,0 +1,54 @@ +2023-04-16 11:11:54 - r - INFO: - Hyperparameters: +2023-04-16 11:11:54 - r - INFO: - ================================================================================ +2023-04-16 11:11:54 - r - INFO: - Name Value Type +2023-04-16 11:11:54 - r - INFO: - env_name gym +2023-04-16 11:11:54 - r - INFO: - new_step_api 1 +2023-04-16 11:11:54 - r - INFO: - wrapper None +2023-04-16 11:11:54 - r - INFO: - render 0 +2023-04-16 11:11:54 - r - INFO: - render_mode None +2023-04-16 11:11:54 - r - INFO: - algo_name TD3_BC +2023-04-16 11:11:54 - r - INFO: - mode train +2023-04-16 11:11:54 - r - INFO: - mp_backend mp +2023-04-16 11:11:54 - r - INFO: - seed 1 +2023-04-16 11:11:54 - r - INFO: - device cuda +2023-04-16 11:11:54 - r - INFO: - train_eps 1 +2023-04-16 11:11:54 - r - INFO: - test_eps 10 +2023-04-16 11:11:54 - r - INFO: - eval_eps 5 +2023-04-16 11:11:54 - r - INFO: - eval_per_episode 1 +2023-04-16 11:11:54 - r - INFO: - max_steps 200 +2023-04-16 11:11:54 - r - INFO: - load_checkpoint 0 +2023-04-16 11:11:54 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-16 11:11:54 - r - INFO: - show_fig 0 +2023-04-16 11:11:54 - r - INFO: - save_fig 1 +2023-04-16 11:11:54 - r - INFO: - explore_steps 1000 +2023-04-16 11:11:54 - r - INFO: - policy_freq 2 +2023-04-16 11:11:54 - r - INFO: - actor_lr 0.0003 +2023-04-16 11:11:54 - r - INFO: - critic_lr 0.0003 +2023-04-16 11:11:54 - r - INFO: - actor_hidden_dim 256 +2023-04-16 11:11:54 - r - INFO: - critic_hidden_dim 256 +2023-04-16 11:11:54 - r - INFO: - gamma 0.99 +2023-04-16 11:11:54 - r - INFO: - tau 0.005 +2023-04-16 11:11:54 - r - INFO: - policy_noise 0.2 +2023-04-16 11:11:54 - r - INFO: - expl_noise 0.1 +2023-04-16 11:11:54 - r - INFO: - noise_clip 0.5 +2023-04-16 11:11:54 - r - INFO: - batch_size 100 +2023-04-16 11:11:54 - r - INFO: - buffer_size 1000000 +2023-04-16 11:11:54 - r - INFO: - alpha 5 +2023-04-16 11:11:54 - r - INFO: - lmbda 1 +2023-04-16 11:11:54 - r - INFO: - train_iterations 1500 +2023-04-16 11:11:54 - r - INFO: - normalize 0 +2023-04-16 11:11:54 - r - INFO: - expert_path tasks/Collect_gym_TD3_20230416-111040/traj/traj.pkl +2023-04-16 11:11:54 - r - INFO: - id Pendulum-v1 +2023-04-16 11:11:54 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_BC_20230416-111154 +2023-04-16 11:11:54 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_BC_20230416-111154/results +2023-04-16 11:11:54 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_BC_20230416-111154/logs +2023-04-16 11:11:54 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_BC_20230416-111154/traj +2023-04-16 11:11:54 - r - INFO: - video_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_BC_20230416-111154/videos +2023-04-16 11:11:54 - r - INFO: - ================================================================================ +2023-04-16 11:11:54 - r - INFO: - action_bound: 2.0 +2023-04-16 11:11:54 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-16 11:11:58 - r - INFO: - Start training! +2023-04-16 11:11:58 - r - INFO: - Env: gym, Algorithm: TD3_BC, Device: cuda +2023-04-16 11:28:22 - r - INFO: - Episode: 1/1, Reward: 0.000, Step: 0 +2023-04-16 11:28:25 - r - INFO: - Current episode 1 has the best eval reward: -0.477 +2023-04-16 11:28:25 - r - INFO: - Finish training! diff --git a/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/actor.pth b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..d01f0446b73b2992fa96229d97626fda87daf727 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b872a4af1157564eebb890940a46d0f1df2d2c1fc369f98cfe0d6424fc43a301 +size 270167 diff --git a/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/critic_1.pth b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/critic_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..b4410954afd07ce999fbdaf1dc4ecd8c785cddda --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/critic_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0427328c4fbb4d5cf265d77ac129550e7c291cfd776d483b85f38b6b743f91d +size 271191 diff --git a/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/critic_2.pth b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/critic_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..629a8140417ea7113f3248704110b8bbd3ece107 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/models/critic_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31df1a65697a1573a5def537c431e5f284bd32ea3c05d5071d9e06b13a176b38 +size 271191 diff --git a/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/results/learning_curve.png b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..39af68285c5f69853da271655d8b36ad031b0c6b Binary files /dev/null and b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/results/learning_curve.png differ diff --git a/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/results/res.csv b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..23456c507d5be0a0881755016fb76a0f234224e4 --- /dev/null +++ b/Pendulum-v1/Train_gym_TD3_BC_20230416-111154/results/res.csv @@ -0,0 +1,2 @@ +episodes,rewards,steps +0,0,0 diff --git a/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/config.yaml b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..990ec05ce564c8626ef09cf91b0b718b0d42fafe --- /dev/null +++ b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/config.yaml @@ -0,0 +1,40 @@ +general_cfg: + algo_name: TD3 + device: cpu + env_name: gym + eval_eps: 1 + eval_per_episode: 10 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + mp_backend: mp + n_workers: 4 + new_step_api: true + render: false + render_mode: human + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 200 + wrapper: null +algo_cfg: + actor_hidden_dim: 256 + actor_lr: 0.001 + batch_size: 100 + buffer_size: 1000000 + critic_hidden_dim: 256 + critic_lr: 0.001 + expl_noise: 0.1 + explore_steps: 1000 + gamma: 0.99 + global_best_reward: -1800 + noise_clip: 0.5 + policy_freq: 2 + policy_noise: 0.2 + tau: 0.005 +env_cfg: + id: Pendulum-v1 + new_step_api: true + render_mode: null diff --git a/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/logs/log.txt b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..de31fd5e050b0d71527b81c7e6b1aaaf96cc3bc5 --- /dev/null +++ b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/logs/log.txt @@ -0,0 +1,49 @@ +2023-04-16 21:40:19 - r - INFO: - Hyperparameters: +2023-04-16 21:40:19 - r - INFO: - ================================================================================ +2023-04-16 21:40:19 - r - INFO: - Name Value Type +2023-04-16 21:40:19 - r - INFO: - env_name gym +2023-04-16 21:40:19 - r - INFO: - new_step_api 1 +2023-04-16 21:40:19 - r - INFO: - wrapper None +2023-04-16 21:40:19 - r - INFO: - render 0 +2023-04-16 21:40:19 - r - INFO: - render_mode None +2023-04-16 21:40:19 - r - INFO: - algo_name TD3 +2023-04-16 21:40:19 - r - INFO: - mode train +2023-04-16 21:40:19 - r - INFO: - mp_backend mp +2023-04-16 21:40:19 - r - INFO: - seed 1 +2023-04-16 21:40:19 - r - INFO: - device cpu +2023-04-16 21:40:19 - r - INFO: - train_eps 200 +2023-04-16 21:40:19 - r - INFO: - test_eps 10 +2023-04-16 21:40:19 - r - INFO: - eval_eps 1 +2023-04-16 21:40:19 - r - INFO: - eval_per_episode 10 +2023-04-16 21:40:19 - r - INFO: - max_steps 200 +2023-04-16 21:40:19 - r - INFO: - load_checkpoint 0 +2023-04-16 21:40:19 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2023-04-16 21:40:19 - r - INFO: - show_fig 0 +2023-04-16 21:40:19 - r - INFO: - save_fig 1 +2023-04-16 21:40:19 - r - INFO: - n_workers 4 +2023-04-16 21:40:19 - r - INFO: - explore_steps 1000 +2023-04-16 21:40:19 - r - INFO: - policy_freq 2 +2023-04-16 21:40:19 - r - INFO: - actor_lr 0.001 +2023-04-16 21:40:19 - r - INFO: - critic_lr 0.001 +2023-04-16 21:40:19 - r - INFO: - actor_hidden_dim 256 +2023-04-16 21:40:19 - r - INFO: - critic_hidden_dim 256 +2023-04-16 21:40:19 - r - INFO: - gamma 0.99 +2023-04-16 21:40:19 - r - INFO: - tau 0.005 +2023-04-16 21:40:19 - r - INFO: - policy_noise 0.2 +2023-04-16 21:40:19 - r - INFO: - expl_noise 0.1 +2023-04-16 21:40:19 - r - INFO: - noise_clip 0.5 +2023-04-16 21:40:19 - r - INFO: - batch_size 100 +2023-04-16 21:40:19 - r - INFO: - buffer_size 1000000 +2023-04-16 21:40:19 - r - INFO: - global_best_reward -1800 +2023-04-16 21:40:19 - r - INFO: - id Pendulum-v1 +2023-04-16 21:40:19 - r - INFO: - task_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-214019 +2023-04-16 21:40:19 - r - INFO: - res_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-214019/results +2023-04-16 21:40:19 - r - INFO: - log_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-214019/logs +2023-04-16 21:40:19 - r - INFO: - traj_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-214019/traj +2023-04-16 21:40:19 - r - INFO: - video_dir /home/dingli/joyrl_offline/tasks/Train_gym_TD3_20230416-214019/videos +2023-04-16 21:40:19 - r - INFO: - ================================================================================ +2023-04-16 21:40:19 - r - INFO: - action_bound: 2.0 +2023-04-16 21:40:19 - r - INFO: - n_states: 3, n_actions: 1 +2023-04-16 21:40:19 - r - INFO: - Start training! +2023-04-16 21:40:19 - r - INFO: - Env: gym, Algorithm: TD3, Device: cpu +2023-04-16 22:12:57 - r - INFO: - Finish training! diff --git a/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/actor.pth b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/actor.pth new file mode 100644 index 0000000000000000000000000000000000000000..69cd81f9ba4724e9f8555c6dedc4bab222f9c5d4 --- /dev/null +++ b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/actor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c980b556a5a7de268589d9cb3f0d6753c443ebcf13f413245b3868298b706bd +size 270103 diff --git a/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/critic_1.pth b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/critic_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..dde1a62afe4109ff2066c1ffded39944a2241c9f --- /dev/null +++ b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/critic_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f23d87610b7fa55de3c0021d57f5cee965b2ddeffbfd5faafba5bc580eff9a15 +size 271127 diff --git a/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/critic_2.pth b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/critic_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..fef1f280978433a251648779a34d74f837e6b015 --- /dev/null +++ b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/models/critic_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49eac81686ca615c879ff219962324a4ced1994172109f54d9a7fd0eadd0f20f +size 271127 diff --git a/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/results/learning_curve.png b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..d2eaf9634e94e97c4fc1f7ee8caf7f60cbd5e410 Binary files /dev/null and b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/results/learning_curve.png differ diff --git a/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/results/res.csv b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..78177f7da453ea5f925c30b078658babbd63d2e7 --- /dev/null +++ b/Pendulum-v1/Train_gym_mp_TD3_20230416-214019/results/res.csv @@ -0,0 +1,202 @@ +episodes,rewards +0,-1072.9448384675063 +1,-909.8711787251437 +2,-1532.9813517069765 +3,-879.3261328336463 +4,-1591.153236375872 +5,-879.8182202586569 +6,-1489.8577529676788 +7,-1064.834944425985 +8,-1551.9169181924483 +9,-1703.8796379117107 +10,-869.8436689541852 +11,-1113.665053944186 +12,-1723.3880136434805 +13,-1597.6268606880687 +14,-902.0991604554731 +15,-1725.8806556302386 +16,-1170.9129103639796 +17,-1454.7498068267275 +18,-1700.9415823481365 +19,-1166.8871657217067 +20,-1428.4518810951602 +21,-1100.4574456887694 +22,-1116.467600388714 +23,-1355.7427347502608 +24,-1490.99141992789 +25,-1448.9902172139848 +26,-1283.0289983333184 +27,-1410.3698347594977 +28,-1340.378101732254 +29,-1128.0871441445981 +30,-1347.6619960523549 +31,-1286.5056711203247 +32,-1283.5363666926735 +33,-1303.9251899461797 +34,-1051.031230258747 +35,-1238.7996519157552 +36,-1279.6219150974548 +37,-1427.590409194811 +38,-984.2876706587689 +39,-1169.2721804822497 +40,-1235.9879076351597 +41,-814.1818641761856 +42,-1509.2324782226779 +43,-1111.2386558774397 +44,-1157.6426027857178 +45,-6.9856087333532635 +46,-1512.831822862592 +47,-1034.2480581448617 +48,-1103.3607933792528 +49,-3.7587473507987395 +50,-1512.6142707425745 +51,-951.200471385728 +52,-1512.5058091725261 +53,-1023.1372070178279 +54,-2.5126399225569647 +55,-917.3213421354695 +56,-1510.535138886379 +57,-823.5325188141269 +58,-2.319661115822847 +59,-666.2800698389791 +60,-756.3686391234536 +61,-1507.9254329194957 +62,-1.6968351264929238 +63,-401.7009697722153 +64,-449.3777454793288 +65,-1508.7143235424164 +66,-1.4445441782812496 +67,-129.89607463288004 +68,-429.86002511164776 +69,-1511.382598196328 +70,-1.431755244395173 +71,-128.43126134700674 +72,-274.99303476250105 +73,-1509.7185293144828 +74,-1.7145322936832699 +75,-127.54236659910637 +76,-268.8340517821617 +77,-1508.4549245105636 +78,-1.6600020174560317 +79,-126.72069775223915 +80,-250.15924407848686 +81,-1.447277234436987 +82,-1513.6710885226746 +83,-127.58822537252695 +84,-241.23654351829663 +85,-2.195537513269449 +86,-967.5708916684799 +87,-127.9714430766412 +88,-239.06679722015176 +89,-382.70186933262795 +90,-1.6247973702102902 +91,-129.157319495522 +92,-1.5587740979368212 +93,-239.22660210807007 +94,-249.8385379838881 +95,-128.32449617760543 +96,-238.6474104387132 +97,-1.2846364439171443 +98,-244.32088247228216 +99,-124.63246071326004 +100,-1.5229538221449892 +101,-236.3053144039616 +102,-243.26924355318715 +103,-125.16400737144856 +104,-237.14068010393783 +105,-241.75021429973614 +106,-1.4454788060678272 +107,-124.73919079478374 +108,-238.60776950644436 +109,-1.2768818799952555 +110,-237.09114891314525 +111,-123.43627409905358 +112,-239.10945389392492 +113,-1.5218834672538244 +114,-238.7879650760776 +115,-123.526405150471 +116,-234.43498805767734 +117,-1.6784879185281887 +118,-122.24488384330651 +119,-236.53524850824363 +120,-237.59400181353413 +121,-1.8702390939980649 +122,-121.7592760356512 +123,-234.20641070573603 +124,-235.88749117078123 +125,-121.12473296603382 +126,-2.5152303846605117 +127,-232.9363635008431 +128,-236.29440216175306 +129,-121.78436364902296 +130,-3.5707554297673747 +131,-236.26803185508024 +132,-240.98634028722387 +133,-3.876186876438371 +134,-122.28832346540379 +135,-242.90581323317267 +136,-236.15216729332914 +137,-123.51161950523465 +138,-4.778498969634311 +139,-243.85296258041646 +140,-240.1415463356868 +141,-3.450753086752825 +142,-119.88894451262325 +143,-242.58965953399047 +144,-240.12507988520315 +145,-119.53368195203215 +146,-2.560647622380151 +147,-239.91473026783808 +148,-238.8251418862308 +149,-1.9711086809553888 +150,-119.46489420023975 +151,-240.6229614778444 +152,-237.22953289715494 +153,-119.87374395522095 +154,-2.883408669946736 +155,-240.54276113000373 +156,-237.50950225352184 +157,-118.90586505061383 +158,-2.72514772925897 +159,-241.679632264929 +160,-118.76972463565845 +161,-237.39851544170725 +162,-2.8153439381396637 +163,-240.0488293765369 +164,-120.08670002663122 +165,-238.26440152524086 +166,-240.79536720724906 +167,-3.4048768630382154 +168,-236.2203332690807 +169,-118.87468539888819 +170,-241.7897122522074 +171,-3.5613225427547484 +172,-236.40781715368377 +173,-118.98051940106194 +174,-241.13072758183552 +175,-3.3241864478586733 +176,-233.7141539713175 +177,-121.81189967091031 +178,-238.94906266182787 +179,-3.4757796654361157 +180,-231.71566809521153 +181,-120.40173420108931 +182,-239.35367830011688 +183,-3.1264665317188394 +184,-229.5795597123186 +185,-118.75762746821387 +186,-237.69031568529616 +187,-1.9563314615159402 +188,-227.2294661477471 +189,-116.62081045779091 +190,-0.9101941544864205 +191,-238.2820070842917 +192,-228.8648275623054 +193,-117.86406148446723 +194,-2.8777925872331886 +195,-237.8676778409411 +196,-231.51023664446075 +197,-119.15685383786622 +198,-3.4756682503060814 +199,-237.10006005394453 +200,-229.79531353800593 diff --git a/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/config.yaml b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2a635bf5e7b4f16578533b158ddd9f26d38d6f4 --- /dev/null +++ b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/config.yaml @@ -0,0 +1,25 @@ +general_cfg: + algo_name: QLearning + device: cpu + env_name: Racetrack-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_Racetrack-v0_QLearning_20221128-154935 + max_steps: 200 + mode: test + new_step_api: true + render: true + render_mode: human + save_fig: true + seed: 10 + show_fig: false + test_eps: 20 + train_eps: 400 + wrapper: null +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.9 + lr: 0.1 diff --git a/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/logs/log.txt b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..04fb491167a16cb6d149f4d5ed2da6b536bbc178 --- /dev/null +++ b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/logs/log.txt @@ -0,0 +1,51 @@ +2022-11-28 15:58:53 - r - INFO: - Hyperparameters: +2022-11-28 15:58:53 - r - INFO: - ================================================================================ +2022-11-28 15:58:53 - r - INFO: - Name Value Type +2022-11-28 15:58:53 - r - INFO: - env_name Racetrack-v0 +2022-11-28 15:58:53 - r - INFO: - new_step_api 1 +2022-11-28 15:58:53 - r - INFO: - wrapper None +2022-11-28 15:58:53 - r - INFO: - render 1 +2022-11-28 15:58:53 - r - INFO: - algo_name QLearning +2022-11-28 15:58:53 - r - INFO: - mode test +2022-11-28 15:58:53 - r - INFO: - seed 10 +2022-11-28 15:58:53 - r - INFO: - device cpu +2022-11-28 15:58:53 - r - INFO: - train_eps 400 +2022-11-28 15:58:53 - r - INFO: - test_eps 20 +2022-11-28 15:58:53 - r - INFO: - eval_eps 10 +2022-11-28 15:58:53 - r - INFO: - eval_per_episode 5 +2022-11-28 15:58:53 - r - INFO: - max_steps 200 +2022-11-28 15:58:53 - r - INFO: - load_checkpoint 1 +2022-11-28 15:58:53 - r - INFO: - load_path Train_Racetrack-v0_QLearning_20221128-154935 +2022-11-28 15:58:53 - r - INFO: - show_fig 0 +2022-11-28 15:58:53 - r - INFO: - save_fig 1 +2022-11-28 15:58:53 - r - INFO: - render_mode human +2022-11-28 15:58:53 - r - INFO: - epsilon_start 0.95 +2022-11-28 15:58:53 - r - INFO: - epsilon_end 0.01 +2022-11-28 15:58:53 - r - INFO: - epsilon_decay 300 +2022-11-28 15:58:53 - r - INFO: - gamma 0.9 +2022-11-28 15:58:53 - r - INFO: - lr 0.1 +2022-11-28 15:58:53 - r - INFO: - ================================================================================ +2022-11-28 15:58:54 - r - INFO: - n_states: 4, n_actions: 9 +2022-11-28 15:58:54 - r - INFO: - Start testing! +2022-11-28 15:58:54 - r - INFO: - Env: Racetrack-v0, Algorithm: QLearning, Device: cpu +2022-11-28 15:58:56 - r - INFO: - Episode: 1/20, Reward: 4.000, Step: 6 +2022-11-28 15:58:56 - r - INFO: - Episode: 2/20, Reward: 4.000, Step: 6 +2022-11-28 15:58:59 - r - INFO: - Episode: 3/20, Reward: -13.000, Step: 13 +2022-11-28 15:59:00 - r - INFO: - Episode: 4/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:01 - r - INFO: - Episode: 5/20, Reward: 3.000, Step: 7 +2022-11-28 15:59:03 - r - INFO: - Episode: 6/20, Reward: -13.000, Step: 13 +2022-11-28 15:59:04 - r - INFO: - Episode: 7/20, Reward: 2.000, Step: 8 +2022-11-28 15:59:06 - r - INFO: - Episode: 8/20, Reward: -12.000, Step: 12 +2022-11-28 15:59:07 - r - INFO: - Episode: 9/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:08 - r - INFO: - Episode: 10/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:09 - r - INFO: - Episode: 11/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:10 - r - INFO: - Episode: 12/20, Reward: 3.000, Step: 7 +2022-11-28 15:59:11 - r - INFO: - Episode: 13/20, Reward: 2.000, Step: 8 +2022-11-28 15:59:12 - r - INFO: - Episode: 14/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:13 - r - INFO: - Episode: 15/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:14 - r - INFO: - Episode: 16/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:15 - r - INFO: - Episode: 17/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:16 - r - INFO: - Episode: 18/20, Reward: 4.000, Step: 6 +2022-11-28 15:59:18 - r - INFO: - Episode: 19/20, Reward: 1.000, Step: 9 +2022-11-28 15:59:19 - r - INFO: - Episode: 20/20, Reward: 3.000, Step: 7 +2022-11-28 15:59:19 - r - INFO: - Finish testing! diff --git a/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/models/Qleaning_model.pkl b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/models/Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..eeff794be8cde1412de6fdd7d143921e85c19b4b --- /dev/null +++ b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/models/Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c953dba9e2d336b3ead136902a4e6f2d99d29b66ae3210a485cfef6ac09fa72 +size 156911 diff --git a/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/results/learning_curve.png b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..cfaf3248d838b12263e5d24ed3b22800686b295c Binary files /dev/null and b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/results/learning_curve.png differ diff --git a/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/results/res.csv b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..2691b6e9f56671d9b5ca47165590353c23a6f53b --- /dev/null +++ b/Racetrack-v0/Test_Racetrack-v0_QLearning_20221128-155853/results/res.csv @@ -0,0 +1,21 @@ +episodes,rewards,steps +0,4,6 +1,4,6 +2,-13,13 +3,4,6 +4,3,7 +5,-13,13 +6,2,8 +7,-12,12 +8,4,6 +9,4,6 +10,4,6 +11,3,7 +12,2,8 +13,4,6 +14,4,6 +15,4,6 +16,4,6 +17,4,6 +18,1,9 +19,3,7 diff --git a/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/config.yaml b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4076881be5f04259bbf295c928cf20fd43f1b94e --- /dev/null +++ b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/config.yaml @@ -0,0 +1,24 @@ +general_cfg: + algo_name: QLearning + device: cpu + env_name: Racetrack-v0 + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + new_step_api: true + render: false + save_fig: true + seed: 10 + show_fig: false + test_eps: 20 + train_eps: 400 + wrapper: null +algo_cfg: + epsilon_decay: 300 + epsilon_end: 0.01 + epsilon_start: 0.95 + gamma: 0.9 + lr: 0.1 diff --git a/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/logs/log.txt b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..91278d1f4d6f146ea9e7c883938460871685af4a --- /dev/null +++ b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/logs/log.txt @@ -0,0 +1,445 @@ +2022-11-28 15:49:35 - r - INFO: - Hyperparameters: +2022-11-28 15:49:35 - r - INFO: - ================================================================================ +2022-11-28 15:49:35 - r - INFO: - Name Value Type +2022-11-28 15:49:35 - r - INFO: - env_name Racetrack-v0 +2022-11-28 15:49:35 - r - INFO: - new_step_api 1 +2022-11-28 15:49:35 - r - INFO: - wrapper None +2022-11-28 15:49:35 - r - INFO: - render 0 +2022-11-28 15:49:35 - r - INFO: - algo_name QLearning +2022-11-28 15:49:35 - r - INFO: - mode train +2022-11-28 15:49:35 - r - INFO: - seed 10 +2022-11-28 15:49:35 - r - INFO: - device cpu +2022-11-28 15:49:35 - r - INFO: - train_eps 400 +2022-11-28 15:49:35 - r - INFO: - test_eps 20 +2022-11-28 15:49:35 - r - INFO: - eval_eps 10 +2022-11-28 15:49:35 - r - INFO: - eval_per_episode 5 +2022-11-28 15:49:35 - r - INFO: - max_steps 200 +2022-11-28 15:49:35 - r - INFO: - load_checkpoint 0 +2022-11-28 15:49:35 - r - INFO: - load_path Train_CartPole-v1_DQN_20221026-054757 +2022-11-28 15:49:35 - r - INFO: - show_fig 0 +2022-11-28 15:49:35 - r - INFO: - save_fig 1 +2022-11-28 15:49:35 - r - INFO: - epsilon_start 0.95 +2022-11-28 15:49:35 - r - INFO: - epsilon_end 0.01 +2022-11-28 15:49:35 - r - INFO: - epsilon_decay 300 +2022-11-28 15:49:35 - r - INFO: - gamma 0.9 +2022-11-28 15:49:35 - r - INFO: - lr 0.1 +2022-11-28 15:49:35 - r - INFO: - ================================================================================ +2022-11-28 15:49:36 - r - INFO: - n_states: 4, n_actions: 9 +2022-11-28 15:49:36 - r - INFO: - Start training! +2022-11-28 15:49:36 - r - INFO: - Env: Racetrack-v0, Algorithm: QLearning, Device: cpu +2022-11-28 15:49:36 - r - INFO: - Episode: 1/400, Reward: -850.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 2/400, Reward: -780.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 3/400, Reward: -730.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 4/400, Reward: -650.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 5/400, Reward: -540.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Current episode 5 has the best eval reward: -676.000 +2022-11-28 15:49:36 - r - INFO: - Episode: 6/400, Reward: -560.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 7/400, Reward: -550.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 8/400, Reward: -580.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 9/400, Reward: -550.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 10/400, Reward: -560.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Current episode 10 has the best eval reward: -614.000 +2022-11-28 15:49:36 - r - INFO: - Episode: 11/400, Reward: -560.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 12/400, Reward: -530.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 13/400, Reward: -550.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 14/400, Reward: -570.000, Step: 200 +2022-11-28 15:49:36 - r - INFO: - Episode: 15/400, Reward: -620.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Current episode 15 has the best eval reward: -606.000 +2022-11-28 15:49:37 - r - INFO: - Episode: 16/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 17/400, Reward: -560.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 18/400, Reward: -550.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 19/400, Reward: -540.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 20/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Current episode 20 has the best eval reward: -555.000 +2022-11-28 15:49:37 - r - INFO: - Episode: 21/400, Reward: -550.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 22/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 23/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 24/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 25/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Current episode 25 has the best eval reward: -527.000 +2022-11-28 15:49:37 - r - INFO: - Episode: 26/400, Reward: -540.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 27/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 28/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 29/400, Reward: -500.000, Step: 200 +2022-11-28 15:49:37 - r - INFO: - Episode: 30/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Current episode 30 has the best eval reward: -470.000 +2022-11-28 15:49:38 - r - INFO: - Episode: 31/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 32/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 33/400, Reward: -530.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 34/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 35/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 36/400, Reward: -530.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 37/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 38/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 39/400, Reward: -366.000, Step: 156 +2022-11-28 15:49:38 - r - INFO: - Episode: 40/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 41/400, Reward: -412.000, Step: 172 +2022-11-28 15:49:38 - r - INFO: - Episode: 42/400, Reward: -490.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 43/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 44/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Episode: 45/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:38 - r - INFO: - Current episode 45 has the best eval reward: -463.000 +2022-11-28 15:49:38 - r - INFO: - Episode: 46/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 47/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 48/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 49/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 50/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Current episode 50 has the best eval reward: -218.000 +2022-11-28 15:49:39 - r - INFO: - Episode: 51/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 52/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 53/400, Reward: -500.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 54/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 55/400, Reward: -369.000, Step: 169 +2022-11-28 15:49:39 - r - INFO: - Episode: 56/400, Reward: -450.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 57/400, Reward: -540.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 58/400, Reward: -53.000, Step: 33 +2022-11-28 15:49:39 - r - INFO: - Episode: 59/400, Reward: -500.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 60/400, Reward: -450.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 61/400, Reward: -530.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 62/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 63/400, Reward: -500.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 64/400, Reward: -490.000, Step: 200 +2022-11-28 15:49:39 - r - INFO: - Episode: 65/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 66/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 67/400, Reward: -490.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 68/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 69/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 70/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Current episode 70 has the best eval reward: -159.900 +2022-11-28 15:49:40 - r - INFO: - Episode: 71/400, Reward: -500.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 72/400, Reward: -343.000, Step: 153 +2022-11-28 15:49:40 - r - INFO: - Episode: 73/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 74/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 75/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 76/400, Reward: -490.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 77/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 78/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 79/400, Reward: -199.000, Step: 89 +2022-11-28 15:49:40 - r - INFO: - Episode: 80/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 81/400, Reward: -44.000, Step: 24 +2022-11-28 15:49:40 - r - INFO: - Episode: 82/400, Reward: -520.000, Step: 200 +2022-11-28 15:49:40 - r - INFO: - Episode: 83/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 84/400, Reward: -480.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 85/400, Reward: -9.000, Step: 9 +2022-11-28 15:49:41 - r - INFO: - Episode: 86/400, Reward: -450.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 87/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 88/400, Reward: -134.000, Step: 64 +2022-11-28 15:49:41 - r - INFO: - Episode: 89/400, Reward: -60.000, Step: 30 +2022-11-28 15:49:41 - r - INFO: - Episode: 90/400, Reward: -510.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 91/400, Reward: -111.000, Step: 51 +2022-11-28 15:49:41 - r - INFO: - Episode: 92/400, Reward: -99.000, Step: 49 +2022-11-28 15:49:41 - r - INFO: - Episode: 93/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 94/400, Reward: -323.000, Step: 133 +2022-11-28 15:49:41 - r - INFO: - Episode: 95/400, Reward: -218.000, Step: 98 +2022-11-28 15:49:41 - r - INFO: - Episode: 96/400, Reward: -29.000, Step: 19 +2022-11-28 15:49:41 - r - INFO: - Episode: 97/400, Reward: -424.000, Step: 194 +2022-11-28 15:49:41 - r - INFO: - Episode: 98/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 99/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 100/400, Reward: -240.000, Step: 120 +2022-11-28 15:49:41 - r - INFO: - Episode: 101/400, Reward: -17.000, Step: 17 +2022-11-28 15:49:41 - r - INFO: - Episode: 102/400, Reward: -11.000, Step: 11 +2022-11-28 15:49:41 - r - INFO: - Episode: 103/400, Reward: -83.000, Step: 43 +2022-11-28 15:49:41 - r - INFO: - Episode: 104/400, Reward: -460.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 105/400, Reward: -11.000, Step: 11 +2022-11-28 15:49:41 - r - INFO: - Episode: 106/400, Reward: -470.000, Step: 200 +2022-11-28 15:49:41 - r - INFO: - Episode: 107/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:41 - r - INFO: - Episode: 108/400, Reward: -85.000, Step: 45 +2022-11-28 15:49:41 - r - INFO: - Episode: 109/400, Reward: -204.000, Step: 104 +2022-11-28 15:49:41 - r - INFO: - Episode: 110/400, Reward: -139.000, Step: 59 +2022-11-28 15:49:42 - r - INFO: - Current episode 110 has the best eval reward: -114.800 +2022-11-28 15:49:42 - r - INFO: - Episode: 111/400, Reward: -39.000, Step: 29 +2022-11-28 15:49:42 - r - INFO: - Episode: 112/400, Reward: -58.000, Step: 38 +2022-11-28 15:49:42 - r - INFO: - Episode: 113/400, Reward: -200.000, Step: 90 +2022-11-28 15:49:42 - r - INFO: - Episode: 114/400, Reward: -105.000, Step: 45 +2022-11-28 15:49:42 - r - INFO: - Episode: 115/400, Reward: -107.000, Step: 47 +2022-11-28 15:49:42 - r - INFO: - Episode: 116/400, Reward: -179.000, Step: 79 +2022-11-28 15:49:42 - r - INFO: - Episode: 117/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 118/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 119/400, Reward: -37.000, Step: 27 +2022-11-28 15:49:42 - r - INFO: - Episode: 120/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:42 - r - INFO: - Current episode 120 has the best eval reward: -17.200 +2022-11-28 15:49:42 - r - INFO: - Episode: 121/400, Reward: -61.000, Step: 31 +2022-11-28 15:49:42 - r - INFO: - Episode: 122/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 123/400, Reward: -21.000, Step: 21 +2022-11-28 15:49:42 - r - INFO: - Episode: 124/400, Reward: -40.000, Step: 30 +2022-11-28 15:49:42 - r - INFO: - Episode: 125/400, Reward: -49.000, Step: 29 +2022-11-28 15:49:42 - r - INFO: - Episode: 126/400, Reward: -12.000, Step: 12 +2022-11-28 15:49:42 - r - INFO: - Episode: 127/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 128/400, Reward: -140.000, Step: 70 +2022-11-28 15:49:42 - r - INFO: - Episode: 129/400, Reward: -13.000, Step: 13 +2022-11-28 15:49:42 - r - INFO: - Episode: 130/400, Reward: -115.000, Step: 55 +2022-11-28 15:49:42 - r - INFO: - Episode: 131/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 132/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 133/400, Reward: -20.000, Step: 20 +2022-11-28 15:49:42 - r - INFO: - Episode: 134/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 135/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 136/400, Reward: -17.000, Step: 17 +2022-11-28 15:49:42 - r - INFO: - Episode: 137/400, Reward: -61.000, Step: 31 +2022-11-28 15:49:42 - r - INFO: - Episode: 138/400, Reward: -19.000, Step: 19 +2022-11-28 15:49:42 - r - INFO: - Episode: 139/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:42 - r - INFO: - Episode: 140/400, Reward: -16.000, Step: 16 +2022-11-28 15:49:42 - r - INFO: - Current episode 140 has the best eval reward: -3.800 +2022-11-28 15:49:42 - r - INFO: - Episode: 141/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 142/400, Reward: -13.000, Step: 13 +2022-11-28 15:49:42 - r - INFO: - Episode: 143/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 144/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:42 - r - INFO: - Episode: 145/400, Reward: -108.000, Step: 58 +2022-11-28 15:49:42 - r - INFO: - Episode: 146/400, Reward: -136.000, Step: 76 +2022-11-28 15:49:42 - r - INFO: - Episode: 147/400, Reward: -99.000, Step: 49 +2022-11-28 15:49:42 - r - INFO: - Episode: 148/400, Reward: -69.000, Step: 39 +2022-11-28 15:49:42 - r - INFO: - Episode: 149/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 150/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 151/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 152/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 153/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 154/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 155/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 156/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 157/400, Reward: -140.000, Step: 70 +2022-11-28 15:49:42 - r - INFO: - Episode: 158/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 159/400, Reward: -17.000, Step: 17 +2022-11-28 15:49:42 - r - INFO: - Episode: 160/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:42 - r - INFO: - Episode: 161/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 162/400, Reward: -31.000, Step: 21 +2022-11-28 15:49:42 - r - INFO: - Episode: 163/400, Reward: -18.000, Step: 18 +2022-11-28 15:49:42 - r - INFO: - Episode: 164/400, Reward: -88.000, Step: 48 +2022-11-28 15:49:42 - r - INFO: - Episode: 165/400, Reward: -1.000, Step: 11 +2022-11-28 15:49:42 - r - INFO: - Episode: 166/400, Reward: -32.000, Step: 22 +2022-11-28 15:49:42 - r - INFO: - Episode: 167/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 168/400, Reward: -34.000, Step: 24 +2022-11-28 15:49:42 - r - INFO: - Episode: 169/400, Reward: -102.000, Step: 52 +2022-11-28 15:49:42 - r - INFO: - Episode: 170/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Current episode 170 has the best eval reward: -3.800 +2022-11-28 15:49:42 - r - INFO: - Episode: 171/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 172/400, Reward: -13.000, Step: 13 +2022-11-28 15:49:42 - r - INFO: - Episode: 173/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 174/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 175/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:42 - r - INFO: - Episode: 176/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 177/400, Reward: -37.000, Step: 27 +2022-11-28 15:49:42 - r - INFO: - Episode: 178/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 179/400, Reward: -45.000, Step: 25 +2022-11-28 15:49:42 - r - INFO: - Episode: 180/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 181/400, Reward: -27.000, Step: 17 +2022-11-28 15:49:42 - r - INFO: - Episode: 182/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 183/400, Reward: -42.000, Step: 22 +2022-11-28 15:49:42 - r - INFO: - Episode: 184/400, Reward: -12.000, Step: 12 +2022-11-28 15:49:42 - r - INFO: - Episode: 185/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 186/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:42 - r - INFO: - Episode: 187/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 188/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 189/400, Reward: -29.000, Step: 19 +2022-11-28 15:49:42 - r - INFO: - Episode: 190/400, Reward: -19.000, Step: 19 +2022-11-28 15:49:42 - r - INFO: - Episode: 191/400, Reward: -19.000, Step: 19 +2022-11-28 15:49:42 - r - INFO: - Episode: 192/400, Reward: -19.000, Step: 19 +2022-11-28 15:49:42 - r - INFO: - Episode: 193/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 194/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:42 - r - INFO: - Episode: 195/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 196/400, Reward: -147.000, Step: 77 +2022-11-28 15:49:42 - r - INFO: - Episode: 197/400, Reward: -48.000, Step: 28 +2022-11-28 15:49:42 - r - INFO: - Episode: 198/400, Reward: -1.000, Step: 11 +2022-11-28 15:49:42 - r - INFO: - Episode: 199/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 200/400, Reward: -76.000, Step: 36 +2022-11-28 15:49:42 - r - INFO: - Episode: 201/400, Reward: 0.000, Step: 10 +2022-11-28 15:49:42 - r - INFO: - Episode: 202/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 203/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 204/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 205/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:42 - r - INFO: - Current episode 205 has the best eval reward: 1.600 +2022-11-28 15:49:42 - r - INFO: - Episode: 206/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:42 - r - INFO: - Episode: 207/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 208/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:42 - r - INFO: - Episode: 209/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:42 - r - INFO: - Episode: 210/400, Reward: -49.000, Step: 29 +2022-11-28 15:49:43 - r - INFO: - Episode: 211/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 212/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 213/400, Reward: -17.000, Step: 17 +2022-11-28 15:49:43 - r - INFO: - Episode: 214/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 215/400, Reward: -13.000, Step: 13 +2022-11-28 15:49:43 - r - INFO: - Episode: 216/400, Reward: -13.000, Step: 13 +2022-11-28 15:49:43 - r - INFO: - Episode: 217/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 218/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 219/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 220/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 221/400, Reward: -17.000, Step: 17 +2022-11-28 15:49:43 - r - INFO: - Episode: 222/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 223/400, Reward: -14.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 224/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:43 - r - INFO: - Episode: 225/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 226/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 227/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 228/400, Reward: -31.000, Step: 21 +2022-11-28 15:49:43 - r - INFO: - Episode: 229/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 230/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 231/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 232/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 233/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:43 - r - INFO: - Episode: 234/400, Reward: -14.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 235/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 236/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 237/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 238/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 239/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 240/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 241/400, Reward: -21.000, Step: 21 +2022-11-28 15:49:43 - r - INFO: - Episode: 242/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 243/400, Reward: -27.000, Step: 17 +2022-11-28 15:49:43 - r - INFO: - Episode: 244/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 245/400, Reward: -16.000, Step: 16 +2022-11-28 15:49:43 - r - INFO: - Episode: 246/400, Reward: -32.000, Step: 22 +2022-11-28 15:49:43 - r - INFO: - Episode: 247/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 248/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 249/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 250/400, Reward: -12.000, Step: 12 +2022-11-28 15:49:43 - r - INFO: - Episode: 251/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 252/400, Reward: -49.000, Step: 29 +2022-11-28 15:49:43 - r - INFO: - Episode: 253/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 254/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 255/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 256/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 257/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 258/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 259/400, Reward: -54.000, Step: 34 +2022-11-28 15:49:43 - r - INFO: - Episode: 260/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:43 - r - INFO: - Episode: 261/400, Reward: -34.000, Step: 24 +2022-11-28 15:49:43 - r - INFO: - Episode: 262/400, Reward: -18.000, Step: 18 +2022-11-28 15:49:43 - r - INFO: - Episode: 263/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 264/400, Reward: -45.000, Step: 25 +2022-11-28 15:49:43 - r - INFO: - Episode: 265/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 266/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 267/400, Reward: -30.000, Step: 20 +2022-11-28 15:49:43 - r - INFO: - Episode: 268/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 269/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 270/400, Reward: -16.000, Step: 16 +2022-11-28 15:49:43 - r - INFO: - Episode: 271/400, Reward: -44.000, Step: 24 +2022-11-28 15:49:43 - r - INFO: - Episode: 272/400, Reward: -4.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 273/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 274/400, Reward: -14.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 275/400, Reward: -29.000, Step: 19 +2022-11-28 15:49:43 - r - INFO: - Episode: 276/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 277/400, Reward: -21.000, Step: 21 +2022-11-28 15:49:43 - r - INFO: - Episode: 278/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 279/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 280/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 281/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 282/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 283/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 284/400, Reward: -14.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 285/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 286/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 287/400, Reward: -34.000, Step: 24 +2022-11-28 15:49:43 - r - INFO: - Episode: 288/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 289/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 290/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Current episode 290 has the best eval reward: 3.300 +2022-11-28 15:49:43 - r - INFO: - Episode: 291/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 292/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 293/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 294/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 295/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 296/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 297/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 298/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 299/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 300/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 301/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 302/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 303/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 304/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 305/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 306/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 307/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 308/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 309/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 310/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 311/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 312/400, Reward: -19.000, Step: 19 +2022-11-28 15:49:43 - r - INFO: - Episode: 313/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 314/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 315/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 316/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 317/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 318/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 319/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 320/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 321/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 322/400, Reward: -29.000, Step: 19 +2022-11-28 15:49:43 - r - INFO: - Episode: 323/400, Reward: -14.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 324/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 325/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 326/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 327/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 328/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 329/400, Reward: 1.000, Step: 9 +2022-11-28 15:49:43 - r - INFO: - Episode: 330/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 331/400, Reward: -11.000, Step: 11 +2022-11-28 15:49:43 - r - INFO: - Episode: 332/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 333/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 334/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 335/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 336/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 337/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:43 - r - INFO: - Episode: 338/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 339/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 340/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 341/400, Reward: -16.000, Step: 16 +2022-11-28 15:49:43 - r - INFO: - Episode: 342/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 343/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 344/400, Reward: -39.000, Step: 29 +2022-11-28 15:49:43 - r - INFO: - Episode: 345/400, Reward: -28.000, Step: 18 +2022-11-28 15:49:43 - r - INFO: - Episode: 346/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 347/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 348/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 349/400, Reward: -17.000, Step: 17 +2022-11-28 15:49:43 - r - INFO: - Episode: 350/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 351/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 352/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 353/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 354/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 355/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 356/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 357/400, Reward: 1.000, Step: 9 +2022-11-28 15:49:43 - r - INFO: - Episode: 358/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 359/400, Reward: -13.000, Step: 13 +2022-11-28 15:49:43 - r - INFO: - Episode: 360/400, Reward: -32.000, Step: 22 +2022-11-28 15:49:43 - r - INFO: - Episode: 361/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 362/400, Reward: -14.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 363/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 364/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 365/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 366/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 367/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 368/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 369/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 370/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 371/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 372/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:43 - r - INFO: - Episode: 373/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 374/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 375/400, Reward: -19.000, Step: 19 +2022-11-28 15:49:43 - r - INFO: - Episode: 376/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 377/400, Reward: -14.000, Step: 14 +2022-11-28 15:49:43 - r - INFO: - Episode: 378/400, Reward: -35.000, Step: 25 +2022-11-28 15:49:43 - r - INFO: - Episode: 379/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 380/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 381/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 382/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 383/400, Reward: -32.000, Step: 22 +2022-11-28 15:49:43 - r - INFO: - Episode: 384/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 385/400, Reward: -13.000, Step: 13 +2022-11-28 15:49:43 - r - INFO: - Episode: 386/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 387/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:43 - r - INFO: - Episode: 388/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 389/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 390/400, Reward: 0.000, Step: 10 +2022-11-28 15:49:43 - r - INFO: - Episode: 391/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 392/400, Reward: 4.000, Step: 6 +2022-11-28 15:49:43 - r - INFO: - Episode: 393/400, Reward: -36.000, Step: 26 +2022-11-28 15:49:43 - r - INFO: - Episode: 394/400, Reward: -35.000, Step: 25 +2022-11-28 15:49:43 - r - INFO: - Episode: 395/400, Reward: -15.000, Step: 15 +2022-11-28 15:49:43 - r - INFO: - Episode: 396/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 397/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 398/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Episode: 399/400, Reward: 2.000, Step: 8 +2022-11-28 15:49:43 - r - INFO: - Episode: 400/400, Reward: 3.000, Step: 7 +2022-11-28 15:49:43 - r - INFO: - Finish training! diff --git a/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/models/Qleaning_model.pkl b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/models/Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..eeff794be8cde1412de6fdd7d143921e85c19b4b --- /dev/null +++ b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/models/Qleaning_model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c953dba9e2d336b3ead136902a4e6f2d99d29b66ae3210a485cfef6ac09fa72 +size 156911 diff --git a/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/results/learning_curve.png b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..48a3313ca1a253d29a4888ee2beccf399445b9f2 Binary files /dev/null and b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/results/learning_curve.png differ diff --git a/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/results/res.csv b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..cb51e08190c872ec6b2e3f9d9f885b2d3e322116 --- /dev/null +++ b/Racetrack-v0/Train_Racetrack-v0_QLearning_20221128-154935/results/res.csv @@ -0,0 +1,401 @@ +episodes,rewards,steps +0,-850,200 +1,-780,200 +2,-730,200 +3,-650,200 +4,-540,200 +5,-560,200 +6,-550,200 +7,-580,200 +8,-550,200 +9,-560,200 +10,-560,200 +11,-530,200 +12,-550,200 +13,-570,200 +14,-620,200 +15,-510,200 +16,-560,200 +17,-550,200 +18,-540,200 +19,-520,200 +20,-550,200 +21,-520,200 +22,-520,200 +23,-520,200 +24,-510,200 +25,-540,200 +26,-510,200 +27,-460,200 +28,-500,200 +29,-520,200 +30,-470,200 +31,-520,200 +32,-530,200 +33,-470,200 +34,-480,200 +35,-530,200 +36,-510,200 +37,-470,200 +38,-366,156 +39,-510,200 +40,-412,172 +41,-490,200 +42,-480,200 +43,-470,200 +44,-470,200 +45,-460,200 +46,-510,200 +47,-480,200 +48,-470,200 +49,-460,200 +50,-510,200 +51,-510,200 +52,-500,200 +53,-470,200 +54,-369,169 +55,-450,200 +56,-540,200 +57,-53,33 +58,-500,200 +59,-450,200 +60,-530,200 +61,-480,200 +62,-500,200 +63,-490,200 +64,-520,200 +65,-460,200 +66,-490,200 +67,-510,200 +68,-480,200 +69,-480,200 +70,-500,200 +71,-343,153 +72,-470,200 +73,-460,200 +74,-480,200 +75,-490,200 +76,-470,200 +77,-480,200 +78,-199,89 +79,-480,200 +80,-44,24 +81,-520,200 +82,-460,200 +83,-480,200 +84,-9,9 +85,-450,200 +86,-460,200 +87,-134,64 +88,-60,30 +89,-510,200 +90,-111,51 +91,-99,49 +92,-470,200 +93,-323,133 +94,-218,98 +95,-29,19 +96,-424,194 +97,-470,200 +98,-470,200 +99,-240,120 +100,-17,17 +101,-11,11 +102,-83,43 +103,-460,200 +104,-11,11 +105,-470,200 +106,4,6 +107,-85,45 +108,-204,104 +109,-139,59 +110,-39,29 +111,-58,38 +112,-200,90 +113,-105,45 +114,-107,47 +115,-179,79 +116,4,6 +117,4,6 +118,-37,27 +119,2,8 +120,-61,31 +121,4,6 +122,-21,21 +123,-40,30 +124,-49,29 +125,-12,12 +126,4,6 +127,-140,70 +128,-13,13 +129,-115,55 +130,4,6 +131,4,6 +132,-20,20 +133,4,6 +134,4,6 +135,-17,17 +136,-61,31 +137,-19,19 +138,-15,15 +139,-16,16 +140,4,6 +141,-13,13 +142,4,6 +143,-15,15 +144,-108,58 +145,-136,76 +146,-99,49 +147,-69,39 +148,4,6 +149,3,7 +150,4,6 +151,3,7 +152,4,6 +153,3,7 +154,4,6 +155,3,7 +156,-140,70 +157,4,6 +158,-17,17 +159,-15,15 +160,3,7 +161,-31,21 +162,-18,18 +163,-88,48 +164,-1,11 +165,-32,22 +166,4,6 +167,-34,24 +168,-102,52 +169,4,6 +170,3,7 +171,-13,13 +172,3,7 +173,4,6 +174,2,8 +175,3,7 +176,-37,27 +177,3,7 +178,-45,25 +179,4,6 +180,-27,17 +181,4,6 +182,-42,22 +183,-12,12 +184,3,7 +185,3,7 +186,4,6 +187,4,6 +188,-29,19 +189,-19,19 +190,-19,19 +191,-19,19 +192,4,6 +193,-15,15 +194,4,6 +195,-147,77 +196,-48,28 +197,-1,11 +198,4,6 +199,-76,36 +200,0,10 +201,4,6 +202,4,6 +203,4,6 +204,-15,15 +205,2,8 +206,4,6 +207,2,8 +208,4,6 +209,-49,29 +210,3,7 +211,4,6 +212,-17,17 +213,4,6 +214,-13,13 +215,-13,13 +216,2,8 +217,4,6 +218,4,6 +219,4,6 +220,-17,17 +221,4,6 +222,-14,14 +223,-15,15 +224,4,6 +225,4,6 +226,3,7 +227,-31,21 +228,4,6 +229,4,6 +230,4,6 +231,4,6 +232,-15,15 +233,-14,14 +234,3,7 +235,4,6 +236,4,6 +237,2,8 +238,3,7 +239,4,6 +240,-21,21 +241,3,7 +242,-27,17 +243,4,6 +244,-16,16 +245,-32,22 +246,4,6 +247,4,6 +248,4,6 +249,-12,12 +250,4,6 +251,-49,29 +252,3,7 +253,2,8 +254,3,7 +255,4,6 +256,4,6 +257,3,7 +258,-54,34 +259,-15,15 +260,-34,24 +261,-18,18 +262,2,8 +263,-45,25 +264,3,7 +265,3,7 +266,-30,20 +267,4,6 +268,4,6 +269,-16,16 +270,-44,24 +271,-4,14 +272,3,7 +273,-14,14 +274,-29,19 +275,4,6 +276,-21,21 +277,4,6 +278,4,6 +279,3,7 +280,4,6 +281,4,6 +282,3,7 +283,-14,14 +284,4,6 +285,3,7 +286,-34,24 +287,4,6 +288,4,6 +289,4,6 +290,3,7 +291,4,6 +292,4,6 +293,3,7 +294,4,6 +295,4,6 +296,2,8 +297,4,6 +298,4,6 +299,4,6 +300,3,7 +301,3,7 +302,2,8 +303,4,6 +304,2,8 +305,4,6 +306,2,8 +307,3,7 +308,4,6 +309,4,6 +310,3,7 +311,-19,19 +312,4,6 +313,2,8 +314,4,6 +315,4,6 +316,4,6 +317,4,6 +318,3,7 +319,3,7 +320,3,7 +321,-29,19 +322,-14,14 +323,4,6 +324,2,8 +325,4,6 +326,2,8 +327,3,7 +328,1,9 +329,4,6 +330,-11,11 +331,2,8 +332,4,6 +333,3,7 +334,4,6 +335,4,6 +336,-15,15 +337,3,7 +338,4,6 +339,4,6 +340,-16,16 +341,3,7 +342,3,7 +343,-39,29 +344,-28,18 +345,3,7 +346,4,6 +347,4,6 +348,-17,17 +349,3,7 +350,4,6 +351,4,6 +352,3,7 +353,3,7 +354,3,7 +355,4,6 +356,1,9 +357,3,7 +358,-13,13 +359,-32,22 +360,4,6 +361,-14,14 +362,3,7 +363,4,6 +364,4,6 +365,3,7 +366,4,6 +367,4,6 +368,3,7 +369,4,6 +370,3,7 +371,-15,15 +372,4,6 +373,4,6 +374,-19,19 +375,4,6 +376,-14,14 +377,-35,25 +378,4,6 +379,4,6 +380,3,7 +381,4,6 +382,-32,22 +383,4,6 +384,-13,13 +385,4,6 +386,-15,15 +387,3,7 +388,2,8 +389,0,10 +390,4,6 +391,4,6 +392,-36,26 +393,-35,25 +394,-15,15 +395,2,8 +396,3,7 +397,3,7 +398,2,8 +399,3,7 diff --git a/theAlley/Test_theAlley_VI_20221123-095820/config.yaml b/theAlley/Test_theAlley_VI_20221123-095820/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2142a2f9ec180589b0e8e0b5f184b8d4b3cb104 --- /dev/null +++ b/theAlley/Test_theAlley_VI_20221123-095820/config.yaml @@ -0,0 +1,18 @@ +general_cfg: + algo_name: VI + device: cpu + env_name: theAlley + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: true + load_path: Train_theAlley_VI_20221122-215228 + max_steps: 200 + mode: test + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + gamma: 0.95 + lr: 0.0001 diff --git a/theAlley/Test_theAlley_VI_20221123-095820/logs/log.txt b/theAlley/Test_theAlley_VI_20221123-095820/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..69796e8bfbfc8f4a6481f2e448991617294f6fa3 --- /dev/null +++ b/theAlley/Test_theAlley_VI_20221123-095820/logs/log.txt @@ -0,0 +1,14 @@ +2022-11-23 09:58:34 - r - INFO: - n_states: 13, n_actions: 4 +2022-11-23 09:58:34 - r - INFO: - Start testing! +2022-11-23 09:58:34 - r - INFO: - Env: theAlley, Algorithm: VI, Device: cpu +2022-11-23 09:58:34 - r - INFO: - Episode: 1/10, Reward: 10.000, Step: 15 +2022-11-23 09:58:34 - r - INFO: - Episode: 2/10, Reward: 10.000, Step: 15 +2022-11-23 09:58:34 - r - INFO: - Episode: 3/10, Reward: 10.000, Step: 15 +2022-11-23 09:58:34 - r - INFO: - Episode: 4/10, Reward: -5.000, Step: 7 +2022-11-23 09:58:34 - r - INFO: - Episode: 5/10, Reward: 10.000, Step: 17 +2022-11-23 09:58:34 - r - INFO: - Episode: 6/10, Reward: 10.000, Step: 17 +2022-11-23 09:58:34 - r - INFO: - Episode: 7/10, Reward: 10.000, Step: 14 +2022-11-23 09:58:34 - r - INFO: - Episode: 8/10, Reward: 10.000, Step: 13 +2022-11-23 09:58:34 - r - INFO: - Episode: 9/10, Reward: -5.000, Step: 5 +2022-11-23 09:58:34 - r - INFO: - Episode: 10/10, Reward: 10.000, Step: 13 +2022-11-23 09:58:34 - r - INFO: - Finish testing! diff --git a/theAlley/Test_theAlley_VI_20221123-095820/models/checkpoint.pt b/theAlley/Test_theAlley_VI_20221123-095820/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2179a82522b6ffc4666c178c12c9179f70782e83 --- /dev/null +++ b/theAlley/Test_theAlley_VI_20221123-095820/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8be0c5bd82e71035750ff212a940e63e4cac89a31921192ca1d89d410c3ddf7f +size 687 diff --git a/theAlley/Test_theAlley_VI_20221123-095820/results/learning_curve.png b/theAlley/Test_theAlley_VI_20221123-095820/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..31c1824082fe740404aec82b4ffe567cf71f7675 Binary files /dev/null and b/theAlley/Test_theAlley_VI_20221123-095820/results/learning_curve.png differ diff --git a/theAlley/Test_theAlley_VI_20221123-095820/results/res.csv b/theAlley/Test_theAlley_VI_20221123-095820/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..69d14062891f6fabafc00b0fa478d090e5b963d1 --- /dev/null +++ b/theAlley/Test_theAlley_VI_20221123-095820/results/res.csv @@ -0,0 +1,11 @@ +episodes,rewards,steps +0,10.0,15 +1,10.0,15 +2,10.0,15 +3,-5.0,7 +4,10.0,17 +5,10.0,17 +6,10.0,14 +7,10.0,13 +8,-5.0,5 +9,10.0,13 diff --git a/theAlley/Train_theAlley_VI_20221122-215228/config.yaml b/theAlley/Train_theAlley_VI_20221122-215228/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfdced481c2b454dbbdaac9f3f2ea582b2a37925 --- /dev/null +++ b/theAlley/Train_theAlley_VI_20221122-215228/config.yaml @@ -0,0 +1,18 @@ +general_cfg: + algo_name: VI + device: cpu + env_name: theAlley + eval_eps: 10 + eval_per_episode: 5 + load_checkpoint: false + load_path: Train_CartPole-v1_DQN_20221026-054757 + max_steps: 200 + mode: train + save_fig: true + seed: 1 + show_fig: false + test_eps: 10 + train_eps: 100 +algo_cfg: + gamma: 0.95 + lr: 0.0001 diff --git a/theAlley/Train_theAlley_VI_20221122-215228/logs/log.txt b/theAlley/Train_theAlley_VI_20221122-215228/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..2f8ec90e92963cdf14169b491f9e7e983a92d438 --- /dev/null +++ b/theAlley/Train_theAlley_VI_20221122-215228/logs/log.txt @@ -0,0 +1,108 @@ +2022-11-22 21:53:05 - r - INFO: - n_states: 13, n_actions: 4 +2022-11-22 21:53:05 - r - INFO: - Start training! +2022-11-22 21:53:05 - r - INFO: - Env: theAlley, Algorithm: VI, Device: cpu +2022-11-22 21:53:05 - r - INFO: - Episode: 1/100, Reward: 10.000, Step: 27 +2022-11-22 21:53:05 - r - INFO: - Episode: 2/100, Reward: -5.000, Step: 11 +2022-11-22 21:53:05 - r - INFO: - Episode: 3/100, Reward: -5.000, Step: 12 +2022-11-22 21:53:05 - r - INFO: - Episode: 4/100, Reward: -5.000, Step: 8 +2022-11-22 21:53:05 - r - INFO: - Episode: 5/100, Reward: 10.000, Step: 18 +2022-11-22 21:53:05 - r - INFO: - Current episode 5 has the best eval reward: 4.000 +2022-11-22 21:53:06 - r - INFO: - Episode: 6/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 7/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 8/100, Reward: -5.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 9/100, Reward: 10.000, Step: 12 +2022-11-22 21:53:06 - r - INFO: - Episode: 10/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 11/100, Reward: 10.000, Step: 17 +2022-11-22 21:53:06 - r - INFO: - Episode: 12/100, Reward: 10.000, Step: 16 +2022-11-22 21:53:06 - r - INFO: - Episode: 13/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 14/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 15/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Current episode 15 has the best eval reward: 5.500 +2022-11-22 21:53:06 - r - INFO: - Episode: 16/100, Reward: 10.000, Step: 12 +2022-11-22 21:53:06 - r - INFO: - Episode: 17/100, Reward: 10.000, Step: 16 +2022-11-22 21:53:06 - r - INFO: - Episode: 18/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 19/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 20/100, Reward: 10.000, Step: 17 +2022-11-22 21:53:06 - r - INFO: - Current episode 20 has the best eval reward: 5.500 +2022-11-22 21:53:06 - r - INFO: - Episode: 21/100, Reward: -5.000, Step: 11 +2022-11-22 21:53:06 - r - INFO: - Episode: 22/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 23/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 24/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 25/100, Reward: -5.000, Step: 9 +2022-11-22 21:53:06 - r - INFO: - Current episode 25 has the best eval reward: 8.500 +2022-11-22 21:53:06 - r - INFO: - Episode: 26/100, Reward: -5.000, Step: 16 +2022-11-22 21:53:06 - r - INFO: - Episode: 27/100, Reward: -5.000, Step: 10 +2022-11-22 21:53:06 - r - INFO: - Episode: 28/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 29/100, Reward: -5.000, Step: 11 +2022-11-22 21:53:06 - r - INFO: - Episode: 30/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 31/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 32/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 33/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 34/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 35/100, Reward: -5.000, Step: 8 +2022-11-22 21:53:06 - r - INFO: - Episode: 36/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 37/100, Reward: -5.000, Step: 11 +2022-11-22 21:53:06 - r - INFO: - Episode: 38/100, Reward: -5.000, Step: 8 +2022-11-22 21:53:06 - r - INFO: - Episode: 39/100, Reward: -5.000, Step: 11 +2022-11-22 21:53:06 - r - INFO: - Episode: 40/100, Reward: -5.000, Step: 6 +2022-11-22 21:53:06 - r - INFO: - Episode: 41/100, Reward: 10.000, Step: 16 +2022-11-22 21:53:06 - r - INFO: - Episode: 42/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 43/100, Reward: 10.000, Step: 16 +2022-11-22 21:53:06 - r - INFO: - Episode: 44/100, Reward: 10.000, Step: 17 +2022-11-22 21:53:06 - r - INFO: - Episode: 45/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 46/100, Reward: -5.000, Step: 8 +2022-11-22 21:53:06 - r - INFO: - Episode: 47/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 48/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 49/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 50/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 51/100, Reward: -5.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 52/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 53/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 54/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 55/100, Reward: 10.000, Step: 12 +2022-11-22 21:53:06 - r - INFO: - Episode: 56/100, Reward: -5.000, Step: 9 +2022-11-22 21:53:06 - r - INFO: - Episode: 57/100, Reward: 10.000, Step: 12 +2022-11-22 21:53:06 - r - INFO: - Episode: 58/100, Reward: -5.000, Step: 10 +2022-11-22 21:53:06 - r - INFO: - Episode: 59/100, Reward: 10.000, Step: 19 +2022-11-22 21:53:06 - r - INFO: - Episode: 60/100, Reward: -5.000, Step: 6 +2022-11-22 21:53:06 - r - INFO: - Episode: 61/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 62/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 63/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 64/100, Reward: 10.000, Step: 20 +2022-11-22 21:53:06 - r - INFO: - Episode: 65/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 66/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 67/100, Reward: -5.000, Step: 11 +2022-11-22 21:53:06 - r - INFO: - Episode: 68/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 69/100, Reward: -5.000, Step: 7 +2022-11-22 21:53:06 - r - INFO: - Episode: 70/100, Reward: -5.000, Step: 11 +2022-11-22 21:53:06 - r - INFO: - Episode: 71/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 72/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 73/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 74/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 75/100, Reward: -5.000, Step: 9 +2022-11-22 21:53:06 - r - INFO: - Episode: 76/100, Reward: -5.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 77/100, Reward: -5.000, Step: 10 +2022-11-22 21:53:06 - r - INFO: - Episode: 78/100, Reward: 10.000, Step: 19 +2022-11-22 21:53:06 - r - INFO: - Episode: 79/100, Reward: 10.000, Step: 17 +2022-11-22 21:53:06 - r - INFO: - Episode: 80/100, Reward: 10.000, Step: 19 +2022-11-22 21:53:06 - r - INFO: - Episode: 81/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 82/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 83/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 84/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 85/100, Reward: -5.000, Step: 9 +2022-11-22 21:53:06 - r - INFO: - Episode: 86/100, Reward: 10.000, Step: 18 +2022-11-22 21:53:06 - r - INFO: - Episode: 87/100, Reward: -5.000, Step: 6 +2022-11-22 21:53:06 - r - INFO: - Episode: 88/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 89/100, Reward: 10.000, Step: 12 +2022-11-22 21:53:06 - r - INFO: - Episode: 90/100, Reward: 10.000, Step: 16 +2022-11-22 21:53:06 - r - INFO: - Episode: 91/100, Reward: 10.000, Step: 15 +2022-11-22 21:53:06 - r - INFO: - Episode: 92/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 93/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Episode: 94/100, Reward: 10.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 95/100, Reward: 10.000, Step: 16 +2022-11-22 21:53:06 - r - INFO: - Episode: 96/100, Reward: -5.000, Step: 5 +2022-11-22 21:53:06 - r - INFO: - Episode: 97/100, Reward: -5.000, Step: 6 +2022-11-22 21:53:06 - r - INFO: - Episode: 98/100, Reward: -5.000, Step: 13 +2022-11-22 21:53:06 - r - INFO: - Episode: 99/100, Reward: 10.000, Step: 12 +2022-11-22 21:53:06 - r - INFO: - Episode: 100/100, Reward: 10.000, Step: 14 +2022-11-22 21:53:06 - r - INFO: - Finish training! diff --git a/theAlley/Train_theAlley_VI_20221122-215228/models/checkpoint.pt b/theAlley/Train_theAlley_VI_20221122-215228/models/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..2179a82522b6ffc4666c178c12c9179f70782e83 --- /dev/null +++ b/theAlley/Train_theAlley_VI_20221122-215228/models/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8be0c5bd82e71035750ff212a940e63e4cac89a31921192ca1d89d410c3ddf7f +size 687 diff --git a/theAlley/Train_theAlley_VI_20221122-215228/results/learning_curve.png b/theAlley/Train_theAlley_VI_20221122-215228/results/learning_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..2266dda785ca085d24b329b452a9789c43f349a2 Binary files /dev/null and b/theAlley/Train_theAlley_VI_20221122-215228/results/learning_curve.png differ diff --git a/theAlley/Train_theAlley_VI_20221122-215228/results/res.csv b/theAlley/Train_theAlley_VI_20221122-215228/results/res.csv new file mode 100644 index 0000000000000000000000000000000000000000..9bbaf1ec3ad7fd9ed47f22856a3b3ba2b88d87b4 --- /dev/null +++ b/theAlley/Train_theAlley_VI_20221122-215228/results/res.csv @@ -0,0 +1,101 @@ +episodes,rewards,steps +0,10.0,27 +1,-5.0,11 +2,-5.0,12 +3,-5.0,8 +4,10.0,18 +5,10.0,13 +6,10.0,13 +7,-5.0,13 +8,10.0,12 +9,10.0,14 +10,10.0,17 +11,10.0,16 +12,10.0,13 +13,10.0,13 +14,10.0,14 +15,10.0,12 +16,10.0,16 +17,10.0,13 +18,10.0,15 +19,10.0,17 +20,-5.0,11 +21,10.0,15 +22,-5.0,5 +23,10.0,14 +24,-5.0,9 +25,-5.0,16 +26,-5.0,10 +27,10.0,14 +28,-5.0,11 +29,10.0,14 +30,10.0,15 +31,10.0,13 +32,-5.0,5 +33,10.0,15 +34,-5.0,8 +35,10.0,14 +36,-5.0,11 +37,-5.0,8 +38,-5.0,11 +39,-5.0,6 +40,10.0,16 +41,-5.0,5 +42,10.0,16 +43,10.0,17 +44,10.0,13 +45,-5.0,8 +46,10.0,14 +47,10.0,13 +48,10.0,15 +49,10.0,13 +50,-5.0,13 +51,10.0,14 +52,10.0,15 +53,10.0,14 +54,10.0,12 +55,-5.0,9 +56,10.0,12 +57,-5.0,10 +58,10.0,19 +59,-5.0,6 +60,10.0,15 +61,10.0,14 +62,-5.0,5 +63,10.0,20 +64,10.0,15 +65,10.0,15 +66,-5.0,11 +67,-5.0,5 +68,-5.0,7 +69,-5.0,11 +70,10.0,14 +71,10.0,13 +72,10.0,14 +73,-5.0,5 +74,-5.0,9 +75,-5.0,13 +76,-5.0,10 +77,10.0,19 +78,10.0,17 +79,10.0,19 +80,10.0,13 +81,-5.0,5 +82,10.0,14 +83,10.0,14 +84,-5.0,9 +85,10.0,18 +86,-5.0,6 +87,10.0,14 +88,10.0,12 +89,10.0,16 +90,10.0,15 +91,-5.0,5 +92,10.0,14 +93,10.0,13 +94,10.0,16 +95,-5.0,5 +96,-5.0,6 +97,-5.0,13 +98,10.0,12 +99,10.0,14