10M steps, increased learning rate with linear scheduler

SoccerTwos.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8171b9f22f3de0dfac49f68c4df948d5cda20dc11b6a2d73b5e244d153896472
+oid sha256:03619a542b658ecd6cbb3d9622e7ceedf7c91bffb499d3ffe4379e73cb8d99f1
 size 1768747

SoccerTwos/SoccerTwos-10000180.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03619a542b658ecd6cbb3d9622e7ceedf7c91bffb499d3ffe4379e73cb8d99f1
+size 1768747

SoccerTwos/SoccerTwos-10000180.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abf2617282af0cec18607ee4d7db88dc65920459a612f766625aea1c31f81354
+size 28430482

SoccerTwos/SoccerTwos-8499872.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68806773524a03c2b728334643013dc61551c5ee0dd1b314355ef476498f18fd
+size 1768747

SoccerTwos/SoccerTwos-8499872.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e7b9bfc9296dd277014c73db2b9017c4f4384d5e3dd1cfc6f40b5f44ebbd8e
+size 28430358

SoccerTwos/SoccerTwos-8999958.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea22ee0f34553843b26385e0bb2e31e752694338074fdd570ced047da073ccef
+size 1768747

SoccerTwos/SoccerTwos-8999958.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be92998cb948b03f6129c0b1feb970f0562c17f05ef30514f3fa983d8f77dd29
+size 28430358

SoccerTwos/SoccerTwos-9499928.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a4eac4ea0e6859aa55e743e799fc735d6948a74e70bf36eaefb0a6e1be2e9ac
+size 1768747

SoccerTwos/SoccerTwos-9499928.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fabcfb5c36354cb1f7a91df2e04b5bcb112d4ae4f6d00b43f0f67cb90a9ef37
+size 28430358

SoccerTwos/SoccerTwos-9999962.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03619a542b658ecd6cbb3d9622e7ceedf7c91bffb499d3ffe4379e73cb8d99f1
+size 1768747

SoccerTwos/SoccerTwos-9999962.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce8a20a48cc08457607c17781129ded606114221477e352ce2b6e89586d8e619
+size 28430358

SoccerTwos/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:722fe59ebe4db88e2dd4d2191dfc0ba389035dbf97e91c45a728de4768562749
+oid sha256:64a0f0b918bbd5ee299916d5d441286fa3b232800e3a12ef7d593c1f13f8a502
 size 28429366

SoccerTwos/events.out.tfevents.1703855990.nilss-mbp.lan.63432.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec9843a985611ef4ed491f19e429eb314a3583aa551fac477561c944aac4a300
+size 10069153

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 2048, "buffer_size": 20480, "learning_rate": 0.0005, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 5000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 250000, "swap_steps": 2000, "window": 10, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos.app", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 2048, "buffer_size": 20480, "learning_rate": 0.0005, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 10000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 250000, "swap_steps": 2000, "window": 10, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos.app", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -37,7 +37,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 5000000
+    max_steps: 10000000
     time_horizon: 1000
     summary_freq: 10000
     threaded: false

run_logs/Player-0.log

@@ -10,7 +10,7 @@ NullGfxDevice:
     Renderer: Null Device
     Vendor:   Unity Technologies
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.034 seconds
+- Completed reload, in  0.035 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -35,7 +35,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.204709 ms
+UnloadTime: 0.254209 ms
 Registered Communicator in Agent.
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 TOO LONG
@@ -475,74 +475,13 @@ TOO LONG
 TOO LONG
 TOO LONG
 TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-Thread 0x1711cf000 may have been prematurely finalized
+Thread 0x16f8ef000 may have been prematurely finalized
 Setting up 6 worker threads for Enlighten.
 Memory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [8.0 KB-16.0 KB]: 94692 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [8.0 KB-16.0 KB]: 188671 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.1 MB
@@ -570,7 +509,7 @@ Memory Statistics:
     [ALLOC_TEMP_Job.Worker 0]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0 B
+      Peak Allocated Bytes 0.7 KB
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 10]
       Initial Block Size 32.0 KB
@@ -605,7 +544,7 @@ Memory Statistics:
     [ALLOC_TEMP_Job.Worker 3]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0.7 KB
+      Peak Allocated Bytes 0 B
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 12]
       Initial Block Size 32.0 KB
@@ -713,22 +652,22 @@ Memory Statistics:
       Peak Allocated Bytes 0 B
       Overflow Count 0
 [ALLOC_DEFAULT] Dual Thread Allocator
-  Peak main deferred allocation count 6
+  Peak main deferred allocation count 4
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [4.0 MB-8.0 MB]: 94693 frames
+      Peak usage frame count: [4.0 MB-8.0 MB]: 188672 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 5.0 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 94693 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 188672 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 17.4 MB
+      Peak Allocated memory 17.5 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -757,13 +696,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 94692 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 188671 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 65.6 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [128.0 KB-256.0 KB]: 94693 frames
+      Peak usage frame count: [128.0 KB-256.0 KB]: 188672 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 173.5 KB
@@ -775,13 +714,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 94693 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 188672 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 0.6 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 94692 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 188671 frames, [2.0 MB-4.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 3.4 MB
@@ -793,13 +732,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 94693 frames
+      Peak usage frame count: [0-1.0 KB]: 188672 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 94692 frames, [2.0 KB-4.0 KB]: 1 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 188671 frames, [2.0 KB-4.0 KB]: 1 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 2.2 KB

run_logs/timers.json

@@ -2,201 +2,201 @@
     "name": "root",
     "gauges": {
         "SoccerTwos.Policy.Entropy.mean": {
-            "value": 1.755637288093567,
-            "min": 1.7151814699172974,
-            "max": 3.295724630355835,
-            "count": 499
+            "value": 1.1532409191131592,
+            "min": 1.1196668148040771,
+            "max": 3.2957353591918945,
+            "count": 1000
         },
         "SoccerTwos.Policy.Entropy.sum": {
-            "value": 36910.51953125,
-            "min": 13612.6572265625,
-            "max": 172905.75,
-            "count": 499
+            "value": 23987.41015625,
+            "min": 7871.72021484375,
+            "max": 112492.0390625,
+            "count": 1000
         },
         "SoccerTwos.Environment.EpisodeLength.mean": {
-            "value": 63.6125,
-            "min": 39.645161290322584,
+            "value": 62.79746835443038,
+            "min": 45.00917431192661,
             "max": 999.0,
-            "count": 499
+            "count": 1000
         },
         "SoccerTwos.Environment.EpisodeLength.sum": {
-            "value": 20356.0,
-            "min": 10692.0,
-            "max": 31500.0,
-            "count": 499
+            "value": 19844.0,
+            "min": 7992.0,
+            "max": 30788.0,
+            "count": 1000
         },
         "SoccerTwos.Self-play.ELO.mean": {
-            "value": 1540.6404213531655,
-            "min": 1197.4957713356807,
-            "max": 1576.4143309371482,
-            "count": 478
+            "value": 1709.0109510622178,
+            "min": 1190.391640367494,
+            "max": 1720.9898581771342,
+            "count": 991
         },
         "SoccerTwos.Self-play.ELO.sum": {
-            "value": 246502.4674165065,
-            "min": 2399.421175774324,
-            "max": 363079.2013600959,
-            "count": 478
+            "value": 270023.7302678304,
+            "min": 2381.3820137520165,
+            "max": 367918.0238814893,
+            "count": 991
         },
         "SoccerTwos.Step.mean": {
-            "value": 4999972.0,
-            "min": 9254.0,
-            "max": 4999972.0,
-            "count": 500
+            "value": 9999962.0,
+            "min": 9412.0,
+            "max": 9999962.0,
+            "count": 1000
         },
         "SoccerTwos.Step.sum": {
-            "value": 4999972.0,
-            "min": 9254.0,
-            "max": 4999972.0,
-            "count": 500
+            "value": 9999962.0,
+            "min": 9412.0,
+            "max": 9999962.0,
+            "count": 1000
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.mean": {
-            "value": -0.034482039511203766,
-            "min": -0.10746785998344421,
-            "max": 0.16136549413204193,
-            "count": 500
+            "value": 0.01338563859462738,
+            "min": -0.11655034869909286,
+            "max": 0.16845153272151947,
+            "count": 1000
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.sum": {
-            "value": -5.517126083374023,
-            "min": -22.675718307495117,
-            "max": 27.432022094726562,
-            "count": 500
+            "value": 2.1149308681488037,
+            "min": -22.494216918945312,
+            "max": 23.403825759887695,
+            "count": 1000
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.mean": {
-            "value": -0.034504301846027374,
-            "min": -0.11580751091241837,
-            "max": 0.1590351015329361,
-            "count": 500
+            "value": 0.014068040996789932,
+            "min": -0.11802836507558823,
+            "max": 0.17341575026512146,
+            "count": 1000
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.sum": {
-            "value": -5.520688533782959,
-            "min": -24.43538475036621,
-            "max": 26.952106475830078,
-            "count": 500
+            "value": 2.222750425338745,
+            "min": -22.422487258911133,
+            "max": 22.71746253967285,
+            "count": 1000
         },
         "SoccerTwos.Environment.CumulativeReward.mean": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 500
+            "count": 1000
         },
         "SoccerTwos.Environment.CumulativeReward.sum": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 500
+            "count": 1000
         },
         "SoccerTwos.Policy.ExtrinsicReward.mean": {
-            "value": -0.08047500103712082,
-            "min": -0.5921090895479376,
-            "max": 0.4576799988746643,
-            "count": 500
+            "value": -0.0220329112644437,
+            "min": -0.5286444425582886,
+            "max": 0.40512195447596105,
+            "count": 1000
         },
         "SoccerTwos.Policy.ExtrinsicReward.sum": {
-            "value": -12.876000165939331,
-            "min": -66.35360014438629,
-            "max": 60.00919944047928,
-            "count": 500
+            "value": -3.4811999797821045,
+            "min": -57.29039967060089,
+            "max": 47.71400010585785,
+            "count": 1000
         },
         "SoccerTwos.Environment.GroupCumulativeReward.mean": {
-            "value": -0.08047500103712082,
-            "min": -0.5921090895479376,
-            "max": 0.4576799988746643,
-            "count": 500
+            "value": -0.0220329112644437,
+            "min": -0.5286444425582886,
+            "max": 0.40512195447596105,
+            "count": 1000
         },
         "SoccerTwos.Environment.GroupCumulativeReward.sum": {
-            "value": -12.876000165939331,
-            "min": -66.35360014438629,
-            "max": 60.00919944047928,
-            "count": 500
+            "value": -3.4811999797821045,
+            "min": -57.29039967060089,
+            "max": 47.71400010585785,
+            "count": 1000
         },
         "SoccerTwos.IsTraining.mean": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 500
+            "count": 1000
         },
         "SoccerTwos.IsTraining.sum": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 500
+            "count": 1000
         },
         "SoccerTwos.Losses.PolicyLoss.mean": {
-            "value": 0.015034021192695946,
-            "min": 0.010941779592152064,
-            "max": 0.024873137117053072,
-            "count": 239
+            "value": 0.018708063105683927,
+            "min": 0.01079763230518438,
+            "max": 0.0237867681561814,
+            "count": 482
         },
         "SoccerTwos.Losses.PolicyLoss.sum": {
-            "value": 0.015034021192695946,
-            "min": 0.010941779592152064,
-            "max": 0.024873137117053072,
-            "count": 239
+            "value": 0.018708063105683927,
+            "min": 0.01079763230518438,
+            "max": 0.0237867681561814,
+            "count": 482
         },
         "SoccerTwos.Losses.ValueLoss.mean": {
-            "value": 0.11739198019107183,
-            "min": 3.1021232795562052e-06,
-            "max": 0.1251884085436662,
-            "count": 239
+            "value": 0.10804408763845762,
+            "min": 0.0008606591868253115,
+            "max": 0.1229787160952886,
+            "count": 482
         },
         "SoccerTwos.Losses.ValueLoss.sum": {
-            "value": 0.11739198019107183,
-            "min": 3.1021232795562052e-06,
-            "max": 0.1251884085436662,
-            "count": 239
+            "value": 0.10804408763845762,
+            "min": 0.0008606591868253115,
+            "max": 0.1229787160952886,
+            "count": 482
         },
         "SoccerTwos.Losses.BaselineLoss.mean": {
-            "value": 0.11775108724832535,
-            "min": 3.304618659664508e-06,
-            "max": 0.1258082126577695,
-            "count": 239
+            "value": 0.10806638523936271,
+            "min": 0.0008650216366125581,
+            "max": 0.12314595381418864,
+            "count": 482
         },
         "SoccerTwos.Losses.BaselineLoss.sum": {
-            "value": 0.11775108724832535,
-            "min": 3.304618659664508e-06,
-            "max": 0.1258082126577695,
-            "count": 239
+            "value": 0.10806638523936271,
+            "min": 0.0008650216366125581,
+            "max": 0.12314595381418864,
+            "count": 482
         },
         "SoccerTwos.Policy.LearningRate.mean": {
-            "value": 1.0132997973599917e-06,
-            "min": 1.0132997973599917e-06,
-            "max": 0.0004968980006203998,
-            "count": 239
+            "value": 7.12599857500005e-07,
+            "min": 7.12599857500005e-07,
+            "max": 0.0004986521002695799,
+            "count": 482
         },
         "SoccerTwos.Policy.LearningRate.sum": {
-            "value": 1.0132997973599917e-06,
-            "min": 1.0132997973599917e-06,
-            "max": 0.0004968980006203998,
-            "count": 239
+            "value": 7.12599857500005e-07,
+            "min": 7.12599857500005e-07,
+            "max": 0.0004986521002695799,
+            "count": 482
         },
         "SoccerTwos.Policy.Epsilon.mean": {
-            "value": 0.10020264000000001,
-            "min": 0.10020264000000001,
-            "max": 0.19937960000000005,
-            "count": 239
+            "value": 0.10014250000000001,
+            "min": 0.10014250000000001,
+            "max": 0.19973042,
+            "count": 482
         },
         "SoccerTwos.Policy.Epsilon.sum": {
-            "value": 0.10020264000000001,
-            "min": 0.10020264000000001,
-            "max": 0.19937960000000005,
-            "count": 239
+            "value": 0.10014250000000001,
+            "min": 0.10014250000000001,
+            "max": 0.19973042,
+            "count": 482
         },
         "SoccerTwos.Policy.Beta.mean": {
-            "value": 2.0111735999999927e-05,
-            "min": 2.0111735999999927e-05,
-            "max": 0.004969042040000001,
-            "count": 239
+            "value": 1.7110750000000053e-05,
+            "min": 1.7110750000000053e-05,
+            "max": 0.004986547957999999,
+            "count": 482
         },
         "SoccerTwos.Policy.Beta.sum": {
-            "value": 2.0111735999999927e-05,
-            "min": 2.0111735999999927e-05,
-            "max": 0.004969042040000001,
-            "count": 239
+            "value": 1.7110750000000053e-05,
+            "min": 1.7110750000000053e-05,
+            "max": 0.004986547957999999,
+            "count": 482
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1703848764",
+        "start_time_seconds": "1703855988",
         "python_version": "3.10.12 (main, Jul  5 2023, 15:02:25) [Clang 14.0.6 ]",
         "command_line_arguments": "/opt/homebrew/Caskroom/miniconda/base/envs/rl/bin/mlagents-learn ./config/poca/SoccerTwos.yaml --env=./training-envs-executables/SoccerTwos.app --run-id=SoccerTwos --no-graphics --force",
         "mlagents_version": "1.1.0.dev0",
@@ -204,103 +204,103 @@
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "2.1.2",
         "numpy_version": "1.23.5",
-        "end_time_seconds": "1703853889"
+        "end_time_seconds": "1703866208"
     },
-    "total": 5124.430119083001,
+    "total": 10219.73770537498,
     "count": 1,
-    "self": 0.19001916600973345,
+    "self": 0.19493024999974295,
     "children": {
         "run_training.setup": {
-            "total": 0.012270416977116838,
+            "total": 0.013058999989880249,
             "count": 1,
-            "self": 0.012270416977116838
+            "self": 0.013058999989880249
         },
         "TrainerController.start_learning": {
-            "total": 5124.227829500014,
+            "total": 10219.529716124991,
             "count": 1,
-            "self": 2.531355981802335,
+            "self": 4.421711311646504,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 1.5887895020714495,
-                    "count": 20,
-                    "self": 1.5887895020714495
+                    "total": 1.822008292976534,
+                    "count": 40,
+                    "self": 1.822008292976534
                 },
                 "TrainerController.advance": {
-                    "total": 5119.955040642148,
-                    "count": 341985,
-                    "self": 2.185286336200079,
+                    "total": 10213.13215393637,
+                    "count": 681034,
+                    "self": 3.9921176846837625,
                     "children": {
                         "env_step": {
-                            "total": 2303.891035468987,
-                            "count": 341985,
-                            "self": 1941.7491068352829,
+                            "total": 4570.680127619009,
+                            "count": 681034,
+                            "self": 3851.7968664654472,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 360.6655651805049,
-                                    "count": 341985,
-                                    "self": 11.17105897743022,
+                                    "total": 715.9787979857938,
+                                    "count": 681034,
+                                    "self": 21.6306824052881,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 349.4945062030747,
-                                            "count": 631296,
-                                            "self": 349.4945062030747
+                                            "total": 694.3481155805057,
+                                            "count": 1257830,
+                                            "self": 694.3481155805057
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 1.4763634531991556,
-                                    "count": 341985,
+                                    "total": 2.9044631677679718,
+                                    "count": 681034,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 5119.23122701954,
-                                            "count": 341985,
+                                            "total": 10211.220314579754,
+                                            "count": 681034,
                                             "is_parallel": true,
-                                            "self": 3492.6540978672565,
+                                            "self": 6963.624994570651,
                                             "children": {
                                                 "steps_from_proto": {
-                                                    "total": 0.013060167053481564,
-                                                    "count": 40,
+                                                    "total": 0.025804257107665762,
+                                                    "count": 80,
                                                     "is_parallel": true,
-                                                    "self": 0.0021915972174610943,
+                                                    "self": 0.00424826008384116,
                                                     "children": {
                                                         "_process_rank_one_or_two_observation": {
-                                                            "total": 0.01086856983602047,
-                                                            "count": 160,
+                                                            "total": 0.021555997023824602,
+                                                            "count": 320,
                                                             "is_parallel": true,
-                                                            "self": 0.01086856983602047
+                                                            "self": 0.021555997023824602
                                                         }
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 1626.5640689852298,
-                                                    "count": 341985,
+                                                    "total": 3247.569515751995,
+                                                    "count": 681034,
                                                     "is_parallel": true,
-                                                    "self": 78.99318619386759,
+                                                    "self": 157.49633839310263,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 37.23996753204847,
-                                                            "count": 341985,
+                                                            "total": 75.07220318031614,
+                                                            "count": 681034,
                                                             "is_parallel": true,
-                                                            "self": 37.23996753204847
+                                                            "self": 75.07220318031614
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 1310.1512493426271,
-                                                            "count": 341985,
+                                                            "total": 2617.271251162514,
+                                                            "count": 681034,
                                                             "is_parallel": true,
-                                                            "self": 1310.1512493426271
+                                                            "self": 2617.271251162514
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 200.17966591668664,
-                                                            "count": 683970,
+                                                            "total": 397.72972301606205,
+                                                            "count": 1362068,
                                                             "is_parallel": true,
-                                                            "self": 30.950994182756403,
+                                                            "self": 61.088766790606314,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 169.22867173393024,
-                                                                    "count": 2735880,
+                                                                    "total": 336.64095622545574,
+                                                                    "count": 5448272,
                                                                     "is_parallel": true,
-                                                                    "self": 169.22867173393024
+                                                                    "self": 336.64095622545574
                                                                 }
                                                             }
                                                         }
@@ -313,31 +313,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 2813.8787188369606,
-                            "count": 341985,
-                            "self": 21.16630858287681,
+                            "total": 5638.459908632678,
+                            "count": 681034,
+                            "self": 41.62850201164838,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 556.1312195862702,
-                                    "count": 341985,
-                                    "self": 554.5936779602489,
+                                    "total": 1114.3295763299102,
+                                    "count": 681034,
+                                    "self": 1111.2247817909229,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 1.537541626021266,
-                                            "count": 10,
-                                            "self": 1.537541626021266
+                                            "total": 3.104794538987335,
+                                            "count": 20,
+                                            "self": 3.104794538987335
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 2236.5811906678136,
-                                    "count": 239,
-                                    "self": 242.41541127837263,
+                                    "total": 4482.501830291119,
+                                    "count": 482,
+                                    "self": 476.7318739280745,
                                     "children": {
                                         "TorchPOCAOptimizer.update": {
-                                            "total": 1994.165779389441,
-                                            "count": 7185,
-                                            "self": 1994.165779389441
+                                            "total": 4005.769956363045,
+                                            "count": 14469,
+                                            "self": 4005.769956363045
                                         }
                                     }
                                 }
@@ -346,19 +346,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 3.3300602808594704e-07,
+                    "total": 2.500019036233425e-07,
                     "count": 1,
-                    "self": 3.3300602808594704e-07
+                    "self": 2.500019036233425e-07
                 },
                 "TrainerController._save_models": {
-                    "total": 0.15264304098673165,
+                    "total": 0.15384233399527147,
                     "count": 1,
-                    "self": 0.000656957970932126,
+                    "self": 0.0008283340139314532,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.15198608301579952,
+                            "total": 0.15301399998134002,
                             "count": 1,
-                            "self": 0.15198608301579952
+                            "self": 0.15301399998134002
                         }
                     }
                 }

run_logs/training_status.json

@@ -2,59 +2,59 @@
     "SoccerTwos": {
         "checkpoints": [
             {
-                "steps": 3499990,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-3499990.onnx",
+                "steps": 8499872,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-8499872.onnx",
                 "reward": 0.0,
-                "creation_time": 1703852337.048543,
+                "creation_time": 1703864672.0303428,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-3499990.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-8499872.pt"
                 ]
             },
             {
-                "steps": 3999988,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-3999988.onnx",
+                "steps": 8999958,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-8999958.onnx",
                 "reward": 0.0,
-                "creation_time": 1703852850.532505,
+                "creation_time": 1703865186.172404,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-3999988.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-8999958.pt"
                 ]
             },
             {
-                "steps": 4499985,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-4499985.onnx",
+                "steps": 9499928,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-9499928.onnx",
                 "reward": 0.0,
-                "creation_time": 1703853379.401021,
+                "creation_time": 1703865694.776364,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-4499985.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-9499928.pt"
                 ]
             },
             {
-                "steps": 4999972,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-4999972.onnx",
+                "steps": 9999962,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-9999962.onnx",
                 "reward": 0.0,
-                "creation_time": 1703853888.732385,
+                "creation_time": 1703866208.272885,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-4999972.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-9999962.pt"
                 ]
             },
             {
-                "steps": 5000376,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-5000376.onnx",
+                "steps": 10000180,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-10000180.onnx",
                 "reward": 0.0,
-                "creation_time": 1703853888.9599922,
+                "creation_time": 1703866208.445204,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-5000376.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-10000180.pt"
                 ]
             }
         ],
-        "elo": 1543.402385883174,
+        "elo": 1709.4646768149448,
         "final_checkpoint": {
-            "steps": 5000376,
+            "steps": 10000180,
             "file_path": "results/SoccerTwos/SoccerTwos.onnx",
             "reward": 0.0,
-            "creation_time": 1703853888.9599922,
+            "creation_time": 1703866208.445204,
             "auxillary_file_paths": [
-                "results/SoccerTwos/SoccerTwos/SoccerTwos-5000376.pt"
+                "results/SoccerTwos/SoccerTwos/SoccerTwos-10000180.pt"
             ]
         }
     },