5M steps, increased learning rate with linear scheduler

SoccerTwos.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a927605bab3756880322ffbd82591e17b9bb154c4fdfc7aa53dec2b81fee87f6
+oid sha256:8171b9f22f3de0dfac49f68c4df948d5cda20dc11b6a2d73b5e244d153896472
 size 1768747

SoccerTwos/SoccerTwos-3499990.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53f40873d5390b4f523c95fc167e4b6e1cef2c2d6e19be275f7f026ba0ae53be
+size 1768747

SoccerTwos/SoccerTwos-3499990.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53624c57bd7bd02e1b93d99843f9ab552bd414561ad8bde5506456b170726264
+size 28430358

SoccerTwos/SoccerTwos-3999988.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ee7944532872486f1b57c530c1c53e1c20fe5ad038b7e57d0825b24f0d3f030
+size 1768747

SoccerTwos/SoccerTwos-3999988.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:898d2e3151f633fc67f4a768874fac685fbf912ba93e0f1bb7dba1a48b292994
+size 28430358

SoccerTwos/SoccerTwos-4499985.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a069f767798732937b0ef8a83ee0455b175fe805b479fece70ece051a56a8746
+size 1768747

SoccerTwos/SoccerTwos-4499985.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5bfc75a2b680e1048d6fb76b860c5faad46dd965cde640a14b3bd61badc0dee
+size 28430358

SoccerTwos/SoccerTwos-4999972.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8171b9f22f3de0dfac49f68c4df948d5cda20dc11b6a2d73b5e244d153896472
+size 1768747

SoccerTwos/SoccerTwos-4999972.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f9baa0ff638555050fea9bd46515cf5d23335f5f978bb7d4403a56726907844
+size 28430358

SoccerTwos/SoccerTwos-5000376.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8171b9f22f3de0dfac49f68c4df948d5cda20dc11b6a2d73b5e244d153896472
+size 1768747

SoccerTwos/SoccerTwos-5000376.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32c3920c0b57d2ed7866fd28964c598624be7d4afd1fdc3e5019d8310c1bb00b
+size 28430358

SoccerTwos/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e9770185d2047554a219e6bc1797189e286d95b1f64eebe9cecf9b2ce127d76
+oid sha256:722fe59ebe4db88e2dd4d2191dfc0ba389035dbf97e91c45a728de4768562749
 size 28429366

SoccerTwos/events.out.tfevents.1703848765.nilss-mbp.lan.62191.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:957cee33c212d35c435af44dedc2e16d5b6c65963df108e63c326080cf90a00a
+size 4771353

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 2048, "buffer_size": 20480, "learning_rate": 0.0005, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 50000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 250000, "swap_steps": 2000, "window": 10, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos.app", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 2048, "buffer_size": 20480, "learning_rate": 0.0005, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 5000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 250000, "swap_steps": 2000, "window": 10, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos.app", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -37,7 +37,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 50000000
+    max_steps: 5000000
     time_horizon: 1000
     summary_freq: 10000
     threaded: false

run_logs/Player-0.log

@@ -10,7 +10,7 @@ NullGfxDevice:
     Renderer: Null Device
     Vendor:   Unity Technologies
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.033 seconds
+- Completed reload, in  0.034 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -35,7 +35,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.190958 ms
+UnloadTime: 0.204709 ms
 Registered Communicator in Agent.
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 TOO LONG
@@ -290,13 +290,259 @@ TOO LONG
 TOO LONG
 TOO LONG
 TOO LONG
-Thread 0x16f507000 may have been prematurely finalized
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+TOO LONG
+Thread 0x1711cf000 may have been prematurely finalized
 Setting up 6 worker threads for Enlighten.
 Memory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [8.0 KB-16.0 KB]: 327785 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [8.0 KB-16.0 KB]: 94692 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.1 MB
@@ -359,7 +605,7 @@ Memory Statistics:
     [ALLOC_TEMP_Job.Worker 3]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0 B
+      Peak Allocated Bytes 0.7 KB
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 12]
       Initial Block Size 32.0 KB
@@ -414,7 +660,7 @@ Memory Statistics:
     [ALLOC_TEMP_Job.Worker 8]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0.7 KB
+      Peak Allocated Bytes 0 B
       Overflow Count 0
     [ALLOC_TEMP_OSX HID Input]
       Initial Block Size 64.0 KB
@@ -467,22 +713,22 @@ Memory Statistics:
       Peak Allocated Bytes 0 B
       Overflow Count 0
 [ALLOC_DEFAULT] Dual Thread Allocator
-  Peak main deferred allocation count 5
+  Peak main deferred allocation count 6
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [4.0 MB-8.0 MB]: 327786 frames
+      Peak usage frame count: [4.0 MB-8.0 MB]: 94693 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 5.2 MB
+      Peak Allocated memory 5.0 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 327786 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 94693 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 17.2 MB
+      Peak Allocated memory 17.4 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -511,13 +757,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 327785 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 94692 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 65.6 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [128.0 KB-256.0 KB]: 327786 frames
+      Peak usage frame count: [128.0 KB-256.0 KB]: 94693 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 173.5 KB
@@ -529,13 +775,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 327786 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 94693 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 0.6 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 327785 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 94692 frames, [2.0 MB-4.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 3.4 MB
@@ -547,13 +793,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 327786 frames
+      Peak usage frame count: [0-1.0 KB]: 94693 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 327785 frames, [2.0 KB-4.0 KB]: 1 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 94692 frames, [2.0 KB-4.0 KB]: 1 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 2.2 KB

run_logs/timers.json

@@ -2,201 +2,201 @@
     "name": "root",
     "gauges": {
         "SoccerTwos.Policy.Entropy.mean": {
-            "value": 1.3300622701644897,
-            "min": 1.266438364982605,
-            "max": 3.29573655128479,
-            "count": 1740
+            "value": 1.755637288093567,
+            "min": 1.7151814699172974,
+            "max": 3.295724630355835,
+            "count": 499
         },
         "SoccerTwos.Policy.Entropy.sum": {
-            "value": 27069.427734375,
-            "min": 21218.80078125,
-            "max": 106789.578125,
-            "count": 1740
+            "value": 36910.51953125,
+            "min": 13612.6572265625,
+            "max": 172905.75,
+            "count": 499
         },
         "SoccerTwos.Environment.EpisodeLength.mean": {
-            "value": 89.76363636363637,
-            "min": 40.675213675213676,
+            "value": 63.6125,
+            "min": 39.645161290322584,
             "max": 999.0,
-            "count": 1740
+            "count": 499
         },
         "SoccerTwos.Environment.EpisodeLength.sum": {
-            "value": 19748.0,
-            "min": 9376.0,
-            "max": 31520.0,
-            "count": 1740
+            "value": 20356.0,
+            "min": 10692.0,
+            "max": 31500.0,
+            "count": 499
         },
         "SoccerTwos.Self-play.ELO.mean": {
-            "value": 1673.6429207825179,
-            "min": 1196.5457435177627,
-            "max": 1726.4687385612406,
-            "count": 1739
+            "value": 1540.6404213531655,
+            "min": 1197.4957713356807,
+            "max": 1576.4143309371482,
+            "count": 478
         },
         "SoccerTwos.Self-play.ELO.sum": {
-            "value": 184100.72128607697,
-            "min": 2394.535852594916,
-            "max": 376556.50433398865,
-            "count": 1739
+            "value": 246502.4674165065,
+            "min": 2399.421175774324,
+            "max": 363079.2013600959,
+            "count": 478
         },
         "SoccerTwos.Step.mean": {
-            "value": 17399884.0,
-            "min": 9778.0,
-            "max": 17399884.0,
-            "count": 1740
+            "value": 4999972.0,
+            "min": 9254.0,
+            "max": 4999972.0,
+            "count": 500
         },
         "SoccerTwos.Step.sum": {
-            "value": 17399884.0,
-            "min": 9778.0,
-            "max": 17399884.0,
-            "count": 1740
+            "value": 4999972.0,
+            "min": 9254.0,
+            "max": 4999972.0,
+            "count": 500
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.mean": {
-            "value": -0.04988090693950653,
-            "min": -0.13098840415477753,
-            "max": 0.16840921342372894,
-            "count": 1740
+            "value": -0.034482039511203766,
+            "min": -0.10746785998344421,
+            "max": 0.16136549413204193,
+            "count": 500
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.sum": {
-            "value": -5.536780834197998,
-            "min": -19.894561767578125,
-            "max": 21.182220458984375,
-            "count": 1740
+            "value": -5.517126083374023,
+            "min": -22.675718307495117,
+            "max": 27.432022094726562,
+            "count": 500
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.mean": {
-            "value": -0.049323659390211105,
-            "min": -0.13105210661888123,
-            "max": 0.17191989719867706,
-            "count": 1740
+            "value": -0.034504301846027374,
+            "min": -0.11580751091241837,
+            "max": 0.1590351015329361,
+            "count": 500
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.sum": {
-            "value": -5.474925994873047,
-            "min": -20.791091918945312,
-            "max": 21.83549690246582,
-            "count": 1740
+            "value": -5.520688533782959,
+            "min": -24.43538475036621,
+            "max": 26.952106475830078,
+            "count": 500
         },
         "SoccerTwos.Environment.CumulativeReward.mean": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 1740
+            "count": 500
         },
         "SoccerTwos.Environment.CumulativeReward.sum": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 1740
+            "count": 500
         },
         "SoccerTwos.Policy.ExtrinsicReward.mean": {
-            "value": -0.19373873762182287,
-            "min": -0.6153846153846154,
-            "max": 0.42934736766313253,
-            "count": 1740
+            "value": -0.08047500103712082,
+            "min": -0.5921090895479376,
+            "max": 0.4576799988746643,
+            "count": 500
         },
         "SoccerTwos.Policy.ExtrinsicReward.sum": {
-            "value": -21.50499987602234,
-            "min": -55.30279994010925,
-            "max": 52.12959986925125,
-            "count": 1740
+            "value": -12.876000165939331,
+            "min": -66.35360014438629,
+            "max": 60.00919944047928,
+            "count": 500
         },
         "SoccerTwos.Environment.GroupCumulativeReward.mean": {
-            "value": -0.19373873762182287,
-            "min": -0.6153846153846154,
-            "max": 0.42934736766313253,
-            "count": 1740
+            "value": -0.08047500103712082,
+            "min": -0.5921090895479376,
+            "max": 0.4576799988746643,
+            "count": 500
         },
         "SoccerTwos.Environment.GroupCumulativeReward.sum": {
-            "value": -21.50499987602234,
-            "min": -55.30279994010925,
-            "max": 52.12959986925125,
-            "count": 1740
+            "value": -12.876000165939331,
+            "min": -66.35360014438629,
+            "max": 60.00919944047928,
+            "count": 500
         },
         "SoccerTwos.IsTraining.mean": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 1740
+            "count": 500
         },
         "SoccerTwos.IsTraining.sum": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 1740
+            "count": 500
         },
         "SoccerTwos.Losses.PolicyLoss.mean": {
-            "value": 0.018441027875026824,
-            "min": 0.011074252808854605,
-            "max": 0.029008984806326528,
-            "count": 843
+            "value": 0.015034021192695946,
+            "min": 0.010941779592152064,
+            "max": 0.024873137117053072,
+            "count": 239
         },
         "SoccerTwos.Losses.PolicyLoss.sum": {
-            "value": 0.018441027875026824,
-            "min": 0.011074252808854605,
-            "max": 0.029008984806326528,
-            "count": 843
+            "value": 0.015034021192695946,
+            "min": 0.010941779592152064,
+            "max": 0.024873137117053072,
+            "count": 239
         },
         "SoccerTwos.Losses.ValueLoss.mean": {
-            "value": 0.08094811787207921,
-            "min": 0.0011656492541078478,
-            "max": 0.11928065071503322,
-            "count": 843
+            "value": 0.11739198019107183,
+            "min": 3.1021232795562052e-06,
+            "max": 0.1251884085436662,
+            "count": 239
         },
         "SoccerTwos.Losses.ValueLoss.sum": {
-            "value": 0.08094811787207921,
-            "min": 0.0011656492541078478,
-            "max": 0.11928065071503322,
-            "count": 843
+            "value": 0.11739198019107183,
+            "min": 3.1021232795562052e-06,
+            "max": 0.1251884085436662,
+            "count": 239
         },
         "SoccerTwos.Losses.BaselineLoss.mean": {
-            "value": 0.08201082050800323,
-            "min": 0.0011800630503178884,
-            "max": 0.12257990539073944,
-            "count": 843
+            "value": 0.11775108724832535,
+            "min": 3.304618659664508e-06,
+            "max": 0.1258082126577695,
+            "count": 239
         },
         "SoccerTwos.Losses.BaselineLoss.sum": {
-            "value": 0.08201082050800323,
-            "min": 0.0011800630503178884,
-            "max": 0.12257990539073944,
-            "count": 843
+            "value": 0.11775108724832535,
+            "min": 3.304618659664508e-06,
+            "max": 0.1258082126577695,
+            "count": 239
         },
         "SoccerTwos.Policy.LearningRate.mean": {
-            "value": 0.0003260040147992041,
-            "min": 0.0003260040147992041,
-            "max": 0.0004997152400569519,
-            "count": 843
+            "value": 1.0132997973599917e-06,
+            "min": 1.0132997973599917e-06,
+            "max": 0.0004968980006203998,
+            "count": 239
         },
         "SoccerTwos.Policy.LearningRate.sum": {
-            "value": 0.0003260040147992041,
-            "min": 0.0003260040147992041,
-            "max": 0.0004997152400569519,
-            "count": 843
+            "value": 1.0132997973599917e-06,
+            "min": 1.0132997973599917e-06,
+            "max": 0.0004968980006203998,
+            "count": 239
         },
         "SoccerTwos.Policy.Epsilon.mean": {
-            "value": 0.16520079599999996,
-            "min": 0.16520079599999996,
-            "max": 0.19994304799999996,
-            "count": 843
+            "value": 0.10020264000000001,
+            "min": 0.10020264000000001,
+            "max": 0.19937960000000005,
+            "count": 239
         },
         "SoccerTwos.Policy.Epsilon.sum": {
-            "value": 0.16520079599999996,
-            "min": 0.16520079599999996,
-            "max": 0.19994304799999996,
-            "count": 843
+            "value": 0.10020264000000001,
+            "min": 0.10020264000000001,
+            "max": 0.19937960000000005,
+            "count": 239
         },
         "SoccerTwos.Policy.Beta.mean": {
-            "value": 0.003263519720399999,
-            "min": 0.003263519720399999,
-            "max": 0.004997158095199998,
-            "count": 843
+            "value": 2.0111735999999927e-05,
+            "min": 2.0111735999999927e-05,
+            "max": 0.004969042040000001,
+            "count": 239
         },
         "SoccerTwos.Policy.Beta.sum": {
-            "value": 0.003263519720399999,
-            "min": 0.003263519720399999,
-            "max": 0.004997158095199998,
-            "count": 843
+            "value": 2.0111735999999927e-05,
+            "min": 2.0111735999999927e-05,
+            "max": 0.004969042040000001,
+            "count": 239
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1703778636",
+        "start_time_seconds": "1703848764",
         "python_version": "3.10.12 (main, Jul  5 2023, 15:02:25) [Clang 14.0.6 ]",
         "command_line_arguments": "/opt/homebrew/Caskroom/miniconda/base/envs/rl/bin/mlagents-learn ./config/poca/SoccerTwos.yaml --env=./training-envs-executables/SoccerTwos.app --run-id=SoccerTwos --no-graphics --force",
         "mlagents_version": "1.1.0.dev0",
@@ -204,103 +204,103 @@
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "2.1.2",
         "numpy_version": "1.23.5",
-        "end_time_seconds": "1703795928"
+        "end_time_seconds": "1703853889"
     },
-    "total": 17291.35947341699,
+    "total": 5124.430119083001,
     "count": 1,
-    "self": 10.002615667006467,
+    "self": 0.19001916600973345,
     "children": {
         "run_training.setup": {
-            "total": 0.014216332987416536,
+            "total": 0.012270416977116838,
             "count": 1,
-            "self": 0.014216332987416536
+            "self": 0.012270416977116838
         },
         "TrainerController.start_learning": {
-            "total": 17281.342641416995,
+            "total": 5124.227829500014,
             "count": 1,
-            "self": 7.727816474769497,
+            "self": 2.531355981802335,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 1.8024594139424153,
-                    "count": 70,
-                    "self": 1.8024594139424153
+                    "total": 1.5887895020714495,
+                    "count": 20,
+                    "self": 1.5887895020714495
                 },
                 "TrainerController.advance": {
-                    "total": 17271.652083277266,
-                    "count": 1196889,
-                    "self": 7.361157477425877,
+                    "total": 5119.955040642148,
+                    "count": 341985,
+                    "self": 2.185286336200079,
                     "children": {
                         "env_step": {
-                            "total": 7843.07346421259,
-                            "count": 1196889,
-                            "self": 6619.223402722797,
+                            "total": 2303.891035468987,
+                            "count": 341985,
+                            "self": 1941.7491068352829,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 1218.7477199516725,
-                                    "count": 1196889,
-                                    "self": 36.26535058821901,
+                                    "total": 360.6655651805049,
+                                    "count": 341985,
+                                    "self": 11.17105897743022,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 1182.4823693634535,
-                                            "count": 2185268,
-                                            "self": 1182.4823693634535
+                                            "total": 349.4945062030747,
+                                            "count": 631296,
+                                            "self": 349.4945062030747
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 5.102341538120527,
-                                    "count": 1196888,
+                                    "total": 1.4763634531991556,
+                                    "count": 341985,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 17268.48108889186,
-                                            "count": 1196888,
+                                            "total": 5119.23122701954,
+                                            "count": 341985,
                                             "is_parallel": true,
-                                            "self": 11671.18532553385,
+                                            "self": 3492.6540978672565,
                                             "children": {
                                                 "steps_from_proto": {
-                                                    "total": 0.044110004091635346,
-                                                    "count": 140,
+                                                    "total": 0.013060167053481564,
+                                                    "count": 40,
                                                     "is_parallel": true,
-                                                    "self": 0.0071852429246064276,
+                                                    "self": 0.0021915972174610943,
                                                     "children": {
                                                         "_process_rank_one_or_two_observation": {
-                                                            "total": 0.03692476116702892,
-                                                            "count": 560,
+                                                            "total": 0.01086856983602047,
+                                                            "count": 160,
                                                             "is_parallel": true,
-                                                            "self": 0.03692476116702892
+                                                            "self": 0.01086856983602047
                                                         }
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 5597.2516533539165,
-                                                    "count": 1196888,
+                                                    "total": 1626.5640689852298,
+                                                    "count": 341985,
                                                     "is_parallel": true,
-                                                    "self": 272.46567574935034,
+                                                    "self": 78.99318619386759,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 129.84849359397776,
-                                                            "count": 1196888,
+                                                            "total": 37.23996753204847,
+                                                            "count": 341985,
                                                             "is_parallel": true,
-                                                            "self": 129.84849359397776
+                                                            "self": 37.23996753204847
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 4506.445674063638,
-                                                            "count": 1196888,
+                                                            "total": 1310.1512493426271,
+                                                            "count": 341985,
                                                             "is_parallel": true,
-                                                            "self": 4506.445674063638
+                                                            "self": 1310.1512493426271
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 688.4918099469505,
-                                                            "count": 2393776,
+                                                            "total": 200.17966591668664,
+                                                            "count": 683970,
                                                             "is_parallel": true,
-                                                            "self": 106.13120424517547,
+                                                            "self": 30.950994182756403,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 582.3606057017751,
-                                                                    "count": 9575104,
+                                                                    "total": 169.22867173393024,
+                                                                    "count": 2735880,
                                                                     "is_parallel": true,
-                                                                    "self": 582.3606057017751
+                                                                    "self": 169.22867173393024
                                                                 }
                                                             }
                                                         }
@@ -313,31 +313,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 9421.21746158725,
-                            "count": 1196888,
-                            "self": 69.65316009536036,
+                            "total": 2813.8787188369606,
+                            "count": 341985,
+                            "self": 21.16630858287681,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 1950.7394902148517,
-                                    "count": 1196888,
-                                    "self": 1945.4922155466338,
+                                    "total": 556.1312195862702,
+                                    "count": 341985,
+                                    "self": 554.5936779602489,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 5.2472746682178695,
-                                            "count": 34,
-                                            "self": 5.2472746682178695
+                                            "total": 1.537541626021266,
+                                            "count": 10,
+                                            "self": 1.537541626021266
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 7400.824811277038,
-                                    "count": 843,
-                                    "self": 830.3490734959778,
+                                    "total": 2236.5811906678136,
+                                    "count": 239,
+                                    "self": 242.41541127837263,
                                     "children": {
                                         "TorchPOCAOptimizer.update": {
-                                            "total": 6570.47573778106,
-                                            "count": 25299,
-                                            "self": 6570.47573778106
+                                            "total": 1994.165779389441,
+                                            "count": 7185,
+                                            "self": 1994.165779389441
                                         }
                                     }
                                 }
@@ -346,19 +346,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 3.33995558321476e-07,
+                    "total": 3.3300602808594704e-07,
                     "count": 1,
-                    "self": 3.33995558321476e-07
+                    "self": 3.3300602808594704e-07
                 },
                 "TrainerController._save_models": {
-                    "total": 0.1602819170220755,
+                    "total": 0.15264304098673165,
                     "count": 1,
-                    "self": 0.0015421260322909802,
+                    "self": 0.000656957970932126,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.15873979098978452,
+                            "total": 0.15198608301579952,
                             "count": 1,
-                            "self": 0.15873979098978452
+                            "self": 0.15198608301579952
                         }
                     }
                 }

run_logs/training_status.json

@@ -2,59 +2,59 @@
     "SoccerTwos": {
         "checkpoints": [
             {
-                "steps": 15499904,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-15499904.onnx",
+                "steps": 3499990,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-3499990.onnx",
                 "reward": 0.0,
-                "creation_time": 1703794033.6399949,
+                "creation_time": 1703852337.048543,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-15499904.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-3499990.pt"
                 ]
             },
             {
-                "steps": 15999958,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-15999958.onnx",
+                "steps": 3999988,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-3999988.onnx",
                 "reward": 0.0,
-                "creation_time": 1703794534.1326778,
+                "creation_time": 1703852850.532505,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-15999958.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-3999988.pt"
                 ]
             },
             {
-                "steps": 16499979,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-16499979.onnx",
+                "steps": 4499985,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-4499985.onnx",
                 "reward": 0.0,
-                "creation_time": 1703795023.9592068,
+                "creation_time": 1703853379.401021,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-16499979.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-4499985.pt"
                 ]
             },
             {
-                "steps": 16999852,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-16999852.onnx",
+                "steps": 4999972,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-4999972.onnx",
                 "reward": 0.0,
-                "creation_time": 1703795515.021195,
+                "creation_time": 1703853888.732385,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-16999852.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-4999972.pt"
                 ]
             },
             {
-                "steps": 17409136,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-17409136.onnx",
+                "steps": 5000376,
+                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-5000376.onnx",
                 "reward": 0.0,
-                "creation_time": 1703795918.066219,
+                "creation_time": 1703853888.9599922,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-17409136.pt"
+                    "results/SoccerTwos/SoccerTwos/SoccerTwos-5000376.pt"
                 ]
             }
         ],
-        "elo": 1673.7859725087915,
+        "elo": 1543.402385883174,
         "final_checkpoint": {
-            "steps": 17409136,
+            "steps": 5000376,
             "file_path": "results/SoccerTwos/SoccerTwos.onnx",
             "reward": 0.0,
-            "creation_time": 1703795918.066219,
+            "creation_time": 1703853888.9599922,
             "auxillary_file_paths": [
-                "results/SoccerTwos/SoccerTwos/SoccerTwos-17409136.pt"
+                "results/SoccerTwos/SoccerTwos/SoccerTwos-5000376.pt"
             ]
         }
     },