Longer training

SoccerTwos.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2caa488caba8dee8e1630df2ccdb3235c2475f660113ef70a3cd044e47d10a14
+oid sha256:4d46086f1af51da059118a32a87efbfcf969e05de3df0828f63709fd54926cfb
 size 1768747

SoccerTwos/SoccerTwos-15999812.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6402359f9cfc31562c165152e960f1dda357621ab62a40f0e7992538d55dc44
+size 1768747

SoccerTwos/SoccerTwos-15999812.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7815cf04ef7fc1dfca90aeae14fbf801163deb00ef8233fce07c28447d351556
+size 28432146

SoccerTwos/SoccerTwos-16499964.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2424b56a08fc3b77c1217555fb841bddc3d50ad61c66690e9a012c01d3d7cd7c
+size 1768747

SoccerTwos/SoccerTwos-16499964.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2889fd750d021f097e024946adac9767f5587af7b8fe0a9cbd4220c50d55661
+size 28432146

SoccerTwos/SoccerTwos-16999954.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7195cc22cf8f71bc93fa9891e978d32313e6ebe0233f1c03b6eec258aceaaeea
+size 1768747

SoccerTwos/SoccerTwos-16999954.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34d5cce3fc5ee695ed35700107082e491e03eb3ea82e4c41bb4af07c812a00a6
+size 28432146

SoccerTwos/SoccerTwos-17499994.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:632a80b878eebbe39255c8ecb0efb2d3391b3694260860b02a2fa83e7fe444e2
+size 1768747

SoccerTwos/SoccerTwos-17499994.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97ff0fa21b36a355b56903a1ea307a65354fff544c7d5625343aecee7b8dfcb9
+size 28432146

SoccerTwos/SoccerTwos-17703666.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d46086f1af51da059118a32a87efbfcf969e05de3df0828f63709fd54926cfb
+size 1768747

SoccerTwos/SoccerTwos-17703666.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d47ce8bede9c665ec2bbec3538c6b4da4a20009b01576335256328d820ff54cb
+size 28432146

SoccerTwos/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f987d040b7b0c5b5c05edd24953b04691c2698eae980879fb78a97dc2748ad5c
+oid sha256:567af6a83a22848b16f6b2b2b28d9dfaab6859a604e35d0547b760eb27042428
 size 28431030

SoccerTwos/events.out.tfevents.1718514435.pop-os.93470.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1747741a2fe3b622c93d0c9e66b4e4131baa1a93fcd90c4150871113a1dae99
+size 17904863

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 2048, "buffer_size": 20480, "learning_rate": 0.0003, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "constant", "beta_schedule": "constant", "epsilon_schedule": "constant"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 10000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 200000, "swap_steps": 2000, "window": 10, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos/SoccerTwos.x86_64", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"SoccerTwos": {"trainer_type": "poca", "hyperparameters": {"batch_size": 2048, "buffer_size": 20480, "learning_rate": 0.0003, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "learning_rate_schedule": "constant", "beta_schedule": "constant", "epsilon_schedule": "constant"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 50000000, "time_horizon": 1000, "summary_freq": 10000, "threaded": false, "self_play": {"save_steps": 50000, "team_change": 200000, "swap_steps": 2000, "window": 10, "play_against_latest_model_ratio": 0.5, "initial_elo": 1200.0}, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/SoccerTwos/SoccerTwos.x86_64", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SoccerTwos50M", "initialize_from": null, "load_model": false, "resume": false, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -37,7 +37,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 10000000
+    max_steps: 50000000
     time_horizon: 1000
     summary_freq: 10000
     threaded: false
@@ -71,11 +71,11 @@ engine_settings:
   no_graphics_monitor: false
 environment_parameters: null
 checkpoint_settings:
-  run_id: SoccerTwos
+  run_id: SoccerTwos50M
   initialize_from: null
   load_model: false
   resume: false
-  force: true
+  force: false
   train_model: false
   inference: false
   results_dir: results

run_logs/Player-0.log

@@ -2,8 +2,6 @@ Mono path[0] = '/home/ivan/Code/rl/hf/unit7/ml-agents/training-envs-executables/
 Mono config path = '/home/ivan/Code/rl/hf/unit7/ml-agents/training-envs-executables/SoccerTwos/SoccerTwos_Data/MonoBleedingEdge/etc'
 Preloaded 'lib_burst_generated.so'
 Preloaded 'libgrpc_csharp_ext.x64.so'
-PlayerPrefs - Creating folder: /home/ivan/.config/unity3d/Unity Technologies/SoccerTwos
-Unable to load player prefs
 Initialize engine version: 2021.3.14f1 (eee1884e7226)
 [Subsystems] Discovering subsystems at path /home/ivan/Code/rl/hf/unit7/ml-agents/training-envs-executables/SoccerTwos/SoccerTwos_Data/UnitySubsystems
 Forcing GfxDevice: Null
@@ -13,7 +11,7 @@ NullGfxDevice:
     Renderer: Null Device
     Vendor:   Unity Technologies
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.057 seconds
+- Completed reload, in  0.055 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -38,7 +36,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.433668 ms
+UnloadTime: 0.439302 ms
 Registered Communicator in Agent.
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 requesting resize 84 x 84
@@ -508,412 +506,12 @@ TOO LONG
 TOO LONG
 TOO LONG
 TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
-TOO LONG
 Setting up 4 worker threads for Enlighten.
 Memory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [8.0 KB-16.0 KB]: 189475 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [8.0 KB-16.0 KB]: 333814 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.1 MB
@@ -1059,22 +657,22 @@ Memory Statistics:
       Peak Allocated Bytes 0 B
       Overflow Count 0
 [ALLOC_DEFAULT] Dual Thread Allocator
-  Peak main deferred allocation count 3
+  Peak main deferred allocation count 6
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [4.0 MB-8.0 MB]: 189476 frames
+      Peak usage frame count: [4.0 MB-8.0 MB]: 333815 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 4.9 MB
+      Peak Allocated memory 4.7 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 189476 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 333815 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 17.3 MB
+      Peak Allocated memory 17.5 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -1103,13 +701,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 189475 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 333814 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 65.6 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [128.0 KB-256.0 KB]: 189476 frames
+      Peak usage frame count: [128.0 KB-256.0 KB]: 333815 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 173.5 KB
@@ -1121,13 +719,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 189476 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 333815 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 0.6 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 189475 frames, [4.0 MB-8.0 MB]: 1 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 333814 frames, [4.0 MB-8.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 2
       Peak Allocated memory 4.4 MB
@@ -1139,13 +737,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 189476 frames
+      Peak usage frame count: [0-1.0 KB]: 333815 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 189475 frames, [2.0 KB-4.0 KB]: 1 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 333814 frames, [2.0 KB-4.0 KB]: 1 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 2.2 KB

run_logs/timers.json

@@ -2,261 +2,261 @@
     "name": "root",
     "gauges": {
         "SoccerTwos.Policy.Entropy.mean": {
-            "value": 1.619049310684204,
-            "min": 1.619049310684204,
-            "max": 3.2956883907318115,
-            "count": 1000
+            "value": 1.5035665035247803,
+            "min": 1.470587968826294,
+            "max": 3.295746326446533,
+            "count": 1770
         },
         "SoccerTwos.Policy.Entropy.sum": {
-            "value": 31396.603515625,
-            "min": 18282.5,
-            "max": 115885.4609375,
-            "count": 1000
+            "value": 28242.9921875,
+            "min": 27409.775390625,
+            "max": 120267.0,
+            "count": 1770
         },
         "SoccerTwos.Environment.EpisodeLength.mean": {
-            "value": 46.660377358490564,
-            "min": 45.66355140186916,
+            "value": 60.34567901234568,
+            "min": 39.3739837398374,
             "max": 999.0,
-            "count": 1000
+            "count": 1770
         },
         "SoccerTwos.Environment.EpisodeLength.sum": {
-            "value": 19784.0,
-            "min": 9156.0,
-            "max": 31876.0,
-            "count": 1000
+            "value": 19552.0,
+            "min": 14212.0,
+            "max": 28212.0,
+            "count": 1770
         },
         "SoccerTwos.Self-play.ELO.mean": {
-            "value": 1583.2908266567183,
-            "min": 1174.4644358660264,
-            "max": 1600.7497911642322,
-            "count": 990
+            "value": 1544.5104403214739,
+            "min": 1180.9941894213423,
+            "max": 1603.3526931779911,
+            "count": 1761
         },
         "SoccerTwos.Self-play.ELO.sum": {
-            "value": 335657.6552512243,
-            "min": 2370.8536334395567,
-            "max": 339084.33217810746,
-            "count": 990
+            "value": 250210.69133207877,
+            "min": 2363.2477491938266,
+            "max": 392660.82324914914,
+            "count": 1761
         },
         "SoccerTwos.Step.mean": {
-            "value": 9999982.0,
-            "min": 9954.0,
-            "max": 9999982.0,
-            "count": 1000
+            "value": 17699978.0,
+            "min": 9122.0,
+            "max": 17699978.0,
+            "count": 1770
         },
         "SoccerTwos.Step.sum": {
-            "value": 9999982.0,
-            "min": 9954.0,
-            "max": 9999982.0,
-            "count": 1000
+            "value": 17699978.0,
+            "min": 9122.0,
+            "max": 17699978.0,
+            "count": 1770
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.mean": {
-            "value": -0.00607228372246027,
-            "min": -0.11751111596822739,
-            "max": 0.21167558431625366,
-            "count": 1000
+            "value": -0.028001178056001663,
+            "min": -0.1374976485967636,
+            "max": 0.2069934755563736,
+            "count": 1770
         },
         "SoccerTwos.Policy.ExtrinsicBaselineEstimate.sum": {
-            "value": -1.2812519073486328,
-            "min": -18.80177879333496,
-            "max": 22.583396911621094,
-            "count": 1000
+            "value": -4.536190986633301,
+            "min": -28.18701934814453,
+            "max": 27.792827606201172,
+            "count": 1770
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.mean": {
-            "value": -0.002361964900046587,
-            "min": -0.11942289024591446,
-            "max": 0.20918215811252594,
-            "count": 1000
+            "value": -0.023671336472034454,
+            "min": -0.1340864896774292,
+            "max": 0.2030305713415146,
+            "count": 1770
         },
         "SoccerTwos.Policy.ExtrinsicValueEstimate.sum": {
-            "value": -0.4983745813369751,
-            "min": -19.08892822265625,
-            "max": 22.341575622558594,
-            "count": 1000
+            "value": -3.834756374359131,
+            "min": -27.168577194213867,
+            "max": 27.650724411010742,
+            "count": 1770
         },
         "SoccerTwos.Environment.CumulativeReward.mean": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 1000
+            "count": 1770
         },
         "SoccerTwos.Environment.CumulativeReward.sum": {
             "value": 0.0,
             "min": 0.0,
             "max": 0.0,
-            "count": 1000
+            "count": 1770
         },
         "SoccerTwos.Policy.ExtrinsicReward.mean": {
-            "value": -0.09993649376512138,
-            "min": -0.6316222217347887,
-            "max": 0.6275224503205747,
-            "count": 1000
+            "value": 0.07485432021411849,
+            "min": -0.6008615379150097,
+            "max": 0.43080000256030065,
+            "count": 1770
         },
         "SoccerTwos.Policy.ExtrinsicReward.sum": {
-            "value": -21.086600184440613,
-            "min": -52.99939978122711,
-            "max": 61.49720013141632,
-            "count": 1000
+            "value": 12.126399874687195,
+            "min": -63.243399918079376,
+            "max": 62.33080017566681,
+            "count": 1770
         },
         "SoccerTwos.Environment.GroupCumulativeReward.mean": {
-            "value": -0.09993649376512138,
-            "min": -0.6316222217347887,
-            "max": 0.6275224503205747,
-            "count": 1000
+            "value": 0.07485432021411849,
+            "min": -0.6008615379150097,
+            "max": 0.43080000256030065,
+            "count": 1770
         },
         "SoccerTwos.Environment.GroupCumulativeReward.sum": {
-            "value": -21.086600184440613,
-            "min": -52.99939978122711,
-            "max": 61.49720013141632,
-            "count": 1000
+            "value": 12.126399874687195,
+            "min": -63.243399918079376,
+            "max": 62.33080017566681,
+            "count": 1770
         },
         "SoccerTwos.IsTraining.mean": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 1000
+            "count": 1770
         },
         "SoccerTwos.IsTraining.sum": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 1000
+            "count": 1770
         },
         "SoccerTwos.Losses.PolicyLoss.mean": {
-            "value": 0.02385382152472933,
-            "min": 0.010645401534687456,
-            "max": 0.02385382152472933,
-            "count": 480
+            "value": 0.013383328518830239,
+            "min": 0.010278040572908745,
+            "max": 0.02610366263737281,
+            "count": 857
         },
         "SoccerTwos.Losses.PolicyLoss.sum": {
-            "value": 0.02385382152472933,
-            "min": 0.010645401534687456,
-            "max": 0.02385382152472933,
-            "count": 480
+            "value": 0.013383328518830239,
+            "min": 0.010278040572908745,
+            "max": 0.02610366263737281,
+            "count": 857
         },
         "SoccerTwos.Losses.ValueLoss.mean": {
-            "value": 0.1010389839609464,
-            "min": 0.00015815873339306564,
-            "max": 0.11575197329123815,
-            "count": 480
+            "value": 0.1036595510939757,
+            "min": 8.419977239100262e-05,
+            "max": 0.12884489769736926,
+            "count": 857
         },
         "SoccerTwos.Losses.ValueLoss.sum": {
-            "value": 0.1010389839609464,
-            "min": 0.00015815873339306564,
-            "max": 0.11575197329123815,
-            "count": 480
+            "value": 0.1036595510939757,
+            "min": 8.419977239100262e-05,
+            "max": 0.12884489769736926,
+            "count": 857
         },
         "SoccerTwos.Losses.BaselineLoss.mean": {
-            "value": 0.10250727211435635,
-            "min": 0.00015888037111532564,
-            "max": 0.11866774335503578,
-            "count": 480
+            "value": 0.10672147423028946,
+            "min": 8.521313526822875e-05,
+            "max": 0.13119359935323396,
+            "count": 857
         },
         "SoccerTwos.Losses.BaselineLoss.sum": {
-            "value": 0.10250727211435635,
-            "min": 0.00015888037111532564,
-            "max": 0.11866774335503578,
-            "count": 480
+            "value": 0.10672147423028946,
+            "min": 8.521313526822875e-05,
+            "max": 0.13119359935323396,
+            "count": 857
         },
         "SoccerTwos.Policy.LearningRate.mean": {
             "value": 0.0003,
             "min": 0.0003,
             "max": 0.0003,
-            "count": 480
+            "count": 857
         },
         "SoccerTwos.Policy.LearningRate.sum": {
             "value": 0.0003,
             "min": 0.0003,
             "max": 0.0003,
-            "count": 480
+            "count": 857
         },
         "SoccerTwos.Policy.Epsilon.mean": {
             "value": 0.20000000000000007,
-            "min": 0.2,
+            "min": 0.20000000000000007,
             "max": 0.20000000000000007,
-            "count": 480
+            "count": 857
         },
         "SoccerTwos.Policy.Epsilon.sum": {
             "value": 0.20000000000000007,
-            "min": 0.2,
+            "min": 0.20000000000000007,
             "max": 0.20000000000000007,
-            "count": 480
+            "count": 857
         },
         "SoccerTwos.Policy.Beta.mean": {
             "value": 0.005000000000000001,
-            "min": 0.005,
+            "min": 0.005000000000000001,
             "max": 0.005000000000000001,
-            "count": 480
+            "count": 857
         },
         "SoccerTwos.Policy.Beta.sum": {
             "value": 0.005000000000000001,
-            "min": 0.005,
+            "min": 0.005000000000000001,
             "max": 0.005000000000000001,
-            "count": 480
+            "count": 857
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1718479331",
+        "start_time_seconds": "1718514434",
         "python_version": "3.10.14 (main, Jun 14 2024, 18:23:26) [GCC 11.4.0]",
-        "command_line_arguments": "/home/ivan/Code/rl/hf/unit7/u7/bin/mlagents-learn ./config/poca/SoccerTwos.yaml --env=./training-envs-executables/SoccerTwos/SoccerTwos.x86_64 --run-id=SoccerTwos --no-graphics --force",
+        "command_line_arguments": "/home/ivan/Code/rl/hf/unit7/u7/bin/mlagents-learn ./config/poca/SoccerTwos.yaml --env=./training-envs-executables/SoccerTwos/SoccerTwos.x86_64 --run-id=SoccerTwos50M --no-graphics",
         "mlagents_version": "1.1.0.dev0",
         "mlagents_envs_version": "1.1.0.dev0",
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "2.3.1+cu121",
         "numpy_version": "1.23.5",
-        "end_time_seconds": "1718492588"
+        "end_time_seconds": "1718538484"
     },
-    "total": 13257.191734499007,
+    "total": 24049.879396452976,
     "count": 1,
-    "self": 0.21883212600369006,
+    "self": 0.0053754179971292615,
     "children": {
         "run_training.setup": {
-            "total": 0.010704863001592457,
+            "total": 0.011126903991680592,
             "count": 1,
-            "self": 0.010704863001592457
+            "self": 0.011126903991680592
         },
         "TrainerController.start_learning": {
-            "total": 13256.962197510002,
+            "total": 24049.862894130987,
             "count": 1,
-            "self": 11.19704046548577,
+            "self": 19.95134469534969,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 2.3967403589049354,
-                    "count": 50,
-                    "self": 2.3967403589049354
+                    "total": 2.48158151388634,
+                    "count": 89,
+                    "self": 2.48158151388634
                 },
                 "TrainerController.advance": {
-                    "total": 13243.187545907567,
-                    "count": 678304,
-                    "self": 10.965911682753358,
+                    "total": 24027.216803529765,
+                    "count": 1224896,
+                    "self": 20.33166449604323,
                     "children": {
                         "env_step": {
-                            "total": 9734.517115767929,
-                            "count": 678304,
-                            "self": 7381.073501714622,
+                            "total": 17344.032462770992,
+                            "count": 1224896,
+                            "self": 13189.24239976547,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 2346.105229501147,
-                                    "count": 678304,
-                                    "self": 59.2861431860365,
+                                    "total": 4141.220234239125,
+                                    "count": 1224896,
+                                    "self": 104.04888988181483,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 2286.8190863151103,
-                                            "count": 1263196,
-                                            "self": 2286.8190863151103
+                                            "total": 4037.17134435731,
+                                            "count": 2225464,
+                                            "self": 4037.17134435731
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 7.338384552160278,
-                                    "count": 678304,
+                                    "total": 13.569828766398132,
+                                    "count": 1224895,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 13242.747293296328,
-                                            "count": 678304,
+                                            "total": 24019.578487852705,
+                                            "count": 1224895,
                                             "is_parallel": true,
-                                            "self": 7165.139361804351,
+                                            "self": 13159.66805658117,
                                             "children": {
                                                 "run_training.setup": {
                                                     "total": 0.0,
@@ -265,48 +265,48 @@
                                                     "self": 0.0,
                                                     "children": {
                                                         "steps_from_proto": {
-                                                            "total": 0.0017898979713208973,
+                                                            "total": 0.00173658796120435,
                                                             "count": 2,
                                                             "is_parallel": true,
-                                                            "self": 0.00042084691813215613,
+                                                            "self": 0.00040872988756746054,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 0.0013690510531887412,
+                                                                    "total": 0.0013278580736368895,
                                                                     "count": 8,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0013690510531887412
+                                                                    "self": 0.0013278580736368895
                                                                 }
                                                             }
                                                         },
                                                         "UnityEnvironment.step": {
-                                                            "total": 0.0187501460313797,
+                                                            "total": 0.018732728029135615,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.0003802470164373517,
+                                                            "self": 0.0003881650627590716,
                                                             "children": {
                                                                 "UnityEnvironment._generate_step_input": {
-                                                                    "total": 0.0003191750147379935,
+                                                                    "total": 0.000311177980620414,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0003191750147379935
+                                                                    "self": 0.000311177980620414
                                                                 },
                                                                 "communicator.exchange": {
-                                                                    "total": 0.016770449001342058,
+                                                                    "total": 0.01675490802153945,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.016770449001342058
+                                                                    "self": 0.01675490802153945
                                                                 },
                                                                 "steps_from_proto": {
-                                                                    "total": 0.0012802749988622963,
+                                                                    "total": 0.0012784769642166793,
                                                                     "count": 2,
                                                                     "is_parallel": true,
-                                                                    "self": 0.00026854907628148794,
+                                                                    "self": 0.0002583829336799681,
                                                                     "children": {
                                                                         "_process_rank_one_or_two_observation": {
-                                                                            "total": 0.0010117259225808084,
+                                                                            "total": 0.0010200940305367112,
                                                                             "count": 8,
                                                                             "is_parallel": true,
-                                                                            "self": 0.0010117259225808084
+                                                                            "self": 0.0010200940305367112
                                                                         }
                                                                     }
                                                                 }
@@ -315,50 +315,50 @@
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 6077.540349922841,
-                                                    "count": 678303,
+                                                    "total": 10859.790246265533,
+                                                    "count": 1224894,
                                                     "is_parallel": true,
-                                                    "self": 317.6406598382746,
+                                                    "self": 564.7082912233891,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 190.51674898219062,
-                                                            "count": 678303,
+                                                            "total": 331.67259040189674,
+                                                            "count": 1224894,
                                                             "is_parallel": true,
-                                                            "self": 190.51674898219062
+                                                            "self": 331.67259040189674
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 4672.0610915617435,
-                                                            "count": 678303,
+                                                            "total": 8369.044318542641,
+                                                            "count": 1224894,
                                                             "is_parallel": true,
-                                                            "self": 4672.0610915617435
+                                                            "self": 8369.044318542641
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 897.3218495406327,
-                                                            "count": 1356606,
+                                                            "total": 1594.365046097606,
+                                                            "count": 2449788,
                                                             "is_parallel": true,
-                                                            "self": 167.22421341767767,
+                                                            "self": 302.4276817381615,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 730.097636122955,
-                                                                    "count": 5426424,
+                                                                    "total": 1291.9373643594445,
+                                                                    "count": 9799152,
                                                                     "is_parallel": true,
-                                                                    "self": 730.097636122955
+                                                                    "self": 1291.9373643594445
                                                                 }
                                                             }
                                                         }
                                                     }
                                                 },
                                                 "steps_from_proto": {
-                                                    "total": 0.06758156913565472,
-                                                    "count": 98,
+                                                    "total": 0.12018500600242987,
+                                                    "count": 176,
                                                     "is_parallel": true,
-                                                    "self": 0.01229875162243843,
+                                                    "self": 0.022353325912263244,
                                                     "children": {
                                                         "_process_rank_one_or_two_observation": {
-                                                            "total": 0.05528281751321629,
-                                                            "count": 392,
+                                                            "total": 0.09783168009016663,
+                                                            "count": 704,
                                                             "is_parallel": true,
-                                                            "self": 0.05528281751321629
+                                                            "self": 0.09783168009016663
                                                         }
                                                     }
                                                 }
@@ -369,31 +369,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 3497.704518456885,
-                            "count": 678304,
-                            "self": 85.6454694432905,
+                            "total": 6662.85267626273,
+                            "count": 1224895,
+                            "self": 152.00744313228643,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 1227.5400425905827,
-                                    "count": 678304,
-                                    "self": 1223.2639663565205,
+                                    "total": 2579.932920970081,
+                                    "count": 1224895,
+                                    "self": 2573.186581147311,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 4.276076234062202,
-                                            "count": 20,
-                                            "self": 4.276076234062202
+                                            "total": 6.746339822770096,
+                                            "count": 35,
+                                            "self": 6.746339822770096
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 2184.519006423012,
-                                    "count": 480,
-                                    "self": 1017.7918696403503,
+                                    "total": 3930.9123121603625,
+                                    "count": 857,
+                                    "self": 1843.0431646477082,
                                     "children": {
                                         "TorchPOCAOptimizer.update": {
-                                            "total": 1166.7271367826615,
-                                            "count": 14403,
-                                            "self": 1166.7271367826615
+                                            "total": 2087.8691475126543,
+                                            "count": 25710,
+                                            "self": 2087.8691475126543
                                         }
                                     }
                                 }
@@ -402,19 +402,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 7.2503462433815e-07,
+                    "total": 7.069902494549751e-07,
                     "count": 1,
-                    "self": 7.2503462433815e-07
+                    "self": 7.069902494549751e-07
                 },
                 "TrainerController._save_models": {
-                    "total": 0.18087005300913006,
+                    "total": 0.2131636849953793,
                     "count": 1,
-                    "self": 0.0009355859947390854,
+                    "self": 0.000949086039327085,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.17993446701439098,
+                            "total": 0.21221459895605221,
                             "count": 1,
-                            "self": 0.17993446701439098
+                            "self": 0.21221459895605221
                         }
                     }
                 }

run_logs/training_status.json

@@ -2,59 +2,59 @@
     "SoccerTwos": {
         "checkpoints": [
             {
-                "steps": 8499784,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-8499784.onnx",
+                "steps": 15999812,
+                "file_path": "results/SoccerTwos50M/SoccerTwos/SoccerTwos-15999812.onnx",
                 "reward": 0.0,
-                "creation_time": 1718490534.9493773,
+                "creation_time": 1718536164.4444187,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-8499784.pt"
+                    "results/SoccerTwos50M/SoccerTwos/SoccerTwos-15999812.pt"
                 ]
             },
             {
-                "steps": 8999922,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-8999922.onnx",
+                "steps": 16499964,
+                "file_path": "results/SoccerTwos50M/SoccerTwos/SoccerTwos-16499964.onnx",
                 "reward": 0.0,
-                "creation_time": 1718491223.062741,
+                "creation_time": 1718536846.9933026,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-8999922.pt"
+                    "results/SoccerTwos50M/SoccerTwos/SoccerTwos-16499964.pt"
                 ]
             },
             {
-                "steps": 9499922,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-9499922.onnx",
+                "steps": 16999954,
+                "file_path": "results/SoccerTwos50M/SoccerTwos/SoccerTwos-16999954.onnx",
                 "reward": 0.0,
-                "creation_time": 1718491906.3724327,
+                "creation_time": 1718537524.079914,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-9499922.pt"
+                    "results/SoccerTwos50M/SoccerTwos/SoccerTwos-16999954.pt"
                 ]
             },
             {
-                "steps": 9999982,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-9999982.onnx",
+                "steps": 17499994,
+                "file_path": "results/SoccerTwos50M/SoccerTwos/SoccerTwos-17499994.onnx",
                 "reward": 0.0,
-                "creation_time": 1718492588.0111504,
+                "creation_time": 1718538204.5297017,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-9999982.pt"
+                    "results/SoccerTwos50M/SoccerTwos/SoccerTwos-17499994.pt"
                 ]
             },
             {
-                "steps": 10000132,
-                "file_path": "results/SoccerTwos/SoccerTwos/SoccerTwos-10000132.onnx",
+                "steps": 17703666,
+                "file_path": "results/SoccerTwos50M/SoccerTwos/SoccerTwos-17703666.onnx",
                 "reward": 0.0,
-                "creation_time": 1718492588.2083416,
+                "creation_time": 1718538484.650736,
                 "auxillary_file_paths": [
-                    "results/SoccerTwos/SoccerTwos/SoccerTwos-10000132.pt"
+                    "results/SoccerTwos50M/SoccerTwos/SoccerTwos-17703666.pt"
                 ]
             }
         ],
-        "elo": 1582.9704966166294,
+        "elo": 1543.0773588216994,
         "final_checkpoint": {
-            "steps": 10000132,
-            "file_path": "results/SoccerTwos/SoccerTwos.onnx",
+            "steps": 17703666,
+            "file_path": "results/SoccerTwos50M/SoccerTwos.onnx",
             "reward": 0.0,
-            "creation_time": 1718492588.2083416,
+            "creation_time": 1718538484.650736,
             "auxillary_file_paths": [
-                "results/SoccerTwos/SoccerTwos/SoccerTwos-10000132.pt"
+                "results/SoccerTwos50M/SoccerTwos/SoccerTwos-17703666.pt"
             ]
         }
     },