A2C playing HalfCheetahBulletEnv-v0 from https://github.com/sgoodfriend/rl-algo-impls/tree/0760ef7d52b17f30219a27c18ba52c8895025ae3

README.md

@@ -10,7 +10,7 @@ model-index:
   results:
   - metrics:
     - type: mean_reward
-      value: 2088.99 +/- 20.18
+      value: 2092.81 +/- 24.08
       name: mean_reward
     task:
       type: reinforcement-learning
@@ -31,9 +31,9 @@ This model was trained from 3 trainings of **A2C** agents using different initia
 
 | algo   | env                     |   seed |   reward_mean |   reward_std |   eval_episodes | best   | wandb_url                                                                    |
 |:-------|:------------------------|-------:|--------------:|-------------:|----------------:|:-------|:-----------------------------------------------------------------------------|
-| a2c    | HalfCheetahBulletEnv-v0 |      1 |       2088.99 |      20.1777 |              12 | *      | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/dr7qmh4p) |
-| a2c    | HalfCheetahBulletEnv-v0 |      2 |       1888.21 |      15.6873 |              12 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/vdzku1t6) |
-| a2c    | HalfCheetahBulletEnv-v0 |      3 |       1655.66 |      17.1523 |              12 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/krkgtren) |
+| a2c    | HalfCheetahBulletEnv-v0 |      1 |       2092.81 |      24.0751 |              12 | *      | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/dr7qmh4p) |
+| a2c    | HalfCheetahBulletEnv-v0 |      2 |       1903.39 |      19.643  |              12 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/vdzku1t6) |
+| a2c    | HalfCheetahBulletEnv-v0 |      3 |       1654.74 |      25.2782 |              12 |        | [wandb](https://wandb.ai/sgoodfriend/rl-algo-impls-benchmarks/runs/krkgtren) |
 
 
 ### Prerequisites: Weights & Biases (WandB)

replay.meta.json

@@ -1 +1 @@
-{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 4.2.7-0ubuntu0.1 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with gcc 9 (Ubuntu 9.4.0-1ubuntu1~20.04.1)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared\\nlibavutil      56. 31.100 / 56. 31.100\\nlibavcodec     58. 54.100 / 58. 54.100\\nlibavformat    58. 29.100 / 58. 29.100\\nlibavdevice    58.  8.100 / 58.  8.100\\nlibavfilter     7. 57.100 /  7. 57.100\\nlibavresample   4.  0.  0 /  4.  0.  0\\nlibswscale      5.  5.100 /  5.  5.100\\nlibswresample   3.  5.100 /  3.  5.100\\nlibpostproc    55.  5.100 / 55.  5.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "320x240", "-pix_fmt", "rgb24", "-framerate", "60", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "60", "/tmp/tmp10lrlxpd/a2c-HalfCheetahBulletEnv-v0/replay.mp4"]}, "episode": {"r": -285.2018737792969, "l": 1000, "t": 28.757889}}
+{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 5.1.2 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with clang version 14.0.6\\nconfiguration: --prefix=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pl --cc=arm64-apple-darwin20.0.0-clang --cxx=arm64-apple-darwin20.0.0-clang++ --nm=arm64-apple-darwin20.0.0-nm --ar=arm64-apple-darwin20.0.0-ar --disable-doc --disable-openssl --enable-demuxer=dash --enable-hardcoded-tables --enable-libfreetype --enable-libfontconfig --enable-libopenh264 --enable-cross-compile --arch=arm64 --target-os=darwin --cross-prefix=arm64-apple-darwin20.0.0- --host-cc=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/x86_64-apple-darwin13.4.0-clang --enable-neon --enable-gnutls --enable-libmp3lame --enable-libvpx --enable-pthreads --enable-gpl --enable-libx264 --enable-libx265 --enable-libaom --enable-libsvtav1 --enable-libxml2 --enable-pic --enable-shared --disable-static --enable-version3 --enable-zlib --pkg-config=/Users/runner/miniforge3/conda-bld/ffmpeg_1671040513231/_build_env/bin/pkg-config\\nlibavutil      57. 28.100 / 57. 28.100\\nlibavcodec     59. 37.100 / 59. 37.100\\nlibavformat    59. 27.100 / 59. 27.100\\nlibavdevice    59.  7.100 / 59.  7.100\\nlibavfilter     8. 44.100 /  8. 44.100\\nlibswscale      6.  7.100 /  6.  7.100\\nlibswresample   4.  7.100 /  4.  7.100\\nlibpostproc    56.  6.100 / 56.  6.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "320x240", "-pix_fmt", "rgb24", "-framerate", "60", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "60", "/var/folders/9g/my5557_91xddp6lx00nkzly80000gn/T/tmpslj7b99m/a2c-HalfCheetahBulletEnv-v0/replay.mp4"]}, "episode": {"r": 2114.407958984375, "l": 1000, "t": 28.332733}}

replay.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db68503811bba5887e09dc681b392c6f16648fa3e1db2657e95a227feab0bfb0
-size 1184875
+oid sha256:37553323024abda9eb83eaf658aa3e0c304ebb6838c4d53d5f77379171c85be1
+size 936026

runner/env.py

@@ -26,6 +26,7 @@ from wrappers.initial_step_truncate_wrapper import InitialStepTruncateWrapper
 from wrappers.is_vector_env import IsVectorEnv
 from wrappers.noop_env_seed import NoopEnvSeed
 from wrappers.normalize import NormalizeObservation, NormalizeReward
+from wrappers.sync_vector_env_render_compat import SyncVectorEnvRenderCompat
 from wrappers.transpose_image_observation import TransposeImageObservation
 from wrappers.vectorable_wrapper import VecEnv
 from wrappers.video_compat_wrapper import VideoCompatWrapper
@@ -180,6 +181,8 @@ def _make_vec_env(
     else:
         raise ValueError(f"env_type {env_type} unsupported")
     envs = VecEnvClass([make(i) for i in range(n_envs)])
+    if env_type == "gymvec" and vec_env_class == "sync":
+        envs = SyncVectorEnvRenderCompat(envs)
     if training:
         assert tb_writer
         envs = EpisodeStatsWriter(

shared/callbacks/eval_callback.py

@@ -75,7 +75,9 @@ def evaluate(
     print_returns: bool = True,
     ignore_first_episode: bool = False,
 ) -> EpisodesStats:
+    policy.sync_normalization(env)
     policy.eval()
+
     episodes = EvaluateAccumulator(
         env.num_envs, n_episodes, print_returns, ignore_first_episode
     )
@@ -137,7 +139,6 @@ class EvalCallback(Callback):
     def on_step(self, timesteps_elapsed: int = 1) -> bool:
         super().on_step(timesteps_elapsed)
         if self.timesteps_elapsed // self.step_freq >= len(self.stats):
-            self.policy.sync_normalization(self.env)
             self.evaluate()
         return True
 
@@ -176,7 +177,6 @@ class EvalCallback(Callback):
             )
             if strictly_better and self.record_best_videos:
                 assert self.video_env and self.best_video_dir
-                self.policy.sync_normalization(self.video_env)
                 self.best_video_base_path = os.path.join(
                     self.best_video_dir, str(self.timesteps_elapsed)
                 )

wrappers/sync_vector_env_render_compat.py

@@ -0,0 +1,31 @@
+import numpy as np
+
+from gym.vector.sync_vector_env import SyncVectorEnv
+from stable_baselines3.common.vec_env.base_vec_env import tile_images
+from typing import Optional
+
+from wrappers.vectorable_wrapper import (
+    VecotarableWrapper,
+)
+
+
+class SyncVectorEnvRenderCompat(VecotarableWrapper):
+    def __init__(self, env) -> None:
+        super().__init__(env)
+
+    def render(self, mode: str = "human") -> Optional[np.ndarray]:
+        base_env = self.env.unwrapped
+        if isinstance(base_env, SyncVectorEnv):
+            imgs = [env.render(mode="rgb_array") for env in base_env.envs]
+            bigimg = tile_images(imgs)
+            if mode == "human":
+                import cv2
+
+                cv2.imshow("vecenv", bigimg[:, :, ::-1])
+                cv2.waitKey(1)
+            elif mode == "rgb_array":
+                return bigimg
+            else:
+                raise NotImplemented(f"Render mode {mode} is not supported")
+        else:
+            return self.env.render(mode=mode)

wrappers/vec_episode_recorder.py

@@ -1,9 +1,6 @@
 import numpy as np
 
-from gym.vector.sync_vector_env import SyncVectorEnv
 from gym.wrappers.monitoring.video_recorder import VideoRecorder
-from stable_baselines3.common.vec_env.base_vec_env import tile_images
-from typing import Optional
 
 from wrappers.vectorable_wrapper import (
     VecotarableWrapper,
@@ -45,7 +42,7 @@ class VecEpisodeRecorder(VecotarableWrapper):
         self._close_video_recorder()
 
         self.video_recorder = VideoRecorder(
-            SyncVectorEnvRenderCompat(self.env),
+            self.env,
             base_path=self.base_path,
         )
 
@@ -56,25 +53,3 @@ class VecEpisodeRecorder(VecotarableWrapper):
         if self.video_recorder:
             self.video_recorder.close()
         self.video_recorder = None
-
-
-class SyncVectorEnvRenderCompat(VecotarableWrapper):
-    def __init__(self, env) -> None:
-        super().__init__(env)
-
-    def render(self, mode: str = "human") -> Optional[np.ndarray]:
-        base_env = self.env.unwrapped
-        if isinstance(base_env, SyncVectorEnv):
-            imgs = [env.render(mode="rgb_array") for env in base_env.envs]
-            bigimg = tile_images(imgs)
-            if mode == "humnan":
-                import cv2
-
-                cv2.imshow("vecenv", bigimg[:, :, ::-1])
-                cv2.waitKey(1)
-            elif mode == "rgb_array":
-                return bigimg
-            else:
-                raise NotImplemented(f"Render mode {mode} is not supported")
-        else:
-            return self.env.render(mode=mode)