A2C playing AntBulletEnv-v0 from https://github.com/sgoodfriend/rl-algo-impls/tree/2067e21d62fff5db60168687e7d9e89019a8bfc0

a2c-AntBulletEnv-v0.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7fe167fa72a102631922c50e6c755640b932d69be9416b03234f512eeb672602
-size 130079

a2c-AntBulletEnv-v0/_stable_baselines3_version

@@ -1 +0,0 @@
-1.7.0

a2c-AntBulletEnv-v0/data

@@ -1,109 +0,0 @@
-{
-    "policy_class": {
-        ":type:": "<class 'abc.ABCMeta'>",
-        ":serialized:": "gAWVOwAAAAAAAACMIXN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi5wb2xpY2llc5SMEUFjdG9yQ3JpdGljUG9saWN5lJOULg==",
-        "__module__": "stable_baselines3.common.policies",
-        "__doc__": "\n    Policy class for actor-critic algorithms (has both policy and value prediction).\n    Used by A2C, PPO and the likes.\n\n    :param observation_space: Observation space\n    :param action_space: Action space\n    :param lr_schedule: Learning rate schedule (could be constant)\n    :param net_arch: The specification of the policy and value networks.\n    :param activation_fn: Activation function\n    :param ortho_init: Whether to use or not orthogonal initialization\n    :param use_sde: Whether to use State Dependent Exploration or not\n    :param log_std_init: Initial value for the log standard deviation\n    :param full_std: Whether to use (n_features x n_actions) parameters\n        for the std instead of only (n_features,) when using gSDE\n    :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure\n        a positive standard deviation (cf paper). It allows to keep variance\n        above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.\n    :param squash_output: Whether to squash the output using a tanh function,\n        this allows to ensure boundaries when using gSDE.\n    :param features_extractor_class: Features extractor to use.\n    :param features_extractor_kwargs: Keyword arguments\n        to pass to the features extractor.\n    :param share_features_extractor: If True, the features extractor is shared between the policy and value networks.\n    :param normalize_images: Whether to normalize images or not,\n         dividing by 255.0 (True by default)\n    :param optimizer_class: The optimizer to use,\n        ``th.optim.Adam`` by default\n    :param optimizer_kwargs: Additional keyword arguments,\n        excluding the learning rate, to pass to the optimizer\n    ",
-        "__init__": "<function ActorCriticPolicy.__init__ at 0x7ff06ed7c3a0>",
-        "_get_constructor_parameters": "<function ActorCriticPolicy._get_constructor_parameters at 0x7ff06ed7c430>",
-        "reset_noise": "<function ActorCriticPolicy.reset_noise at 0x7ff06ed7c4c0>",
-        "_build_mlp_extractor": "<function ActorCriticPolicy._build_mlp_extractor at 0x7ff06ed7c550>",
-        "_build": "<function ActorCriticPolicy._build at 0x7ff06ed7c5e0>",
-        "forward": "<function ActorCriticPolicy.forward at 0x7ff06ed7c670>",
-        "extract_features": "<function ActorCriticPolicy.extract_features at 0x7ff06ed7c700>",
-        "_get_action_dist_from_latent": "<function ActorCriticPolicy._get_action_dist_from_latent at 0x7ff06ed7c790>",
-        "_predict": "<function ActorCriticPolicy._predict at 0x7ff06ed7c820>",
-        "evaluate_actions": "<function ActorCriticPolicy.evaluate_actions at 0x7ff06ed7c8b0>",
-        "get_distribution": "<function ActorCriticPolicy.get_distribution at 0x7ff06ed7c940>",
-        "predict_values": "<function ActorCriticPolicy.predict_values at 0x7ff06ed7c9d0>",
-        "__abstractmethods__": "frozenset()",
-        "_abc_impl": "<_abc_data object at 0x7ff06ed70f60>"
-    },
-    "verbose": 1,
-    "policy_kwargs": {
-        ":type:": "<class 'dict'>",
-        ":serialized:": "gAWVowAAAAAAAAB9lCiMDGxvZ19zdGRfaW5pdJRK/v///4wKb3J0aG9faW5pdJSJjA9vcHRpbWl6ZXJfY2xhc3OUjBN0b3JjaC5vcHRpbS5ybXNwcm9wlIwHUk1TcHJvcJSTlIwQb3B0aW1pemVyX2t3YXJnc5R9lCiMBWFscGhhlEc/764UeuFHrowDZXBzlEc+5Pi1iONo8YwMd2VpZ2h0X2RlY2F5lEsAdXUu",
-        "log_std_init": -2,
-        "ortho_init": false,
-        "optimizer_class": "<class 'torch.optim.rmsprop.RMSprop'>",
-        "optimizer_kwargs": {
-            "alpha": 0.99,
-            "eps": 1e-05,
-            "weight_decay": 0
-        }
-    },
-    "observation_space": {
-        ":type:": "<class 'gym.spaces.box.Box'>",
-        ":serialized:": "gAWVZwIAAAAAAACMDmd5bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMBl9zaGFwZZRLHIWUjANsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWcAAAAAAAAAAAAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/lGgKSxyFlIwBQ5R0lFKUjARoaWdolGgSKJZwAAAAAAAAAAAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH+UaApLHIWUaBV0lFKUjA1ib3VuZGVkX2JlbG93lGgSKJYcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUaAeMAmIxlImIh5RSlChLA4wBfJROTk5K/////0r/////SwB0lGJLHIWUaBV0lFKUjA1ib3VuZGVkX2Fib3ZllGgSKJYcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUaCFLHIWUaBV0lFKUjApfbnBfcmFuZG9tlE51Yi4=",
-        "dtype": "float32",
-        "_shape": [
-            28
-        ],
-        "low": "[-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf\n -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf]",
-        "high": "[inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf\n inf inf inf inf inf inf inf inf inf inf]",
-        "bounded_below": "[False False False False False False False False False False False False\n False False False False False False False False False False False False\n False False False False]",
-        "bounded_above": "[False False False False False False False False False False False False\n False False False False False False False False False False False False\n False False False False]",
-        "_np_random": null
-    },
-    "action_space": {
-        ":type:": "<class 'gym.spaces.box.Box'>",
-        ":serialized:": "gAWVnwEAAAAAAACMDmd5bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMBl9zaGFwZZRLCIWUjANsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWIAAAAAAAAAAAAIC/AACAvwAAgL8AAIC/AACAvwAAgL8AAIC/AACAv5RoCksIhZSMAUOUdJRSlIwEaGlnaJRoEiiWIAAAAAAAAAAAAIA/AACAPwAAgD8AAIA/AACAPwAAgD8AAIA/AACAP5RoCksIhZRoFXSUUpSMDWJvdW5kZWRfYmVsb3eUaBIolggAAAAAAAAAAQEBAQEBAQGUaAeMAmIxlImIh5RSlChLA4wBfJROTk5K/////0r/////SwB0lGJLCIWUaBV0lFKUjA1ib3VuZGVkX2Fib3ZllGgSKJYIAAAAAAAAAAEBAQEBAQEBlGghSwiFlGgVdJRSlIwKX25wX3JhbmRvbZROdWIu",
-        "dtype": "float32",
-        "_shape": [
-            8
-        ],
-        "low": "[-1. -1. -1. -1. -1. -1. -1. -1.]",
-        "high": "[1. 1. 1. 1. 1. 1. 1. 1.]",
-        "bounded_below": "[ True  True  True  True  True  True  True  True]",
-        "bounded_above": "[ True  True  True  True  True  True  True  True]",
-        "_np_random": null
-    },
-    "n_envs": 4,
-    "num_timesteps": 2000000,
-    "_total_timesteps": 2000000,
-    "_num_timesteps_at_start": 0,
-    "seed": null,
-    "action_noise": null,
-    "start_time": 1674588743994659442,
-    "learning_rate": {
-        ":type:": "<class 'function'>",
-        ":serialized:": "gAWVdgIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAksTQwiIAHwAFABTAJROhZQpjBJwcm9ncmVzc19yZW1haW5pbmeUhZSMHzxpcHl0aG9uLWlucHV0LTEzLWVhYTdkOGY5N2ZkNj6UjAhzY2hlZHVsZZRLBEMCAAGUjA1pbml0aWFsX3ZhbHVllIWUKXSUUpR9lCiMC19fcGFja2FnZV9flE6MCF9fbmFtZV9flIwIX19tYWluX1+UdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHH2UfZQoaBVoDYwMX19xdWFsbmFtZV9flIwhbGluZWFyX3NjaGVkdWxlLjxsb2NhbHM+LnNjaGVkdWxllIwPX19hbm5vdGF0aW9uc19flH2UKIwScHJvZ3Jlc3NfcmVtYWluaW5nlIwIYnVpbHRpbnOUjAVmbG9hdJSTlIwGcmV0dXJulGgpdYwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBaMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP091EE1VHWmFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
-    },
-    "tensorboard_log": null,
-    "lr_schedule": {
-        ":type:": "<class 'function'>",
-        ":serialized:": "gAWVdgIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAksTQwiIAHwAFABTAJROhZQpjBJwcm9ncmVzc19yZW1haW5pbmeUhZSMHzxpcHl0aG9uLWlucHV0LTEzLWVhYTdkOGY5N2ZkNj6UjAhzY2hlZHVsZZRLBEMCAAGUjA1pbml0aWFsX3ZhbHVllIWUKXSUUpR9lCiMC19fcGFja2FnZV9flE6MCF9fbmFtZV9flIwIX19tYWluX1+UdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHH2UfZQoaBVoDYwMX19xdWFsbmFtZV9flIwhbGluZWFyX3NjaGVkdWxlLjxsb2NhbHM+LnNjaGVkdWxllIwPX19hbm5vdGF0aW9uc19flH2UKIwScHJvZ3Jlc3NfcmVtYWluaW5nlIwIYnVpbHRpbnOUjAVmbG9hdJSTlIwGcmV0dXJulGgpdYwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBaMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP091EE1VHWmFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
-    },
-    "_last_obs": {
-        ":type:": "<class 'numpy.ndarray'>",
-        ":serialized:": "gAWVNQIAAAAAAACMEm51bXB5LmNvcmUubnVtZXJpY5SMC19mcm9tYnVmZmVylJOUKJbAAQAAAAAAANkXHT/P+Bu/1A+1PjIvpT89Bh+/zjqjPhDbqT4Dqha/AjV/P+zYsr+uKGE/dj6Xv6Cxv7+R4wc8NxoKv6piD8CimWe/+TfIPi55aD9Zeju9gErvPhI+3L5IS6C/aYMmP8k7cr8P4ve/8OcFwEF3ab/IB8Q+iiKSPjT47T5cWJE/BK7Rvr5ZgD+8Kak900civ+X6jr4MOYU9Y5Shv5Jbtz6m2U++w9apP5QSCT7zfbk/P/O2PzWg8Lp/kpq+P/AOvwa3jb+DbFA/cyMdP3pKXj/JO3K//jAEP/DnBcDGWow/sCkSP3Xwnj5GF+s+0PqgP1LPhL70tAs/D5JdPdH8PDwVeaM+7Rabv2R/mz5gexrADzf+vtnLpj8/PMu+cPeXP8wagb4/bOU/gJVZPfvU8r+XP0C/uvbjvuIAMT9kUDM/yTtyv/4wBD9+tfQ+xlqMP1d1jT2Zgbe+1bHkPva9wj/bEYW/mrFZP9iSDD9GL/G+l7yMP/DjTz9xM7U/HN+OPqTRRr94K0XAr08LO38geb/41IG/tzC3v9ZobD/na868JuUBvnf+tr+WGXi/NfUov0BGhz8P4ve/frX0PkF3ab+UjAVudW1weZSMBWR0eXBllJOUjAJmNJSJiIeUUpQoSwOMATyUTk5OSv////9K/////0sAdJRiSwRLHIaUjAFDlHSUUpQu"
-    },
-    "_last_episode_starts": {
-        ":type:": "<class 'numpy.ndarray'>",
-        ":serialized:": "gAWVdwAAAAAAAACMEm51bXB5LmNvcmUubnVtZXJpY5SMC19mcm9tYnVmZmVylJOUKJYEAAAAAAAAAAAAAACUjAVudW1weZSMBWR0eXBllJOUjAJiMZSJiIeUUpQoSwOMAXyUTk5OSv////9K/////0sAdJRiSwSFlIwBQ5R0lFKULg=="
-    },
-    "_last_original_obs": {
-        ":type:": "<class 'numpy.ndarray'>",
-        ":serialized:": "gAWVNQIAAAAAAACMEm51bXB5LmNvcmUubnVtZXJpY5SMC19mcm9tYnVmZmVylJOUKJbAAQAAAAAAAAAAAAAtoGm1AACAPwAAAAAAAAAAAAAAAAAAAAAAAACA9o2EvQAAAACgRPm/AAAAAMwsiT0AAAAAyVXjPwAAAABeHa69AAAAAFdF5z8AAAAAAcmpPQAAAAApeea/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHEHLNAAAgD8AAAAAAAAAAAAAAAAAAAAAAAAAgEwf2b0AAAAA2Gz1vwAAAADLltm6AAAAAM9Z7D8AAAAA3yVmvQAAAADn69s/AAAAADiBBz0AAAAAw1rrvwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOBNMrYAAIA/AAAAAAAAAAAAAAAAAAAAAAAAAIAyWb08AAAAAEuk/L8AAAAAL1SzvQAAAAByavk/AAAAAP8V470AAAAA/m/iPwAAAAAtweW9AAAAAExT9L8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADP6ZA0AACAPwAAAAAAAAAAAAAAAAAAAAAAAACAfKcNPgAAAAAF3eK/AAAAACbLMT0AAAAARnIAQAAAAAArb0K7AAAAAMWz/z8AAAAANVvXvQAAAAAaNe2/AAAAAAAAAAAAAAAAAAAAAAAAAACUjAVudW1weZSMBWR0eXBllJOUjAJmNJSJiIeUUpQoSwOMATyUTk5OSv////9K/////0sAdJRiSwRLHIaUjAFDlHSUUpQu"
-    },
-    "_episode_num": 0,
-    "use_sde": true,
-    "sde_sample_freq": -1,
-    "_current_progress_remaining": 0.0,
-    "ep_info_buffer": {
-        ":type:": "<class 'collections.deque'>",
-        ":serialized:": "gAWVRAwAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKUKH2UKIwBcpRHQKaK8jRlYlqMAWyUTegDjAF0lEdArJafl8w6AHV9lChoBkdAproSsySFG2gHTegDaAhHQKyeau01IiF1fZQoaAZHQKW/SZpi7TVoB03oA2gIR0Csnr6jnFHbdX2UKGgGR0ClbyI/7iyZaAdN6ANoCEdArKJTlmvnsHV9lChoBkdApXSPAh0QsmgHTegDaAhHQKyiYGeMAFR1fZQoaAZHQKYeObMHKOloB03oA2gIR0Csqia/qPfbdX2UKGgGR0Cl25fAbhm5aAdN6ANoCEdArKp0rAgxJ3V9lChoBkdApGo8rmQr+mgHTegDaAhHQKyt3NGmUGF1fZQoaAZHQKW26Hk92X9oB03oA2gIR0CsreV76YVqdX2UKGgGR0Clpgs1KoQ4aAdN6ANoCEdArLXvPw/gSHV9lChoBkdApZRJfOUt7WgHTegDaAhHQKy2Q7jDKo11fZQoaAZHQKWQJ2alUIdoB03oA2gIR0Csubl7Uoa2dX2UKGgGR0CmLI2nsLOSaAdN6ANoCEdArLnCPU8V6HV9lChoBkdApVqDB9Cu2mgHTegDaAhHQKzBwo1k1/F1fZQoaAZHQKZuXCqp97ZoB03oA2gIR0CswhCEQGwBdX2UKGgGR0Cmlu0j9n9OaAdN6ANoCEdArMWdjbzshXV9lChoBkdApnp/RgJC0GgHTegDaAhHQKzFpoL5RCR1fZQoaAZHQKR8O5J9RaZoB03oA2gIR0CszVechC+ldX2UKGgGR0CltMF41P30aAdN6ANoCEdArM2rY/Vy3nV9lChoBkdApc4PphWo32gHTegDaAhHQKzRWe/5+H91fZQoaAZHQKZrGVUMoc9oB03oA2gIR0Cs0WMKTjebdX2UKGgGR0CmgCTvy9VWaAdN6ANoCEdArNj4dGRV63V9lChoBkdApfwql3yI6GgHTegDaAhHQKzZS31BdD91fZQoaAZHQKZoYRq46OpoB03oA2gIR0Cs3Nb8m8dxdX2UKGgGR0ClFq4sVclgaAdN6ANoCEdArNzf2PDHfnV9lChoBkdApdattVJcxGgHTegDaAhHQKzkmxptaZB1fZQoaAZHQKWUFjVhCt1oB03oA2gIR0Cs5O8LjPv8dX2UKGgGR0CmL6/N7jT8aAdN6ANoCEdArOhYIKMNt3V9lChoBkdApdR+nQ6ZIGgHTegDaAhHQKzoYBRQ7911fZQoaAZHQKYjF24d6s1oB03oA2gIR0Cs8AV6/qPfdX2UKGgGR0ClmIZ9Vmz0aAdN6ANoCEdArPBaKpDNQnV9lChoBkdApk9kpiI+GGgHTegDaAhHQKz0EkN4JNV1fZQoaAZHQKVyiL2HtWxoB03oA2gIR0Cs9Bv5pJwsdX2UKGgGR0CmUOSncclxaAdN6ANoCEdArPvNdVvMr3V9lChoBkdAprML/VAiV2gHTegDaAhHQKz8HtdiUgV1fZQoaAZHQKWHJzcRDkVoB03oA2gIR0Cs/4O6NEPUdX2UKGgGR0Clvp+QlruZaAdN6ANoCEdArP+Lvd/KAHV9lChoBkdAptM4bfgrH2gHTegDaAhHQK0HKjbBXS11fZQoaAZHQKTNTr/sE7poB03oA2gIR0CtB3/xtpEhdX2UKGgGR0ClpbtFjNILaAdN6ANoCEdArQsPj+717XV9lChoBkdApN2DeZXuE2gHTegDaAhHQK0LGJl8PWh1fZQoaAZHQKOCkuKXOW1oB03oA2gIR0CtEulT3qRmdX2UKGgGR0Cl+HYb83uNaAdN6ANoCEdArRM6aPS2IHV9lChoBkdApPy5dKNADGgHTegDaAhHQK0Wr5WzWwx1fZQoaAZHQKZWr4dIXj5oB03oA2gIR0CtFrfCAMDwdX2UKGgGR0CmKUQW3z+WaAdN6ANoCEdArR6OpQ1rI3V9lChoBkdAphbtDtw71mgHTegDaAhHQK0e3Ktga3t1fZQoaAZHQKW7YPTXrdFoB03oA2gIR0CtIkclolD4dX2UKGgGR0Cl5MDps41haAdN6ANoCEdArSJQ/RmbsnV9lChoBkdApiBvW4EwFmgHTegDaAhHQK0p+HEdeY51fZQoaAZHQKQNrjc2zfJoB03oA2gIR0CtKkj28IzFdX2UKGgGR0Cckdajvd/KaAdNggNoCEdArSyY1pCa7XV9lChoBkdApomsPvrnkmgHTegDaAhHQK0tzR4yGi51fZQoaAZHQKZk+Fyq+8JoB03oA2gIR0CtNXu1OTJRdX2UKGgGR0CllYyxqwhXaAdN6ANoCEdArTXIMx46fnV9lChoBkdAphfyzRhMJ2gHTegDaAhHQK04BurIYFd1fZQoaAZHQKSM9KV6eGxoB03oA2gIR0CtOUYlpoK2dX2UKGgGR0CmiWAJTl1baAdN6ANoCEdArUEHCj1wpHV9lChoBkdApgHe0gKWs2gHTegDaAhHQK1BYPxQSBd1fZQoaAZHQKcAthnanJloB03oA2gIR0CtQ791U2k0dX2UKGgGR0CiDy96Tnq3aAdN6ANoCEdArUTnRPXTVnV9lChoBkdApFqvcSGrS2gHTegDaAhHQK1MZh/Aj6h1fZQoaAZHQKXAlP/rB0poB03oA2gIR0CtTLSQHRkVdX2UKGgGR0CjmJprDZUUaAdN6ANoCEdArU7xddE9dXV9lChoBkdAptUnsZ5zHWgHTegDaAhHQK1QI8VYZEV1fZQoaAZHQKLkzDlYEGJoB03oA2gIR0CtWCQob4rSdX2UKGgGR0CeM9MS9M9KaAdN6ANoCEdArVh42GZeA3V9lChoBkdApeqzwMH8j2gHTegDaAhHQK1a0TSsr/d1fZQoaAZHQKYbd2wmmchoB03oA2gIR0CtXAZeJHiFdX2UKGgGR0Cl3GUtRNypaAdN6ANoCEdArWO18qnWKHV9lChoBkdApWpvz+WGAWgHTegDaAhHQK1kBFn7Hhl1fZQoaAZHQKZSKHBUJfJoB03oA2gIR0CtZlCuMdcTdX2UKGgGR0CmpdCiAUcoaAdN6ANoCEdArWeCZnctXnV9lChoBkdAppM5Q53kgmgHTegDaAhHQK1vG8h9srN1fZQoaAZHQKYExGDtgKFoB03oA2gIR0Ctb22CVbA2dX2UKGgGR0CmtFdFWn0kaAdN6ANoCEdArXG/VurIYHV9lChoBkdApfNvitJWemgHTegDaAhHQK1zFY5DJEJ1fZQoaAZHQKXdBBSk0rNoB03oA2gIR0CteqxdY4hmdX2UKGgGR0CmhG8kdFOPaAdN6ANoCEdArXr6QvHtGHV9lChoBkdApU/SYPXkHWgHTegDaAhHQK19RnVXmvJ1fZQoaAZHQKWUeXk5p8FoB03oA2gIR0Ctfnhun/DMdX2UKGgGR0CkjzIt16mgaAdN6ANoCEdArYYPdCVrynV9lChoBkdApPtMZ75VO2gHTegDaAhHQK2GYNAkcCJ1fZQoaAZHQKZMGafjCHhoB03oA2gIR0CtiLaOYIBzdX2UKGgGR0Cl0eULMLWqaAdN6ANoCEdArYnyg7HQyHV9lChoBkdApd5MyN4qw2gHTegDaAhHQK2Rs0iQkop1fZQoaAZHQKVcAo5xR2toB03oA2gIR0CtkgYjB2wFdX2UKGgGR0ClzucPFvQ4aAdN6ANoCEdArZRV0NjLCHV9lChoBkdApVq6v7m+02gHTegDaAhHQK2VmQjD8+B1fZQoaAZHQKSTCldC3PRoB03oA2gIR0CtnXYwqRU4dX2UKGgGR0Cl5iNfw7T2aAdN6ANoCEdArZ3F+iJwbXV9lChoBkdApdJXDYRNAWgHTegDaAhHQK2gGQlKK511fZQoaAZHQKZjJvMKTjhoB03oA2gIR0CtoWfvWpZPdX2UKGgGR0Cllc9fsu3+aAdN6ANoCEdAralP+bVjJHV9lChoBkdApTLXMSsbN2gHTegDaAhHQK2pqDU3GXJ1fZQoaAZHQKYQnPVNHpdoB03oA2gIR0Ctq/a9kBjndX2UKGgGR0Cmo4jRlYlqaAdN6ANoCEdAra09Tgl4T3V9lChoBkdApNBz6JqIrWgHTegDaAhHQK2051jAi3Z1fZQoaAZHQKXI+y5Zr59oB03oA2gIR0CttTxWkrPMdX2UKGgGR0CharHoPkJbaAdNVgNoCEdArbXjyUcGT3VlLg=="
-    },
-    "ep_success_buffer": {
-        ":type:": "<class 'collections.deque'>",
-        ":serialized:": "gAWVIAAAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKULg=="
-    },
-    "_n_updates": 62500,
-    "n_steps": 8,
-    "gamma": 0.99,
-    "gae_lambda": 0.9,
-    "ent_coef": 0.0,
-    "vf_coef": 0.4,
-    "max_grad_norm": 0.5,
-    "normalize_advantage": false
-}

a2c-AntBulletEnv-v0/policy.optimizer.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ae34a234a13e78e7764685d17296460fb4b030a1e520b8dddb06e1151b45c170
-size 56190

a2c-AntBulletEnv-v0/policy.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e8d436595959204150e7bb98d9534582e5b1f9994536c261ca00a695c7ce203c
-size 56958

a2c-AntBulletEnv-v0/pytorch_variables.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d030ad8db708280fcae77d87e973102039acd23a11bdecc3db8eb6c0ac940ee1
-size 431

a2c-AntBulletEnv-v0/system_info.txt

@@ -1,7 +0,0 @@
-- OS: Linux-5.10.147+-x86_64-with-glibc2.29 # 1 SMP Sat Dec 10 16:00:40 UTC 2022
-- Python: 3.8.10
-- Stable-Baselines3: 1.7.0
-- PyTorch: 1.13.1+cu116
-- GPU Enabled: True
-- Numpy: 1.21.6
-- Gym: 0.21.0

a2c/a2c.py

@@ -1,201 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from dataclasses import asdict, dataclass, field
-from time import perf_counter
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import List, Optional, Sequence, NamedTuple, TypeVar
-
-from shared.algorithm import Algorithm
-from shared.callbacks.callback import Callback
-from shared.gae import compute_advantage, compute_rtg_and_advantage
-from shared.policy.on_policy import ActorCritic
-from shared.schedule import schedule, update_learning_rate
-from shared.stats import log_scalars
-from shared.trajectory import Trajectory, TrajectoryAccumulator
-from wrappers.vectorable_wrapper import (
-    VecEnv,
-    VecEnvObs,
-    single_observation_space,
-    single_action_space,
-)
-
-A2CSelf = TypeVar("A2CSelf", bound="A2C")
-
-
-class A2C(Algorithm):
-    def __init__(
-        self,
-        policy: ActorCritic,
-        env: VecEnv,
-        device: torch.device,
-        tb_writer: SummaryWriter,
-        learning_rate: float = 7e-4,
-        learning_rate_decay: str = "none",
-        n_steps: int = 5,
-        gamma: float = 0.99,
-        gae_lambda: float = 1.0,
-        ent_coef: float = 0.0,
-        ent_coef_decay: str = "none",
-        vf_coef: float = 0.5,
-        max_grad_norm: float = 0.5,
-        rms_prop_eps: float = 1e-5,
-        use_rms_prop: bool = True,
-        sde_sample_freq: int = -1,
-        normalize_advantage: bool = False,
-    ) -> None:
-        super().__init__(policy, env, device, tb_writer)
-        self.policy = policy
-
-        self.lr_schedule = schedule(learning_rate_decay, learning_rate)
-        if use_rms_prop:
-            self.optimizer = torch.optim.RMSprop(
-                policy.parameters(), lr=learning_rate, eps=rms_prop_eps
-            )
-        else:
-            self.optimizer = torch.optim.Adam(policy.parameters(), lr=learning_rate)
-
-        self.n_steps = n_steps
-
-        self.gamma = gamma
-        self.gae_lambda = gae_lambda
-
-        self.vf_coef = vf_coef
-        self.ent_coef_schedule = schedule(ent_coef_decay, ent_coef)
-        self.max_grad_norm = max_grad_norm
-
-        self.sde_sample_freq = sde_sample_freq
-        self.normalize_advantage = normalize_advantage
-
-    def learn(
-        self: A2CSelf, total_timesteps: int, callback: Optional[Callback] = None
-    ) -> A2CSelf:
-        epoch_dim = (self.n_steps, self.env.num_envs)
-        step_dim = (self.env.num_envs,)
-        obs_space = single_observation_space(self.env)
-        act_space = single_action_space(self.env)
-
-        obs = np.zeros(epoch_dim + obs_space.shape, dtype=obs_space.dtype)
-        actions = np.zeros(epoch_dim + act_space.shape, dtype=act_space.dtype)
-        rewards = np.zeros(epoch_dim, dtype=np.float32)
-        episode_starts = np.zeros(epoch_dim, dtype=np.byte)
-        values = np.zeros(epoch_dim, dtype=np.float32)
-        logprobs = np.zeros(epoch_dim, dtype=np.float32)
-
-        next_obs = self.env.reset()
-        next_episode_starts = np.ones(step_dim, dtype=np.byte)
-
-        timesteps_elapsed = 0
-        while timesteps_elapsed < total_timesteps:
-            start_time = perf_counter()
-
-            progress = timesteps_elapsed / total_timesteps
-            ent_coef = self.ent_coef_schedule(progress)
-            learning_rate = self.lr_schedule(progress)
-            update_learning_rate(self.optimizer, learning_rate)
-            log_scalars(
-                self.tb_writer,
-                "charts",
-                {
-                    "ent_coef": ent_coef,
-                    "learning_rate": learning_rate,
-                },
-                timesteps_elapsed,
-            )
-
-            self.policy.eval()
-            self.policy.reset_noise()
-            for s in range(self.n_steps):
-                timesteps_elapsed += self.env.num_envs
-                if self.sde_sample_freq > 0 and s > 0 and s % self.sde_sample_freq == 0:
-                    self.policy.reset_noise()
-
-                obs[s] = next_obs
-                episode_starts[s] = next_episode_starts
-
-                actions[s], values[s], logprobs[s], clamped_action = self.policy.step(
-                    next_obs
-                )
-                next_obs, rewards[s], next_episode_starts, _ = self.env.step(
-                    clamped_action
-                )
-
-            advantages = np.zeros(epoch_dim, dtype=np.float32)
-            last_gae_lam = 0
-            for t in reversed(range(self.n_steps)):
-                if t == self.n_steps - 1:
-                    next_nonterminal = 1.0 - next_episode_starts
-                    next_value = self.policy.value(next_obs)
-                else:
-                    next_nonterminal = 1.0 - episode_starts[t + 1]
-                    next_value = values[t + 1]
-                delta = (
-                    rewards[t] + self.gamma * next_value * next_nonterminal - values[t]
-                )
-                last_gae_lam = (
-                    delta
-                    + self.gamma * self.gae_lambda * next_nonterminal * last_gae_lam
-                )
-                advantages[t] = last_gae_lam
-            returns = advantages + values
-
-            b_obs = torch.tensor(obs.reshape((-1,) + obs_space.shape)).to(self.device)
-            b_actions = torch.tensor(actions.reshape((-1,) + act_space.shape)).to(
-                self.device
-            )
-            b_advantages = torch.tensor(advantages.reshape(-1)).to(self.device)
-            b_returns = torch.tensor(returns.reshape(-1)).to(self.device)
-
-            if self.normalize_advantage:
-                b_advantages = (b_advantages - b_advantages.mean()) / (
-                    b_advantages.std() + 1e-8
-                )
-
-            self.policy.train()
-            logp_a, entropy, v = self.policy(b_obs, b_actions)
-
-            pi_loss = -(b_advantages * logp_a).mean()
-            value_loss = F.mse_loss(b_returns, v)
-            entropy_loss = -entropy.mean()
-
-            loss = pi_loss + self.vf_coef * value_loss + ent_coef * entropy_loss
-
-            self.optimizer.zero_grad()
-            loss.backward()
-            nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)
-            self.optimizer.step()
-
-            y_pred = values.reshape(-1)
-            y_true = returns.reshape(-1)
-            var_y = np.var(y_true).item()
-            explained_var = (
-                np.nan if var_y == 0 else 1 - np.var(y_true - y_pred).item() / var_y
-            )
-
-            end_time = perf_counter()
-            rollout_steps = self.n_steps * self.env.num_envs
-            self.tb_writer.add_scalar(
-                "train/steps_per_second",
-                (rollout_steps) / (end_time - start_time),
-                timesteps_elapsed,
-            )
-
-            log_scalars(
-                self.tb_writer,
-                "losses",
-                {
-                    "loss": loss.item(),
-                    "pi_loss": pi_loss.item(),
-                    "v_loss": value_loss.item(),
-                    "entropy_loss": entropy_loss.item(),
-                    "explained_var": explained_var,
-                },
-                timesteps_elapsed,
-            )
-
-            if callback:
-                callback.on_step(timesteps_elapsed=rollout_steps)
-
-        return self

benchmarks/benchmark_test.sh

@@ -1,32 +0,0 @@
-source benchmarks/train_loop.sh
-
-export WANDB_PROJECT_NAME="rl-algo-impls"
-
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-3}"
-
-ALGOS=(
-    # "vpg"
-    "dqn"
-    # "ppo"
-)
-ENVS=(
-    # Basic
-    "CartPole-v1"
-    "MountainCar-v0"
-    # "MountainCarContinuous-v0"
-    "Acrobot-v1"
-    "LunarLander-v2"
-    # # PyBullet
-    # "HalfCheetahBulletEnv-v0"
-    # "AntBulletEnv-v0"
-    # "HopperBulletEnv-v0"
-    # "Walker2DBulletEnv-v0"
-    # # CarRacing
-    # "CarRacing-v0"
-    # Atari
-    "PongNoFrameskip-v4"
-    "BreakoutNoFrameskip-v4"
-    "SpaceInvadersNoFrameskip-v4"
-    "QbertNoFrameskip-v4"
-)
-train_loop "${ALGOS[*]}" "${ENVS[*]}" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

benchmarks/colab_atari1.sh

@@ -1,5 +0,0 @@
-source benchmarks/train_loop.sh
-ALGOS="ppo"
-ENVS="PongNoFrameskip-v4 BreakoutNoFrameskip-v4"
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-3}"
-train_loop $ALGOS "$ENVS" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

benchmarks/colab_atari2.sh

@@ -1,5 +0,0 @@
-source benchmarks/train_loop.sh
-ALGOS="ppo"
-ENVS="SpaceInvadersNoFrameskip-v4 QbertNoFrameskip-v4"
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-3}"
-train_loop $ALGOS "$ENVS" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

benchmarks/colab_basic.sh

@@ -1,5 +0,0 @@
-source benchmarks/train_loop.sh
-ALGOS="ppo"
-ENVS="CartPole-v1 MountainCar-v0 MountainCarContinuous-v0 Acrobot-v1 LunarLander-v2"
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-3}"
-train_loop $ALGOS "$ENVS" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

benchmarks/colab_benchmark.ipynb

@@ -1,195 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": [],
-      "machine_shape": "hm",
-      "authorship_tag": "ABX9TyOGIH7rqgasim3Sz7b1rpoE",
-      "include_colab_link": true
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    },
-    "gpuClass": "standard",
-    "accelerator": "GPU"
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "view-in-github",
-        "colab_type": "text"
-      },
-      "source": [
-        "<a href=\"https://colab.research.google.com/github/sgoodfriend/rl-algo-impls/blob/main/benchmarks/colab_benchmark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "# [sgoodfriend/rl-algo-impls](https://github.com/sgoodfriend/rl-algo-impls) in Google Colaboratory\n",
-        "## Parameters\n",
-        "\n",
-        "\n",
-        "1.   Wandb\n",
-        "\n"
-      ],
-      "metadata": {
-        "id": "S-tXDWP8WTLc"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from getpass import getpass\n",
-        "import os\n",
-        "os.environ[\"WANDB_API_KEY\"] = getpass(\"Wandb API key to upload metrics, videos, and models: \")"
-      ],
-      "metadata": {
-        "id": "1ZtdYgxWNGwZ"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Setup\n",
-        "Clone [sgoodfriend/rl-algo-impls](https://github.com/sgoodfriend/rl-algo-impls) "
-      ],
-      "metadata": {
-        "id": "bsG35Io0hmKG"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "!git clone https://github.com/sgoodfriend/rl-algo-impls.git"
-      ],
-      "metadata": {
-        "id": "k5ynTV25hdAf"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Installing the correct packages:\n",
-        "\n",
-        "While conda and poetry are generally used for package management, the mismatch in Python versions (3.10 in the project file vs 3.8 in Colab) makes using the package yml files difficult to use. For now, instead I'm going to specify the list of requirements manually below:"
-      ],
-      "metadata": {
-        "id": "jKxGok-ElYQ7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "!apt install python-opengl\n",
-        "!apt install ffmpeg\n",
-        "!apt install xvfb\n",
-        "!apt install swig"
-      ],
-      "metadata": {
-        "id": "nn6EETTc2Ewf"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "%cd /content/rl-algo-impls\n",
-        "!pip install -r colab_requirements.txt"
-      ],
-      "metadata": {
-        "id": "AfZh9rH3yQii"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Run Once Per Runtime"
-      ],
-      "metadata": {
-        "id": "4o5HOLjc4wq7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import wandb\n",
-        "wandb.login()"
-      ],
-      "metadata": {
-        "id": "PCXa5tdS2qFX"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Restart Session beteween runs"
-      ],
-      "metadata": {
-        "id": "AZBZfSUV43JQ"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "from pyvirtualdisplay import Display\n",
-        "\n",
-        "virtual_display = Display(visible=0, size=(1400, 900))\n",
-        "virtual_display.start()"
-      ],
-      "metadata": {
-        "id": "VzemeQJP2NO9"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "The below 5 bash scripts train agents on environments with 3 seeds each:\n",
-        "- colab_basic.sh and colab_pybullet.sh test on a set of basic gym environments and 4 PyBullet environments. Running both together will likely take about 18 hours. This is likely to run into runtime limits for free Colab and Colab Pro, but is fine for Colab Pro+.\n",
-        "- colab_carracing.sh only trains 3 seeds on CarRacing-v0, which takes almost 22 hours on Colab Pro+ on high-RAM, standard GPU.\n",
-        "- colab_atari1.sh and colab_atari2.sh likely need to be run separately because each takes about 19 hours on high-RAM, standard GPU."
-      ],
-      "metadata": {
-        "id": "nSHfna0hLlO1"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%cd /content/rl-algo-impls\n",
-        "os.environ[\"BENCHMARK_MAX_PROCS\"] = str(1) # Can't reliably raise this to 2+, but would make it faster.\n",
-        "!./benchmarks/colab_basic.sh\n",
-        "!./benchmarks/colab_pybullet.sh\n",
-        "# !./benchmarks/colab_carracing.sh\n",
-        "# !./benchmarks/colab_atari1.sh\n",
-        "# !./benchmarks/colab_atari2.sh"
-      ],
-      "metadata": {
-        "id": "07aHYFH1zfXa"
-      },
-      "execution_count": null,
-      "outputs": []
-    }
-  ]
-}

benchmarks/colab_carracing.sh

@@ -1,5 +0,0 @@
-source benchmarks/train_loop.sh
-ALGOS="ppo"
-ENVS="CarRacing-v0"
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-3}"
-train_loop $ALGOS "$ENVS" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

benchmarks/colab_pybullet.sh

@@ -1,5 +0,0 @@
-source benchmarks/train_loop.sh
-ALGOS="ppo"
-ENVS="HalfCheetahBulletEnv-v0 AntBulletEnv-v0 HopperBulletEnv-v0 Walker2DBulletEnv-v0"
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-3}"
-train_loop $ALGOS "$ENVS" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

benchmarks/train_loop.sh

@@ -1,15 +0,0 @@
-train_loop () {
-    local WANDB_TAGS="benchmark_$(git rev-parse --short HEAD) host_$(hostname)"
-    local algo
-    local env
-    local seed
-    local WANDB_PROJECT_NAME="${WANDB_PROJECT_NAME:-rl-algo-impls-benchmarks}"
-    local SEEDS="${SEEDS:-1 2 3}"
-    for algo in $(echo $1); do
-        for env in $(echo $2); do
-            for seed in $SEEDS; do
-                echo python train.py --algo $algo --env $env --seed $seed --pool-size 1 --wandb-tags $WANDB_TAGS --wandb-project-name $WANDB_PROJECT_NAME
-            done
-        done
-    done
-}

colab_enjoy.ipynb

@@ -1,198 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": [],
-      "machine_shape": "hm",
-      "authorship_tag": "ABX9TyN6S7kyJKrM5x0OOiN+CgTc",
-      "include_colab_link": true
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    },
-    "gpuClass": "standard",
-    "accelerator": "GPU"
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "view-in-github",
-        "colab_type": "text"
-      },
-      "source": [
-        "<a href=\"https://colab.research.google.com/github/sgoodfriend/rl-algo-impls/blob/main/colab_enjoy.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "# [sgoodfriend/rl-algo-impls](https://github.com/sgoodfriend/rl-algo-impls) in Google Colaboratory\n",
-        "## Parameters\n",
-        "\n",
-        "\n",
-        "1.   Wandb\n",
-        "\n"
-      ],
-      "metadata": {
-        "id": "S-tXDWP8WTLc"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from getpass import getpass\n",
-        "import os\n",
-        "os.environ[\"WANDB_API_KEY\"] = getpass(\"Wandb API key to upload metrics, videos, and models: \")"
-      ],
-      "metadata": {
-        "id": "1ZtdYgxWNGwZ"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "2. enjoy.py parameters"
-      ],
-      "metadata": {
-        "id": "ao0nAh3MOdN7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "WANDB_RUN_PATH=\"sgoodfriend/rl-algo-impls-benchmarks/rd0lisee\""
-      ],
-      "metadata": {
-        "id": "jKL_NFhVOjSc"
-      },
-      "execution_count": 2,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Setup\n",
-        "Clone [sgoodfriend/rl-algo-impls](https://github.com/sgoodfriend/rl-algo-impls) "
-      ],
-      "metadata": {
-        "id": "bsG35Io0hmKG"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "!git clone https://github.com/sgoodfriend/rl-algo-impls.git"
-      ],
-      "metadata": {
-        "id": "k5ynTV25hdAf"
-      },
-      "execution_count": 3,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Installing the correct packages:\n",
-        "\n",
-        "While conda and poetry are generally used for package management, the mismatch in Python versions (3.10 in the project file vs 3.8 in Colab) makes using the package yml files difficult to use. For now, instead I'm going to specify the list of requirements manually below:"
-      ],
-      "metadata": {
-        "id": "jKxGok-ElYQ7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "!apt install python-opengl\n",
-        "!apt install ffmpeg\n",
-        "!apt install xvfb\n",
-        "!apt install swig"
-      ],
-      "metadata": {
-        "id": "nn6EETTc2Ewf"
-      },
-      "execution_count": 4,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "%cd /content/rl-algo-impls\n",
-        "!pip install -r colab_requirements.txt"
-      ],
-      "metadata": {
-        "id": "AfZh9rH3yQii"
-      },
-      "execution_count": 5,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Run Once Per Runtime"
-      ],
-      "metadata": {
-        "id": "4o5HOLjc4wq7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import wandb\n",
-        "wandb.login()"
-      ],
-      "metadata": {
-        "id": "PCXa5tdS2qFX"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Restart Session beteween runs"
-      ],
-      "metadata": {
-        "id": "AZBZfSUV43JQ"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "from pyvirtualdisplay import Display\n",
-        "\n",
-        "virtual_display = Display(visible=0, size=(1400, 900))\n",
-        "virtual_display.start()"
-      ],
-      "metadata": {
-        "id": "VzemeQJP2NO9"
-      },
-      "execution_count": 7,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%cd /content/rl-algo-impls\n",
-        "!python enjoy.py --wandb-run-path={WANDB_RUN_PATH}"
-      ],
-      "metadata": {
-        "id": "07aHYFH1zfXa"
-      },
-      "execution_count": null,
-      "outputs": []
-    }
-  ]
-}

colab_requirements.txt

@@ -1,14 +0,0 @@
-AutoROM.accept-rom-license >= 0.4.2, < 0.5
-stable-baselines3[extra] >= 1.7.0, < 1.8
-gym[box2d] >= 0.21.0, < 0.22
-pyglet == 1.5.27
-wandb >= 0.13.10, < 0.14
-pyvirtualdisplay == 3.0
-pybullet >= 3.2.5, < 3.3
-tabulate >= 0.9.0, < 0.10
-huggingface-hub >= 0.12.0, < 0.13
-numexpr >= 2.8.4, < 2.9
-gym3 >= 0.3.3, < 0.4
-glfw >= 1.12.0, < 1.13
-procgen >= 0.10.7, < 0.11
-ipython >= 8.10.0, < 8.11

colab_train.ipynb

@@ -1,200 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": [],
-      "machine_shape": "hm",
-      "authorship_tag": "ABX9TyMmemQnx6G7GOnn6XBdjgxY",
-      "include_colab_link": true
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    },
-    "gpuClass": "standard",
-    "accelerator": "GPU"
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "view-in-github",
-        "colab_type": "text"
-      },
-      "source": [
-        "<a href=\"https://colab.research.google.com/github/sgoodfriend/rl-algo-impls/blob/main/colab_train.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "# [sgoodfriend/rl-algo-impls](https://github.com/sgoodfriend/rl-algo-impls) in Google Colaboratory\n",
-        "## Parameters\n",
-        "\n",
-        "\n",
-        "1.   Wandb\n",
-        "\n"
-      ],
-      "metadata": {
-        "id": "S-tXDWP8WTLc"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from getpass import getpass\n",
-        "import os\n",
-        "os.environ[\"WANDB_API_KEY\"] = getpass(\"Wandb API key to upload metrics, videos, and models: \")"
-      ],
-      "metadata": {
-        "id": "1ZtdYgxWNGwZ"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "2. train run parameters"
-      ],
-      "metadata": {
-        "id": "ao0nAh3MOdN7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "ALGO = \"ppo\"\n",
-        "ENV = \"CartPole-v1\"\n",
-        "SEED = 1"
-      ],
-      "metadata": {
-        "id": "jKL_NFhVOjSc"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Setup\n",
-        "Clone [sgoodfriend/rl-algo-impls](https://github.com/sgoodfriend/rl-algo-impls) "
-      ],
-      "metadata": {
-        "id": "bsG35Io0hmKG"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "!git clone https://github.com/sgoodfriend/rl-algo-impls.git"
-      ],
-      "metadata": {
-        "id": "k5ynTV25hdAf"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Installing the correct packages:\n",
-        "\n",
-        "While conda and poetry are generally used for package management, the mismatch in Python versions (3.10 in the project file vs 3.8 in Colab) makes using the package yml files difficult to use. For now, instead I'm going to specify the list of requirements manually below:"
-      ],
-      "metadata": {
-        "id": "jKxGok-ElYQ7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "!apt install python-opengl\n",
-        "!apt install ffmpeg\n",
-        "!apt install xvfb\n",
-        "!apt install swig"
-      ],
-      "metadata": {
-        "id": "nn6EETTc2Ewf"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "%cd /content/rl-algo-impls\n",
-        "!pip install -r colab_requirements.txt"
-      ],
-      "metadata": {
-        "id": "AfZh9rH3yQii"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Run Once Per Runtime"
-      ],
-      "metadata": {
-        "id": "4o5HOLjc4wq7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import wandb\n",
-        "wandb.login()"
-      ],
-      "metadata": {
-        "id": "PCXa5tdS2qFX"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Restart Session beteween runs"
-      ],
-      "metadata": {
-        "id": "AZBZfSUV43JQ"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%%capture\n",
-        "from pyvirtualdisplay import Display\n",
-        "\n",
-        "virtual_display = Display(visible=0, size=(1400, 900))\n",
-        "virtual_display.start()"
-      ],
-      "metadata": {
-        "id": "VzemeQJP2NO9"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%cd /content/rl-algo-impls\n",
-        "!python train.py --algo {ALGO} --env {ENV} --seed {SEED}"
-      ],
-      "metadata": {
-        "id": "07aHYFH1zfXa"
-      },
-      "execution_count": null,
-      "outputs": []
-    }
-  ]
-}

config.json

@@ -1 +0,0 @@
-{"policy_class": {":type:": "<class 'abc.ABCMeta'>", ":serialized:": "gAWVOwAAAAAAAACMIXN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi5wb2xpY2llc5SMEUFjdG9yQ3JpdGljUG9saWN5lJOULg==", "__module__": "stable_baselines3.common.policies", "__doc__": "\n    Policy class for actor-critic algorithms (has both policy and value prediction).\n    Used by A2C, PPO and the likes.\n\n    :param observation_space: Observation space\n    :param action_space: Action space\n    :param lr_schedule: Learning rate schedule (could be constant)\n    :param net_arch: The specification of the policy and value networks.\n    :param activation_fn: Activation function\n    :param ortho_init: Whether to use or not orthogonal initialization\n    :param use_sde: Whether to use State Dependent Exploration or not\n    :param log_std_init: Initial value for the log standard deviation\n    :param full_std: Whether to use (n_features x n_actions) parameters\n        for the std instead of only (n_features,) when using gSDE\n    :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure\n        a positive standard deviation (cf paper). It allows to keep variance\n        above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.\n    :param squash_output: Whether to squash the output using a tanh function,\n        this allows to ensure boundaries when using gSDE.\n    :param features_extractor_class: Features extractor to use.\n    :param features_extractor_kwargs: Keyword arguments\n        to pass to the features extractor.\n    :param share_features_extractor: If True, the features extractor is shared between the policy and value networks.\n    :param normalize_images: Whether to normalize images or not,\n         dividing by 255.0 (True by default)\n    :param optimizer_class: The optimizer to use,\n        ``th.optim.Adam`` by default\n    :param optimizer_kwargs: Additional keyword arguments,\n        excluding the learning rate, to pass to the optimizer\n    ", "__init__": "<function ActorCriticPolicy.__init__ at 0x7ff06ed7c3a0>", "_get_constructor_parameters": "<function ActorCriticPolicy._get_constructor_parameters at 0x7ff06ed7c430>", "reset_noise": "<function ActorCriticPolicy.reset_noise at 0x7ff06ed7c4c0>", "_build_mlp_extractor": "<function ActorCriticPolicy._build_mlp_extractor at 0x7ff06ed7c550>", "_build": "<function ActorCriticPolicy._build at 0x7ff06ed7c5e0>", "forward": "<function ActorCriticPolicy.forward at 0x7ff06ed7c670>", "extract_features": "<function ActorCriticPolicy.extract_features at 0x7ff06ed7c700>", "_get_action_dist_from_latent": "<function ActorCriticPolicy._get_action_dist_from_latent at 0x7ff06ed7c790>", "_predict": "<function ActorCriticPolicy._predict at 0x7ff06ed7c820>", "evaluate_actions": "<function ActorCriticPolicy.evaluate_actions at 0x7ff06ed7c8b0>", "get_distribution": "<function ActorCriticPolicy.get_distribution at 0x7ff06ed7c940>", "predict_values": "<function ActorCriticPolicy.predict_values at 0x7ff06ed7c9d0>", "__abstractmethods__": "frozenset()", "_abc_impl": "<_abc_data object at 0x7ff06ed70f60>"}, "verbose": 1, "policy_kwargs": {":type:": "<class 'dict'>", ":serialized:": "gAWVowAAAAAAAAB9lCiMDGxvZ19zdGRfaW5pdJRK/v///4wKb3J0aG9faW5pdJSJjA9vcHRpbWl6ZXJfY2xhc3OUjBN0b3JjaC5vcHRpbS5ybXNwcm9wlIwHUk1TcHJvcJSTlIwQb3B0aW1pemVyX2t3YXJnc5R9lCiMBWFscGhhlEc/764UeuFHrowDZXBzlEc+5Pi1iONo8YwMd2VpZ2h0X2RlY2F5lEsAdXUu", "log_std_init": -2, "ortho_init": false, "optimizer_class": "<class 'torch.optim.rmsprop.RMSprop'>", "optimizer_kwargs": {"alpha": 0.99, "eps": 1e-05, "weight_decay": 0}}, "observation_space": {":type:": "<class 'gym.spaces.box.Box'>", ":serialized:": "gAWVZwIAAAAAAACMDmd5bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMBl9zaGFwZZRLHIWUjANsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWcAAAAAAAAAAAAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/AACA/wAAgP8AAID/lGgKSxyFlIwBQ5R0lFKUjARoaWdolGgSKJZwAAAAAAAAAAAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH8AAIB/AACAfwAAgH+UaApLHIWUaBV0lFKUjA1ib3VuZGVkX2JlbG93lGgSKJYcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUaAeMAmIxlImIh5RSlChLA4wBfJROTk5K/////0r/////SwB0lGJLHIWUaBV0lFKUjA1ib3VuZGVkX2Fib3ZllGgSKJYcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUaCFLHIWUaBV0lFKUjApfbnBfcmFuZG9tlE51Yi4=", "dtype": "float32", "_shape": [28], "low": "[-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf\n -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf]", "high": "[inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf inf\n inf inf inf inf inf inf inf inf inf inf]", "bounded_below": "[False False False False False False False False False False False False\n False False False False False False False False False False False False\n False False False False]", "bounded_above": "[False False False False False False False False False False False False\n False False False False False False False False False False False False\n False False False False]", "_np_random": null}, "action_space": {":type:": "<class 'gym.spaces.box.Box'>", ":serialized:": "gAWVnwEAAAAAAACMDmd5bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMBl9zaGFwZZRLCIWUjANsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWIAAAAAAAAAAAAIC/AACAvwAAgL8AAIC/AACAvwAAgL8AAIC/AACAv5RoCksIhZSMAUOUdJRSlIwEaGlnaJRoEiiWIAAAAAAAAAAAAIA/AACAPwAAgD8AAIA/AACAPwAAgD8AAIA/AACAP5RoCksIhZRoFXSUUpSMDWJvdW5kZWRfYmVsb3eUaBIolggAAAAAAAAAAQEBAQEBAQGUaAeMAmIxlImIh5RSlChLA4wBfJROTk5K/////0r/////SwB0lGJLCIWUaBV0lFKUjA1ib3VuZGVkX2Fib3ZllGgSKJYIAAAAAAAAAAEBAQEBAQEBlGghSwiFlGgVdJRSlIwKX25wX3JhbmRvbZROdWIu", "dtype": "float32", "_shape": [8], "low": "[-1. -1. -1. -1. -1. -1. -1. -1.]", "high": "[1. 1. 1. 1. 1. 1. 1. 1.]", "bounded_below": "[ True  True  True  True  True  True  True  True]", "bounded_above": "[ True  True  True  True  True  True  True  True]", "_np_random": null}, "n_envs": 4, "num_timesteps": 2000000, "_total_timesteps": 2000000, "_num_timesteps_at_start": 0, "seed": null, "action_noise": null, "start_time": 1674588743994659442, "learning_rate": {":type:": "<class 'function'>", ":serialized:": "gAWVdgIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAksTQwiIAHwAFABTAJROhZQpjBJwcm9ncmVzc19yZW1haW5pbmeUhZSMHzxpcHl0aG9uLWlucHV0LTEzLWVhYTdkOGY5N2ZkNj6UjAhzY2hlZHVsZZRLBEMCAAGUjA1pbml0aWFsX3ZhbHVllIWUKXSUUpR9lCiMC19fcGFja2FnZV9flE6MCF9fbmFtZV9flIwIX19tYWluX1+UdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHH2UfZQoaBVoDYwMX19xdWFsbmFtZV9flIwhbGluZWFyX3NjaGVkdWxlLjxsb2NhbHM+LnNjaGVkdWxllIwPX19hbm5vdGF0aW9uc19flH2UKIwScHJvZ3Jlc3NfcmVtYWluaW5nlIwIYnVpbHRpbnOUjAVmbG9hdJSTlIwGcmV0dXJulGgpdYwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBaMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP091EE1VHWmFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="}, "tensorboard_log": null, "lr_schedule": {":type:": "<class 'function'>", ":serialized:": "gAWVdgIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAksTQwiIAHwAFABTAJROhZQpjBJwcm9ncmVzc19yZW1haW5pbmeUhZSMHzxpcHl0aG9uLWlucHV0LTEzLWVhYTdkOGY5N2ZkNj6UjAhzY2hlZHVsZZRLBEMCAAGUjA1pbml0aWFsX3ZhbHVllIWUKXSUUpR9lCiMC19fcGFja2FnZV9flE6MCF9fbmFtZV9flIwIX19tYWluX1+UdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHH2UfZQoaBVoDYwMX19xdWFsbmFtZV9flIwhbGluZWFyX3NjaGVkdWxlLjxsb2NhbHM+LnNjaGVkdWxllIwPX19hbm5vdGF0aW9uc19flH2UKIwScHJvZ3Jlc3NfcmVtYWluaW5nlIwIYnVpbHRpbnOUjAVmbG9hdJSTlIwGcmV0dXJulGgpdYwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBaMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP091EE1VHWmFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="}, "_last_obs": {":type:": "<class 'numpy.ndarray'>", ":serialized:": "gAWVNQIAAAAAAACMEm51bXB5LmNvcmUubnVtZXJpY5SMC19mcm9tYnVmZmVylJOUKJbAAQAAAAAAANkXHT/P+Bu/1A+1PjIvpT89Bh+/zjqjPhDbqT4Dqha/AjV/P+zYsr+uKGE/dj6Xv6Cxv7+R4wc8NxoKv6piD8CimWe/+TfIPi55aD9Zeju9gErvPhI+3L5IS6C/aYMmP8k7cr8P4ve/8OcFwEF3ab/IB8Q+iiKSPjT47T5cWJE/BK7Rvr5ZgD+8Kak900civ+X6jr4MOYU9Y5Shv5Jbtz6m2U++w9apP5QSCT7zfbk/P/O2PzWg8Lp/kpq+P/AOvwa3jb+DbFA/cyMdP3pKXj/JO3K//jAEP/DnBcDGWow/sCkSP3Xwnj5GF+s+0PqgP1LPhL70tAs/D5JdPdH8PDwVeaM+7Rabv2R/mz5gexrADzf+vtnLpj8/PMu+cPeXP8wagb4/bOU/gJVZPfvU8r+XP0C/uvbjvuIAMT9kUDM/yTtyv/4wBD9+tfQ+xlqMP1d1jT2Zgbe+1bHkPva9wj/bEYW/mrFZP9iSDD9GL/G+l7yMP/DjTz9xM7U/HN+OPqTRRr94K0XAr08LO38geb/41IG/tzC3v9ZobD/na868JuUBvnf+tr+WGXi/NfUov0BGhz8P4ve/frX0PkF3ab+UjAVudW1weZSMBWR0eXBllJOUjAJmNJSJiIeUUpQoSwOMATyUTk5OSv////9K/////0sAdJRiSwRLHIaUjAFDlHSUUpQu"}, "_last_episode_starts": {":type:": "<class 'numpy.ndarray'>", ":serialized:": "gAWVdwAAAAAAAACMEm51bXB5LmNvcmUubnVtZXJpY5SMC19mcm9tYnVmZmVylJOUKJYEAAAAAAAAAAAAAACUjAVudW1weZSMBWR0eXBllJOUjAJiMZSJiIeUUpQoSwOMAXyUTk5OSv////9K/////0sAdJRiSwSFlIwBQ5R0lFKULg=="}, "_last_original_obs": {":type:": "<class 'numpy.ndarray'>", ":serialized:": "gAWVNQIAAAAAAACMEm51bXB5LmNvcmUubnVtZXJpY5SMC19mcm9tYnVmZmVylJOUKJbAAQAAAAAAAAAAAAAtoGm1AACAPwAAAAAAAAAAAAAAAAAAAAAAAACA9o2EvQAAAACgRPm/AAAAAMwsiT0AAAAAyVXjPwAAAABeHa69AAAAAFdF5z8AAAAAAcmpPQAAAAApeea/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHEHLNAAAgD8AAAAAAAAAAAAAAAAAAAAAAAAAgEwf2b0AAAAA2Gz1vwAAAADLltm6AAAAAM9Z7D8AAAAA3yVmvQAAAADn69s/AAAAADiBBz0AAAAAw1rrvwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOBNMrYAAIA/AAAAAAAAAAAAAAAAAAAAAAAAAIAyWb08AAAAAEuk/L8AAAAAL1SzvQAAAAByavk/AAAAAP8V470AAAAA/m/iPwAAAAAtweW9AAAAAExT9L8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADP6ZA0AACAPwAAAAAAAAAAAAAAAAAAAAAAAACAfKcNPgAAAAAF3eK/AAAAACbLMT0AAAAARnIAQAAAAAArb0K7AAAAAMWz/z8AAAAANVvXvQAAAAAaNe2/AAAAAAAAAAAAAAAAAAAAAAAAAACUjAVudW1weZSMBWR0eXBllJOUjAJmNJSJiIeUUpQoSwOMATyUTk5OSv////9K/////0sAdJRiSwRLHIaUjAFDlHSUUpQu"}, "_episode_num": 0, "use_sde": true, "sde_sample_freq": -1, "_current_progress_remaining": 0.0, "ep_info_buffer": {":type:": "<class 'collections.deque'>", ":serialized:": "gAWVRAwAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKUKH2UKIwBcpRHQKaK8jRlYlqMAWyUTegDjAF0lEdArJafl8w6AHV9lChoBkdAproSsySFG2gHTegDaAhHQKyeau01IiF1fZQoaAZHQKW/SZpi7TVoB03oA2gIR0Csnr6jnFHbdX2UKGgGR0ClbyI/7iyZaAdN6ANoCEdArKJTlmvnsHV9lChoBkdApXSPAh0QsmgHTegDaAhHQKyiYGeMAFR1fZQoaAZHQKYeObMHKOloB03oA2gIR0Csqia/qPfbdX2UKGgGR0Cl25fAbhm5aAdN6ANoCEdArKp0rAgxJ3V9lChoBkdApGo8rmQr+mgHTegDaAhHQKyt3NGmUGF1fZQoaAZHQKW26Hk92X9oB03oA2gIR0CsreV76YVqdX2UKGgGR0Clpgs1KoQ4aAdN6ANoCEdArLXvPw/gSHV9lChoBkdApZRJfOUt7WgHTegDaAhHQKy2Q7jDKo11fZQoaAZHQKWQJ2alUIdoB03oA2gIR0Csubl7Uoa2dX2UKGgGR0CmLI2nsLOSaAdN6ANoCEdArLnCPU8V6HV9lChoBkdApVqDB9Cu2mgHTegDaAhHQKzBwo1k1/F1fZQoaAZHQKZuXCqp97ZoB03oA2gIR0CswhCEQGwBdX2UKGgGR0Cmlu0j9n9OaAdN6ANoCEdArMWdjbzshXV9lChoBkdApnp/RgJC0GgHTegDaAhHQKzFpoL5RCR1fZQoaAZHQKR8O5J9RaZoB03oA2gIR0CszVechC+ldX2UKGgGR0CltMF41P30aAdN6ANoCEdArM2rY/Vy3nV9lChoBkdApc4PphWo32gHTegDaAhHQKzRWe/5+H91fZQoaAZHQKZrGVUMoc9oB03oA2gIR0Cs0WMKTjebdX2UKGgGR0CmgCTvy9VWaAdN6ANoCEdArNj4dGRV63V9lChoBkdApfwql3yI6GgHTegDaAhHQKzZS31BdD91fZQoaAZHQKZoYRq46OpoB03oA2gIR0Cs3Nb8m8dxdX2UKGgGR0ClFq4sVclgaAdN6ANoCEdArNzf2PDHfnV9lChoBkdApdattVJcxGgHTegDaAhHQKzkmxptaZB1fZQoaAZHQKWUFjVhCt1oB03oA2gIR0Cs5O8LjPv8dX2UKGgGR0CmL6/N7jT8aAdN6ANoCEdArOhYIKMNt3V9lChoBkdApdR+nQ6ZIGgHTegDaAhHQKzoYBRQ7911fZQoaAZHQKYjF24d6s1oB03oA2gIR0Cs8AV6/qPfdX2UKGgGR0ClmIZ9Vmz0aAdN6ANoCEdArPBaKpDNQnV9lChoBkdApk9kpiI+GGgHTegDaAhHQKz0EkN4JNV1fZQoaAZHQKVyiL2HtWxoB03oA2gIR0Cs9Bv5pJwsdX2UKGgGR0CmUOSncclxaAdN6ANoCEdArPvNdVvMr3V9lChoBkdAprML/VAiV2gHTegDaAhHQKz8HtdiUgV1fZQoaAZHQKWHJzcRDkVoB03oA2gIR0Cs/4O6NEPUdX2UKGgGR0Clvp+QlruZaAdN6ANoCEdArP+Lvd/KAHV9lChoBkdAptM4bfgrH2gHTegDaAhHQK0HKjbBXS11fZQoaAZHQKTNTr/sE7poB03oA2gIR0CtB3/xtpEhdX2UKGgGR0ClpbtFjNILaAdN6ANoCEdArQsPj+717XV9lChoBkdApN2DeZXuE2gHTegDaAhHQK0LGJl8PWh1fZQoaAZHQKOCkuKXOW1oB03oA2gIR0CtEulT3qRmdX2UKGgGR0Cl+HYb83uNaAdN6ANoCEdArRM6aPS2IHV9lChoBkdApPy5dKNADGgHTegDaAhHQK0Wr5WzWwx1fZQoaAZHQKZWr4dIXj5oB03oA2gIR0CtFrfCAMDwdX2UKGgGR0CmKUQW3z+WaAdN6ANoCEdArR6OpQ1rI3V9lChoBkdAphbtDtw71mgHTegDaAhHQK0e3Ktga3t1fZQoaAZHQKW7YPTXrdFoB03oA2gIR0CtIkclolD4dX2UKGgGR0Cl5MDps41haAdN6ANoCEdArSJQ/RmbsnV9lChoBkdApiBvW4EwFmgHTegDaAhHQK0p+HEdeY51fZQoaAZHQKQNrjc2zfJoB03oA2gIR0CtKkj28IzFdX2UKGgGR0Cckdajvd/KaAdNggNoCEdArSyY1pCa7XV9lChoBkdApomsPvrnkmgHTegDaAhHQK0tzR4yGi51fZQoaAZHQKZk+Fyq+8JoB03oA2gIR0CtNXu1OTJRdX2UKGgGR0CllYyxqwhXaAdN6ANoCEdArTXIMx46fnV9lChoBkdAphfyzRhMJ2gHTegDaAhHQK04BurIYFd1fZQoaAZHQKSM9KV6eGxoB03oA2gIR0CtOUYlpoK2dX2UKGgGR0CmiWAJTl1baAdN6ANoCEdArUEHCj1wpHV9lChoBkdApgHe0gKWs2gHTegDaAhHQK1BYPxQSBd1fZQoaAZHQKcAthnanJloB03oA2gIR0CtQ791U2k0dX2UKGgGR0CiDy96Tnq3aAdN6ANoCEdArUTnRPXTVnV9lChoBkdApFqvcSGrS2gHTegDaAhHQK1MZh/Aj6h1fZQoaAZHQKXAlP/rB0poB03oA2gIR0CtTLSQHRkVdX2UKGgGR0CjmJprDZUUaAdN6ANoCEdArU7xddE9dXV9lChoBkdAptUnsZ5zHWgHTegDaAhHQK1QI8VYZEV1fZQoaAZHQKLkzDlYEGJoB03oA2gIR0CtWCQob4rSdX2UKGgGR0CeM9MS9M9KaAdN6ANoCEdArVh42GZeA3V9lChoBkdApeqzwMH8j2gHTegDaAhHQK1a0TSsr/d1fZQoaAZHQKYbd2wmmchoB03oA2gIR0CtXAZeJHiFdX2UKGgGR0Cl3GUtRNypaAdN6ANoCEdArWO18qnWKHV9lChoBkdApWpvz+WGAWgHTegDaAhHQK1kBFn7Hhl1fZQoaAZHQKZSKHBUJfJoB03oA2gIR0CtZlCuMdcTdX2UKGgGR0CmpdCiAUcoaAdN6ANoCEdArWeCZnctXnV9lChoBkdAppM5Q53kgmgHTegDaAhHQK1vG8h9srN1fZQoaAZHQKYExGDtgKFoB03oA2gIR0Ctb22CVbA2dX2UKGgGR0CmtFdFWn0kaAdN6ANoCEdArXG/VurIYHV9lChoBkdApfNvitJWemgHTegDaAhHQK1zFY5DJEJ1fZQoaAZHQKXdBBSk0rNoB03oA2gIR0CteqxdY4hmdX2UKGgGR0CmhG8kdFOPaAdN6ANoCEdArXr6QvHtGHV9lChoBkdApU/SYPXkHWgHTegDaAhHQK19RnVXmvJ1fZQoaAZHQKWUeXk5p8FoB03oA2gIR0Ctfnhun/DMdX2UKGgGR0CkjzIt16mgaAdN6ANoCEdArYYPdCVrynV9lChoBkdApPtMZ75VO2gHTegDaAhHQK2GYNAkcCJ1fZQoaAZHQKZMGafjCHhoB03oA2gIR0CtiLaOYIBzdX2UKGgGR0Cl0eULMLWqaAdN6ANoCEdArYnyg7HQyHV9lChoBkdApd5MyN4qw2gHTegDaAhHQK2Rs0iQkop1fZQoaAZHQKVcAo5xR2toB03oA2gIR0CtkgYjB2wFdX2UKGgGR0ClzucPFvQ4aAdN6ANoCEdArZRV0NjLCHV9lChoBkdApVq6v7m+02gHTegDaAhHQK2VmQjD8+B1fZQoaAZHQKSTCldC3PRoB03oA2gIR0CtnXYwqRU4dX2UKGgGR0Cl5iNfw7T2aAdN6ANoCEdArZ3F+iJwbXV9lChoBkdApdJXDYRNAWgHTegDaAhHQK2gGQlKK511fZQoaAZHQKZjJvMKTjhoB03oA2gIR0CtoWfvWpZPdX2UKGgGR0Cllc9fsu3+aAdN6ANoCEdAralP+bVjJHV9lChoBkdApTLXMSsbN2gHTegDaAhHQK2pqDU3GXJ1fZQoaAZHQKYQnPVNHpdoB03oA2gIR0Ctq/a9kBjndX2UKGgGR0Cmo4jRlYlqaAdN6ANoCEdAra09Tgl4T3V9lChoBkdApNBz6JqIrWgHTegDaAhHQK2051jAi3Z1fZQoaAZHQKXI+y5Zr59oB03oA2gIR0CttTxWkrPMdX2UKGgGR0CharHoPkJbaAdNVgNoCEdArbXjyUcGT3VlLg=="}, "ep_success_buffer": {":type:": "<class 'collections.deque'>", ":serialized:": "gAWVIAAAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKULg=="}, "_n_updates": 62500, "n_steps": 8, "gamma": 0.99, "gae_lambda": 0.9, "ent_coef": 0.0, "vf_coef": 0.4, "max_grad_norm": 0.5, "normalize_advantage": false, "system_info": {"OS": "Linux-5.10.147+-x86_64-with-glibc2.29 # 1 SMP Sat Dec 10 16:00:40 UTC 2022", "Python": "3.8.10", "Stable-Baselines3": "1.7.0", "PyTorch": "1.13.1+cu116", "GPU Enabled": "True", "Numpy": "1.21.6", "Gym": "0.21.0"}}

dqn/dqn.py

@@ -1,182 +0,0 @@
-import copy
-import numpy as np
-import random
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from collections import deque
-from torch.optim import Adam
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import List, NamedTuple, Optional, TypeVar
-
-from dqn.policy import DQNPolicy
-from shared.algorithm import Algorithm
-from shared.callbacks.callback import Callback
-from shared.schedule import linear_schedule
-from wrappers.vectorable_wrapper import VecEnv, VecEnvObs
-
-
-class Transition(NamedTuple):
-    obs: np.ndarray
-    action: np.ndarray
-    reward: float
-    done: bool
-    next_obs: np.ndarray
-
-
-class Batch(NamedTuple):
-    obs: np.ndarray
-    actions: np.ndarray
-    rewards: np.ndarray
-    dones: np.ndarray
-    next_obs: np.ndarray
-
-
-class ReplayBuffer:
-    def __init__(self, num_envs: int, maxlen: int) -> None:
-        self.num_envs = num_envs
-        self.buffer = deque(maxlen=maxlen)
-
-    def add(
-        self,
-        obs: VecEnvObs,
-        action: np.ndarray,
-        reward: np.ndarray,
-        done: np.ndarray,
-        next_obs: VecEnvObs,
-    ) -> None:
-        assert isinstance(obs, np.ndarray)
-        assert isinstance(next_obs, np.ndarray)
-        for i in range(self.num_envs):
-            self.buffer.append(
-                Transition(obs[i], action[i], reward[i], done[i], next_obs[i])
-            )
-
-    def sample(self, batch_size: int) -> Batch:
-        ts = random.sample(self.buffer, batch_size)
-        return Batch(
-            obs=np.array([t.obs for t in ts]),
-            actions=np.array([t.action for t in ts]),
-            rewards=np.array([t.reward for t in ts]),
-            dones=np.array([t.done for t in ts]),
-            next_obs=np.array([t.next_obs for t in ts]),
-        )
-
-    def __len__(self) -> int:
-        return len(self.buffer)
-
-
-DQNSelf = TypeVar("DQNSelf", bound="DQN")
-
-
-class DQN(Algorithm):
-    def __init__(
-        self,
-        policy: DQNPolicy,
-        env: VecEnv,
-        device: torch.device,
-        tb_writer: SummaryWriter,
-        learning_rate: float = 1e-4,
-        buffer_size: int = 1_000_000,
-        learning_starts: int = 50_000,
-        batch_size: int = 32,
-        tau: float = 1.0,
-        gamma: float = 0.99,
-        train_freq: int = 4,
-        gradient_steps: int = 1,
-        target_update_interval: int = 10_000,
-        exploration_fraction: float = 0.1,
-        exploration_initial_eps: float = 1.0,
-        exploration_final_eps: float = 0.05,
-        max_grad_norm: float = 10.0,
-    ) -> None:
-        super().__init__(policy, env, device, tb_writer)
-        self.policy = policy
-
-        self.optimizer = Adam(self.policy.q_net.parameters(), lr=learning_rate)
-
-        self.target_q_net = copy.deepcopy(self.policy.q_net).to(self.device)
-        self.target_q_net.train(False)
-        self.tau = tau
-        self.target_update_interval = target_update_interval
-
-        self.replay_buffer = ReplayBuffer(self.env.num_envs, buffer_size)
-        self.batch_size = batch_size
-
-        self.learning_starts = learning_starts
-        self.train_freq = train_freq
-        self.gradient_steps = gradient_steps
-
-        self.gamma = gamma
-        self.exploration_eps_schedule = linear_schedule(
-            exploration_initial_eps,
-            exploration_final_eps,
-            end_fraction=exploration_fraction,
-        )
-
-        self.max_grad_norm = max_grad_norm
-
-    def learn(
-        self: DQNSelf, total_timesteps: int, callback: Optional[Callback] = None
-    ) -> DQNSelf:
-        self.policy.train(True)
-        obs = self.env.reset()
-        obs = self._collect_rollout(self.learning_starts, obs, 1)
-        learning_steps = total_timesteps - self.learning_starts
-        timesteps_elapsed = 0
-        steps_since_target_update = 0
-        while timesteps_elapsed < learning_steps:
-            progress = timesteps_elapsed / learning_steps
-            eps = self.exploration_eps_schedule(progress)
-            obs = self._collect_rollout(self.train_freq, obs, eps)
-            rollout_steps = self.train_freq
-            timesteps_elapsed += rollout_steps
-            for _ in range(
-                self.gradient_steps if self.gradient_steps > 0 else self.train_freq
-            ):
-                self.train()
-            steps_since_target_update += rollout_steps
-            if steps_since_target_update >= self.target_update_interval:
-                self._update_target()
-                steps_since_target_update = 0
-            if callback:
-                callback.on_step(timesteps_elapsed=rollout_steps)
-        return self
-
-    def train(self) -> None:
-        if len(self.replay_buffer) < self.batch_size:
-            return
-        o, a, r, d, next_o = self.replay_buffer.sample(self.batch_size)
-        o = torch.as_tensor(o, device=self.device)
-        a = torch.as_tensor(a, device=self.device).unsqueeze(1)
-        r = torch.as_tensor(r, dtype=torch.float32, device=self.device)
-        d = torch.as_tensor(d, dtype=torch.long, device=self.device)
-        next_o = torch.as_tensor(next_o, device=self.device)
-
-        with torch.no_grad():
-            target = r + (1 - d) * self.gamma * self.target_q_net(next_o).max(1).values
-        current = self.policy.q_net(o).gather(dim=1, index=a).squeeze(1)
-        loss = F.smooth_l1_loss(current, target)
-
-        self.optimizer.zero_grad()
-        loss.backward()
-        if self.max_grad_norm:
-            nn.utils.clip_grad_norm_(self.policy.q_net.parameters(), self.max_grad_norm)
-        self.optimizer.step()
-
-    def _collect_rollout(self, timesteps: int, obs: VecEnvObs, eps: float) -> VecEnvObs:
-        for _ in range(0, timesteps, self.env.num_envs):
-            action = self.policy.act(obs, eps, deterministic=False)
-            next_obs, reward, done, _ = self.env.step(action)
-            self.replay_buffer.add(obs, action, reward, done, next_obs)
-            obs = next_obs
-        return obs
-
-    def _update_target(self) -> None:
-        for target_param, param in zip(
-            self.target_q_net.parameters(), self.policy.q_net.parameters()
-        ):
-            target_param.data.copy_(
-                self.tau * param.data + (1 - self.tau) * target_param.data
-            )

dqn/policy.py

@@ -1,52 +0,0 @@
-import numpy as np
-import os
-import torch
-
-from typing import Optional, Sequence, TypeVar
-
-from dqn.q_net import QNetwork
-from shared.policy.policy import Policy
-from wrappers.vectorable_wrapper import (
-    VecEnv,
-    VecEnvObs,
-    single_observation_space,
-    single_action_space,
-)
-
-DQNPolicySelf = TypeVar("DQNPolicySelf", bound="DQNPolicy")
-
-
-class DQNPolicy(Policy):
-    def __init__(
-        self,
-        env: VecEnv,
-        hidden_sizes: Sequence[int] = [],
-        cnn_feature_dim: int = 512,
-        cnn_style: str = "nature",
-        cnn_layers_init_orthogonal: Optional[bool] = None,
-        impala_channels: Sequence[int] = (16, 32, 32),
-        **kwargs,
-    ) -> None:
-        super().__init__(env, **kwargs)
-        self.q_net = QNetwork(
-            single_observation_space(env),
-            single_action_space(env),
-            hidden_sizes,
-            cnn_feature_dim=cnn_feature_dim,
-            cnn_style=cnn_style,
-            cnn_layers_init_orthogonal=cnn_layers_init_orthogonal,
-            impala_channels=impala_channels,
-        )
-
-    def act(
-        self, obs: VecEnvObs, eps: float = 0, deterministic: bool = True
-    ) -> np.ndarray:
-        assert eps == 0 if deterministic else eps >= 0
-        if not deterministic and np.random.random() < eps:
-            return np.array(
-                [self.env.action_space.sample() for _ in range(self.env.num_envs)]
-            )
-        else:
-            o = self._as_tensor(obs)
-            with torch.no_grad():
-                return self.q_net(o).argmax(axis=1).cpu().numpy()

dqn/q_net.py

@@ -1,41 +0,0 @@
-import gym
-import torch as th
-import torch.nn as nn
-
-from gym.spaces import Discrete
-from typing import Optional, Sequence, Type
-
-from shared.module.feature_extractor import FeatureExtractor
-from shared.module.module import mlp
-
-
-class QNetwork(nn.Module):
-    def __init__(
-        self,
-        observation_space: gym.Space,
-        action_space: gym.Space,
-        hidden_sizes: Sequence[int] = [],
-        activation: Type[nn.Module] = nn.ReLU,  # Used by stable-baselines3
-        cnn_feature_dim: int = 512,
-        cnn_style: str = "nature",
-        cnn_layers_init_orthogonal: Optional[bool] = None,
-        impala_channels: Sequence[int] = (16, 32, 32),
-    ) -> None:
-        super().__init__()
-        assert isinstance(action_space, Discrete)
-        self._feature_extractor = FeatureExtractor(
-            observation_space,
-            activation,
-            cnn_feature_dim=cnn_feature_dim,
-            cnn_style=cnn_style,
-            cnn_layers_init_orthogonal=cnn_layers_init_orthogonal,
-            impala_channels=impala_channels,
-        )
-        layer_sizes = (
-            (self._feature_extractor.out_dim,) + tuple(hidden_sizes) + (action_space.n,)
-        )
-        self._fc = mlp(layer_sizes, activation)
-
-    def forward(self, obs: th.Tensor) -> th.Tensor:
-        x = self._feature_extractor(obs)
-        return self._fc(x)

hyperparams/a2c.yml

@@ -1,127 +0,0 @@
-CartPole-v1: &cartpole-defaults
-  n_timesteps: !!float 5e5
-  env_hyperparams:
-    n_envs: 8
-
-CartPole-v0:
-  <<: *cartpole-defaults
-
-MountainCar-v0:
-  n_timesteps: !!float 1e6
-  env_hyperparams:
-    n_envs: 16
-    normalize: true
-
-MountainCarContinuous-v0:
-  n_timesteps: !!float 1e5
-  env_hyperparams:
-    n_envs: 4
-    normalize: true
-  # policy_hyperparams:
-  #   use_sde: true
-  #   log_std_init: 0.0
-  #   init_layers_orthogonal: false
-  algo_hyperparams:
-    n_steps: 100
-    sde_sample_freq: 16
-
-Acrobot-v1:
-  n_timesteps: !!float 5e5
-  env_hyperparams:
-    normalize: true
-    n_envs: 16
-
-LunarLander-v2:
-  n_timesteps: !!float 1e6
-  env_hyperparams:
-    n_envs: 8
-    normalize: true
-  algo_hyperparams:
-    n_steps: 5
-    gamma: 0.995
-    learning_rate: !!float 8.3e-4
-    learning_rate_decay: linear
-    ent_coef: !!float 1e-5
-
-BipedalWalker-v3:
-  n_timesteps: !!float 5e6
-  env_hyperparams:
-    n_envs: 16
-    normalize: true
-  policy_hyperparams:
-    use_sde: true
-    log_std_init: -2
-    init_layers_orthogonal: false
-  algo_hyperparams:
-    ent_coef: 0
-    max_grad_norm: 0.5
-    n_steps: 8
-    gae_lambda: 0.9
-    vf_coef: 0.4
-    gamma: 0.99
-    learning_rate: !!float 9.6e-4
-    learning_rate_decay: linear
-
-HalfCheetahBulletEnv-v0: &pybullet-defaults
-  n_timesteps: !!float 2e6
-  env_hyperparams:
-    n_envs: 4
-    normalize: true
-  policy_hyperparams:
-    use_sde: true
-    log_std_init: -2
-    init_layers_orthogonal: false
-  algo_hyperaparms: &pybullet-algo-defaults
-    n_steps: 8
-    ent_coef: 0
-    max_grad_norm: 0.5
-    gae_lambda: 0.9
-    gamma: 0.99
-    vf_coef: 0.4
-    learning_rate: !!float 9.6e-4
-    learning_rate_decay: linear
-
-AntBulletEnv-v0:
-  <<: *pybullet-defaults
-
-Walker2DBulletEnv-v0:
-  <<: *pybullet-defaults
-
-HopperBulletEnv-v0:
-  <<: *pybullet-defaults
-
-CarRacing-v0:
-  n_timesteps: !!float 4e6
-  env_hyperparams:
-    n_envs: 8
-    frame_stack: 4
-  policy_hyperparams:
-    use_sde: true
-    log_std_init: -2
-    init_layers_orthogonal: false
-    activation_fn: relu
-    share_features_extractor: false
-    cnn_feature_dim: 256
-    hidden_sizes: [256]
-  algo_hyperparams:
-    n_steps: 8
-    learning_rate: !!float 5e-5
-    learning_rate_decay: linear
-    gamma: 0.99
-    gae_lambda: 0.95
-    ent_coef: 0
-    sde_sample_freq: 4
-
-_atari: &atari-defaults
-  n_timesteps: !!float 1e7
-  env_hyperparams: &atari-env-defaults
-    n_envs: 16
-    frame_stack: 4
-    no_reward_timeout_steps: 1000
-    no_reward_fire_steps: 500
-    vec_env_class: async
-  policy_hyperparams: &atari-policy-defaults
-    activation_fn: relu
-  algo_hyperparams:
-    ent_coef: 0.01
-    vf_coef: 0.25

hyperparams/dqn.yml

@@ -1,130 +0,0 @@
-CartPole-v1: &cartpole-defaults
-  n_timesteps: !!float 5e4
-  env_hyperparams:
-    rolling_length: 50
-  policy_hyperparams:
-    hidden_sizes: [256, 256]
-  algo_hyperparams:
-    learning_rate: !!float 2.3e-3
-    batch_size: 64
-    buffer_size: 100000
-    learning_starts: 1000
-    gamma: 0.99
-    target_update_interval: 10
-    train_freq: 256
-    gradient_steps: 128
-    exploration_fraction: 0.16
-    exploration_final_eps: 0.04
-  eval_params:
-    step_freq: !!float 1e4
-
-CartPole-v0:
-  <<: *cartpole-defaults
-  n_timesteps: !!float 4e4
-
-MountainCar-v0:
-  n_timesteps: !!float 1.2e5
-  env_hyperparams:
-    rolling_length: 50
-  policy_hyperparams:
-    hidden_sizes: [256, 256]
-  algo_hyperparams:
-    learning_rate: !!float 4e-3
-    batch_size: 128
-    buffer_size: 10000
-    learning_starts: 1000
-    gamma: 0.98
-    target_update_interval: 600
-    train_freq: 16
-    gradient_steps: 8
-    exploration_fraction: 0.2
-    exploration_final_eps: 0.07
-
-Acrobot-v1:
-  n_timesteps: !!float 1e5
-  env_hyperparams:
-    rolling_length: 50
-  policy_hyperparams:
-    hidden_sizes: [256, 256]
-  algo_hyperparams:
-    learning_rate: !!float 6.3e-4
-    batch_size: 128
-    buffer_size: 50000
-    learning_starts: 0
-    gamma: 0.99
-    target_update_interval: 250
-    train_freq: 4
-    gradient_steps: -1
-    exploration_fraction: 0.12
-    exploration_final_eps: 0.1
-
-LunarLander-v2:
-  n_timesteps: !!float 5e5
-  env_hyperparams:
-    rolling_length: 50
-  policy_hyperparams:
-    hidden_sizes: [256, 256]
-  algo_hyperparams:
-    learning_rate: !!float 1e-4
-    batch_size: 256
-    buffer_size: 100000
-    learning_starts: 10000
-    gamma: 0.99
-    target_update_interval: 250
-    train_freq: 8
-    gradient_steps: -1
-    exploration_fraction: 0.12
-    exploration_final_eps: 0.1
-    max_grad_norm: 0.5
-  eval_params:
-    step_freq: 25_000
-
-_atari: &atari-defaults
-  n_timesteps: !!float 1e7
-  env_hyperparams:
-    frame_stack: 4
-    no_reward_timeout_steps: 1_000
-    no_reward_fire_steps: 500
-    n_envs: 8
-    vec_env_class: async
-  algo_hyperparams:
-    buffer_size: 100000
-    learning_rate: !!float 1e-4
-    batch_size: 32
-    learning_starts: 100000
-    target_update_interval: 1000
-    train_freq: 8
-    gradient_steps: 2
-    exploration_fraction: 0.1
-    exploration_final_eps: 0.01
-  eval_params:
-    deterministic: false
-
-PongNoFrameskip-v4:
-  <<: *atari-defaults
-  n_timesteps: !!float 2.5e6
-
-_impala-atari: &impala-atari-defaults
-  <<: *atari-defaults
-  policy_hyperparams:
-    cnn_style: impala
-    cnn_feature_dim: 256
-    init_layers_orthogonal: true
-    cnn_layers_init_orthogonal: false
-
-impala-PongNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: PongNoFrameskip-v4
-  n_timesteps: !!float 2.5e6
-
-impala-BreakoutNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: BreakoutNoFrameskip-v4
-
-impala-SpaceInvadersNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: SpaceInvadersNoFrameskip-v4
-
-impala-QbertNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: QbertNoFrameskip-v4

hyperparams/ppo.yml

@@ -1,383 +0,0 @@
-CartPole-v1: &cartpole-defaults
-  n_timesteps: !!float 1e5
-  env_hyperparams:
-    n_envs: 8
-  algo_hyperparams:
-    n_steps: 32
-    batch_size: 256
-    n_epochs: 20
-    gae_lambda: 0.8
-    gamma: 0.98
-    ent_coef: 0.0
-    learning_rate: 0.001
-    learning_rate_decay: linear
-    clip_range: 0.2
-    clip_range_decay: linear
-  eval_params:
-    step_freq: !!float 2.5e4
-
-CartPole-v0:
-  <<: *cartpole-defaults
-  n_timesteps: !!float 5e4
-
-MountainCar-v0:
-  n_timesteps: !!float 1e6
-  env_hyperparams:
-    normalize: true
-    n_envs: 16
-  algo_hyperparams:
-    n_steps: 16
-    n_epochs: 4
-    gae_lambda: 0.98
-    gamma: 0.99
-    ent_coef: 0.0
-
-MountainCarContinuous-v0:
-  n_timesteps: !!float 1e5
-  env_hyperparams:
-    normalize: true
-    n_envs: 4
-  # policy_hyperparams:
-  #   init_layers_orthogonal: false
-  #   log_std_init: -3.29
-  #   use_sde: true
-  algo_hyperparams:
-    n_steps: 512
-    batch_size: 256
-    n_epochs: 10
-    learning_rate: !!float 7.77e-5
-    ent_coef: 0.01 # 0.00429
-    ent_coef_decay: linear
-    clip_range: 0.1
-    gae_lambda: 0.9
-    max_grad_norm: 5
-    vf_coef: 0.19
-  eval_params:
-    step_freq: 5000
-
-Acrobot-v1:
-  n_timesteps: !!float 1e6
-  env_hyperparams:
-    n_envs: 16
-    normalize: true
-  algo_hyperparams:
-    n_steps: 256
-    n_epochs: 4
-    gae_lambda: 0.94
-    gamma: 0.99
-    ent_coef: 0.0
-
-LunarLander-v2:
-  n_timesteps: !!float 4e6
-  env_hyperparams:
-    n_envs: 16
-  algo_hyperparams:
-    n_steps: 1024
-    batch_size: 64
-    n_epochs: 4
-    gae_lambda: 0.98
-    gamma: 0.999
-    learning_rate: !!float 5e-4
-    learning_rate_decay: linear
-    clip_range: 0.2
-    clip_range_decay: linear
-    ent_coef: 0.01
-    normalize_advantage: false
-
-BipedalWalker-v3:
-  n_timesteps: !!float 10e6
-  env_hyperparams:
-    n_envs: 16
-    normalize: true
-  algo_hyperparams:
-    n_steps: 2048
-    batch_size: 64
-    gae_lambda: 0.95
-    gamma: 0.99
-    n_epochs: 10
-    ent_coef: 0.001
-    learning_rate: !!float 2.5e-4
-    learning_rate_decay: linear
-    clip_range: 0.2
-    clip_range_decay: linear
-
-CarRacing-v0: &carracing-defaults
-  n_timesteps: !!float 4e6
-  env_hyperparams:
-    n_envs: 8
-    frame_stack: 4
-  policy_hyperparams: &carracing-policy-defaults
-    use_sde: true
-    log_std_init: -2
-    init_layers_orthogonal: false
-    activation_fn: relu
-    share_features_extractor: false
-    cnn_feature_dim: 256
-    hidden_sizes: [256]
-  algo_hyperparams:
-    n_steps: 512
-    batch_size: 128
-    n_epochs: 10
-    learning_rate: !!float 1e-4
-    learning_rate_decay: linear
-    gamma: 0.99
-    gae_lambda: 0.95
-    ent_coef: 0.0
-    sde_sample_freq: 4
-    max_grad_norm: 0.5
-    vf_coef: 0.5
-    clip_range: 0.2
-
-impala-CarRacing-v0:
-  <<: *carracing-defaults
-  env_id: CarRacing-v0
-  policy_hyperparams:
-    <<: *carracing-policy-defaults
-    cnn_style: impala
-    init_layers_orthogonal: true
-    cnn_layers_init_orthogonal: false
-    hidden_sizes: []
-
-# BreakoutNoFrameskip-v4
-# PongNoFrameskip-v4
-# SpaceInvadersNoFrameskip-v4
-# QbertNoFrameskip-v4
-_atari: &atari-defaults
-  n_timesteps: !!float 1e7
-  env_hyperparams: &atari-env-defaults
-    n_envs: 8
-    frame_stack: 4
-    no_reward_timeout_steps: 1000
-    no_reward_fire_steps: 500
-    vec_env_class: async
-  policy_hyperparams: &atari-policy-defaults
-    activation_fn: relu
-  algo_hyperparams:
-    n_steps: 128
-    batch_size: 256
-    n_epochs: 4
-    learning_rate: !!float 2.5e-4
-    learning_rate_decay: linear
-    clip_range: 0.1
-    clip_range_decay: linear
-    vf_coef: 0.5
-    ent_coef: 0.01
-  eval_params:
-    deterministic: false
-
-_norm-rewards-atari: &norm-rewards-atari-default
-  <<: *atari-defaults
-  env_hyperparams:
-    <<: *atari-env-defaults
-    clip_atari_rewards: false
-    normalize: true
-    normalize_kwargs:
-      norm_obs: false
-      norm_reward: true
-
-norm-rewards-BreakoutNoFrameskip-v4:
-  <<: *norm-rewards-atari-default
-  env_id: BreakoutNoFrameskip-v4
-
-debug-PongNoFrameskip-v4:
-  <<: *atari-defaults
-  device: cpu
-  env_id: PongNoFrameskip-v4
-  env_hyperparams:
-    <<: *atari-env-defaults
-    vec_env_class: sync
-
-_impala-atari: &impala-atari-defaults
-  <<: *atari-defaults
-  policy_hyperparams:
-    <<: *atari-policy-defaults
-    cnn_style: impala
-    cnn_feature_dim: 256
-    init_layers_orthogonal: true
-    cnn_layers_init_orthogonal: false
-
-impala-PongNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: PongNoFrameskip-v4
-
-impala-BreakoutNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: BreakoutNoFrameskip-v4
-
-impala-SpaceInvadersNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: SpaceInvadersNoFrameskip-v4
-
-impala-QbertNoFrameskip-v4:
-  <<: *impala-atari-defaults
-  env_id: QbertNoFrameskip-v4
-
-HalfCheetahBulletEnv-v0: &pybullet-defaults
-  n_timesteps: !!float 2e6
-  env_hyperparams: &pybullet-env-defaults
-    n_envs: 16
-    normalize: true
-  policy_hyperparams: &pybullet-policy-defaults
-    pi_hidden_sizes: [256, 256]
-    v_hidden_sizes: [256, 256]
-    activation_fn: relu
-  algo_hyperparams: &pybullet-algo-defaults
-    n_steps: 512
-    batch_size: 128
-    n_epochs: 20
-    gamma: 0.99
-    gae_lambda: 0.9
-    ent_coef: 0.0
-    max_grad_norm: 0.5
-    vf_coef: 0.5
-    learning_rate: !!float 3e-5
-    clip_range: 0.4
-
-AntBulletEnv-v0:
-  <<: *pybullet-defaults
-  policy_hyperparams:
-    <<: *pybullet-policy-defaults
-  algo_hyperparams:
-    <<: *pybullet-algo-defaults
-
-Walker2DBulletEnv-v0:
-  <<: *pybullet-defaults
-  algo_hyperparams:
-    <<: *pybullet-algo-defaults
-    clip_range_decay: linear
-
-HopperBulletEnv-v0:
-  <<: *pybullet-defaults
-  algo_hyperparams:
-    <<: *pybullet-algo-defaults
-    clip_range_decay: linear
-
-HumanoidBulletEnv-v0:
-  <<: *pybullet-defaults
-  n_timesteps: !!float 1e7
-  env_hyperparams:
-    <<: *pybullet-env-defaults
-    n_envs: 8
-  policy_hyperparams:
-    <<: *pybullet-policy-defaults
-    # log_std_init: -1
-  algo_hyperparams:
-    <<: *pybullet-algo-defaults
-    n_steps: 2048
-    batch_size: 64
-    n_epochs: 10
-    gae_lambda: 0.95
-    learning_rate: !!float 2.5e-4
-    clip_range: 0.2
-
-_procgen: &procgen-defaults
-  env_hyperparams: &procgen-env-defaults
-    env_type: procgen
-    n_envs: 64
-    # grayscale: false
-    # frame_stack: 4
-    normalize: true # procgen only normalizes reward
-    make_kwargs: &procgen-make-kwargs-defaults
-      num_threads: 8
-  policy_hyperparams: &procgen-policy-defaults
-    activation_fn: relu
-    cnn_style: impala
-    cnn_feature_dim: 256
-    init_layers_orthogonal: true
-    cnn_layers_init_orthogonal: false
-  algo_hyperparams: &procgen-algo-defaults
-    gamma: 0.999
-    gae_lambda: 0.95
-    n_steps: 256
-    batch_size: 2048
-    n_epochs: 3
-    ent_coef: 0.01
-    clip_range: 0.2
-    # clip_range_decay: linear
-    clip_range_vf: 0.2
-    learning_rate: !!float 5e-4
-    # learning_rate_decay: linear
-    vf_coef: 0.5
-  eval_params: &procgen-eval-defaults
-    ignore_first_episode: true
-    # deterministic: false
-    step_freq: !!float 1e5
-
-_procgen-easy: &procgen-easy-defaults
-  <<: *procgen-defaults
-  n_timesteps: !!float 25e6
-  env_hyperparams: &procgen-easy-env-defaults
-    <<: *procgen-env-defaults
-    make_kwargs:
-      <<: *procgen-make-kwargs-defaults
-      distribution_mode: easy
-
-procgen-coinrun-easy: &coinrun-easy-defaults
-  <<: *procgen-easy-defaults
-  env_id: coinrun
-
-debug-procgen-coinrun:
-  <<: *coinrun-easy-defaults
-  device: cpu
-
-procgen-starpilot-easy:
-  <<: *procgen-easy-defaults
-  env_id: starpilot
-
-procgen-bossfight-easy:
-  <<: *procgen-easy-defaults
-  env_id: bossfight
-
-procgen-bigfish-easy:
-  <<: *procgen-easy-defaults
-  env_id: bigfish
-
-_procgen-hard: &procgen-hard-defaults
-  <<: *procgen-defaults
-  n_timesteps: !!float 200e6
-  env_hyperparams: &procgen-hard-env-defaults
-    <<: *procgen-env-defaults
-    n_envs: 256
-    make_kwargs:
-      <<: *procgen-make-kwargs-defaults
-      distribution_mode: hard
-  algo_hyperparams: &procgen-hard-algo-defaults
-    <<: *procgen-algo-defaults
-    batch_size: 8192
-    clip_range_decay: linear
-    learning_rate_decay: linear
-  eval_params:
-    <<: *procgen-eval-defaults
-    step_freq: !!float 5e5
-
-procgen-starpilot-hard: &procgen-starpilot-hard-defaults
-  <<: *procgen-hard-defaults
-  env_id: starpilot
-
-procgen-starpilot-hard-2xIMPALA:
-  <<: *procgen-starpilot-hard-defaults
-  policy_hyperparams:
-    <<: *procgen-policy-defaults
-    impala_channels: [32, 64, 64]
-  algo_hyperparams:
-    <<: *procgen-hard-algo-defaults
-    learning_rate: !!float 3.3e-4
-
-procgen-starpilot-hard-2xIMPALA-fat:
-  <<: *procgen-starpilot-hard-defaults
-  policy_hyperparams:
-    <<: *procgen-policy-defaults
-    impala_channels: [32, 64, 64]
-    cnn_feature_dim: 512
-  algo_hyperparams:
-    <<: *procgen-hard-algo-defaults
-    learning_rate: !!float 2.5e-4
-
-procgen-starpilot-hard-4xIMPALA:
-  <<: *procgen-starpilot-hard-defaults
-  policy_hyperparams:
-    <<: *procgen-policy-defaults
-    impala_channels: [64, 128, 128]
-  algo_hyperparams:
-    <<: *procgen-hard-algo-defaults
-    learning_rate: !!float 2.1e-4

hyperparams/vpg.yml

@@ -1,197 +0,0 @@
-CartPole-v1: &cartpole-defaults
-  n_timesteps: !!float 4e5
-  algo_hyperparams:
-    n_steps: 4096
-    pi_lr: 0.01
-    gamma: 0.99
-    gae_lambda: 1
-    val_lr: 0.01
-    train_v_iters: 80
-  eval_params:
-    step_freq: !!float 2.5e4
-
-CartPole-v0:
-  <<: *cartpole-defaults
-  n_timesteps: !!float 1e5
-  algo_hyperparams:
-    n_steps: 1024
-    pi_lr: 0.01
-    gamma: 0.99
-    gae_lambda: 1
-    val_lr: 0.01
-    train_v_iters: 80
-
-MountainCar-v0:
-  n_timesteps: !!float 1e6
-  env_hyperparams:
-    normalize: true
-    n_envs: 16
-  algo_hyperparams:
-    n_steps: 200
-    pi_lr: 0.005
-    gamma: 0.99
-    gae_lambda: 0.97
-    val_lr: 0.01
-    train_v_iters: 80
-    max_grad_norm: 0.5
-
-MountainCarContinuous-v0:
-  n_timesteps: !!float 3e5
-  env_hyperparams:
-    normalize: true
-    n_envs: 4
-  # policy_hyperparams:
-  #   init_layers_orthogonal: false
-  #   log_std_init: -3.29
-  #   use_sde: true
-  algo_hyperparams:
-    n_steps: 1000
-    pi_lr: !!float 5e-4
-    gamma: 0.99
-    gae_lambda: 0.9
-    val_lr: !!float 1e-3
-    train_v_iters: 80
-    max_grad_norm: 5
-  eval_params:
-    step_freq: 5000
-
-Acrobot-v1:
-  n_timesteps: !!float 2e5
-  algo_hyperparams:
-    n_steps: 2048
-    pi_lr: 0.005
-    gamma: 0.99
-    gae_lambda: 0.97
-    val_lr: 0.01
-    train_v_iters: 80
-    max_grad_norm: 0.5
-
-LunarLander-v2:
-  n_timesteps: !!float 4e6
-  policy_hyperparams:
-    hidden_sizes: [256, 256]
-  algo_hyperparams:
-    n_steps: 2048
-    pi_lr: 0.0001
-    gamma: 0.999
-    gae_lambda: 0.97
-    val_lr: 0.0001
-    train_v_iters: 80
-    max_grad_norm: 0.5
-  eval_params:
-    deterministic: false
-
-BipedalWalker-v3:
-  n_timesteps: !!float 10e6
-  env_hyperparams:
-    n_envs: 16
-    normalize: true
-  policy_hyperparams:
-    hidden_sizes: [256, 256]
-  algo_hyperparams:
-    n_steps: 1600
-    gae_lambda: 0.95
-    gamma: 0.99
-    pi_lr: !!float 1e-4
-    val_lr: !!float 1e-4
-    train_v_iters: 80
-    max_grad_norm: 0.5
-  eval_params:
-    deterministic: false
-
-CarRacing-v0:
-  n_timesteps: !!float 4e6
-  env_hyperparams:
-    frame_stack: 4
-    n_envs: 4
-    vec_env_class: sync
-  policy_hyperparams:
-    use_sde: true
-    log_std_init: -2
-    init_layers_orthogonal: false
-    activation_fn: relu
-    cnn_feature_dim: 256
-    hidden_sizes: [256]
-  algo_hyperparams:
-    n_steps: 1000
-    pi_lr: !!float 5e-5
-    gamma: 0.99
-    gae_lambda: 0.95
-    val_lr: !!float 1e-4
-    train_v_iters: 40
-    max_grad_norm: 0.5
-    sde_sample_freq: 4
-
-HalfCheetahBulletEnv-v0: &pybullet-defaults
-  n_timesteps: !!float 2e6
-  env_hyperparams: &pybullet-env-defaults
-    normalize: true
-  policy_hyperparams: &pybullet-policy-defaults
-    hidden_sizes: [256, 256]
-  algo_hyperparams: &pybullet-algo-defaults
-    n_steps: 4000
-    pi_lr: !!float 3e-4
-    gamma: 0.99
-    gae_lambda: 0.97
-    val_lr: !!float 1e-3
-    train_v_iters: 80
-    max_grad_norm: 0.5
-
-AntBulletEnv-v0:
-  <<: *pybullet-defaults
-  policy_hyperparams:
-    <<: *pybullet-policy-defaults
-    hidden_sizes: [400, 300]
-  algo_hyperparams:
-    <<: *pybullet-algo-defaults
-    pi_lr: !!float 7e-4
-    val_lr: !!float 7e-3
-
-HopperBulletEnv-v0:
-  <<: *pybullet-defaults
-
-Walker2DBulletEnv-v0:
-  <<: *pybullet-defaults
-
-FrozenLake-v1:
-  n_timesteps: !!float 8e5
-  env_params:
-    make_kwargs:
-      map_name: 8x8
-      is_slippery: true
-  policy_hyperparams:
-    hidden_sizes: [64]
-  algo_hyperparams:
-    n_steps: 2048
-    pi_lr: 0.01
-    gamma: 0.99
-    gae_lambda: 0.98
-    val_lr: 0.01
-    train_v_iters: 80
-    max_grad_norm: 0.5
-  eval_params:
-    step_freq: !!float 5e4
-    n_episodes: 10
-    save_best: true
-
-_atari: &atari-defaults
-  n_timesteps: !!float 25e6
-  env_hyperparams:
-    n_envs: 4
-    frame_stack: 4
-    no_reward_timeout_steps: 1000
-    no_reward_fire_steps: 500
-    vec_env_class: async
-  policy_hyperparams:
-    activation_fn: relu
-  algo_hyperparams:
-    n_steps: 2048
-    pi_lr: !!float 5e-5
-    gamma: 0.99
-    gae_lambda: 0.95
-    val_lr: !!float 1e-4
-    train_v_iters: 80
-    max_grad_norm: 0.5
-    ent_coef: 0.01
-  eval_params:
-    deterministic: false

lambda_labs/benchmark.sh

@@ -1,34 +0,0 @@
-source benchmarks/train_loop.sh
-
-# export WANDB_PROJECT_NAME="rl-algo-impls"
-
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-6}"
-
-ALGOS=(
-    # "vpg"
-    # "dqn"
-    # "ppo"
-    "a2c"
-)
-ENVS=(
-    # Basic
-    "CartPole-v1"
-    "MountainCar-v0"
-    "MountainCarContinuous-v0"
-    "Acrobot-v1"
-    "LunarLander-v2"
-    "BipedalWalker-v3"
-    # PyBullet
-    "HalfCheetahBulletEnv-v0"
-    "AntBulletEnv-v0"
-    "HopperBulletEnv-v0"
-    "Walker2DBulletEnv-v0"
-    # CarRacing
-    "CarRacing-v0"
-    # Atari
-    "PongNoFrameskip-v4"
-    "BreakoutNoFrameskip-v4"
-    "SpaceInvadersNoFrameskip-v4"
-    "QbertNoFrameskip-v4"
-)
-train_loop "${ALGOS[*]}" "${ENVS[*]}" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

lambda_labs/impala_atari_benchmark.sh

@@ -1,19 +0,0 @@
-source benchmarks/train_loop.sh
-
-# export WANDB_PROJECT_NAME="rl-algo-impls"
-
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-5}"
-
-ALGOS=(
-    # "vpg"
-    # "dqn"
-    "ppo"
-)
-ENVS=(
-    "impala-PongNoFrameskip-v4"
-    "impala-BreakoutNoFrameskip-v4"
-    "impala-SpaceInvadersNoFrameskip-v4"
-    "impala-QbertNoFrameskip-v4"
-    "impala-CarRacing-v0"
-)
-train_loop "${ALGOS[*]}" "${ENVS[*]}" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

lambda_labs/lambda_requirements.txt

@@ -1,16 +0,0 @@
-scipy >= 1.10.0, < 1.11
-tensorboard >= ^2.11.0, < 2.12
-AutoROM.accept-rom-license >= 0.4.2, < 0.5
-stable-baselines3[extra] >= 1.7.0, < 1.8
-gym[box2d] >= 0.21.0, < 0.22
-pyglet == 1.5.27
-wandb >= 0.13.10, < 0.14
-pyvirtualdisplay == 3.0
-pybullet >= 3.2.5, < 3.3
-tabulate >= 0.9.0, < 0.10
-huggingface-hub >= 0.12.0, < 0.13
-numexpr >= 2.8.4, < 2.9
-gym3 >= 0.3.3, < 0.4
-glfw >= 1.12.0, < 1.13
-procgen >= 0.10.7, < 0.11
-ipython >= 8.10.0, < 8.11

lambda_labs/procgen_benchmark.sh

@@ -1,18 +0,0 @@
-source benchmarks/train_loop.sh
-
-# export WANDB_PROJECT_NAME="rl-algo-impls"
-
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-3}"
-
-ALGOS=(
-    # "vpg"
-    # "dqn"
-    "ppo"
-)
-ENVS=(
-    "procgen-coinrun-easy"
-    "procgen-starpilot-easy"
-    "procgen-bossfight-easy"
-    "procgen-bigfish-easy"
-)
-train_loop "${ALGOS[*]}" "${ENVS[*]}" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

lambda_labs/setup.sh

@@ -1,10 +0,0 @@
-sudo apt update
-sudo apt install -y python-opengl
-sudo apt install -y ffmpeg
-sudo apt install -y xvfb
-sudo apt install -y swig
-
-python3 -m pip install --upgrade pip
-pip install --upgrade torch torchvision torchaudio
-
-pip install --upgrade -r ~/rl-algo-impls/lambda_labs/lambda_requirements.txt

lambda_labs/starpilot_hard_benchmark.sh

@@ -1,16 +0,0 @@
-source benchmarks/train_loop.sh
-
-# export WANDB_PROJECT_NAME="rl-algo-impls"
-
-BENCHMARK_MAX_PROCS="${BENCHMARK_MAX_PROCS:-1}"
-
-ALGOS=(
-    "ppo"
-)
-ENVS=(
-    "procgen-starpilot-hard"
-    "procgen-starpilot-hard-2xIMPALA"
-    "procgen-starpilot-hard-2xIMPALA-fat"
-    "procgen-starpilot-hard-4xIMPALA"
-)
-train_loop "${ALGOS[*]}" "${ENVS[*]}" | xargs -I CMD -P $BENCHMARK_MAX_PROCS bash -c CMD

ppo/ppo.py

@@ -1,349 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from dataclasses import asdict, dataclass, field
-from time import perf_counter
-from torch.optim import Adam
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import List, Optional, NamedTuple, TypeVar
-
-from shared.algorithm import Algorithm
-from shared.callbacks.callback import Callback
-from shared.gae import compute_advantage, compute_rtg_and_advantage
-from shared.policy.on_policy import ActorCritic
-from shared.schedule import constant_schedule, linear_schedule, update_learning_rate
-from shared.trajectory import Trajectory, TrajectoryAccumulator
-from wrappers.vectorable_wrapper import VecEnv, VecEnvObs
-
-
-@dataclass
-class PPOTrajectory(Trajectory):
-    logp_a: List[float] = field(default_factory=list)
-
-    def add(
-        self,
-        obs: np.ndarray,
-        act: np.ndarray,
-        next_obs: np.ndarray,
-        rew: float,
-        terminated: bool,
-        v: float,
-        logp_a: float,
-    ):
-        super().add(obs, act, next_obs, rew, terminated, v)
-        self.logp_a.append(logp_a)
-
-
-class PPOTrajectoryAccumulator(TrajectoryAccumulator):
-    def __init__(self, num_envs: int) -> None:
-        super().__init__(num_envs, PPOTrajectory)
-
-    def step(
-        self,
-        obs: VecEnvObs,
-        action: np.ndarray,
-        next_obs: VecEnvObs,
-        reward: np.ndarray,
-        done: np.ndarray,
-        val: np.ndarray,
-        logp_a: np.ndarray,
-    ) -> None:
-        super().step(obs, action, next_obs, reward, done, val, logp_a)
-
-
-class TrainStepStats(NamedTuple):
-    loss: float
-    pi_loss: float
-    v_loss: float
-    entropy_loss: float
-    approx_kl: float
-    clipped_frac: float
-    val_clipped_frac: float
-
-
-@dataclass
-class TrainStats:
-    loss: float
-    pi_loss: float
-    v_loss: float
-    entropy_loss: float
-    approx_kl: float
-    clipped_frac: float
-    val_clipped_frac: float
-    explained_var: float
-
-    def __init__(self, step_stats: List[TrainStepStats], explained_var: float) -> None:
-        self.loss = np.mean([s.loss for s in step_stats]).item()
-        self.pi_loss = np.mean([s.pi_loss for s in step_stats]).item()
-        self.v_loss = np.mean([s.v_loss for s in step_stats]).item()
-        self.entropy_loss = np.mean([s.entropy_loss for s in step_stats]).item()
-        self.approx_kl = np.mean([s.approx_kl for s in step_stats]).item()
-        self.clipped_frac = np.mean([s.clipped_frac for s in step_stats]).item()
-        self.val_clipped_frac = np.mean([s.val_clipped_frac for s in step_stats]).item()
-        self.explained_var = explained_var
-
-    def write_to_tensorboard(self, tb_writer: SummaryWriter, global_step: int) -> None:
-        for name, value in asdict(self).items():
-            tb_writer.add_scalar(f"losses/{name}", value, global_step=global_step)
-
-    def __repr__(self) -> str:
-        return " | ".join(
-            [
-                f"Loss: {round(self.loss, 2)}",
-                f"Pi L: {round(self.pi_loss, 2)}",
-                f"V L: {round(self.v_loss, 2)}",
-                f"E L: {round(self.entropy_loss, 2)}",
-                f"Apx KL Div: {round(self.approx_kl, 2)}",
-                f"Clip Frac: {round(self.clipped_frac, 2)}",
-                f"Val Clip Frac: {round(self.val_clipped_frac, 2)}",
-            ]
-        )
-
-
-PPOSelf = TypeVar("PPOSelf", bound="PPO")
-
-
-class PPO(Algorithm):
-    def __init__(
-        self,
-        policy: ActorCritic,
-        env: VecEnv,
-        device: torch.device,
-        tb_writer: SummaryWriter,
-        learning_rate: float = 3e-4,
-        learning_rate_decay: str = "none",
-        n_steps: int = 2048,
-        batch_size: int = 64,
-        n_epochs: int = 10,
-        gamma: float = 0.99,
-        gae_lambda: float = 0.95,
-        clip_range: float = 0.2,
-        clip_range_decay: str = "none",
-        clip_range_vf: Optional[float] = None,
-        clip_range_vf_decay: str = "none",
-        normalize_advantage: bool = True,
-        ent_coef: float = 0.0,
-        ent_coef_decay: str = "none",
-        vf_coef: float = 0.5,
-        ppo2_vf_coef_halving: bool = False,
-        max_grad_norm: float = 0.5,
-        update_rtg_between_epochs: bool = False,
-        sde_sample_freq: int = -1,
-    ) -> None:
-        super().__init__(policy, env, device, tb_writer)
-        self.policy = policy
-
-        self.gamma = gamma
-        self.gae_lambda = gae_lambda
-        self.optimizer = Adam(self.policy.parameters(), lr=learning_rate, eps=1e-7)
-        self.lr_schedule = (
-            linear_schedule(learning_rate, 0)
-            if learning_rate_decay == "linear"
-            else constant_schedule(learning_rate)
-        )
-        self.max_grad_norm = max_grad_norm
-        self.clip_range_schedule = (
-            linear_schedule(clip_range, 0)
-            if clip_range_decay == "linear"
-            else constant_schedule(clip_range)
-        )
-        self.clip_range_vf_schedule = None
-        if clip_range_vf:
-            self.clip_range_vf_schedule = (
-                linear_schedule(clip_range_vf, 0)
-                if clip_range_vf_decay == "linear"
-                else constant_schedule(clip_range_vf)
-            )
-        self.normalize_advantage = normalize_advantage
-        self.ent_coef_schedule = (
-            linear_schedule(ent_coef, 0)
-            if ent_coef_decay == "linear"
-            else constant_schedule(ent_coef)
-        )
-        self.vf_coef = vf_coef
-        self.ppo2_vf_coef_halving = ppo2_vf_coef_halving
-
-        self.n_steps = n_steps
-        self.batch_size = batch_size
-        self.n_epochs = n_epochs
-        self.sde_sample_freq = sde_sample_freq
-
-        self.update_rtg_between_epochs = update_rtg_between_epochs
-
-    def learn(
-        self: PPOSelf,
-        total_timesteps: int,
-        callback: Optional[Callback] = None,
-    ) -> PPOSelf:
-        obs = self.env.reset()
-        ts_elapsed = 0
-        while ts_elapsed < total_timesteps:
-            start_time = perf_counter()
-            accumulator = self._collect_trajectories(obs)
-            rollout_steps = self.n_steps * self.env.num_envs
-            ts_elapsed += rollout_steps
-            progress = ts_elapsed / total_timesteps
-            train_stats = self.train(accumulator.all_trajectories, progress, ts_elapsed)
-            train_stats.write_to_tensorboard(self.tb_writer, ts_elapsed)
-            end_time = perf_counter()
-            self.tb_writer.add_scalar(
-                "train/steps_per_second",
-                rollout_steps / (end_time - start_time),
-                ts_elapsed,
-            )
-            if callback:
-                callback.on_step(timesteps_elapsed=rollout_steps)
-
-        return self
-
-    def _collect_trajectories(self, obs: VecEnvObs) -> PPOTrajectoryAccumulator:
-        self.policy.eval()
-        accumulator = PPOTrajectoryAccumulator(self.env.num_envs)
-        self.policy.reset_noise()
-        for i in range(self.n_steps):
-            if self.sde_sample_freq > 0 and i > 0 and i % self.sde_sample_freq == 0:
-                self.policy.reset_noise()
-            action, value, logp_a, clamped_action = self.policy.step(obs)
-            next_obs, reward, done, _ = self.env.step(clamped_action)
-            accumulator.step(obs, action, next_obs, reward, done, value, logp_a)
-            obs = next_obs
-        return accumulator
-
-    def train(
-        self, trajectories: List[PPOTrajectory], progress: float, timesteps_elapsed: int
-    ) -> TrainStats:
-        self.policy.train()
-        learning_rate = self.lr_schedule(progress)
-        update_learning_rate(self.optimizer, learning_rate)
-        self.tb_writer.add_scalar(
-            "charts/learning_rate",
-            self.optimizer.param_groups[0]["lr"],
-            timesteps_elapsed,
-        )
-
-        pi_clip = self.clip_range_schedule(progress)
-        self.tb_writer.add_scalar("charts/pi_clip", pi_clip, timesteps_elapsed)
-        if self.clip_range_vf_schedule:
-            v_clip = self.clip_range_vf_schedule(progress)
-            self.tb_writer.add_scalar("charts/v_clip", v_clip, timesteps_elapsed)
-        else:
-            v_clip = None
-        ent_coef = self.ent_coef_schedule(progress)
-        self.tb_writer.add_scalar("charts/ent_coef", ent_coef, timesteps_elapsed)
-
-        obs = torch.as_tensor(
-            np.concatenate([np.array(t.obs) for t in trajectories]), device=self.device
-        )
-        act = torch.as_tensor(
-            np.concatenate([np.array(t.act) for t in trajectories]), device=self.device
-        )
-        rtg, adv = compute_rtg_and_advantage(
-            trajectories, self.policy, self.gamma, self.gae_lambda, self.device
-        )
-        orig_v = torch.as_tensor(
-            np.concatenate([np.array(t.v) for t in trajectories]), device=self.device
-        )
-        orig_logp_a = torch.as_tensor(
-            np.concatenate([np.array(t.logp_a) for t in trajectories]),
-            device=self.device,
-        )
-
-        step_stats = []
-        for _ in range(self.n_epochs):
-            step_stats.clear()
-            if self.update_rtg_between_epochs:
-                rtg, adv = compute_rtg_and_advantage(
-                    trajectories, self.policy, self.gamma, self.gae_lambda, self.device
-                )
-            else:
-                adv = compute_advantage(
-                    trajectories, self.policy, self.gamma, self.gae_lambda, self.device
-                )
-            idxs = torch.randperm(len(obs))
-            for i in range(0, len(obs), self.batch_size):
-                mb_idxs = idxs[i : i + self.batch_size]
-                mb_adv = adv[mb_idxs]
-                if self.normalize_advantage:
-                    mb_adv = (mb_adv - mb_adv.mean(-1)) / (mb_adv.std(-1) + 1e-8)
-                step_stats.append(
-                    self._train_step(
-                        pi_clip,
-                        v_clip,
-                        ent_coef,
-                        obs[mb_idxs],
-                        act[mb_idxs],
-                        rtg[mb_idxs],
-                        mb_adv,
-                        orig_v[mb_idxs],
-                        orig_logp_a[mb_idxs],
-                    )
-                )
-
-        y_pred, y_true = orig_v.cpu().numpy(), rtg.cpu().numpy()
-        var_y = np.var(y_true).item()
-        explained_var = (
-            np.nan if var_y == 0 else 1 - np.var(y_true - y_pred).item() / var_y
-        )
-
-        return TrainStats(step_stats, explained_var)
-
-    def _train_step(
-        self,
-        pi_clip: float,
-        v_clip: Optional[float],
-        ent_coef: float,
-        obs: torch.Tensor,
-        act: torch.Tensor,
-        rtg: torch.Tensor,
-        adv: torch.Tensor,
-        orig_v: torch.Tensor,
-        orig_logp_a: torch.Tensor,
-    ) -> TrainStepStats:
-        logp_a, entropy, v = self.policy(obs, act)
-        logratio = logp_a - orig_logp_a
-        ratio = torch.exp(logratio)
-        clip_ratio = torch.clamp(ratio, min=1 - pi_clip, max=1 + pi_clip)
-        pi_loss = torch.maximum(-ratio * adv, -clip_ratio * adv).mean()
-
-        v_loss_unclipped = (v - rtg) ** 2
-        if v_clip:
-            v_loss_clipped = (
-                orig_v + torch.clamp(v - orig_v, -v_clip, v_clip) - rtg
-            ) ** 2
-            v_loss = torch.max(v_loss_unclipped, v_loss_clipped).mean()
-        else:
-            v_loss = v_loss_unclipped.mean()
-        if self.ppo2_vf_coef_halving:
-            v_loss *= 0.5
-
-        entropy_loss = -entropy.mean()
-
-        loss = pi_loss + ent_coef * entropy_loss + self.vf_coef * v_loss
-
-        self.optimizer.zero_grad()
-        loss.backward()
-        nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)
-        self.optimizer.step()
-
-        with torch.no_grad():
-            approx_kl = ((ratio - 1) - logratio).mean().cpu().numpy().item()
-            clipped_frac = (
-                ((ratio - 1).abs() > pi_clip).float().mean().cpu().numpy().item()
-            )
-            val_clipped_frac = (
-                (((v - orig_v).abs() > v_clip).float().mean().cpu().numpy().item())
-                if v_clip
-                else 0
-            )
-
-        return TrainStepStats(
-            loss.item(),
-            pi_loss.item(),
-            v_loss.item(),
-            entropy_loss.item(),
-            approx_kl,
-            clipped_frac,
-            val_clipped_frac,
-        )

publish/markdown_format.py

@@ -1,210 +0,0 @@
-import os
-import pandas as pd
-import wandb.apis.public
-import yaml
-
-from collections import defaultdict
-from dataclasses import dataclass, asdict
-from typing import Any, Dict, Iterable, List, NamedTuple, Optional, TypeVar
-from urllib.parse import urlparse
-
-from runner.evaluate import Evaluation
-
-EvaluationRowSelf = TypeVar("EvaluationRowSelf", bound="EvaluationRow")
-
-
-@dataclass
-class EvaluationRow:
-    algo: str
-    env: str
-    seed: Optional[int]
-    reward_mean: float
-    reward_std: float
-    eval_episodes: int
-    best: str
-    wandb_url: str
-
-    @staticmethod
-    def data_frame(rows: List[EvaluationRowSelf]) -> pd.DataFrame:
-        results = defaultdict(list)
-        for r in rows:
-            for k, v in asdict(r).items():
-                results[k].append(v)
-        return pd.DataFrame(results)
-
-
-class EvalTableData(NamedTuple):
-    run: wandb.apis.public.Run
-    evaluation: Evaluation
-
-
-def evaluation_table(table_data: Iterable[EvalTableData]) -> str:
-    best_stats = sorted(
-        [d.evaluation.stats for d in table_data], key=lambda r: r.score, reverse=True
-    )[0]
-    table_data = sorted(table_data, key=lambda d: d.evaluation.config.seed() or 0)
-    rows = [
-        EvaluationRow(
-            config.algo,
-            config.env_id,
-            config.seed(),
-            stats.score.mean,
-            stats.score.std,
-            len(stats),
-            "*" if stats == best_stats else "",
-            f"[wandb]({r.url})",
-        )
-        for (r, (_, stats, config)) in table_data
-    ]
-    df = EvaluationRow.data_frame(rows)
-    return df.to_markdown(index=False)
-
-
-def github_project_link(github_url: str) -> str:
-    return f"[{urlparse(github_url).path}]({github_url})"
-
-
-def header_section(algo: str, env: str, github_url: str, wandb_report_url: str) -> str:
-    algo_caps = algo.upper()
-    lines = [
-        f"# **{algo_caps}** Agent playing **{env}**",
-        f"This is a trained model of a **{algo_caps}** agent playing **{env}** using "
-        f"the {github_project_link(github_url)} repo.",
-        f"All models trained at this commit can be found at {wandb_report_url}.",
-    ]
-    return "\n\n".join(lines)
-
-
-def github_tree_link(github_url: str, commit_hash: Optional[str]) -> str:
-    if not commit_hash:
-        return github_project_link(github_url)
-    return f"[{commit_hash[:7]}]({github_url}/tree/{commit_hash})"
-
-
-def results_section(
-    table_data: List[EvalTableData], algo: str, github_url: str, commit_hash: str
-) -> str:
-    # type: ignore
-    lines = [
-        "## Training Results",
-        f"This model was trained from {len(table_data)} trainings of **{algo.upper()}** "
-        + "agents using different initial seeds. "
-        + f"These agents were trained by checking out "
-        + f"{github_tree_link(github_url, commit_hash)}. "
-        + "The best and last models were kept from each training. "
-        + "This submission has loaded the best models from each training, reevaluates "
-        + "them, and selects the best model from these latest evaluations (mean - std).",
-    ]
-    lines.append(evaluation_table(table_data))
-    return "\n\n".join(lines)
-
-
-def prerequisites_section() -> str:
-    return """
-### Prerequisites: Weights & Biases (WandB)
-Training and benchmarking assumes you have a Weights & Biases project to upload runs to.
-By default training goes to a rl-algo-impls project while benchmarks go to
-rl-algo-impls-benchmarks. During training and benchmarking runs, videos of the best
-models and the model weights are uploaded to WandB.
-
-Before doing anything below, you'll need to create a wandb account and run `wandb
-login`.
-"""
-
-
-def usage_section(github_url: str, run_path: str, commit_hash: str) -> str:
-    return f"""
-## Usage
-{urlparse(github_url).path}: {github_url}
-
-Note: While the model state dictionary and hyperaparameters are saved, the latest
-implementation could be sufficiently different to not be able to reproduce similar
-results. You might need to checkout the commit the agent was trained on:
-{github_tree_link(github_url, commit_hash)}.
-```
-# Downloads the model, sets hyperparameters, and runs agent for 3 episodes
-python enjoy.py --wandb-run-path={run_path}
-```
-
-Setup hasn't been completely worked out yet, so you might be best served by using Google
-Colab starting from the
-[colab_enjoy.ipynb](https://github.com/sgoodfriend/rl-algo-impls/blob/main/colab_enjoy.ipynb)
-notebook.
-"""
-
-
-def training_setion(
-    github_url: str, commit_hash: str, algo: str, env: str, seed: Optional[int]
-) -> str:
-    return f"""
-## Training
-If you want the highest chance to reproduce these results, you'll want to checkout the
-commit the agent was trained on: {github_tree_link(github_url, commit_hash)}. While
-training is deterministic, different hardware will give different results.
-
-```
-python train.py --algo {algo} --env {env} {'--seed ' + str(seed) if seed is not None else ''}
-```
-
-Setup hasn't been completely worked out yet, so you might be best served by using Google
-Colab starting from the
-[colab_train.ipynb](https://github.com/sgoodfriend/rl-algo-impls/blob/main/colab_train.ipynb)
-notebook.
-"""
-
-
-def benchmarking_section(report_url: str) -> str:
-    return f"""
-## Benchmarking (with Lambda Labs instance)
-This and other models from {report_url} were generated by running a script on a Lambda
-Labs instance. In a Lambda Labs instance terminal:
-```
-git clone git@github.com:sgoodfriend/rl-algo-impls.git
-cd rl-algo-impls
-bash ./lambda_labs/setup.sh
-wandb login
-bash ./lambda_labs/benchmark.sh
-```
-
-### Alternative: Google Colab Pro+
-As an alternative,
-[colab_benchmark.ipynb](https://github.com/sgoodfriend/rl-algo-impls/tree/main/benchmarks#:~:text=colab_benchmark.ipynb),
-can be used. However, this requires a Google Colab Pro+ subscription and running across
-4 separate instances because otherwise running all jobs will exceed the 24-hour limit.
-"""
-
-
-def hyperparams_section(run_config: Dict[str, Any]) -> str:
-    return f"""
-## Hyperparameters
-This isn't exactly the format of hyperparams in {os.path.join("hyperparams",
-run_config["algo"] + ".yml")}, but instead the Wandb Run Config. However, it's very
-close and has some additional data:
-```
-{yaml.dump(run_config)}
-```
-"""
-
-
-def model_card_text(
-    algo: str,
-    env: str,
-    github_url: str,
-    commit_hash: str,
-    wandb_report_url: str,
-    table_data: List[EvalTableData],
-    best_eval: EvalTableData,
-) -> str:
-    run, (_, _, config) = best_eval
-    run_path = "/".join(run.path)
-    return "\n\n".join(
-        [
-            header_section(algo, env, github_url, wandb_report_url),
-            results_section(table_data, algo, github_url, commit_hash),
-            prerequisites_section(),
-            usage_section(github_url, run_path, commit_hash),
-            training_setion(github_url, commit_hash, algo, env, config.seed()),
-            benchmarking_section(wandb_report_url),
-            hyperparams_section(run.config),
-        ]
-    )

replay.meta.json

@@ -1 +1 @@
-{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 4.2.7-0ubuntu0.1 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with gcc 9 (Ubuntu 9.4.0-1ubuntu1~20.04.1)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared\\nlibavutil      56. 31.100 / 56. 31.100\\nlibavcodec     58. 54.100 / 58. 54.100\\nlibavformat    58. 29.100 / 58. 29.100\\nlibavdevice    58.  8.100 / 58.  8.100\\nlibavfilter     7. 57.100 /  7. 57.100\\nlibavresample   4.  0.  0 /  4.  0.  0\\nlibswscale      5.  5.100 /  5.  5.100\\nlibswresample   3.  5.100 /  3.  5.100\\nlibpostproc    55.  5.100 / 55.  5.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "320x240", "-pix_fmt", "rgb24", "-framerate", "60", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "60", "/tmp/tmpfm3xc0eb/a2c-AntBulletEnv-v0/replay.mp4"]}, "episode": {"r": 3004.433349609375, "l": 1000, "t": 29.277596}}
+{"content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 4.2.7-0ubuntu0.1 Copyright (c) 2000-2022 the FFmpeg developers\\nbuilt with gcc 9 (Ubuntu 9.4.0-1ubuntu1~20.04.1)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared\\nlibavutil      56. 31.100 / 56. 31.100\\nlibavcodec     58. 54.100 / 58. 54.100\\nlibavformat    58. 29.100 / 58. 29.100\\nlibavdevice    58.  8.100 / 58.  8.100\\nlibavfilter     7. 57.100 /  7. 57.100\\nlibavresample   4.  0.  0 /  4.  0.  0\\nlibswscale      5.  5.100 /  5.  5.100\\nlibswresample   3.  5.100 /  3.  5.100\\nlibpostproc    55.  5.100 / 55.  5.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-f", "rawvideo", "-s:v", "320x240", "-pix_fmt", "rgb24", "-framerate", "60", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-r", "60", "/tmp/tmppc8ru6js/a2c-AntBulletEnv-v0/replay.mp4"]}, "episode": {"r": 3004.433349609375, "l": 1000, "t": 28.542343}}

results.json

@@ -1 +0,0 @@
-{"mean_reward": 2992.133482391329, "std_reward": 55.53945472831181, "is_deterministic": true, "n_eval_episodes": 10, "eval_datetime": "2023-01-24T20:22:24.991332"}

rl_algo_impls/benchmark_publish.py

@@ -54,8 +54,8 @@ def benchmark_publish() -> None:
         "--virtual-display", action="store_true", help="Use headless virtual display"
     )
     # parser.set_defaults(
-    #     wandb_tags=["benchmark_e47a44c", "host_129-146-2-230"],
-    #     wandb_report_url="https://api.wandb.ai/links/sgoodfriend/v4wd7cp5",
+    #     wandb_tags=["benchmark_2067e21", "host_155-248-199-228"],
+    #     wandb_report_url="https://api.wandb.ai/links/sgoodfriend/09frjfcs",
     #     envs=[],
     #     exclude_envs=[],
     # )

rl_algo_impls/huggingface_publish.py

@@ -162,6 +162,7 @@ def publish(
             path_in_repo="",
             commit_message=f"{algo.upper()} playing {env_id} from {github_url}/tree/{commit_hash}",
             token=huggingface_token,
+            delete_patterns="*",
         )
         print(f"Pushed model to the hub: {repo_url}")
 

runner/config.py

@@ -1,155 +0,0 @@
-import os
-
-from datetime import datetime
-from dataclasses import dataclass
-from typing import Any, Dict, NamedTuple, Optional, TypedDict, Union
-
-
-@dataclass
-class RunArgs:
-    algo: str
-    env: str
-    seed: Optional[int] = None
-    use_deterministic_algorithms: bool = True
-
-
-class EnvHyperparams(NamedTuple):
-    env_type: str = "gymvec"
-    n_envs: int = 1
-    frame_stack: int = 1
-    make_kwargs: Optional[Dict[str, Any]] = None
-    no_reward_timeout_steps: Optional[int] = None
-    no_reward_fire_steps: Optional[int] = None
-    vec_env_class: str = "sync"
-    normalize: bool = False
-    normalize_kwargs: Optional[Dict[str, Any]] = None
-    rolling_length: int = 100
-    train_record_video: bool = False
-    video_step_interval: Union[int, float] = 1_000_000
-    initial_steps_to_truncate: Optional[int] = None
-    clip_atari_rewards: bool = True
-
-
-class Hyperparams(TypedDict, total=False):
-    device: str
-    n_timesteps: Union[int, float]
-    env_hyperparams: Dict[str, Any]
-    policy_hyperparams: Dict[str, Any]
-    algo_hyperparams: Dict[str, Any]
-    eval_params: Dict[str, Any]
-
-
-@dataclass
-class Config:
-    args: RunArgs
-    hyperparams: Hyperparams
-    root_dir: str
-    run_id: str = datetime.now().isoformat()
-
-    def seed(self, training: bool = True) -> Optional[int]:
-        seed = self.args.seed
-        if training or seed is None:
-            return seed
-        return seed + self.env_hyperparams.get("n_envs", 1)
-
-    @property
-    def device(self) -> str:
-        return self.hyperparams.get("device", "auto")
-
-    @property
-    def n_timesteps(self) -> int:
-        return int(self.hyperparams.get("n_timesteps", 100_000))
-
-    @property
-    def env_hyperparams(self) -> Dict[str, Any]:
-        return self.hyperparams.get("env_hyperparams", {})
-
-    @property
-    def policy_hyperparams(self) -> Dict[str, Any]:
-        return self.hyperparams.get("policy_hyperparams", {})
-
-    @property
-    def algo_hyperparams(self) -> Dict[str, Any]:
-        return self.hyperparams.get("algo_hyperparams", {})
-
-    @property
-    def eval_params(self) -> Dict[str, Any]:
-        return self.hyperparams.get("eval_params", {})
-
-    @property
-    def algo(self) -> str:
-        return self.args.algo
-
-    @property
-    def env_id(self) -> str:
-        return self.hyperparams.get("env_id") or self.args.env
-
-    def model_name(self, include_seed: bool = True) -> str:
-        # Use arg env name instead of environment name
-        parts = [self.algo, self.args.env]
-        if include_seed and self.args.seed is not None:
-            parts.append(f"S{self.args.seed}")
-
-        # Assume that the custom arg name already has the necessary information
-        if not self.hyperparams.get("env_id"):
-            make_kwargs = self.env_hyperparams.get("make_kwargs", {})
-            if make_kwargs:
-                for k, v in make_kwargs.items():
-                    if type(v) == bool and v:
-                        parts.append(k)
-                    elif type(v) == int and v:
-                        parts.append(f"{k}{v}")
-                    else:
-                        parts.append(str(v))
-
-        return "-".join(parts)
-
-    @property
-    def run_name(self) -> str:
-        parts = [self.model_name(), self.run_id]
-        return "-".join(parts)
-
-    @property
-    def saved_models_dir(self) -> str:
-        return os.path.join(self.root_dir, "saved_models")
-
-    @property
-    def downloaded_models_dir(self) -> str:
-        return os.path.join(self.root_dir, "downloaded_models")
-
-    def model_dir_name(
-        self,
-        best: bool = False,
-        extension: str = "",
-    ) -> str:
-        return self.model_name() + ("-best" if best else "") + extension
-
-    def model_dir_path(self, best: bool = False, downloaded: bool = False) -> str:
-        return os.path.join(
-            self.saved_models_dir if not downloaded else self.downloaded_models_dir,
-            self.model_dir_name(best=best),
-        )
-
-    @property
-    def runs_dir(self) -> str:
-        return os.path.join(self.root_dir, "runs")
-
-    @property
-    def tensorboard_summary_path(self) -> str:
-        return os.path.join(self.runs_dir, self.run_name)
-
-    @property
-    def logs_path(self) -> str:
-        return os.path.join(self.runs_dir, f"log.yml")
-
-    @property
-    def videos_dir(self) -> str:
-        return os.path.join(self.root_dir, "videos")
-
-    @property
-    def video_prefix(self) -> str:
-        return os.path.join(self.videos_dir, self.model_name())
-
-    @property
-    def best_videos_dir(self) -> str:
-        return os.path.join(self.videos_dir, f"{self.model_name()}-best")

runner/env.py

@@ -1,284 +0,0 @@
-import gym
-import numpy as np
-import os
-
-from gym.vector.async_vector_env import AsyncVectorEnv
-from gym.vector.sync_vector_env import SyncVectorEnv
-from gym.wrappers.resize_observation import ResizeObservation
-from gym.wrappers.gray_scale_observation import GrayScaleObservation
-from gym.wrappers.frame_stack import FrameStack
-from stable_baselines3.common.atari_wrappers import (
-    MaxAndSkipEnv,
-    NoopResetEnv,
-)
-from stable_baselines3.common.vec_env.dummy_vec_env import DummyVecEnv
-from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv
-from stable_baselines3.common.vec_env.vec_normalize import VecNormalize
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import Callable, Optional
-
-from runner.config import Config, EnvHyperparams
-from shared.policy.policy import VEC_NORMALIZE_FILENAME
-from wrappers.atari_wrappers import EpisodicLifeEnv, FireOnLifeStarttEnv, ClipRewardEnv
-from wrappers.episode_record_video import EpisodeRecordVideo
-from wrappers.episode_stats_writer import EpisodeStatsWriter
-from wrappers.initial_step_truncate_wrapper import InitialStepTruncateWrapper
-from wrappers.is_vector_env import IsVectorEnv
-from wrappers.noop_env_seed import NoopEnvSeed
-from wrappers.normalize import NormalizeObservation, NormalizeReward
-from wrappers.sync_vector_env_render_compat import SyncVectorEnvRenderCompat
-from wrappers.transpose_image_observation import TransposeImageObservation
-from wrappers.vectorable_wrapper import VecEnv
-from wrappers.video_compat_wrapper import VideoCompatWrapper
-
-
-def make_env(
-    config: Config,
-    hparams: EnvHyperparams,
-    training: bool = True,
-    render: bool = False,
-    normalize_load_path: Optional[str] = None,
-    tb_writer: Optional[SummaryWriter] = None,
-) -> VecEnv:
-    if hparams.env_type == "procgen":
-        return _make_procgen_env(
-            config,
-            hparams,
-            training=training,
-            render=render,
-            normalize_load_path=normalize_load_path,
-            tb_writer=tb_writer,
-        )
-    elif hparams.env_type in {"sb3vec", "gymvec"}:
-        return _make_vec_env(
-            config,
-            hparams,
-            training=training,
-            render=render,
-            normalize_load_path=normalize_load_path,
-            tb_writer=tb_writer,
-        )
-    else:
-        raise ValueError(f"env_type {hparams.env_type} not supported")
-
-
-def make_eval_env(
-    config: Config,
-    hparams: EnvHyperparams,
-    override_n_envs: Optional[int] = None,
-    **kwargs,
-) -> VecEnv:
-    kwargs = kwargs.copy()
-    kwargs["training"] = False
-    if override_n_envs is not None:
-        hparams_kwargs = hparams._asdict()
-        hparams_kwargs["n_envs"] = override_n_envs
-        if override_n_envs == 1:
-            hparams_kwargs["vec_env_class"] = "sync"
-        hparams = EnvHyperparams(**hparams_kwargs)
-    return make_env(config, hparams, **kwargs)
-
-
-def _make_vec_env(
-    config: Config,
-    hparams: EnvHyperparams,
-    training: bool = True,
-    render: bool = False,
-    normalize_load_path: Optional[str] = None,
-    tb_writer: Optional[SummaryWriter] = None,
-) -> VecEnv:
-    (
-        env_type,
-        n_envs,
-        frame_stack,
-        make_kwargs,
-        no_reward_timeout_steps,
-        no_reward_fire_steps,
-        vec_env_class,
-        normalize,
-        normalize_kwargs,
-        rolling_length,
-        train_record_video,
-        video_step_interval,
-        initial_steps_to_truncate,
-        clip_atari_rewards,
-    ) = hparams
-
-    if "BulletEnv" in config.env_id:
-        import pybullet_envs
-
-    spec = gym.spec(config.env_id)
-    seed = config.seed(training=training)
-
-    make_kwargs = make_kwargs.copy() if make_kwargs is not None else {}
-    if "BulletEnv" in config.env_id and render:
-        make_kwargs["render"] = True
-    if "CarRacing" in config.env_id:
-        make_kwargs["verbose"] = 0
-    if "procgen" in config.env_id:
-        if not render:
-            make_kwargs["render_mode"] = "rgb_array"
-
-    def make(idx: int) -> Callable[[], gym.Env]:
-        def _make() -> gym.Env:
-            env = gym.make(config.env_id, **make_kwargs)
-            env = gym.wrappers.RecordEpisodeStatistics(env)
-            env = VideoCompatWrapper(env)
-            if training and train_record_video and idx == 0:
-                env = EpisodeRecordVideo(
-                    env,
-                    config.video_prefix,
-                    step_increment=n_envs,
-                    video_step_interval=int(video_step_interval),
-                )
-            if training and initial_steps_to_truncate:
-                env = InitialStepTruncateWrapper(
-                    env, idx * initial_steps_to_truncate // n_envs
-                )
-            if "AtariEnv" in spec.entry_point:  # type: ignore
-                env = NoopResetEnv(env, noop_max=30)
-                env = MaxAndSkipEnv(env, skip=4)
-                env = EpisodicLifeEnv(env, training=training)
-                action_meanings = env.unwrapped.get_action_meanings()
-                if "FIRE" in action_meanings:  # type: ignore
-                    env = FireOnLifeStarttEnv(env, action_meanings.index("FIRE"))
-                if clip_atari_rewards:
-                    env = ClipRewardEnv(env, training=training)
-                env = ResizeObservation(env, (84, 84))
-                env = GrayScaleObservation(env, keep_dim=False)
-                env = FrameStack(env, frame_stack)
-            elif "CarRacing" in config.env_id:
-                env = ResizeObservation(env, (64, 64))
-                env = GrayScaleObservation(env, keep_dim=False)
-                env = FrameStack(env, frame_stack)
-            elif "procgen" in config.env_id:
-                # env = GrayScaleObservation(env, keep_dim=False)
-                env = NoopEnvSeed(env)
-                env = TransposeImageObservation(env)
-                if frame_stack > 1:
-                    env = FrameStack(env, frame_stack)
-
-            if no_reward_timeout_steps:
-                from wrappers.no_reward_timeout import NoRewardTimeout
-
-                env = NoRewardTimeout(
-                    env, no_reward_timeout_steps, n_fire_steps=no_reward_fire_steps
-                )
-
-            if seed is not None:
-                env.seed(seed + idx)
-                env.action_space.seed(seed + idx)
-                env.observation_space.seed(seed + idx)
-
-            return env
-
-        return _make
-
-    if env_type == "sb3vec":
-        VecEnvClass = {"sync": DummyVecEnv, "async": SubprocVecEnv}[vec_env_class]
-    elif env_type == "gymvec":
-        VecEnvClass = {"sync": SyncVectorEnv, "async": AsyncVectorEnv}[vec_env_class]
-    else:
-        raise ValueError(f"env_type {env_type} unsupported")
-    envs = VecEnvClass([make(i) for i in range(n_envs)])
-    if env_type == "gymvec" and vec_env_class == "sync":
-        envs = SyncVectorEnvRenderCompat(envs)
-    if training:
-        assert tb_writer
-        envs = EpisodeStatsWriter(
-            envs, tb_writer, training=training, rolling_length=rolling_length
-        )
-    if normalize:
-        normalize_kwargs = normalize_kwargs or {}
-        if env_type == "sb3vec":
-            if normalize_load_path:
-                envs = VecNormalize.load(
-                    os.path.join(normalize_load_path, VEC_NORMALIZE_FILENAME),
-                    envs,  # type: ignore
-                )
-            else:
-                envs = VecNormalize(
-                    envs,  # type: ignore
-                    training=training,
-                    **normalize_kwargs,
-                )
-            if not training:
-                envs.norm_reward = False
-        else:
-            if normalize_kwargs.get("norm_obs", True):
-                envs = NormalizeObservation(
-                    envs, training=training, clip=normalize_kwargs.get("clip_obs", 10.0)
-                )
-            if training and normalize_kwargs.get("norm_reward", True):
-                envs = NormalizeReward(
-                    envs,
-                    training=training,
-                    clip=normalize_kwargs.get("clip_reward", 10.0),
-                )
-    return envs
-
-
-def _make_procgen_env(
-    config: Config,
-    hparams: EnvHyperparams,
-    training: bool = True,
-    render: bool = False,
-    normalize_load_path: Optional[str] = None,
-    tb_writer: Optional[SummaryWriter] = None,
-) -> VecEnv:
-    from gym3 import ViewerWrapper, ExtractDictObWrapper
-    from procgen.env import ProcgenGym3Env, ToBaselinesVecEnv
-
-    (
-        _,
-        n_envs,
-        frame_stack,
-        make_kwargs,
-        _,  # no_reward_timeout_steps
-        _,  # no_reward_fire_steps
-        _,  # vec_env_class
-        normalize,
-        normalize_kwargs,
-        rolling_length,
-        _,  # train_record_video
-        _,  # video_step_interval
-        _,  # initial_steps_to_truncate
-        _,  # clip_atari_rewards
-    ) = hparams
-
-    seed = config.seed(training=training)
-
-    make_kwargs = make_kwargs or {}
-    make_kwargs["render_mode"] = "rgb_array"
-    if seed is not None:
-        make_kwargs["rand_seed"] = seed
-
-    envs = ProcgenGym3Env(n_envs, config.env_id, **make_kwargs)
-    envs = ExtractDictObWrapper(envs, key="rgb")
-    if render:
-        envs = ViewerWrapper(envs, info_key="rgb")
-    envs = ToBaselinesVecEnv(envs)
-    envs = IsVectorEnv(envs)
-    # TODO: Handle Grayscale and/or FrameStack
-    envs = TransposeImageObservation(envs)
-
-    envs = gym.wrappers.RecordEpisodeStatistics(envs)
-
-    if seed is not None:
-        envs.action_space.seed(seed)
-        envs.observation_space.seed(seed)
-
-    if training:
-        assert tb_writer
-        envs = EpisodeStatsWriter(
-            envs, tb_writer, training=training, rolling_length=rolling_length
-        )
-    if normalize and training:
-        normalize_kwargs = normalize_kwargs or {}
-        envs = gym.wrappers.NormalizeReward(envs)
-        clip_obs = normalize_kwargs.get("clip_reward", 10.0)
-        envs = gym.wrappers.TransformReward(
-            envs, lambda r: np.clip(r, -clip_obs, clip_obs)
-        )
-
-    return envs  # type: ignore

runner/evaluate.py

@@ -1,103 +0,0 @@
-import os
-import shutil
-
-from dataclasses import dataclass
-from typing import NamedTuple, Optional
-
-from runner.env import make_eval_env
-from runner.config import Config, EnvHyperparams, RunArgs
-from runner.running_utils import (
-    load_hyperparams,
-    set_seeds,
-    get_device,
-    make_policy,
-)
-from shared.callbacks.eval_callback import evaluate
-from shared.policy.policy import Policy
-from shared.stats import EpisodesStats
-
-
-@dataclass
-class EvalArgs(RunArgs):
-    render: bool = True
-    best: bool = True
-    n_envs: Optional[int] = 1
-    n_episodes: int = 3
-    deterministic_eval: Optional[bool] = None
-    no_print_returns: bool = False
-    wandb_run_path: Optional[str] = None
-
-
-class Evaluation(NamedTuple):
-    policy: Policy
-    stats: EpisodesStats
-    config: Config
-
-
-def evaluate_model(args: EvalArgs, root_dir: str) -> Evaluation:
-    if args.wandb_run_path:
-        import wandb
-
-        api = wandb.Api()
-        run = api.run(args.wandb_run_path)
-        hyperparams = run.config
-
-        args.algo = hyperparams["algo"]
-        args.env = hyperparams["env"]
-        args.seed = hyperparams.get("seed", None)
-        args.use_deterministic_algorithms = hyperparams.get(
-            "use_deterministic_algorithms", True
-        )
-
-        config = Config(args, hyperparams, root_dir)
-        model_path = config.model_dir_path(best=args.best, downloaded=True)
-
-        model_archive_name = config.model_dir_name(best=args.best, extension=".zip")
-        run.file(model_archive_name).download()
-        if os.path.isdir(model_path):
-            shutil.rmtree(model_path)
-        shutil.unpack_archive(model_archive_name, model_path)
-        os.remove(model_archive_name)
-    else:
-        hyperparams = load_hyperparams(args.algo, args.env, root_dir)
-
-        config = Config(args, hyperparams, root_dir)
-        model_path = config.model_dir_path(best=args.best)
-
-    print(args)
-
-    set_seeds(args.seed, args.use_deterministic_algorithms)
-
-    env = make_eval_env(
-        config,
-        EnvHyperparams(**config.env_hyperparams),
-        override_n_envs=args.n_envs,
-        render=args.render,
-        normalize_load_path=model_path,
-    )
-    device = get_device(config.device, env)
-    policy = make_policy(
-        args.algo,
-        env,
-        device,
-        load_path=model_path,
-        **config.policy_hyperparams,
-    ).eval()
-
-    deterministic = (
-        args.deterministic_eval
-        if args.deterministic_eval is not None
-        else config.eval_params.get("deterministic", True)
-    )
-    return Evaluation(
-        policy,
-        evaluate(
-            env,
-            policy,
-            args.n_episodes,
-            render=args.render,
-            deterministic=deterministic,
-            print_returns=not args.no_print_returns,
-        ),
-        config,
-    )

runner/running_utils.py

@@ -1,195 +0,0 @@
-import argparse
-import gym
-import json
-import matplotlib.pyplot as plt
-import numpy as np
-import os
-import random
-import torch
-import torch.backends.cudnn
-import yaml
-
-from gym.spaces import Box, Discrete
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import Dict, Optional, Type, Union
-
-from runner.config import Hyperparams
-from shared.algorithm import Algorithm
-from shared.callbacks.eval_callback import EvalCallback
-from shared.policy.on_policy import ActorCritic
-from shared.policy.policy import Policy
-
-from a2c.a2c import A2C
-from dqn.dqn import DQN
-from dqn.policy import DQNPolicy
-from ppo.ppo import PPO
-from vpg.vpg import VanillaPolicyGradient
-from vpg.policy import VPGActorCritic
-from wrappers.vectorable_wrapper import VecEnv, single_observation_space
-
-ALGOS: Dict[str, Type[Algorithm]] = {
-    "dqn": DQN,
-    "vpg": VanillaPolicyGradient,
-    "ppo": PPO,
-    "a2c": A2C,
-}
-POLICIES: Dict[str, Type[Policy]] = {
-    "dqn": DQNPolicy,
-    "vpg": VPGActorCritic,
-    "ppo": ActorCritic,
-    "a2c": ActorCritic,
-}
-
-HYPERPARAMS_PATH = "hyperparams"
-
-
-def base_parser(multiple: bool = True) -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--algo",
-        default=["dqn"],
-        type=str,
-        choices=list(ALGOS.keys()),
-        nargs="+" if multiple else 1,
-        help="Abbreviation(s) of algorithm(s)",
-    )
-    parser.add_argument(
-        "--env",
-        default=["CartPole-v1"],
-        type=str,
-        nargs="+" if multiple else 1,
-        help="Name of environment(s) in gym",
-    )
-    parser.add_argument(
-        "--seed",
-        default=[1],
-        type=int,
-        nargs="*" if multiple else "?",
-        help="Seeds to run experiment. Unset will do one run with no set seed",
-    )
-    parser.add_argument(
-        "--use-deterministic-algorithms",
-        default=True,
-        type=bool,
-        help="If seed set, set torch.use_deterministic_algorithms",
-    )
-    return parser
-
-
-def load_hyperparams(algo: str, env_id: str, root_path: str) -> Hyperparams:
-    hyperparams_path = os.path.join(root_path, HYPERPARAMS_PATH, f"{algo}.yml")
-    with open(hyperparams_path, "r") as f:
-        hyperparams_dict = yaml.safe_load(f)
-
-    if env_id in hyperparams_dict:
-        return hyperparams_dict[env_id]
-
-    if "BulletEnv" in env_id:
-        import pybullet_envs
-    spec = gym.spec(env_id)
-    if "AtariEnv" in str(spec.entry_point) and "_atari" in hyperparams_dict:
-        return hyperparams_dict["_atari"]
-    else:
-        raise ValueError(f"{env_id} not specified in {algo} hyperparameters file")
-
-
-def get_device(device: str, env: VecEnv) -> torch.device:
-    # cuda by default
-    if device == "auto":
-        device = "cuda"
-    # Apple MPS is a second choice (sometimes)
-    if device == "cuda" and not torch.cuda.is_available():
-        device = "mps"
-    # If no MPS, fallback to cpu
-    if device == "mps" and not torch.backends.mps.is_available():
-        device = "cpu"
-    # Simple environments like Discreet and 1-D Boxes might also be better
-    # served with the CPU.
-    if device == "mps":
-        obs_space = single_observation_space(env)
-        if isinstance(obs_space, Discrete):
-            device = "cpu"
-        elif isinstance(obs_space, Box) and len(obs_space.shape) == 1:
-            device = "cpu"
-    print(f"Device: {device}")
-    return torch.device(device)
-
-
-def set_seeds(seed: Optional[int], use_deterministic_algorithms: bool) -> None:
-    if seed is None:
-        return
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    torch.backends.cudnn.benchmark = False
-    torch.use_deterministic_algorithms(use_deterministic_algorithms)
-    os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
-    # Stop warning and it would introduce stochasticity if I was using TF
-    os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
-
-
-def make_policy(
-    algo: str,
-    env: VecEnv,
-    device: torch.device,
-    load_path: Optional[str] = None,
-    **kwargs,
-) -> Policy:
-    policy = POLICIES[algo](env, **kwargs).to(device)
-    if load_path:
-        policy.load(load_path)
-    return policy
-
-
-def plot_eval_callback(callback: EvalCallback, tb_writer: SummaryWriter, run_name: str):
-    figure = plt.figure()
-    cumulative_steps = [
-        (idx + 1) * callback.step_freq for idx in range(len(callback.stats))
-    ]
-    plt.plot(
-        cumulative_steps,
-        [s.score.mean for s in callback.stats],
-        "b-",
-        label="mean",
-    )
-    plt.plot(
-        cumulative_steps,
-        [s.score.mean - s.score.std for s in callback.stats],
-        "g--",
-        label="mean-std",
-    )
-    plt.fill_between(
-        cumulative_steps,
-        [s.score.min for s in callback.stats],  # type: ignore
-        [s.score.max for s in callback.stats],  # type: ignore
-        facecolor="cyan",
-        label="range",
-    )
-    plt.xlabel("Steps")
-    plt.ylabel("Score")
-    plt.legend()
-    plt.title(f"Eval {run_name}")
-    tb_writer.add_figure("eval", figure)
-
-
-Scalar = Union[bool, str, float, int, None]
-
-
-def hparam_dict(
-    hyperparams: Hyperparams, args: Dict[str, Union[Scalar, list]]
-) -> Dict[str, Scalar]:
-    flattened = args.copy()
-    for k, v in flattened.items():
-        if isinstance(v, list):
-            flattened[k] = json.dumps(v)
-    for k, v in hyperparams.items():
-        if isinstance(v, dict):
-            for sk, sv in v.items():
-                key = f"{k}/{sk}"
-                if isinstance(sv, dict) or isinstance(sv, list):
-                    flattened[key] = str(sv)
-                else:
-                    flattened[key] = sv
-        else:
-            flattened[k] = v  # type: ignore
-    return flattened  # type: ignore

runner/train.py

@@ -1,141 +0,0 @@
-# Support for PyTorch mps mode (https://pytorch.org/docs/stable/notes/mps.html)
-import os
-
-os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
-
-import dataclasses
-import shutil
-import wandb
-import yaml
-
-from dataclasses import dataclass
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import Any, Dict, Optional, Sequence
-
-from shared.callbacks.eval_callback import EvalCallback
-from runner.config import Config, EnvHyperparams, RunArgs
-from runner.env import make_env, make_eval_env
-from runner.running_utils import (
-    ALGOS,
-    load_hyperparams,
-    set_seeds,
-    get_device,
-    make_policy,
-    plot_eval_callback,
-    hparam_dict,
-)
-from shared.stats import EpisodesStats
-
-
-@dataclass
-class TrainArgs(RunArgs):
-    wandb_project_name: Optional[str] = None
-    wandb_entity: Optional[str] = None
-    wandb_tags: Sequence[str] = dataclasses.field(default_factory=list)
-
-
-def train(args: TrainArgs):
-    print(args)
-    hyperparams = load_hyperparams(args.algo, args.env, os.getcwd())
-    print(hyperparams)
-    config = Config(args, hyperparams, os.getcwd())
-
-    wandb_enabled = args.wandb_project_name
-    if wandb_enabled:
-        wandb.tensorboard.patch(
-            root_logdir=config.tensorboard_summary_path, pytorch=True
-        )
-        wandb.init(
-            project=args.wandb_project_name,
-            entity=args.wandb_entity,
-            config=hyperparams,  # type: ignore
-            name=config.run_name,
-            monitor_gym=True,
-            save_code=True,
-            tags=args.wandb_tags,
-        )
-        wandb.config.update(args)
-
-    tb_writer = SummaryWriter(config.tensorboard_summary_path)
-
-    set_seeds(args.seed, args.use_deterministic_algorithms)
-
-    env = make_env(
-        config, EnvHyperparams(**config.env_hyperparams), tb_writer=tb_writer
-    )
-    device = get_device(config.device, env)
-    policy = make_policy(args.algo, env, device, **config.policy_hyperparams)
-    algo = ALGOS[args.algo](policy, env, device, tb_writer, **config.algo_hyperparams)
-
-    num_parameters = policy.num_parameters()
-    num_trainable_parameters = policy.num_trainable_parameters()
-    if wandb_enabled:
-        wandb.run.summary["num_parameters"] = num_parameters
-        wandb.run.summary["num_trainable_parameters"] = num_trainable_parameters
-    else:
-        print(
-            f"num_parameters = {num_parameters} ; "
-            f"num_trainable_parameters = {num_trainable_parameters}"
-        )
-
-    eval_env = make_eval_env(config, EnvHyperparams(**config.env_hyperparams))
-    record_best_videos = config.eval_params.get("record_best_videos", True)
-    callback = EvalCallback(
-        policy,
-        eval_env,
-        tb_writer,
-        best_model_path=config.model_dir_path(best=True),
-        **config.eval_params,
-        video_env=make_eval_env(
-            config, EnvHyperparams(**config.env_hyperparams), override_n_envs=1
-        )
-        if record_best_videos
-        else None,
-        best_video_dir=config.best_videos_dir,
-    )
-    algo.learn(config.n_timesteps, callback=callback)
-
-    policy.save(config.model_dir_path(best=False))
-
-    eval_stats = callback.evaluate(n_episodes=10, print_returns=True)
-
-    plot_eval_callback(callback, tb_writer, config.run_name)
-
-    log_dict: Dict[str, Any] = {
-        "eval": eval_stats._asdict(),
-    }
-    if callback.best:
-        log_dict["best_eval"] = callback.best._asdict()
-    log_dict.update(hyperparams)
-    log_dict.update(vars(args))
-    with open(config.logs_path, "a") as f:
-        yaml.dump({config.run_name: log_dict}, f)
-
-    best_eval_stats: EpisodesStats = callback.best  # type: ignore
-    tb_writer.add_hparams(
-        hparam_dict(hyperparams, vars(args)),
-        {
-            "hparam/best_mean": best_eval_stats.score.mean,
-            "hparam/best_result": best_eval_stats.score.mean
-            - best_eval_stats.score.std,
-            "hparam/last_mean": eval_stats.score.mean,
-            "hparam/last_result": eval_stats.score.mean - eval_stats.score.std,
-        },
-        None,
-        config.run_name,
-    )
-
-    tb_writer.close()
-
-    if wandb_enabled:
-        shutil.make_archive(
-            os.path.join(wandb.run.dir, config.model_dir_name()),
-            "zip",
-            config.model_dir_path(),
-        )
-        shutil.make_archive(
-            os.path.join(wandb.run.dir, config.model_dir_name(best=True)),
-            "zip",
-            config.model_dir_path(best=True),
-        )
-        wandb.finish()

shared/algorithm.py

@@ -1,35 +0,0 @@
-import gym
-import torch
-
-from abc import ABC, abstractmethod
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import List, Optional, TypeVar
-
-from shared.callbacks.callback import Callback
-from shared.policy.policy import Policy
-from wrappers.vectorable_wrapper import VecEnv
-
-AlgorithmSelf = TypeVar("AlgorithmSelf", bound="Algorithm")
-
-
-class Algorithm(ABC):
-    @abstractmethod
-    def __init__(
-        self,
-        policy: Policy,
-        env: VecEnv,
-        device: torch.device,
-        tb_writer: SummaryWriter,
-        **kwargs,
-    ) -> None:
-        super().__init__()
-        self.policy = policy
-        self.env = env
-        self.device = device
-        self.tb_writer = tb_writer
-
-    @abstractmethod
-    def learn(
-        self: AlgorithmSelf, total_timesteps: int, callback: Optional[Callback] = None
-    ) -> AlgorithmSelf:
-        ...

shared/callbacks/callback.py

@@ -1,12 +0,0 @@
-from abc import ABC, abstractmethod
-
-
-class Callback(ABC):
-
-    def __init__(self) -> None:
-        super().__init__()
-        self.timesteps_elapsed = 0
-
-    def on_step(self, timesteps_elapsed: int = 1) -> bool:
-        self.timesteps_elapsed += timesteps_elapsed
-        return True

shared/callbacks/eval_callback.py

@@ -1,199 +0,0 @@
-import itertools
-import numpy as np
-import os
-
-from time import perf_counter
-from torch.utils.tensorboard.writer import SummaryWriter
-from typing import List, Optional, Union
-
-from shared.callbacks.callback import Callback
-from shared.policy.policy import Policy
-from shared.stats import Episode, EpisodeAccumulator, EpisodesStats
-from wrappers.vec_episode_recorder import VecEpisodeRecorder
-from wrappers.vectorable_wrapper import VecEnv
-
-
-class EvaluateAccumulator(EpisodeAccumulator):
-    def __init__(
-        self,
-        num_envs: int,
-        goal_episodes: int,
-        print_returns: bool = True,
-        ignore_first_episode: bool = False,
-    ):
-        super().__init__(num_envs)
-        self.completed_episodes_by_env_idx = [[] for _ in range(num_envs)]
-        self.goal_episodes_per_env = int(np.ceil(goal_episodes / num_envs))
-        self.print_returns = print_returns
-        if ignore_first_episode:
-            first_done = set()
-
-            def should_record_done(idx: int) -> bool:
-                has_done_first_episode = idx in first_done
-                first_done.add(idx)
-                return has_done_first_episode
-
-            self.should_record_done = should_record_done
-        else:
-            self.should_record_done = lambda idx: True
-
-    def on_done(self, ep_idx: int, episode: Episode) -> None:
-        if (
-            self.should_record_done(ep_idx)
-            and len(self.completed_episodes_by_env_idx[ep_idx])
-            >= self.goal_episodes_per_env
-        ):
-            return
-        self.completed_episodes_by_env_idx[ep_idx].append(episode)
-        if self.print_returns:
-            print(
-                f"Episode {len(self)} | "
-                f"Score {episode.score} | "
-                f"Length {episode.length}"
-            )
-
-    def __len__(self) -> int:
-        return sum(len(ce) for ce in self.completed_episodes_by_env_idx)
-
-    @property
-    def episodes(self) -> List[Episode]:
-        return list(itertools.chain(*self.completed_episodes_by_env_idx))
-
-    def is_done(self) -> bool:
-        return all(
-            len(ce) == self.goal_episodes_per_env
-            for ce in self.completed_episodes_by_env_idx
-        )
-
-
-def evaluate(
-    env: VecEnv,
-    policy: Policy,
-    n_episodes: int,
-    render: bool = False,
-    deterministic: bool = True,
-    print_returns: bool = True,
-    ignore_first_episode: bool = False,
-) -> EpisodesStats:
-    policy.sync_normalization(env)
-    policy.eval()
-
-    episodes = EvaluateAccumulator(
-        env.num_envs, n_episodes, print_returns, ignore_first_episode
-    )
-
-    obs = env.reset()
-    while not episodes.is_done():
-        act = policy.act(obs, deterministic=deterministic)
-        obs, rew, done, _ = env.step(act)
-        episodes.step(rew, done)
-        if render:
-            env.render()
-    stats = EpisodesStats(episodes.episodes)
-    if print_returns:
-        print(stats)
-    return stats
-
-
-class EvalCallback(Callback):
-    def __init__(
-        self,
-        policy: Policy,
-        env: VecEnv,
-        tb_writer: SummaryWriter,
-        best_model_path: Optional[str] = None,
-        step_freq: Union[int, float] = 50_000,
-        n_episodes: int = 10,
-        save_best: bool = True,
-        deterministic: bool = True,
-        record_best_videos: bool = True,
-        video_env: Optional[VecEnv] = None,
-        best_video_dir: Optional[str] = None,
-        max_video_length: int = 3600,
-        ignore_first_episode: bool = False,
-    ) -> None:
-        super().__init__()
-        self.policy = policy
-        self.env = env
-        self.tb_writer = tb_writer
-        self.best_model_path = best_model_path
-        self.step_freq = int(step_freq)
-        self.n_episodes = n_episodes
-        self.save_best = save_best
-        self.deterministic = deterministic
-        self.stats: List[EpisodesStats] = []
-        self.best = None
-
-        self.record_best_videos = record_best_videos
-        assert video_env or not record_best_videos
-        self.video_env = video_env
-        assert best_video_dir or not record_best_videos
-        self.best_video_dir = best_video_dir
-        if best_video_dir:
-            os.makedirs(best_video_dir, exist_ok=True)
-        self.max_video_length = max_video_length
-        self.best_video_base_path = None
-
-        self.ignore_first_episode = ignore_first_episode
-
-    def on_step(self, timesteps_elapsed: int = 1) -> bool:
-        super().on_step(timesteps_elapsed)
-        if self.timesteps_elapsed // self.step_freq >= len(self.stats):
-            self.evaluate()
-        return True
-
-    def evaluate(
-        self, n_episodes: Optional[int] = None, print_returns: Optional[bool] = None
-    ) -> EpisodesStats:
-        start_time = perf_counter()
-        eval_stat = evaluate(
-            self.env,
-            self.policy,
-            n_episodes or self.n_episodes,
-            deterministic=self.deterministic,
-            print_returns=print_returns or False,
-            ignore_first_episode=self.ignore_first_episode,
-        )
-        end_time = perf_counter()
-        self.tb_writer.add_scalar(
-            "eval/steps_per_second",
-            eval_stat.length.sum() / (end_time - start_time),
-            self.timesteps_elapsed,
-        )
-        self.policy.train(True)
-        print(f"Eval Timesteps: {self.timesteps_elapsed} | {eval_stat}")
-
-        self.stats.append(eval_stat)
-
-        if not self.best or eval_stat >= self.best:
-            strictly_better = not self.best or eval_stat > self.best
-            self.best = eval_stat
-            if self.save_best:
-                assert self.best_model_path
-                self.policy.save(self.best_model_path)
-                print("Saved best model")
-            self.best.write_to_tensorboard(
-                self.tb_writer, "best_eval", self.timesteps_elapsed
-            )
-            if strictly_better and self.record_best_videos:
-                assert self.video_env and self.best_video_dir
-                self.best_video_base_path = os.path.join(
-                    self.best_video_dir, str(self.timesteps_elapsed)
-                )
-                video_wrapped = VecEpisodeRecorder(
-                    self.video_env,
-                    self.best_video_base_path,
-                    max_video_length=self.max_video_length,
-                )
-                video_stats = evaluate(
-                    video_wrapped,
-                    self.policy,
-                    1,
-                    deterministic=self.deterministic,
-                    print_returns=False,
-                )
-                print(f"Saved best video: {video_stats}")
-
-        eval_stat.write_to_tensorboard(self.tb_writer, "eval", self.timesteps_elapsed)
-
-        return eval_stat

shared/gae.py

@@ -1,67 +0,0 @@
-import numpy as np
-import torch
-
-from typing import NamedTuple, Sequence
-
-from shared.policy.on_policy import OnPolicy
-from shared.trajectory import Trajectory
-
-
-class RtgAdvantage(NamedTuple):
-    rewards_to_go: torch.Tensor
-    advantage: torch.Tensor
-
-
-def discounted_cumsum(x: np.ndarray, gamma: float) -> np.ndarray:
-    dc = x.copy()
-    for i in reversed(range(len(x) - 1)):
-        dc[i] += gamma * dc[i + 1]
-    return dc
-
-
-def compute_advantage(
-    trajectories: Sequence[Trajectory],
-    policy: OnPolicy,
-    gamma: float,
-    gae_lambda: float,
-    device: torch.device,
-) -> torch.Tensor:
-    advantage = []
-    for traj in trajectories:
-        last_val = 0
-        if not traj.terminated and traj.next_obs is not None:
-            last_val = policy.value(traj.next_obs)
-        rew = np.append(np.array(traj.rew), last_val)
-        v = np.append(np.array(traj.v), last_val)
-        deltas = rew[:-1] + gamma * v[1:] - v[:-1]
-        advantage.append(discounted_cumsum(deltas, gamma * gae_lambda))
-    return torch.as_tensor(
-        np.concatenate(advantage), dtype=torch.float32, device=device
-    )
-
-
-def compute_rtg_and_advantage(
-    trajectories: Sequence[Trajectory],
-    policy: OnPolicy,
-    gamma: float,
-    gae_lambda: float,
-    device: torch.device,
-) -> RtgAdvantage:
-    rewards_to_go = []
-    advantages = []
-    for traj in trajectories:
-        last_val = 0
-        if not traj.terminated and traj.next_obs is not None:
-            last_val = policy.value(traj.next_obs)
-        rew = np.append(np.array(traj.rew), last_val)
-        v = np.append(np.array(traj.v), last_val)
-        deltas = rew[:-1] + gamma * v[1:] - v[:-1]
-        adv = discounted_cumsum(deltas, gamma * gae_lambda)
-        advantages.append(adv)
-        rewards_to_go.append(v[:-1] + adv)
-    return RtgAdvantage(
-        torch.as_tensor(
-            np.concatenate(rewards_to_go), dtype=torch.float32, device=device
-        ),
-        torch.as_tensor(np.concatenate(advantages), dtype=torch.float32, device=device),
-    )

shared/module/feature_extractor.py

@@ -1,215 +0,0 @@
-import gym
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from abc import ABC, abstractmethod
-from gym.spaces import Box, Discrete
-from stable_baselines3.common.preprocessing import get_flattened_obs_dim
-from typing import Dict, Optional, Sequence, Type
-
-from shared.module.module import layer_init
-
-
-class CnnFeatureExtractor(nn.Module, ABC):
-    @abstractmethod
-    def __init__(
-        self,
-        in_channels: int,
-        activation: Type[nn.Module] = nn.ReLU,
-        init_layers_orthogonal: Optional[bool] = None,
-        **kwargs,
-    ) -> None:
-        super().__init__()
-
-
-class NatureCnn(CnnFeatureExtractor):
-    """
-    CNN from DQN Nature paper: Mnih, Volodymyr, et al.
-    "Human-level control through deep reinforcement learning."
-    Nature 518.7540 (2015): 529-533.
-    """
-
-    def __init__(
-        self,
-        in_channels: int,
-        activation: Type[nn.Module] = nn.ReLU,
-        init_layers_orthogonal: Optional[bool] = None,
-        **kwargs,
-    ) -> None:
-        if init_layers_orthogonal is None:
-            init_layers_orthogonal = True
-        super().__init__(in_channels, activation, init_layers_orthogonal)
-        self.cnn = nn.Sequential(
-            layer_init(
-                nn.Conv2d(in_channels, 32, kernel_size=8, stride=4),
-                init_layers_orthogonal,
-            ),
-            activation(),
-            layer_init(
-                nn.Conv2d(32, 64, kernel_size=4, stride=2),
-                init_layers_orthogonal,
-            ),
-            activation(),
-            layer_init(
-                nn.Conv2d(64, 64, kernel_size=3, stride=1),
-                init_layers_orthogonal,
-            ),
-            activation(),
-            nn.Flatten(),
-        )
-
-    def forward(self, obs: torch.Tensor) -> torch.Tensor:
-        return self.cnn(obs)
-
-
-class ResidualBlock(nn.Module):
-    def __init__(
-        self,
-        channels: int,
-        activation: Type[nn.Module] = nn.ReLU,
-        init_layers_orthogonal: bool = False,
-    ) -> None:
-        super().__init__()
-        self.residual = nn.Sequential(
-            activation(),
-            layer_init(
-                nn.Conv2d(channels, channels, 3, padding=1), init_layers_orthogonal
-            ),
-            activation(),
-            layer_init(
-                nn.Conv2d(channels, channels, 3, padding=1), init_layers_orthogonal
-            ),
-        )
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        return x + self.residual(x)
-
-
-class ConvSequence(nn.Module):
-    def __init__(
-        self,
-        in_channels: int,
-        out_channels: int,
-        activation: Type[nn.Module] = nn.ReLU,
-        init_layers_orthogonal: bool = False,
-    ) -> None:
-        super().__init__()
-        self.seq = nn.Sequential(
-            layer_init(
-                nn.Conv2d(in_channels, out_channels, 3, padding=1),
-                init_layers_orthogonal,
-            ),
-            nn.MaxPool2d(3, stride=2, padding=1),
-            ResidualBlock(out_channels, activation, init_layers_orthogonal),
-            ResidualBlock(out_channels, activation, init_layers_orthogonal),
-        )
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        return self.seq(x)
-
-
-class ImpalaCnn(CnnFeatureExtractor):
-    """
-    IMPALA-style CNN architecture
-    """
-
-    def __init__(
-        self,
-        in_channels: int,
-        activation: Type[nn.Module] = nn.ReLU,
-        init_layers_orthogonal: Optional[bool] = None,
-        impala_channels: Sequence[int] = (16, 32, 32),
-        **kwargs,
-    ) -> None:
-        if init_layers_orthogonal is None:
-            init_layers_orthogonal = False
-        super().__init__(in_channels, activation, init_layers_orthogonal)
-        sequences = []
-        for out_channels in impala_channels:
-            sequences.append(
-                ConvSequence(
-                    in_channels, out_channels, activation, init_layers_orthogonal
-                )
-            )
-            in_channels = out_channels
-        sequences.extend(
-            [
-                activation(),
-                nn.Flatten(),
-            ]
-        )
-        self.seq = nn.Sequential(*sequences)
-
-    def forward(self, obs: torch.Tensor) -> torch.Tensor:
-        return self.seq(obs)
-
-
-CNN_EXTRACTORS_BY_STYLE: Dict[str, Type[CnnFeatureExtractor]] = {
-    "nature": NatureCnn,
-    "impala": ImpalaCnn,
-}
-
-
-class FeatureExtractor(nn.Module):
-    def __init__(
-        self,
-        obs_space: gym.Space,
-        activation: Type[nn.Module],
-        init_layers_orthogonal: bool = False,
-        cnn_feature_dim: int = 512,
-        cnn_style: str = "nature",
-        cnn_layers_init_orthogonal: Optional[bool] = None,
-        impala_channels: Sequence[int] = (16, 32, 32),
-    ) -> None:
-        super().__init__()
-        if isinstance(obs_space, Box):
-            # Conv2D: (channels, height, width)
-            if len(obs_space.shape) == 3:
-                cnn = CNN_EXTRACTORS_BY_STYLE[cnn_style](
-                    obs_space.shape[0],
-                    activation,
-                    init_layers_orthogonal=cnn_layers_init_orthogonal,
-                    impala_channels=impala_channels,
-                )
-
-                def preprocess(obs: torch.Tensor) -> torch.Tensor:
-                    if len(obs.shape) == 3:
-                        obs = obs.unsqueeze(0)
-                    return obs.float() / 255.0
-
-                with torch.no_grad():
-                    cnn_out = cnn(preprocess(torch.as_tensor(obs_space.sample())))
-                self.preprocess = preprocess
-                self.feature_extractor = nn.Sequential(
-                    cnn,
-                    layer_init(
-                        nn.Linear(cnn_out.shape[1], cnn_feature_dim),
-                        init_layers_orthogonal,
-                    ),
-                    activation(),
-                )
-                self.out_dim = cnn_feature_dim
-            elif len(obs_space.shape) == 1:
-
-                def preprocess(obs: torch.Tensor) -> torch.Tensor:
-                    if len(obs.shape) == 1:
-                        obs = obs.unsqueeze(0)
-                    return obs.float()
-
-                self.preprocess = preprocess
-                self.feature_extractor = nn.Flatten()
-                self.out_dim = get_flattened_obs_dim(obs_space)
-            else:
-                raise ValueError(f"Unsupported observation space: {obs_space}")
-        elif isinstance(obs_space, Discrete):
-            self.preprocess = lambda x: F.one_hot(x, obs_space.n).float()
-            self.feature_extractor = nn.Flatten()
-            self.out_dim = obs_space.n
-        else:
-            raise NotImplementedError
-
-    def forward(self, obs: torch.Tensor) -> torch.Tensor:
-        if self.preprocess:
-            obs = self.preprocess(obs)
-        return self.feature_extractor(obs)