sdpkjc commited on
Commit
0c33876
1 Parent(s): 79eed1f

pushing model

Browse files
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ videos/HalfCheetah-v4__td3_continuous_action__3__1702937634-eval/rl-video-episode-8.mp4 filter=lfs diff=lfs merge=lfs -text
37
+ videos/HalfCheetah-v4__td3_continuous_action__3__1702937634-eval/rl-video-episode-1.mp4 filter=lfs diff=lfs merge=lfs -text
38
+ videos/HalfCheetah-v4__td3_continuous_action__3__1702937634-eval/rl-video-episode-0.mp4 filter=lfs diff=lfs merge=lfs -text
39
+ replay.mp4 filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - HalfCheetah-v4
4
+ - deep-reinforcement-learning
5
+ - reinforcement-learning
6
+ - custom-implementation
7
+ library_name: cleanrl
8
+ model-index:
9
+ - name: TD3
10
+ results:
11
+ - task:
12
+ type: reinforcement-learning
13
+ name: reinforcement-learning
14
+ dataset:
15
+ name: HalfCheetah-v4
16
+ type: HalfCheetah-v4
17
+ metrics:
18
+ - type: mean_reward
19
+ value: 10126.55 +/- 2210.99
20
+ name: mean_reward
21
+ verified: false
22
+ ---
23
+
24
+ # (CleanRL) **TD3** Agent Playing **HalfCheetah-v4**
25
+
26
+ This is a trained model of a TD3 agent playing HalfCheetah-v4.
27
+ The model was trained by using [CleanRL](https://github.com/vwxyzjn/cleanrl) and the most up-to-date training code can be
28
+ found [here](https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/td3_continuous_action.py).
29
+
30
+ ## Get Started
31
+
32
+ To use this model, please install the `cleanrl` package with the following command:
33
+
34
+ ```
35
+ pip install "cleanrl[td3_continuous_action]"
36
+ python -m cleanrl_utils.enjoy --exp-name td3_continuous_action --env-id HalfCheetah-v4
37
+ ```
38
+
39
+ Please refer to the [documentation](https://docs.cleanrl.dev/get-started/zoo/) for more detail.
40
+
41
+
42
+ ## Command to reproduce the training
43
+
44
+ ```bash
45
+ curl -OL https://huggingface.co/sdpkjc/HalfCheetah-v4-td3_continuous_action-seed3/raw/main/td3_continuous_action.py
46
+ curl -OL https://huggingface.co/sdpkjc/HalfCheetah-v4-td3_continuous_action-seed3/raw/main/pyproject.toml
47
+ curl -OL https://huggingface.co/sdpkjc/HalfCheetah-v4-td3_continuous_action-seed3/raw/main/poetry.lock
48
+ poetry install --all-extras
49
+ python td3_continuous_action.py --save-model --upload-model --hf-entity sdpkjc --env-id HalfCheetah-v4 --seed 3 --track
50
+ ```
51
+
52
+ # Hyperparameters
53
+ ```python
54
+ {'batch_size': 256,
55
+ 'buffer_size': 1000000,
56
+ 'capture_video': False,
57
+ 'cuda': True,
58
+ 'env_id': 'HalfCheetah-v4',
59
+ 'exp_name': 'td3_continuous_action',
60
+ 'exploration_noise': 0.1,
61
+ 'gamma': 0.99,
62
+ 'hf_entity': 'sdpkjc',
63
+ 'learning_rate': 0.0003,
64
+ 'learning_starts': 25000.0,
65
+ 'noise_clip': 0.5,
66
+ 'policy_frequency': 2,
67
+ 'policy_noise': 0.2,
68
+ 'save_model': True,
69
+ 'seed': 3,
70
+ 'tau': 0.005,
71
+ 'torch_deterministic': True,
72
+ 'total_timesteps': 1000000,
73
+ 'track': True,
74
+ 'upload_model': True,
75
+ 'wandb_entity': None,
76
+ 'wandb_project_name': 'cleanRL'}
77
+ ```
78
+
events.out.tfevents.1702937642.4090-171.273890.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:308bafe8f5a53dc8229a9f95f2689508bbd67caae3351afdd312ac7a7e30d92d
3
+ size 3878521
poetry.lock ADDED
The diff for this file is too large to render. See raw diff
 
pyproject.toml ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [tool.poetry]
2
+ name = "cleanrl"
3
+ version = "1.1.0"
4
+ description = "High-quality single file implementation of Deep Reinforcement Learning algorithms with research-friendly features"
5
+ authors = ["Costa Huang <costa.huang@outlook.com>"]
6
+ packages = [
7
+ { include = "cleanrl" },
8
+ { include = "cleanrl_utils" },
9
+ ]
10
+ keywords = ["reinforcement", "machine", "learning", "research"]
11
+ license="MIT"
12
+ readme = "README.md"
13
+
14
+ [tool.poetry.dependencies]
15
+ python = ">=3.7.1,<3.11"
16
+ tensorboard = "^2.10.0"
17
+ wandb = "^0.13.11"
18
+ gym = "0.23.1"
19
+ torch = ">=1.12.1"
20
+ stable-baselines3 = "1.2.0"
21
+ gymnasium = ">=0.28.1"
22
+ moviepy = "^1.0.3"
23
+ pygame = "2.1.0"
24
+ huggingface-hub = "^0.11.1"
25
+ rich = "<12.0"
26
+ tenacity = "^8.2.2"
27
+
28
+ ale-py = {version = "0.7.4", optional = true}
29
+ AutoROM = {extras = ["accept-rom-license"], version = "^0.4.2", optional = true}
30
+ opencv-python = {version = "^4.6.0.66", optional = true}
31
+ procgen = {version = "^0.10.7", optional = true}
32
+ pytest = {version = "^7.1.3", optional = true}
33
+ mujoco = {version = "<=2.3.3", optional = true}
34
+ imageio = {version = "^2.14.1", optional = true}
35
+ free-mujoco-py = {version = "^2.1.6", optional = true}
36
+ mkdocs-material = {version = "^8.4.3", optional = true}
37
+ markdown-include = {version = "^0.7.0", optional = true}
38
+ openrlbenchmark = {version = "^0.1.1b4", optional = true}
39
+ jax = {version = "^0.3.17", optional = true}
40
+ jaxlib = {version = "^0.3.15", optional = true}
41
+ flax = {version = "^0.6.0", optional = true}
42
+ optuna = {version = "^3.0.1", optional = true}
43
+ optuna-dashboard = {version = "^0.7.2", optional = true}
44
+ envpool = {version = "^0.6.4", optional = true}
45
+ PettingZoo = {version = "1.18.1", optional = true}
46
+ SuperSuit = {version = "3.4.0", optional = true}
47
+ multi-agent-ale-py = {version = "0.1.11", optional = true}
48
+ boto3 = {version = "^1.24.70", optional = true}
49
+ awscli = {version = "^1.25.71", optional = true}
50
+ shimmy = {version = ">=1.0.0", extras = ["dm-control"], optional = true}
51
+
52
+ [tool.poetry.group.dev.dependencies]
53
+ pre-commit = "^2.20.0"
54
+
55
+
56
+ [tool.poetry.group.isaacgym]
57
+ optional = true
58
+ [tool.poetry.group.isaacgym.dependencies]
59
+ isaacgymenvs = {git = "https://github.com/vwxyzjn/IsaacGymEnvs.git", rev = "poetry", python = ">=3.7.1,<3.10"}
60
+ isaacgym = {path = "cleanrl/ppo_continuous_action_isaacgym/isaacgym", develop = true}
61
+
62
+
63
+ [build-system]
64
+ requires = ["poetry-core"]
65
+ build-backend = "poetry.core.masonry.api"
66
+
67
+ [tool.poetry.extras]
68
+ atari = ["ale-py", "AutoROM", "opencv-python"]
69
+ procgen = ["procgen"]
70
+ plot = ["pandas", "seaborn"]
71
+ pytest = ["pytest"]
72
+ mujoco = ["mujoco", "imageio"]
73
+ mujoco_py = ["free-mujoco-py"]
74
+ jax = ["jax", "jaxlib", "flax"]
75
+ docs = ["mkdocs-material", "markdown-include", "openrlbenchmark"]
76
+ envpool = ["envpool"]
77
+ optuna = ["optuna", "optuna-dashboard"]
78
+ pettingzoo = ["PettingZoo", "SuperSuit", "multi-agent-ale-py"]
79
+ cloud = ["boto3", "awscli"]
80
+ dm_control = ["shimmy", "mujoco"]
81
+
82
+ # dependencies for algorithm variant (useful when you want to run a specific algorithm)
83
+ dqn = []
84
+ dqn_atari = ["ale-py", "AutoROM", "opencv-python"]
85
+ dqn_jax = ["jax", "jaxlib", "flax"]
86
+ dqn_atari_jax = [
87
+ "ale-py", "AutoROM", "opencv-python", # atari
88
+ "jax", "jaxlib", "flax" # jax
89
+ ]
90
+ c51 = []
91
+ c51_atari = ["ale-py", "AutoROM", "opencv-python"]
92
+ c51_jax = ["jax", "jaxlib", "flax"]
93
+ c51_atari_jax = [
94
+ "ale-py", "AutoROM", "opencv-python", # atari
95
+ "jax", "jaxlib", "flax" # jax
96
+ ]
97
+ ppo_atari_envpool_xla_jax_scan = [
98
+ "ale-py", "AutoROM", "opencv-python", # atari
99
+ "jax", "jaxlib", "flax", # jax
100
+ "envpool", # envpool
101
+ ]
102
+ qdagger_dqn_atari_impalacnn = [
103
+ "ale-py", "AutoROM", "opencv-python"
104
+ ]
105
+ qdagger_dqn_atari_jax_impalacnn = [
106
+ "ale-py", "AutoROM", "opencv-python", # atari
107
+ "jax", "jaxlib", "flax", # jax
108
+ ]
replay.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:515452975106276ac46619da6a0bd3b131e74c4dc376b8c85639c55b9fc93b8f
3
+ size 1360021
td3_continuous_action.cleanrl_model ADDED
Binary file (872 kB). View file
 
td3_continuous_action.py ADDED
@@ -0,0 +1,300 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # docs and experiment results can be found at https://docs.cleanrl.dev/rl-algorithms/td3/#td3_continuous_actionpy
2
+ import argparse
3
+ import os
4
+ import random
5
+ import time
6
+ from distutils.util import strtobool
7
+
8
+ import gymnasium as gym
9
+ import numpy as np
10
+ import torch
11
+ import torch.nn as nn
12
+ import torch.nn.functional as F
13
+ import torch.optim as optim
14
+ from stable_baselines3.common.buffers import ReplayBuffer
15
+ from torch.utils.tensorboard import SummaryWriter
16
+
17
+
18
+ def parse_args():
19
+ # fmt: off
20
+ parser = argparse.ArgumentParser()
21
+ parser.add_argument("--exp-name", type=str, default=os.path.basename(__file__).rstrip(".py"),
22
+ help="the name of this experiment")
23
+ parser.add_argument("--seed", type=int, default=1,
24
+ help="seed of the experiment")
25
+ parser.add_argument("--torch-deterministic", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
26
+ help="if toggled, `torch.backends.cudnn.deterministic=False`")
27
+ parser.add_argument("--cuda", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
28
+ help="if toggled, cuda will be enabled by default")
29
+ parser.add_argument("--track", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
30
+ help="if toggled, this experiment will be tracked with Weights and Biases")
31
+ parser.add_argument("--wandb-project-name", type=str, default="cleanRL",
32
+ help="the wandb's project name")
33
+ parser.add_argument("--wandb-entity", type=str, default=None,
34
+ help="the entity (team) of wandb's project")
35
+ parser.add_argument("--capture-video", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
36
+ help="whether to capture videos of the agent performances (check out `videos` folder)")
37
+ parser.add_argument("--save-model", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
38
+ help="whether to save model into the `runs/{run_name}` folder")
39
+ parser.add_argument("--upload-model", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
40
+ help="whether to upload the saved model to huggingface")
41
+ parser.add_argument("--hf-entity", type=str, default="",
42
+ help="the user or org name of the model repository from the Hugging Face Hub")
43
+
44
+ # Algorithm specific arguments
45
+ parser.add_argument("--env-id", type=str, default="Hopper-v4",
46
+ help="the id of the environment")
47
+ parser.add_argument("--total-timesteps", type=int, default=1000000,
48
+ help="total timesteps of the experiments")
49
+ parser.add_argument("--learning-rate", type=float, default=3e-4,
50
+ help="the learning rate of the optimizer")
51
+ parser.add_argument("--buffer-size", type=int, default=int(1e6),
52
+ help="the replay memory buffer size")
53
+ parser.add_argument("--gamma", type=float, default=0.99,
54
+ help="the discount factor gamma")
55
+ parser.add_argument("--tau", type=float, default=0.005,
56
+ help="target smoothing coefficient (default: 0.005)")
57
+ parser.add_argument("--batch-size", type=int, default=256,
58
+ help="the batch size of sample from the reply memory")
59
+ parser.add_argument("--policy-noise", type=float, default=0.2,
60
+ help="the scale of policy noise")
61
+ parser.add_argument("--exploration-noise", type=float, default=0.1,
62
+ help="the scale of exploration noise")
63
+ parser.add_argument("--learning-starts", type=int, default=25e3,
64
+ help="timestep to start learning")
65
+ parser.add_argument("--policy-frequency", type=int, default=2,
66
+ help="the frequency of training policy (delayed)")
67
+ parser.add_argument("--noise-clip", type=float, default=0.5,
68
+ help="noise clip parameter of the Target Policy Smoothing Regularization")
69
+ args = parser.parse_args()
70
+ # fmt: on
71
+ return args
72
+
73
+
74
+ def make_env(env_id, seed, idx, capture_video, run_name):
75
+ def thunk():
76
+ if capture_video and idx == 0:
77
+ env = gym.make(env_id, render_mode="rgb_array")
78
+ env = gym.wrappers.RecordVideo(env, f"videos/{run_name}")
79
+ else:
80
+ env = gym.make(env_id)
81
+ env = gym.wrappers.RecordEpisodeStatistics(env)
82
+ env.action_space.seed(seed)
83
+ return env
84
+
85
+ return thunk
86
+
87
+
88
+ # ALGO LOGIC: initialize agent here:
89
+ class QNetwork(nn.Module):
90
+ def __init__(self, env):
91
+ super().__init__()
92
+ self.fc1 = nn.Linear(np.array(env.single_observation_space.shape).prod() + np.prod(env.single_action_space.shape), 256)
93
+ self.fc2 = nn.Linear(256, 256)
94
+ self.fc3 = nn.Linear(256, 1)
95
+
96
+ def forward(self, x, a):
97
+ x = torch.cat([x, a], 1)
98
+ x = F.relu(self.fc1(x))
99
+ x = F.relu(self.fc2(x))
100
+ x = self.fc3(x)
101
+ return x
102
+
103
+
104
+ class Actor(nn.Module):
105
+ def __init__(self, env):
106
+ super().__init__()
107
+ self.fc1 = nn.Linear(np.array(env.single_observation_space.shape).prod(), 256)
108
+ self.fc2 = nn.Linear(256, 256)
109
+ self.fc_mu = nn.Linear(256, np.prod(env.single_action_space.shape))
110
+ # action rescaling
111
+ self.register_buffer(
112
+ "action_scale", torch.tensor((env.action_space.high - env.action_space.low) / 2.0, dtype=torch.float32)
113
+ )
114
+ self.register_buffer(
115
+ "action_bias", torch.tensor((env.action_space.high + env.action_space.low) / 2.0, dtype=torch.float32)
116
+ )
117
+
118
+ def forward(self, x):
119
+ x = F.relu(self.fc1(x))
120
+ x = F.relu(self.fc2(x))
121
+ x = torch.tanh(self.fc_mu(x))
122
+ return x * self.action_scale + self.action_bias
123
+
124
+
125
+ if __name__ == "__main__":
126
+ import stable_baselines3 as sb3
127
+
128
+ if sb3.__version__ < "2.0":
129
+ raise ValueError(
130
+ """Ongoing migration: run the following command to install the new dependencies:
131
+ poetry run pip install "stable_baselines3==2.0.0a1"
132
+ """
133
+ )
134
+
135
+ args = parse_args()
136
+ run_name = f"{args.env_id}__{args.exp_name}__{args.seed}__{int(time.time())}"
137
+ if args.track:
138
+ import wandb
139
+
140
+ wandb.init(
141
+ project=args.wandb_project_name,
142
+ entity=args.wandb_entity,
143
+ sync_tensorboard=True,
144
+ config=vars(args),
145
+ name=run_name,
146
+ monitor_gym=True,
147
+ save_code=True,
148
+ )
149
+ writer = SummaryWriter(f"runs/{run_name}")
150
+ writer.add_text(
151
+ "hyperparameters",
152
+ "|param|value|\n|-|-|\n%s" % ("\n".join([f"|{key}|{value}|" for key, value in vars(args).items()])),
153
+ )
154
+
155
+ # TRY NOT TO MODIFY: seeding
156
+ random.seed(args.seed)
157
+ np.random.seed(args.seed)
158
+ torch.manual_seed(args.seed)
159
+ torch.backends.cudnn.deterministic = args.torch_deterministic
160
+
161
+ device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")
162
+
163
+ # env setup
164
+ envs = gym.vector.SyncVectorEnv([make_env(args.env_id, args.seed, 0, args.capture_video, run_name)])
165
+ assert isinstance(envs.single_action_space, gym.spaces.Box), "only continuous action space is supported"
166
+
167
+ actor = Actor(envs).to(device)
168
+ qf1 = QNetwork(envs).to(device)
169
+ qf2 = QNetwork(envs).to(device)
170
+ qf1_target = QNetwork(envs).to(device)
171
+ qf2_target = QNetwork(envs).to(device)
172
+ target_actor = Actor(envs).to(device)
173
+ target_actor.load_state_dict(actor.state_dict())
174
+ qf1_target.load_state_dict(qf1.state_dict())
175
+ qf2_target.load_state_dict(qf2.state_dict())
176
+ q_optimizer = optim.Adam(list(qf1.parameters()) + list(qf2.parameters()), lr=args.learning_rate)
177
+ actor_optimizer = optim.Adam(list(actor.parameters()), lr=args.learning_rate)
178
+
179
+ envs.single_observation_space.dtype = np.float32
180
+ rb = ReplayBuffer(
181
+ args.buffer_size,
182
+ envs.single_observation_space,
183
+ envs.single_action_space,
184
+ device,
185
+ handle_timeout_termination=False,
186
+ )
187
+ start_time = time.time()
188
+
189
+ # TRY NOT TO MODIFY: start the game
190
+ obs, _ = envs.reset(seed=args.seed)
191
+ for global_step in range(args.total_timesteps):
192
+ # ALGO LOGIC: put action logic here
193
+ if global_step < args.learning_starts:
194
+ actions = np.array([envs.single_action_space.sample() for _ in range(envs.num_envs)])
195
+ else:
196
+ with torch.no_grad():
197
+ actions = actor(torch.Tensor(obs).to(device))
198
+ actions += torch.normal(0, actor.action_scale * args.exploration_noise)
199
+ actions = actions.cpu().numpy().clip(envs.single_action_space.low, envs.single_action_space.high)
200
+
201
+ # TRY NOT TO MODIFY: execute the game and log data.
202
+ next_obs, rewards, terminations, truncations, infos = envs.step(actions)
203
+
204
+ # TRY NOT TO MODIFY: record rewards for plotting purposes
205
+ if "final_info" in infos:
206
+ for info in infos["final_info"]:
207
+ print(f"global_step={global_step}, episodic_return={info['episode']['r']}")
208
+ writer.add_scalar("charts/episodic_return", info["episode"]["r"], global_step)
209
+ writer.add_scalar("charts/episodic_length", info["episode"]["l"], global_step)
210
+ break
211
+
212
+ # TRY NOT TO MODIFY: save data to reply buffer; handle `final_observation`
213
+ real_next_obs = next_obs.copy()
214
+ for idx, trunc in enumerate(truncations):
215
+ if trunc:
216
+ real_next_obs[idx] = infos["final_observation"][idx]
217
+ rb.add(obs, real_next_obs, actions, rewards, terminations, infos)
218
+
219
+ # TRY NOT TO MODIFY: CRUCIAL step easy to overlook
220
+ obs = next_obs
221
+
222
+ # ALGO LOGIC: training.
223
+ if global_step > args.learning_starts:
224
+ data = rb.sample(args.batch_size)
225
+ with torch.no_grad():
226
+ clipped_noise = (torch.randn_like(data.actions, device=device) * args.policy_noise).clamp(
227
+ -args.noise_clip, args.noise_clip
228
+ ) * target_actor.action_scale
229
+
230
+ next_state_actions = (target_actor(data.next_observations) + clipped_noise).clamp(
231
+ envs.single_action_space.low[0], envs.single_action_space.high[0]
232
+ )
233
+ qf1_next_target = qf1_target(data.next_observations, next_state_actions)
234
+ qf2_next_target = qf2_target(data.next_observations, next_state_actions)
235
+ min_qf_next_target = torch.min(qf1_next_target, qf2_next_target)
236
+ next_q_value = data.rewards.flatten() + (1 - data.dones.flatten()) * args.gamma * (min_qf_next_target).view(-1)
237
+
238
+ qf1_a_values = qf1(data.observations, data.actions).view(-1)
239
+ qf2_a_values = qf2(data.observations, data.actions).view(-1)
240
+ qf1_loss = F.mse_loss(qf1_a_values, next_q_value)
241
+ qf2_loss = F.mse_loss(qf2_a_values, next_q_value)
242
+ qf_loss = qf1_loss + qf2_loss
243
+
244
+ # optimize the model
245
+ q_optimizer.zero_grad()
246
+ qf_loss.backward()
247
+ q_optimizer.step()
248
+
249
+ if global_step % args.policy_frequency == 0:
250
+ actor_loss = -qf1(data.observations, actor(data.observations)).mean()
251
+ actor_optimizer.zero_grad()
252
+ actor_loss.backward()
253
+ actor_optimizer.step()
254
+
255
+ # update the target network
256
+ for param, target_param in zip(actor.parameters(), target_actor.parameters()):
257
+ target_param.data.copy_(args.tau * param.data + (1 - args.tau) * target_param.data)
258
+ for param, target_param in zip(qf1.parameters(), qf1_target.parameters()):
259
+ target_param.data.copy_(args.tau * param.data + (1 - args.tau) * target_param.data)
260
+ for param, target_param in zip(qf2.parameters(), qf2_target.parameters()):
261
+ target_param.data.copy_(args.tau * param.data + (1 - args.tau) * target_param.data)
262
+
263
+ if global_step % 100 == 0:
264
+ writer.add_scalar("losses/qf1_values", qf1_a_values.mean().item(), global_step)
265
+ writer.add_scalar("losses/qf2_values", qf2_a_values.mean().item(), global_step)
266
+ writer.add_scalar("losses/qf1_loss", qf1_loss.item(), global_step)
267
+ writer.add_scalar("losses/qf2_loss", qf2_loss.item(), global_step)
268
+ writer.add_scalar("losses/qf_loss", qf_loss.item() / 2.0, global_step)
269
+ writer.add_scalar("losses/actor_loss", actor_loss.item(), global_step)
270
+ print("SPS:", int(global_step / (time.time() - start_time)))
271
+ writer.add_scalar("charts/SPS", int(global_step / (time.time() - start_time)), global_step)
272
+
273
+ if args.save_model:
274
+ model_path = f"runs/{run_name}/{args.exp_name}.cleanrl_model"
275
+ torch.save((actor.state_dict(), qf1.state_dict(), qf2.state_dict()), model_path)
276
+ print(f"model saved to {model_path}")
277
+ from cleanrl_utils.evals.td3_eval import evaluate
278
+
279
+ episodic_returns = evaluate(
280
+ model_path,
281
+ make_env,
282
+ args.env_id,
283
+ eval_episodes=10,
284
+ run_name=f"{run_name}-eval",
285
+ Model=(Actor, QNetwork),
286
+ device=device,
287
+ exploration_noise=args.exploration_noise,
288
+ )
289
+ for idx, episodic_return in enumerate(episodic_returns):
290
+ writer.add_scalar("eval/episodic_return", episodic_return, idx)
291
+
292
+ if args.upload_model:
293
+ from cleanrl_utils.huggingface import push_to_hub
294
+
295
+ repo_name = f"{args.env_id}-{args.exp_name}-seed{args.seed}"
296
+ repo_id = f"{args.hf_entity}/{repo_name}" if args.hf_entity else repo_name
297
+ push_to_hub(args, episodic_returns, repo_id, "TD3", f"runs/{run_name}", f"videos/{run_name}-eval")
298
+
299
+ envs.close()
300
+ writer.close()
videos/HalfCheetah-v4__td3_continuous_action__3__1702937634-eval/rl-video-episode-0.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:062bb5f78ee05493227a9b245abae975cc360d27b8949f25297251ade8ee8d26
3
+ size 1375846
videos/HalfCheetah-v4__td3_continuous_action__3__1702937634-eval/rl-video-episode-1.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:623d7869ca3caa43d89c602d362ab6768b28aed12d13695f9d0619d08a18c192
3
+ size 1375364
videos/HalfCheetah-v4__td3_continuous_action__3__1702937634-eval/rl-video-episode-8.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:515452975106276ac46619da6a0bd3b131e74c4dc376b8c85639c55b9fc93b8f
3
+ size 1360021