Updated dqn_pacmanv5_run1.ipynb to final version prior to training.

notebooks/dqn_pacmanv5_run1.ipynb

@@ -29,20 +29,32 @@
     "import torch as th\n",
     "import numpy as np\n",
     "\n",
-    "# Model Config\n",
-    "EVAL_CALLBACK_FREQ = 1_000\n",
-    "VIDEO_CALLBACK_FREQ = 2_000\n",
+    "# =====File names=====\n",
+    "MODEL_FILE_NAME = \"ALE-Pacman-v5\"\n",
+    "BUFFER_FILE_NAME = \"dqn_replay_buffer_pacman_v1\"\n",
+    "POLICY_FILE_NAME = \"dqn_policy_pacman_v1\"\n",
+    "\n",
+    "# =====Model Config=====\n",
+    "# Evaluate in tenths\n",
+    "EVAL_CALLBACK_FREQ = 150_000\n",
+    "# Record in quarters (the last one won't record, will have to do manually)\n",
+    "VIDEO_CALLBACK_FREQ = 250_000\n",
     "FRAMESKIP = 4\n",
-    "NUM_TIMESTEPS = 10_000\n",
+    "NUM_TIMESTEPS = 1_500_000\n",
     "\n",
-    "# Hyperparams\n",
-    "EXPLORATION_FRACTION = 0.35\n",
+    "# =====Hyperparams=====\n",
+    "EXPLORATION_FRACTION = 0.3\n",
+    "# Buffer size needs to be less than about 60k in order to save it in a Kaggle instance\n",
     "BUFFER_SIZE = 60_000\n",
     "BATCH_SIZE = 8\n",
-    "LEARNING_STARTS = 1_000\n",
-    "LEARNING_RATE = 0.0001\n",
+    "LEARNING_STARTS = 50_000\n",
+    "LEARNING_RATE = 0.0002\n",
     "GAMMA = 0.999\n",
-    "FINAL_EPSILON = 0.1"
+    "FINAL_EPSILON = 0.1\n",
+    "# Target Update Interval is set to 10k by default and looks like it is set to \n",
+    "# 4 in the Nature paper. This is a large discrepency and makes me wonder if it \n",
+    "# is something different or measured differently...\n",
+    "TARGET_UPDATE_INTERVAL = 1_000"
    ]
   },
   {
@@ -245,7 +257,9 @@
     "    gamma=GAMMA,\n",
     "    learning_starts=LEARNING_STARTS,\n",
     "    learning_rate=LEARNING_RATE,\n",
-    "    tensorboard_log=\"./\")"
+    "    target_update_interval=TARGET_UPDATE_INTERVAL,\n",
+    "    tensorboard_log=\"./\",\n",
+    "    )"
    ]
   },
   {
@@ -278,7 +292,7 @@
    "outputs": [],
    "source": [
     "# Train the model\n",
-    "# model.learn(total_timesteps=NUM_TIMESTEPS, callback=callback_list, tb_log_name=\"./tb/\")"
+    "model.learn(total_timesteps=NUM_TIMESTEPS, callback=callback_list, tb_log_name=\"./tb/\")"
    ]
   },
   {
@@ -288,24 +302,10 @@
    "outputs": [],
    "source": [
     "# Save the model, policy, and replay buffer for future loading and training\n",
-    "model.save(\"ALE-Pacman-v5\")\n",
-    "model.save_replay_buffer(\"dqn_replay_buffer_pacman\")\n",
-    "model.policy.save(\"dqn_policy_pacman\")"
+    "model.save(MODEL_FILE_NAME)\n",
+    "model.save_replay_buffer(BUFFER_FILE_NAME)\n",
+    "model.policy.save(POLICY_FILE_NAME)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {