DQN_Phoenix.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TP_DQN_ATARI.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.6"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "gNNCCQUxGKGd"
      },
      "source": [
        "mygame='PhoenixDeterministic-v4' "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMBLJoRWfsQk"
      },
      "source": [
        "##Installation des dépendances\n",
        "\n",
        "on change de méthode pour afficher les trajectoires dans colab. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9PnGUJf8RX9G",
        "outputId": "c4f07839-9908-4221-9952-b57b1def58b4"
      },
      "source": [
        "!apt-get update\n",
        "!apt-get install --with-fix-missing -y xvfb x11-utils"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rIgn:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [Wa\r                                                                               \rHit:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [Co\r0% [2 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\r                                                                               \rHit:3 http://security.ubuntu.com/ubuntu bionic-security InRelease\n",
            "\r0% [2 InRelease gpgv 3,626 B] [Waiting for headers] [Waiting for headers] [Wait\r                                                                               \rIgn:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "\r                                                                               \r0% [2 InRelease gpgv 3,626 B] [Waiting for headers] [Waiting for headers]\r                                                                         \rHit:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "\r0% [2 InRelease gpgv 3,626 B] [Waiting for headers] [Waiting for headers]\r                                                                         \rHit:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "\r0% [2 InRelease gpgv 3,626 B] [Waiting for headers] [Waiting for headers]\r                                                                         \rHit:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "Hit:8 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n",
            "Hit:10 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:11 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n",
            "Hit:12 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Reading package lists... Done\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "x11-utils is already the newest version (7.7+3build1).\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.8).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 15 not upgraded.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PVT5Vo-gTmQR",
        "outputId": "5f6974d5-e59a-4e5f-8b46-33e274086d71"
      },
      "source": [
        "!pip install pyvirtualdisplay==0.2.* \\\n",
        "             PyOpenGL==3.1.* \\\n",
        "             PyOpenGL-accelerate==3.1.*"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pyvirtualdisplay==0.2.* in /usr/local/lib/python3.6/dist-packages (0.2.5)\n",
            "Requirement already satisfied: PyOpenGL==3.1.* in /usr/local/lib/python3.6/dist-packages (3.1.5)\n",
            "Requirement already satisfied: PyOpenGL-accelerate==3.1.* in /usr/local/lib/python3.6/dist-packages (3.1.5)\n",
            "Requirement already satisfied: EasyProcess in /usr/local/lib/python3.6/dist-packages (from pyvirtualdisplay==0.2.*) (0.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rzwqt_SJTrep",
        "outputId": "e2b895c4-ecfd-442b-8e9a-314b4858ae63"
      },
      "source": [
        "!pip install gym[atari]==0.17.*"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: gym[atari]==0.17.* in /usr/local/lib/python3.6/dist-packages (0.17.3)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari]==0.17.*) (1.5.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari]==0.17.*) (1.3.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym[atari]==0.17.*) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym[atari]==0.17.*) (1.19.5)\n",
            "Requirement already satisfied: atari-py~=0.2.0; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari]==0.17.*) (0.2.6)\n",
            "Requirement already satisfied: Pillow; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari]==0.17.*) (7.0.0)\n",
            "Requirement already satisfied: opencv-python; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari]==0.17.*) (4.1.2.30)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[atari]==0.17.*) (0.16.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from atari-py~=0.2.0; extra == \"atari\"->gym[atari]==0.17.*) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SEsvtd_wUAGG",
        "outputId": "c3a90279-de1e-4be6-a72b-d8aab9623813"
      },
      "source": [
        "!echo $DISPLAY"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHdYnF9wUHEQ"
      },
      "source": [
        "import pyvirtualdisplay\n",
        "\n",
        "\n",
        "_display = pyvirtualdisplay.Display(visible=False,  # use False with Xvfb\n",
        "                                    size=(1400, 900))\n",
        "_ = _display.start()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BJfOgnk1UI_p",
        "outputId": "e6fc4f16-671a-41d3-c406-fd880d21624c"
      },
      "source": [
        "!echo $DISPLAY"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ":1007\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "62qeUDpNhriT"
      },
      "source": [
        "Pour tester un agent, on appellera la fonction *record_video*."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BUVdTRYaUjan"
      },
      "source": [
        "import typing\n",
        "\n",
        "import gym\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from IPython import display\n",
        "\n",
        "\n",
        "# represent states as arrays and actions as ints\n",
        "State = np.ndarray\n",
        "Action = int\n",
        "\n",
        "# agent is just a function! \n",
        "Agent = typing.Callable[[State], Action]\n",
        "\n",
        "\n",
        "def uniform_random_policy(state: State,\n",
        "                          number_actions: int,\n",
        "                          random_state: np.random.RandomState) -> Action:\n",
        "    \"\"\"Select an action at random from the set of feasible actions.\"\"\"\n",
        "    feasible_actions = np.arange(number_actions)\n",
        "    probs = np.ones(number_actions) / number_actions\n",
        "    action = random_state.choice(feasible_actions, p=probs)\n",
        "    return action\n",
        "\n",
        "\n",
        "def make_random_agent(number_actions: int,\n",
        "                      random_state: np.random.RandomState = None) -> Agent:\n",
        "    \"\"\"Factory for creating an Agent.\"\"\"\n",
        "    _random_state = np.random.RandomState() if random_state is None else random_state\n",
        "    return lambda state: uniform_random_policy(state, number_actions, _random_state)\n",
        "\n",
        "# create the Gym environment\n",
        "bo = gym.make(mygame)\n",
        "# create an agent\n",
        "random_agent = make_random_agent(bo.action_space.n, random_state=None)\n",
        "bo.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sc_oktjLVQI9"
      },
      "source": [
        "#!nvidia-smi "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KVuxOLlmgJSz"
      },
      "source": [
        "## Prétraitement des images / de l'environnement\n",
        "\n",
        "On utilisera un réseau de neurones à convolutions pour lire directement les images fournies par un émulateur de console ATARI 2600.\n",
        "\n",
        "Toutefois, on applique quelque transformations aux images reçues (suivant Mnih et al. 2015)\n",
        "1. chaque action est appliquée 4 fois de suite (pour simuler un temps de réaction humain). On somme les récompenses des 4 états intermédiaires, et on renvoie une seule image qui est obtenue en appliquant la fonction sur chaque pixel (à partir des 4 imaegs intermédiaires) \n",
        "2. on redimenssionne l'image rectangulaire en un carré de 84x84 pixels. On transformera les couleurs en rééls compris entre 0 et 1.\n",
        "3. chaque image est concaténée aux 3 précédentes pour que le réseau ait accès à une notion de mouvement des objets sur l'écran."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bLlPpGd_vGRX"
      },
      "source": [
        "# Taken from \n",
        "# https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On/blob/master/Chapter06/lib/wrappers.py\n",
        "\n",
        "import cv2\n",
        "import numpy as np\n",
        "import collections\n",
        "\n",
        "class FireResetEnv(gym.Wrapper):\n",
        "    def __init__(self, env=None):\n",
        "        super(FireResetEnv, self).__init__(env)\n",
        "        assert env.unwrapped.get_action_meanings()[1] == 'FIRE'\n",
        "        assert len(env.unwrapped.get_action_meanings()) >= 3\n",
        "\n",
        "    def step(self, action):\n",
        "        return self.env.step(action)\n",
        "\n",
        "    def reset(self):\n",
        "        self.env.reset()\n",
        "        obs, _, done, _ = self.env.step(1)\n",
        "        if done:\n",
        "            self.env.reset()\n",
        "        obs, _, done, _ = self.env.step(2)\n",
        "        if done:\n",
        "            self.env.reset()\n",
        "        return obs\n",
        "\n",
        "class MaxAndSkipEnv(gym.Wrapper):\n",
        "    def __init__(self, env=None, skip=4):\n",
        "        super(MaxAndSkipEnv, self).__init__(env)\n",
        "        # most recent raw observations (for max pooling across time steps)\n",
        "        self._obs_buffer = collections.deque(maxlen=2)\n",
        "        self._skip = skip\n",
        "\n",
        "    def step(self, action):\n",
        "        total_reward = 0.0\n",
        "        done = None\n",
        "        for _ in range(self._skip):\n",
        "            obs, reward, done, info = self.env.step(action)\n",
        "            self._obs_buffer.append(obs)\n",
        "            total_reward += reward\n",
        "            if done:\n",
        "                break\n",
        "        max_frame = np.max(np.stack(self._obs_buffer), axis=0)\n",
        "        return max_frame, total_reward, done, info\n",
        "\n",
        "    def reset(self):\n",
        "        self._obs_buffer.clear()\n",
        "        obs = self.env.reset()\n",
        "        self._obs_buffer.append(obs)\n",
        "        return obs\n",
        "\n",
        "\n",
        "class ProcessFrame84(gym.ObservationWrapper):\n",
        "    def __init__(self, env=None):\n",
        "        super(ProcessFrame84, self).__init__(env)\n",
        "        self.observation_space = gym.spaces.Box(low=0, high=255, shape=(84, 84, 1), dtype=np.uint8)\n",
        "\n",
        "    def observation(self, obs):\n",
        "        return ProcessFrame84.process(obs)\n",
        "\n",
        "    @staticmethod\n",
        "    def process(frame):\n",
        "        if frame.size == 210 * 160 * 3:\n",
        "            img = np.reshape(frame, [210, 160, 3]).astype(np.float32)\n",
        "        elif frame.size == 250 * 160 * 3:\n",
        "            img = np.reshape(frame, [250, 160, 3]).astype(np.float32)\n",
        "        else:\n",
        "            assert False, \"Unknown resolution.\"\n",
        "        img = img[:, :, 0] * 0.299 + img[:, :, 1] * 0.587 + img[:, :, 2] * 0.114\n",
        "        resized_screen = cv2.resize(img, (84, 110), interpolation=cv2.INTER_AREA)\n",
        "        x_t = resized_screen[18:102, :]\n",
        "        x_t = np.reshape(x_t, [84, 84, 1])\n",
        "        return x_t.astype(np.uint8)\n",
        "\n",
        "\n",
        "class BufferWrapper(gym.ObservationWrapper):\n",
        "    def __init__(self, env, n_steps, dtype=np.float32):\n",
        "        super(BufferWrapper, self).__init__(env)\n",
        "        self.dtype = dtype\n",
        "        old_space = env.observation_space\n",
        "        self.observation_space = gym.spaces.Box(old_space.low.repeat(n_steps, axis=0),\n",
        "                                                old_space.high.repeat(n_steps, axis=0), dtype=dtype)\n",
        "\n",
        "    def reset(self):\n",
        "        self.buffer = np.zeros_like(self.observation_space.low, dtype=self.dtype)\n",
        "        return self.observation(self.env.reset())\n",
        "\n",
        "    def observation(self, observation):\n",
        "        self.buffer[:-1] = self.buffer[1:]\n",
        "        self.buffer[-1] = observation\n",
        "        return self.buffer\n",
        "\n",
        "\n",
        "class ImageToPyTorch(gym.ObservationWrapper):\n",
        "    def __init__(self, env):\n",
        "        super(ImageToPyTorch, self).__init__(env)\n",
        "        old_shape = self.observation_space.shape\n",
        "        self.observation_space = gym.spaces.Box(low=0.0, high=1.0, shape=(old_shape[-1], \n",
        "                                old_shape[0], old_shape[1]), dtype=np.float32)\n",
        "\n",
        "    def observation(self, observation):\n",
        "        return np.moveaxis(observation, 2, 0)\n",
        "\n",
        "\n",
        "class Uint8Frame(gym.ObservationWrapper):\n",
        "    def observation(self, obs):\n",
        "        return (np.array(obs,dtype=np.uint8))\n",
        "\n",
        "def make_env(env_name):\n",
        "    env = gym.make(env_name)\n",
        "    #env = MaxAndSkipEnv(env)\n",
        "    #env = FireResetEnv(env)\n",
        "    env = ProcessFrame84(env)\n",
        "    env = ImageToPyTorch(env)\n",
        "    env = BufferWrapper(env, 4)\n",
        "    return Uint8Frame(env)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "614q5vBaYj05"
      },
      "source": [
        "#import pour les réseaux de neurones\n",
        "import torch\n",
        "import torch.nn as nn        # Pytorch neural network package\n",
        "import torch.optim as optim  # Pytorch optimization package\n",
        "\n",
        "#device = torch.device(\"cuda\")\n",
        "device = torch.device(\"cpu\")\n",
        "\n",
        "#pour les logs\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "%load_ext tensorboard"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6nM-erRaYkp7"
      },
      "source": [
        "# Taken from \n",
        "# https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On/blob/master/Chapter06/lib/dqn_model.py\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "class NetworkDQN(nn.Module):\n",
        "    def __init__(self, input_shape, n_actions):\n",
        "        super(NetworkDQN, self).__init__()\n",
        "\n",
        "        self.conv = nn.Sequential(\n",
        "            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
        "            nn.ReLU()\n",
        "        )\n",
        "\n",
        "        conv_out_size = self._get_conv_out(input_shape)\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(conv_out_size, 512),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(512, n_actions)\n",
        "        )\n",
        "\n",
        "    def _get_conv_out(self, shape):\n",
        "        o = self.conv(torch.zeros(1, *shape))\n",
        "        return int(np.prod(o.size()))\n",
        "\n",
        "    def forward(self, x):\n",
        "        conv_out = self.conv(x).view(x.size(0), -1)\n",
        "        return self.fc(conv_out)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mvYWYPgHY6pq",
        "outputId": "d9120559-5dbf-496c-b0b2-354a41af023f"
      },
      "source": [
        "test_env = make_env(mygame)\n",
        "print(test_env.observation_space.shape, test_env.action_space.n)\n",
        "print(test_env.unwrapped.get_action_meanings())\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(4, 84, 84) 8\n",
            "['NOOP', 'FIRE', 'RIGHT', 'LEFT', 'DOWN', 'RIGHTFIRE', 'LEFTFIRE', 'DOWNFIRE']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BILlFM6f94FY"
      },
      "source": [
        "La fonction record_video prend:\n",
        "1. un agent qui définit une méthode *\\_\\_call\\_\\_* qui effectue une action\n",
        "2. le nom du répertoire où écrire les fichiers\n",
        "3. un timeout (en secondes) pour les agents qui ne font rien.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7KioaGO-1gOm"
      },
      "source": [
        "import time\n",
        "\n",
        "\n",
        "def record_video(current_agent, dirname, timeout=120):\n",
        "\n",
        "  FPS=25 #ou 30 ou 60 ??\n",
        "   \n",
        "  _display = pyvirtualdisplay.Display(visible=False, size=(1400, 900))\n",
        "  _ = _display.start()\n",
        "\n",
        "  record_folder= dirname\n",
        "  visualize=True\n",
        "\n",
        "  env = make_env(mygame)\n",
        "  if record_folder:\n",
        "    env = gym.wrappers.Monitor(env, record_folder, force=True)\n",
        "\n",
        "  state = env.reset()\n",
        "  total_reward = 0.0\n",
        "\n",
        "\n",
        "  steps = 0\n",
        "  while True:\n",
        "        if steps > timeout * FPS:\n",
        "          break\n",
        "        steps +=1\n",
        "        start_ts = time.time()\n",
        "        if visualize:\n",
        "            env.render()\n",
        "        \n",
        "        action = current_agent(state)\n",
        "\n",
        "        state, reward, done, _ = env.step(action)\n",
        "        total_reward += reward\n",
        "        if done:\n",
        "            break\n",
        "        if visualize:\n",
        "            delta = 1/FPS - (time.time() - start_ts)\n",
        "            if delta > 0:\n",
        "                time.sleep(delta)\n",
        "  print(\"Total reward: %.2f\" % total_reward)\n",
        "\n",
        "  if record_folder:\n",
        "        env.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CkEW3SfwcMRg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d4c4e392-3210-48f2-d246-c3df0d76af80"
      },
      "source": [
        "record_video(random_agent, \"video_random\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total reward: 420.00\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7pywBrt6Ddqp"
      },
      "source": [
        "#Version *classique*\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2rr2l_5_oxdT"
      },
      "source": [
        "class QLearningApproxNN1(nn.Module):\n",
        "  def __init__(self, env, device):\n",
        "    super(self.__class__, self).__init__()\n",
        "    \n",
        "    self.device = device\n",
        "    self.net = NetworkDQN(env.observation_space.shape, env.action_space.n).to(device)\n",
        "\n",
        "  def reset(self, env):\n",
        "    env.reset()\n",
        "\n",
        "  def get_epsilon_greedy_action(self, env, state, epsilon):\n",
        "    if torch.rand((1,)).item() < epsilon:\n",
        "      return torch.randint(0,env.action_space.n,(1,)).item()\n",
        "    else:\n",
        "      return torch.argmax(self.net(state)).item()\n",
        "\n",
        "  def compute_policy(self, env, maxit, epsilon_start=0.8, epsilon_end=0.05, gamma=0.99, alpha=0.001):\n",
        "    tot_rewards = 0.0\n",
        "    losses = []\n",
        "\n",
        "    self.net.train()\n",
        "    optimizer = torch.optim.Adam(self.net.parameters(), lr=alpha)\n",
        "\n",
        "    for m in range(maxit):\n",
        "      done = False\n",
        "      state = env.reset()\n",
        "      tot_reward = 0.0\n",
        "      \n",
        "      epsilon = epsilon_start - m * (epsilon_start - epsilon_end)  /maxit\n",
        "\n",
        "      while not done:\n",
        "        #choix de l'action\n",
        "        tstate = torch.tensor([state], dtype=torch.float, device=self.device)\n",
        "        action = self.get_epsilon_greedy_action(env, tstate, epsilon)\n",
        "\n",
        "        ostate, reward, done, _ = env.step(action)\n",
        "        tot_reward += reward\n",
        "\n",
        "        #calcul de l'erreur\n",
        "        if done:\n",
        "          estimate = reward\n",
        "        else:\n",
        "          next_state = torch.tensor([ostate], dtype=torch.float, device=self.device)\n",
        "          estimate = (reward + gamma*torch.max(self.net.forward(next_state))).detach()\n",
        "\n",
        "        error = torch.square(estimate - self.net.forward(tstate)[0][action])\n",
        "\n",
        "        #calcul du gradient de l'erreur, descente de gradient, remise à zéro des gradients \n",
        "        error.backward()\n",
        "        optimizer.step()\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        state = ostate\n",
        "\n",
        "      tot_rewards += tot_reward\n",
        "\n",
        "      if (m+1)%10 == 0:\n",
        "        avgr = tot_rewards/m\n",
        "        print(m+1, avgr, epsilon) #, self.parameters)\n",
        "      \n",
        "    self.net.eval()\n",
        "      \n",
        "  #implémente la politique gloutonne\n",
        "  def __call__(self, obs):\n",
        "    state = torch.tensor(obs, dtype=torch.float, device=self.device)\n",
        "    return torch.argmax(self.net.forward(state)).item()\n",
        "  \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qqbTVyQRKb4A"
      },
      "source": [
        "agent = QLearningApproxNN1(test_env, device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i-PKVEXZKoJN"
      },
      "source": [
        "#agent.compute_policy(test_env, 100)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G_NFDGBOOAPF"
      },
      "source": [
        "#record_video(agent, \"old\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SCT9q8mqyqXi"
      },
      "source": [
        "#Tampon d'exemples\n",
        "\n",
        "On implémente un tampon qui va stocker des nouveaux exemples (via la fonction *push*) et permetre d'en lire aléatoirement (via la fonction *sample*):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NDnYRjb4N6mN"
      },
      "source": [
        "class ReplayMemory(object):\n",
        "\n",
        "    def __init__(self, capacity):\n",
        "        self.capacity = capacity\n",
        "        self.memory = []\n",
        "        self.position = 0\n",
        "\n",
        "    def push(self, *args):\n",
        "        \"\"\"Saves a SARSA transition.\"\"\"\n",
        "        if len(self.memory) < self.capacity:\n",
        "            self.memory.append(None)\n",
        "        self.memory[self.position] = args\n",
        "        self.position = (self.position + 1) % self.capacity\n",
        "\n",
        "    def sample(self, batch_size):\n",
        "        return random.sample(self.memory, batch_size)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.memory)\n",
        "\n",
        "    def reset(self):\n",
        "        self.memory = []\n",
        "        self.position = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NZueVTNorT3O"
      },
      "source": [
        "## DQN avec tampon d'exemples et réseau cible\n",
        "\n",
        "Dans cette partie, on va implémenter le DQN:\n",
        "- réseau cible. \n",
        "  - L'estimation Bellman est calculée par un réseau cible. Ce réseau est initialisé par copie du réseau à apprendre.\n",
        "  - Puis toutes les *target_update* mises à jour du réseau à apprendre, on le recopie dans le réseau cible.\n",
        "- tampon d'exemples\n",
        "  - l'apprentissage génère des transitions de trajectoires qui sont stockées dans un tampon (classe *ReplayMemory*).\n",
        "  - Pour mettre à jour le réseau, on tire aléatoirement des exemples depuis le tampon (pas directement des trajectoires)\n",
        "  - Avant de commencer à apprendre, on remplit le tampon (avec *burnin* transitions)\n",
        "\n",
        "- autres\n",
        "  - la mise à jour des paramètres du réseau se fait par une descente de gradients avec des lots de taille 32. La fonction d'erreur est la *différence au carré* avec l'estimation. On fait une mise à jour quand on ajouté 4 transitions au tampon.\n",
        "  - on sauvegarde le modèle toutes les 1000 mises à jour. On utilisera les fonctions *load/save* de pytorch, ainsi que les *state_dict*.\n",
        "  - pendant l'apprentissage les trajectoires sont générées par une politique $\\varepsilon$-gloutonne. La valeur d'$\\varepsilon$ diminue linéairement entre *epsilon_start* et *epsilon_end* à chaque mise à jour des paramètres sur les premières 1000000, puis reste constante à *epsilon_end*.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tSoumLbGHLaS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc942115-18be-4c32-e7a5-a5d941fdad17"
      },
      "source": [
        "import random\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "prefix = \"/content/gdrive/My Drive/\" \n",
        "\n",
        "\n",
        "import random\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "prefix = \"/content/gdrive/My Drive/\" \n",
        "\n",
        "class QLearningApproxNN(nn.Module):\n",
        "  def __init__(self, env, device):\n",
        "    super(self.__class__, self).__init__()\n",
        "    \n",
        "    self.device = device\n",
        "    self.net = NetworkDQN(env.observation_space.shape, env.action_space.n).to(device)\n",
        "\n",
        "  def reset(self):\n",
        "    self.net = NetworkDQN(env.observation_space.shape, env.action_space.n).to(self.device)\n",
        "\n",
        "  def get_epsilon_greedy_action(self, env, state, epsilon):\n",
        "    if torch.rand((1,)).item() < epsilon:\n",
        "      return torch.randint(0,env.action_space.n,(1,)).item()\n",
        "    else:\n",
        "      return torch.argmax(self.net(state)).item()\n",
        "\n",
        "  def compute_policy(self, env, maxit, epsilon_start=1.0, epsilon_end=0.05, gamma=0.99, alpha=2.5e-4, \n",
        "                     replay_size=700000,\n",
        "                     update_target = 10000, label=None, save=None):\n",
        "    tot_rewards = 0.0\n",
        "    tot_lengths = 0.0\n",
        "\n",
        "    optimizer = torch.optim.Adam(self.net.parameters(), lr=alpha)\n",
        "    #optimizer = torch.optim.SGD(self.net.parameters(), lr=alpha)\n",
        "\n",
        "    target = NetworkDQN(env.observation_space.shape, env.action_space.n).to(device)\n",
        "    target.load_state_dict(self.net.state_dict())\n",
        "  \n",
        "    replay = ReplayMemory(replay_size)\n",
        "\n",
        "    #nombre de mises à jour\n",
        "    num_param_updates = 0\n",
        "    #nombre de transitions générées\n",
        "    steps = 0\n",
        "  \n",
        "    if save:\n",
        "      checkpoint = torch.load(save)\n",
        "      self.net.load_state_dict(checkpoint['net_state_dict'])\n",
        "      target.load_state_dict(checkpoint['target_state_dict'])\n",
        "      optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "      epsilon_start = checkpoint['epsilon_start']\n",
        "      epsilon_end = checkpoint['epsilon_end']\n",
        "      num_param_updates = checkpoint['num_param_updates']\n",
        "\n",
        "    epsilon = epsilon_start\n",
        "\n",
        "    #on génère d'abord suffisamment d'exemples pour échantillonner \"aléatoirement\"\n",
        "    while len(replay) < (replay_size/5):\n",
        "      state = env.reset()\n",
        "      done = False\n",
        "      while not done:\n",
        "        action = torch.randint(0, env.action_space.n, (1,)).item()\n",
        "        next_state,reward,done,_ = env.step(action)\n",
        "        replay.push(state, action, reward, next_state if not done else None)\n",
        "        state = next_state\n",
        "        \n",
        "    #####\n",
        "    print(\"let's learn!\")\n",
        "\n",
        "    #Q-Learning commence!\n",
        "\n",
        "    for m in range(maxit):\n",
        "      #intialisation pour une nouvelle trajectoire\n",
        "\n",
        "      state = env.reset()\n",
        "      tot_reward = 0\n",
        "      done = False\n",
        "\n",
        "      c = 0 #nombre de transitions générées pour cette trajectoire\n",
        "      while not done:\n",
        "        c += 1\n",
        "        steps += 1\n",
        "\n",
        "        #choix d'une action + step + ajout de la transition à replay\n",
        "        tstate = torch.tensor([state]) / 255.0\n",
        "        action = self.get_epsilon_greedy_action(env, tstate, epsilon)\n",
        "        next_state,reward,done,_ = env.step(action)\n",
        "        replay.push(state, action, reward, next_state if not done else None)\n",
        "        state = next_state\n",
        "        tot_reward += reward\n",
        "\n",
        "        if (steps+1) % 4 == 0:\n",
        "\n",
        "          batch_size = 32\n",
        "\n",
        "          #creation d'un batch de taille 32\n",
        "\n",
        "          states,actions,rewards,next_states = zip(*replay.sample(batch_size))\n",
        "          state_values = self.net(torch.tensor(states) / 255.0)\n",
        "          actions = torch.tensor(actions)\n",
        "          state_action_values = state_values.gather(1, actions.unsqueeze(1)).squeeze()\n",
        "\n",
        "          rewards = torch.tensor(rewards)\n",
        "\n",
        "          next_non_final_states = torch.tensor([ns for ns in next_states if ns is not None]) / 255.0\n",
        "          non_finals = torch.tensor([s is not None for s in next_states], dtype = torch.bool)\n",
        "          \n",
        "          next_states_values = torch.zeros_like(state_action_values)\n",
        "          next_states_values[non_finals] = target(next_non_final_states).max(1)[0].detach()\n",
        "\n",
        "          expected_gain = rewards + gamma * next_states_values\n",
        "\n",
        "          #calcul de l'erreur moyenne pour ce batch\n",
        "          error = torch.square(expected_gain - state_action_values).mean()\n",
        "\n",
        "          #mise à jour par descente de gradient (via optimizer) + RaZ des gradients \n",
        "          error.backward()\n",
        "          optimizer.step()\n",
        "          optimizer.zero_grad()\n",
        "\n",
        "          num_param_updates += 1\n",
        "          epsilon = max(epsilon_end, epsilon_start - num_param_updates * (epsilon_start - epsilon_end)  /1000000)\n",
        "\n",
        "\n",
        "          #MaJ périodique du réseau cible par copie\n",
        "          if num_param_updates % update_target == 0:\n",
        "            target.load_state_dict(self.net.state_dict())\n",
        "\n",
        "          #sauvegarde périodique du modèle\n",
        "          if ((num_param_updates+1)%5000 == 0)  or ((m+1 == maxit) and done):\n",
        "            print(F\"{num_param_updates+1} updates\")\n",
        "            torch.save({\n",
        "              'net_state_dict': self.net.state_dict(),\n",
        "              'target_state_dict': target.state_dict(),\n",
        "              'optimizer_state_dict': optimizer.state_dict(),\n",
        "              'epsilon_start': epsilon_start,\n",
        "              'epsilon_end': epsilon_end,\n",
        "              'num_param_updates': num_param_updates,\n",
        "              \n",
        "            }, prefix + ((\"save\" if label is None else F\"save_{label}\") + F\"_{num_param_updates+1}.pt\"))\n",
        "\n",
        "        state = next_state\n",
        "\n",
        "      #la trajectoire est finie\n",
        "      tot_rewards += tot_reward\n",
        "      tot_lengths += c\n",
        "\n",
        "      avgr = tot_rewards/(m+1)\n",
        "      avgl = tot_lengths/(m+1)\n",
        "      print(m+1, avgr, avgl, epsilon)\n",
        "\n",
        "\n",
        "  #politique gloutonne\n",
        "  def __call__(self, obs):\n",
        "    state = torch.tensor([obs], dtype=torch.float, device=self.device) / 255.0\n",
        "    return torch.argmax(self.net(state)).item()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n",
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRBaZlt3aD8t"
      },
      "source": [
        "agent1 = QLearningApproxNN(test_env, device)\n",
        "agent1.compute_policy(test_env, 10000, epsilon_start=1.0, epsilon_end=0.05, label=\"1\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eJao7dcA7EG7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9bf35635-fcd3-49fd-9e20-46830d215fc7"
      },
      "source": [
        "cp = torch.load(prefix+\"save_3_675000.pt\", device)\n",
        "agent1 = QLearningApproxNN(test_env, device)\n",
        "agent1.net.load_state_dict(cp['net_state_dict'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CoIfoAWpaJlN"
      },
      "source": [
        "record_video(agent1,\"video675k\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bd26AhcWZ_hc"
      },
      "source": [
        "agent1 = QLearningApproxNN(test_env, device)\n",
        "agent1.compute_policy(test_env, 1000, label=\"3\", save=prefix+\"save_3_1495000.pt\", replay_size=100000)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}