td-tabular-general.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# General RL\n",
    "\n",
    "#### OpenAI gym environments are used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import gym\n",
    "import gym.spaces\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-270-e78188d7eb4a>, line 35)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-270-e78188d7eb4a>\"\u001b[1;36m, line \u001b[1;32m35\u001b[0m\n\u001b[1;33m    return self.epsilon = max(self.epsilon * self.eps_decay, self.eps_min) # e <- e_i\u001b[0m\n\u001b[1;37m                        ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "class Agent:\n",
    "\n",
    "    def __init__(self, nA=6, alpha = 0.1 , gamma = 1, eps_start = 1, eps_decay = 0.9, eps_min = 0.001, sars = 2):\n",
    "        \"\"\" Initialize agent.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        - nA: number of actions available to the agent\n",
    "        - alpha: update rate \n",
    "        - gamma: discount rate\n",
    "        - epsilon: starting epsilon value\n",
    "        - eps_decay: epsilon decay value (linear decay)\n",
    "        - eps_min: the min epsilon\n",
    "        - sars: the update method (2: sarsamax, 3: expected sars, 4: sars, 0-1: mixture of sarsamax & expectedsars)\n",
    "        \"\"\"\n",
    "        self.nA = nA\n",
    "        self.Q = defaultdict(lambda: np.zeros(self.nA), dtype=np.float64)\n",
    "        self.alpha = alpha\n",
    "        self.gamma = gamma\n",
    "        self.epsilon = eps_start\n",
    "        self.eps_decay = eps_decay\n",
    "        self.eps_min = eps_min\n",
    "        self.sars = sars \n",
    "    \n",
    "    def update_epsilon(self): # epsilon update\n",
    "        \"\"\" Updates e-greedy(Q) beginning of each episode\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "\n",
    "        Returns\n",
    "        =======\n",
    "        - epsilon: updated epsilon value\n",
    "        \"\"\"\n",
    "        return self.epsilon = np.max(self.epsilon * self.eps_decay, self.eps_min) # e <- e_i\n",
    "\n",
    "\n",
    "    def select_action(self, state, epsilon): #policy\n",
    "        \"\"\" Given the state, select an action in e-greedy(Q) way\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        - state: the current state of the environment\n",
    "        - epsilon: the epsilon of e-greedy\n",
    "\n",
    "        Returns\n",
    "        =======\n",
    "        - action: an integer, compatible with the task's action space\n",
    "        - policy_s: current policy for given state\n",
    "        \"\"\"\n",
    "        policy_s = np.ones(self.nA, dtype=np.float64) * (epsilon / self.nA)\n",
    "        best_a = np.argmax(self.Q[state])\n",
    "        policy_s[best_a] = 1 - epsilon + (epsilon / self.nA)\n",
    "        action = np.random.choice(np.arange(self.nA), p=policy_s)\n",
    "        return action, policy_s\n",
    "      \n",
    "    def step(self, state, action, reward, next_state, done, policy_s): #update step\n",
    "        \"\"\" Update the agent's knowledge-> state-value function (Q), using the most recently sampled tuple.\n",
    "            \n",
    "        Params\n",
    "        ======\n",
    "        - state: the previous state of the environment\n",
    "        - action: the agent's previous choice of action\n",
    "        - reward: last reward received\n",
    "        - next_state: the current state of the environment\n",
    "        - done: whether the episode is complete (True or False)\n",
    "        - policy_s: the current policy\n",
    "        \"\"\"\n",
    "        Q_update = 0\n",
    "        if not done: # if done next state Q value is 0\n",
    "            if self.sars == 0: # sarsamax\n",
    "                Q_update = np.max(self.Q[next_state])\n",
    "            \n",
    "            elif self.sars == 1: #expected sarsa\n",
    "                Q_update= np.dot(self.Q[next_state], policy_s) \n",
    "                \n",
    "        old_Q = self.Q[state][action]\n",
    "        self.Q[state][action] = old_Q + (self.alpha * (reward + (self.gamma * Q_update) - old_Q))        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Monitor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "import sys\n",
    "import math\n",
    "import numpy as np\n",
    "\n",
    "def interact(env, agent, num_episodes=20000, window=100):\n",
    "    \"\"\" Monitor agent's performance.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "    - env: instance of OpenAI Gym's Taxi-v1 environment\n",
    "    - agent: instance of class Agent (see Agent.py for details)\n",
    "    - num_episodes: number of episodes of agent-environment interaction\n",
    "    - window: number of episodes to consider when calculating average rewards\n",
    "\n",
    "    Returns\n",
    "    =======\n",
    "    - avg_rewards: deque containing average rewards\n",
    "    - best_avg_reward: largest value in the avg_rewards deque\n",
    "    \"\"\"\n",
    "    # initialize average rewards\n",
    "    avg_rewards = deque(maxlen=num_episodes)\n",
    "    # initialize best average reward\n",
    "    best_avg_reward = -math.inf\n",
    "    # initialize monitor for most recent rewards\n",
    "    samp_rewards = deque(maxlen=window)\n",
    "    # for each episode\n",
    "    for i_episode in range(1, num_episodes+1):\n",
    "        \n",
    "        epsilon = agent.update_epsilon() # e = e_i\n",
    "        #agent.alpha = agent.alpha * 0.9999\n",
    "        # begin the episode\n",
    "        state = env.reset()\n",
    "        # initialize the sampled reward\n",
    "        samp_reward = 0\n",
    "        while True:\n",
    "            # agent selects an action\n",
    "            action, policy_s = agent.select_action(state, epsilon)\n",
    "            # agent performs the selected action\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            # agent performs internal updates based on sampled experience\n",
    "            agent.step(state, action, reward, next_state, done, policy_s)\n",
    "            # update the sampled reward\n",
    "            samp_reward += reward\n",
    "            # update the state (s <- s') to next time step\n",
    "            state = next_state\n",
    "            if done:\n",
    "                # save final sampled reward\n",
    "                samp_rewards.append(samp_reward)\n",
    "                break\n",
    "        if (i_episode >= window): ## moving average\n",
    "            # get average reward from last 100 episodes\n",
    "            avg_reward = np.mean(samp_rewards)\n",
    "            # append to deque\n",
    "            avg_rewards.append(avg_reward)\n",
    "            # update best average reward\n",
    "            if avg_reward > best_avg_reward:\n",
    "                best_avg_reward = avg_reward\n",
    "                \n",
    "        #if i_episode % 1000 == 1:\n",
    "        #    print('alpha', agent.alpha)\n",
    "        #    print('epsilon', agent.epsilon)\n",
    "        \n",
    "        # monitor the progress\n",
    "        print(\"\\rEpisode {}/{} || Best average reward {}\".format(i_episode, num_episodes, best_avg_reward), end=\"\")\n",
    "        sys.stdout.flush()\n",
    "        # check if task is solved (according to OpenAI Gym)\n",
    "        if best_avg_reward >= 9.7: ## Taxi specific\n",
    "            print('\\nEnvironment solved in {} episodes.'.format(i_episode), end=\"\")\n",
    "            break\n",
    "        if i_episode == num_episodes: print('\\n')\n",
    "        # plot performance\n",
    "    plt.plot(np.linspace(0,num_episodes,len(avg_rewards),endpoint=False),np.asarray(avg_rewards))\n",
    "    plt.xlabel('Episode Number')\n",
    "    plt.ylabel('Average Reward (Over Next %d Episodes)' % window)\n",
    "    plt.show()\n",
    "    return avg_rewards, best_avg_reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main of the project "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 20000/20000 || Best average reward 9.226\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEKCAYAAADTgGjXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm8XHV9//HX+y7ZF8hqgISEVYIiSwQsP7UIyKICrqC1WmmLWPxp7SYIbdEWW/VntdQFqNWqBSkuICog0AqKSjAbJIQtCQlJCNn3u83y+f1xzk0mN7PdZe5Mct/Px2Med+Z7zsz5zJm55zvfXRGBmZlZXzTVOwAzMztwORMxM7M+cyZiZmZ95kzEzMz6zJmImZn1mTMRMzPrM2ciZmbWZ85EzMysz5yJmJlZn7XUO4BamzRpUsycObPeYZiZHTDmz5+/KSImV7PvQZ+JzJw5k3nz5tU7DDOzA4akVdXu6+osMzPrM2ciZmbWZxWrsyRNAc4CDgPagSXAvIjI1zg2MzNrcCUzEUlnA9cAE4CFwAZgBHApcLSkHwBfjIgdgxGomZk1nnIlkYuAP42IF3tukNQCvBU4D/hhjWIzM7MGVzITiYi/LrMtC9xdk4jMzOyAUbFhXdLHJY1T4j8kLZD05sEIzszMGls1vbOuSNs93gxMBj4E/HNNozoIbd7VycvbO8jlk+WIu7J5trdnKj4vk8uzsyNDPn3ejo4MT6zeRlc2T0TwjV+t4G/vXsKGnR1s2NHBdXct5v4l6+jI5OjK5snlgwUvbuXZl3eSywdd2TwdmRwAndkcm3Z1ArC9LcPn7n+GP/yPuVRaMnnlpt2s3dZOPh90ZnN0ZHIs27CLjkyOZ1/eyfKNu4gIdnTsfX9rt7Xz8TsW8pvlmwBo78qRzSV9MyKCXD54aVs7m3Z10pXNs/DFrSx4cSv5fLBxZycLXtxKRNDelcS+YWcH81Zu4aGl61m3vZ2/vPMJPn//M3vOU6FcPtjZkSEiWL+jg5WbdrOtrYtVm3cTESzbsIvvPraK3Z1ZALbs7tpzvO3tGbbs7trndTuzOba3ZejM5li9pY3tbZk9n2sxndnkvUYEmVyexWu273nvAFt3d9GRyRERdGRy/Gb5Jh5bsXnP9p6fRyaXJ5O+XuG2rmyezmxuz7ks3PbUS9v58kPPsXZbO9vbMuzqzO73/MLXX7lpN+t3dJDN5fnlcxv3+65mcnm27k7O4dKXduz5TgFkc3mu/dGTfO3hZWRyeR59fhO3zV1FRLB8464957L72N3PLfW96/7c7py3muUbdwGQTz/Tws9ld2d2n9fuvm1vz5DN5WnryrJpVydtXdl9Xr/7OdvbMzy4dD2L12wnlw+yuTybdnXyxOpt3DZ3FXNXbGblpt1s2NlBPh97PrMVG3eRyRXvZ7Rq826Wbdi1570vfHErAItWb+PTP3mKR5/ftOe7ExE88/IONuzsYEdHhvauHNvbMzz10nYyufye7xAk14Hvp+ejK5unK7vv8V/c3Mb9S9YVjWmgqdIFQ9KTEXGSpH8FHo6IuyQtjIhTBiXCfpozZ07UerBhRNCZzSPB8dffz8jWZtoL/qmqNaylab8vQ7dRw5oZO6KF9Ts6+xturw1vaaKzRFz9IUFrUxNdJf4B+2rC6GFAclEod3GH8ue8mOYmlX3NMcNb2NWZLbm9N46aPJqNOzrZmb7eiNYmOjID/zl0v6dXjBvByzs6yu5b6bs9elgzu7sqf/ePmzqG59bvKrqtpUlkK3xu5Rw2fgQvbS//PiaNGc6UscNZum5g+wXNnDiKlZvbOHRUK1vbKv9IHEjDmpP/pWnjR7BuewfDWppY+unzaWnu/UgOSfMjYk41+1YzYn2+pAeAWcC1ksYC7t6b2tWZ5d03/5anC76Mvc1Ajpw4ilWb24pezF51+DjeeNxknlm3k/95ZgMA554whYee3rDPfmfMmsDcF7Zw1OTRHDKylS27u/j946fwy+c3MrK1mctfO53vPb6a8SNbWbutnUwuz5bdXXsyh1mTRvM35x/PI89tZPnGXfxu5Vaam8TvHT2R4S1N+xzv9FkT2NmR3fOeR7Q2ceGrpnHXwrWcdcxEfr0s+RVdeIE+fupY3nTCFJ57OXkfrz58PKcdeSg7O7L8ZvkmBIwc1szYEa0sWr2Nc0+YwpK1Ozhq8mhGtDazra2LF7e0sa0tw+SxwznnhCkMb2kmm8uzoyPL3BWbOeuYSfxgwRq27O7a59z0vGAdN3UMx04Zy6Qxw/jVsk1MHjOcM2ZN4IcL1rKtrYu3nDSNO+etYXhLE00SJ0wby4IXtwHsudhKsG57B+eeMIWNOzt5Ys12zj5+Mi3NTTy4dH3Jz/qtJ00D4KdPFv+V+JojxjNqWAvzV21lxcbdHDV5NK8cPZaOTJ4ZE0axdls7U8YO54EixzhlxiFs2d3FhNHDWJjG223ciBbeM2c6Kze38cSabWzc2cmMCaN4cUsbx04ZwzMv7+TlHR1MnzCSGRNGMWpYC9PGj+C59TsZ2dpMJhc8umwTI1qbaM/kmDVpNDMmjGJHR4ZzT5jKF37+LACzDxtHVzbPG45LZsz40YK1nDBtLONHDmNbW9ee7/CmXV0cOTE5fgSMHd5CWyYpRb1i/AhGtjbz/Ia9n9mI1iYmjRnOa2dO4K6Fa5k0ZjiHjmrl1UeM50cL1u7zXjuyeY6ZMoZlG3YxcfQwjp06hu3te7+vE0YPY9Ouzj2lcEi+qyceNo6ZE0dz18K1e87Zjo4sp844hPNmv4Jv/GoF0yeMYndndp/YWptFJpdkes1N4rzZU9myu4v5q7Zy5MRRbNjRSXsmx/vPnMH9S9azaVcnZx8/mVHDWli/o4N5q7buea0ZE0bR0iymjR/B6GEtPLFmG+t3dHLe7Kn7fK9e+YqxnHnURO5etJZtbRlmTxsHwNJ1O1i3vYMzZk3gnacdQZNU9Hs2kKopiTQBJwMrImKbpInA4RHxZM2jGwC1LIlc8pVHeWLN9n3Sur+8X77sZC495fA+vW5EoEH48K2xPfvyTo6cOIoRrc31DqVudndmGdHaTHPTgff/kMvHARk3DHxJJIDZJF16PwOMJhkvUheSLgD+FWgGvhERdWufKcxAvnPF6Xt+ffWXMxADOP4VY+sdQt2NHn7gTu93oGYgvVVNZdnXgNcB700f7wS+WrOIypDUnB77QpKM7b2SZtcjlkee2wjAebOn8sI/XTRgGYiZ2YGkmkzkjIi4GugAiIitwLCaRlXa6cCyiFgREV3AHcAl9Qjko7cvAOBTF53gkoOZDVnVZCKZtAQQAJImU7+G9cOB1QWP16Rpg25nR9JjZtak0fU4vJlZQ6gmE7kJuAuYIulG4FHgszWNqrRiP/n36xkg6UpJ8yTN27hx44AH8dz6nQP+mmZmB6KKrVYRcZuk+cA5JBfxSyPi6ZpHVtwaYHrB4yOAl3ruFBG3ArdC0jtroIN485d+CcB7T58x0C9tZnZAKTeL74SChxuA7xVui4gttQyshN8Bx0qaBawFLgfeV4c4APjURa+s16HNzBpCuZLIfJKqIgEzgK3p/UOAF0kGHw6qiMhK+ijwc5Iuvt+MiKcGOw6AsSNaGDuitR6HNjNrGOVm8Z0FIOlm4J6IuDd9fCFw7uCEVzSue4F763h8YG/DupnZUFZNw/pruzMQgIi4D3hj7UJqbGu2ttc7BDOzhlHNcNBNkq4H/oukeuv9wObyTzl4dU++95HfP7rOkZiZ1V81JZH3kkwBfxfJQlRT2Dt6fcj52sPLADjskJF1jsTMrP6q6eK7Bfi4pHFAPiKKz988RNw5bw0AR04YVedIzMzqr5qVDV8taSGwGHhK0nxJr6p9aI3tlBmH1DsEM7O6q6Y66xbgLyLiyIg4EvhL0oF8Q83cdLW5U2Yc4u69ZmZUl4mMjohfdD+IiIdJpoMfci679TGA/Rb8MTMbqqrpnbVC0t8C300fvx94oXYhNabCNZRPPGxcHSMxM2sc1ZREriDpnfUjkh5ak4AP1TKoRlS45OrPPvb6OkZiZtY4qumdtRX4GOxZFGp0RAzs6vYHgB3tGQBOOmJ8nSMxM2sc1fTOul3SOEmjgaeAZyX9de1Dayw70mlOPnHecXWOxMyscVRTnTU7LXlcSjJn1QzgD2saVQOavyqZtPiQke6VZWbWrZpMpFVSK0km8uOIyFBkIaiD3WfvfQaAQ0fVa2VgM7PGU+04kZUk3Xp/KelIYMi1iXQbPbyaDm1mZkNDNQ3rN5EskdttlaSzaxdSYxveWk2+a2Y2NJRb2fD9EfFfkv6ixC7/UqOYGtLpMyfw+MotjPNIdTOzPcqVRLpHpY8djEAa3bCWJk71fFlmZvsot7LhLenfTw9eOI2rrSvLqGFuDzEzK1TNOJGjJP1E0kZJGyT9WNJRgxFcI2nryjFyWHO9wzAzayjVtBLfDtwJTAMOA74PfK+WQTWi9kyO0c5EzMz2UU0mooj4bkRk01v3MrlDyu7OHCNdnWVmto9qroq/kHQNcAdJ5nEZ8DNJE2DPyocHvfauLKNcEjEz20c1mchl6d8P90i/giRTOejbRyKCtkzOmYiZWQ/VDDacNRiBNLLObJ4I3LBuZtZDyTYRSX9TcP/dPbZ9tpZBNZq2rhwAo1qdiZiZFSrXsH55wf1re2y7oAaxNKzdnck08B4nYma2r3KZiErcL/b4oNaeSUsiw10SMTMrVC4TiRL3iz0eMJJukLRW0qL0dlHBtmslLZP0rKTzaxVDT3uqs9wmYma2j3L1M6+RtIOk1DEyvU/6eESN4/pSRPy/wgRJs0mq2E4kGfT4kKTjIiJX41ho60qqs0a2ujrLzKxQubmzGu1n9yXAHRHRCbwgaRlwOvDbWh+43SURM7OiGnVxjI9KelLSNyUdmqYdDqwu2GdNmlZzrs4yMyuuLpmIpIckLSlyuwT4OnA0cDKwDvhi99OKvFTRthlJV0qaJ2nexo0b+x3vnuosZyJmZvuoSyV/RJxbzX6S/h34afpwDTC9YPMRwEslXv9W4FaAOXPm9LsTwN6SiNtEzMwKVTMV/FRJp0o6RdLUWgckaVrBw7cDS9L79wCXSxouaRZwLPB4reMBV2eZmZVSbnnck4GbgfHA2jT5CEnbgD+LiAU1iunz6bEDWEk6Z1dEPCXpTmApkAWuHoyeWZA0rDcJhrc0ahOSmVl9lKuf+U/gwxExtzBR0pnAt4DX1CKgiPjDMttuBG6sxXHLaevKMWpYC9KQGmNpZlZRuZ/Wo3tmIAAR8Rh7118fEtq6sm5UNzMrolxJ5D5JPwO+w96utdOBDwD31zqwRrJo9TZiyC3DZWZWWbnBhh+TdCHJIL/DSbrYrgG+GhH3DlJ8DWHquBE88/LOeodhZtZwyvZZjYj7gPsGKZaG9pojxtc7BDOzhlNuPZHxkv5Z0tOSNqe3p9O0QwYzyHrL5PK0NrtnlplZT+WujHcCW4GzI2JiREwEzga2Ad8fjOAaRTYXtDS7Z5aZWU/lMpGZEfG5iHi5OyEiXo6IfwZm1D60xpHJuyRiZlZMuSvjKkl/UzhKPR29/kn2nQjxoJfNBS1NLomYmfVULhO5DJgIPCJpi6QtwMPABOA9gxBbw8jk8rS4JGJmtp9yXXy3Ap9Mb0NaNh+0uk3EzGw/ffp5LelDAx1II1u2YRfb2zP1DsPMrOH0tY7m0wMaxQHg18s21zsEM7OGU24W3ydLbQJqPiV8ozlq0pCaLszMrCrlRqxPBc4nGStSSMBvahZRg4kIJHjLSdMq72xmNsSUy0R+CoyJiEU9N0h6uGYRNZhsPoiAYe6dZWa2n3K9s/64zLb31SacxtOZzQMwzAtSmZntx1fGCrqciZiZleQrYwXORMzMSvOVsYI9mYjbRMzM9lPxyijpc9WkHay6ci6JmJmVUs2V8bwiaRcOdCCNqiOTq3cIZmYNq9yiVB+RtBh4paQnC24vAIsHL8T6ettXHgXg43fs19PZzGzIKzdO5HaSpXH/CbimIH1nRGypaVQNZMrY4azf0cksj1g3M9tPuXEi24Htkv4jIlYVbpP0wYj4ds2jawDdi1GdMn1IrQhsZlaVatpE/k7S1yWNThel+gnwtloH1ih2pLP3jhvZWudIzMwaTzWZyBuB5cAi4FHg9oh4V02jaiA7O7OAMxEzs2KqyUQOBc4gyUg6gSMlDZkVmiKSv+NGlGs+MjMbmqrJRB4D7ouIC4DXAocBv+7PQSW9W9JTkvKS5vTYdq2kZZKelXR+QfoFadoySdfs/6q15ZKImdn+qvl5fW5EvAgQEe3AxyS9oZ/HXQK8A7ilMFHSbOBy4ESSzOohScelm79KMmZlDfA7SfdExNJ+xlG18c5EzMz2U00mslrS+4GjIuIzkmYAHf05aEQ8DVCkVuwS4I6I6ARekLQMOD3dtiwiVqTPuyPdd9AykXEjnImYmfVUTXXW14DXAe9NH+8kKRXUwuHA6oLHa9K0UumDZtxIt4mYmfVUzZXxjIg4VdJCgIjYKmlYpSdJegh4RZFN10XEj0s9rUhaUDyzizLHvhK4EmDGjBkVIq2OSyJmZvurJhPJSGomvWhLmgzkKz0pIs7tQzxrgOkFj48AXkrvl0ovduxbgVsB5syZUzKz6Y1Rw5oH4mXMzA4q1VRn3QTcBUyRdCPJWJHP1iiee4DLJQ2XNAs4Fngc+B1wrKRZaSno8nTfQTPSmYiZ2X4qlkQi4jZJ84FzSKqbLu1uGO8rSW8H/g2YDPxM0qKIOD8inpJ0J0mDeRa4OiJy6XM+CvwcaAa+GRFP9SeG3hrR4kzEzKynqlqLI+IZ4JmBOmhE3EVSuim27UbgxiLp9wL3DlQMvdXUNGTGV5qZVa1kJiJpJ3sbr1VwvwUYFhHurmRmNsSVbBOJiLERMS69jSXpUnsj8DLwr4MVYL1NHTe83iGYmTWsiqUJSYcAfw58gGSNkddGxOZaB9YoXn34eCaO7tfYSjOzg1a56qxJwF8ClwHfBE5J1xgZUjK5oLXZ7SFmZsWUK4msAjYC3wLagD8unKYkIv6ltqE1hmw+T0tzNT2hzcyGnnKZyBfY25g+dhBiaUiZXNDinllmZkWVWx73hkGMo2FlcnnGDHdHNDOzYlxPU0HWJREzs5KciVTQnsnR7EzEzKyospmIpCZJ7xmsYBrRsg27eOjpDfUOw8ysIZXNRCIiD3x0kGIxM7MDTDXVWQ9K+itJ0yVN6L7VPDIzM2t41XQ7uiL9e3VBWgBHDXw4ZmZ2IKlmKvhZgxFII8rnk2EyV599dJ0jMTNrTBWrsySNknS9pFvTx8dKemvtQ6u/TD5ZwHHUMI8TMTMrppo2kW8BXcDvpY/XAP9Ys4gaSCaXlEQ8d5aZWXHVZCJHR8TngQxARLSTrC9y0MvmkpJIq+fOMjMrqpqrY5ekkaTzaEk6GuisaVQNoivNRDwBo5lZcdVU9t8A3A9Ml3QbcBbwRzWMqWFku6uzPGLdzKyoij+xI+IB4B0kGcf3gDkR8XBtw2oMi9cmy6f8aOHaOkdiZtaYqlnZ8B6SzOOeiNhd+5Aax8jWZgDOmOWxlWZmxVRT2f9F4PXAUknfl/QuSSNqHFdDuH3uiwAsWr2tzpGYmTWmagYbPgI8IqkZeBPwpyTL5Y6rcWx1t7WtC4CubL7OkZiZNaaquh2lvbPeCVwFvBb4di2DahQnTEvyyTccN7nOkZiZNaZq2kT+GzgTuA/4KvBwOrvvQe8108cD8JZXT6tzJGZmjamaLr7fAt4XEblaB9NoMtm0i2+Lx4mYmRVTNhORNIVkupM/kRTAUuBrEbF+MIKrt+7Bhh4nYmZWXMmf2JLOAn5HMlL9O8B/pZvmptv6TNK7JT0lKS9pTkH6TEntkhalt5sLtp0mabGkZZJuklTzK3vG056YmZVVriTyReDSiFhYkPZjSXcBtwBn9OO4S0gGMN5SZNvyiDi5SPrXgSuBx4B7gQtI2mlqJrNn2hOXRMzMiin3E3tcjwwEgIhYBIztz0Ej4umIeLba/SVNS+P5bUR0l4wu7U8M1ejIJJnIiHTQoZmZ7atcJiJJhxZJnFDhef01S9JCSY9Ien2adjjJFPTd1qRpRUm6UtI8SfM2btzY50A6szmam+TqLDOzEspdHb8EPCDpjZLGprffJ6lC+lKlF5b0kKQlRW6XlHnaOmBGRJwC/AVwu6RxFJ96Pkq9SETcGhFzImLO5Ml9H+PRmckz3D2zzMxKKtkmEhG3SnoJ+AfgRJKL9lLgHyPiJ5VeOCLO7W0wEdFJOs18RMyXtBw4jqTkcUTBrkcAL/X29XurM+tMxMysnLJdfCPip8BPBykWJE0GtkRETtJRwLHAiojYImmnpDOBucAHgH+rdTxd2TzDW9weYmZWSrkuvten7R+ltr+pr2utS3q7pDXA64CfSfp5uukNwJOSngB+AFwVEVvSbR8BvgEsA5ZT455ZkLSJDG91ScTMrJRyJZHFwE8kdQALgI3ACJLSwcnAQ8Bn+3LQiLgLuKtI+g+BH5Z4zjzgVX05Xl89u34Xqza3DeYhzcwOKOXaRH5MMi7kWJLVDKcBO0gGHV6ZrrV+UHt63Y56h2Bm1tCqmQr+eeD5QYjFzMwOMK7wNzOzPnMmYmZmfeZMxMzM+qxkm4ikf6P8qPCP1SQiMzM7YJQricwD5pN06z2VpHH9eZLuvUNugSozM9tfuS6+3waQ9EfA2RGRSR/fDDwwKNGZmVlDq6ZN5DD2nfp9TJp2UEtmnDczs3KqWWP9n4GFkn6RPn4jcEPNImoQ+TQP+cS5x9U3EDOzBlZpjXWRTG9yH3tXMrwmIl6udWD1lktzES8lYmZWWqVZfEPS3RFxGvDjQYqpIeTT6qymJi+Na2ZWSjW/sx+T9NqaR9Jg9pRE5EzEzKyUatpEzgY+LGkVsJtklcGIiJNqGlmd5aK7OsuZiJlZKdVkIhfWPIoGlE9LIk0uiZiZlVTNLL6rACRNIRl4OCTsbVh3JmJmVkrFNhFJF0t6HngBeARYySCsKlhvOTesm5lVVE3D+j8AZwLPRcQs4Bzg1zWNqgHk88lfN6ybmZVWTSaSiYjNQJOkpoj4Bcn8WQe1vQ3rdQ7EzKyBVdOwvk3SGOCXwG2SNgDZ2oZVf25YNzOrrJrf2ZcAbcAngPuB5cDbahlUI8jkkvosT6FlZlZaNZnIZcDREZGNiG9HxE1p9dZB7e5FLwHwzV+/UOdIzMwaVzXVWTOB90uaSbLGyK+AX0XEotqFVX83/c/zADzz8s46R2Jm1rgqlkQi4u8i4k3AicCjwF+TLFZ1UPv0xScC8I0PzKlzJGZmjatiSUTS9cBZJOuILAT+iqQ0clAbOyI5NcdOHVPnSMzMGlc11VnvIOmN9TOSwYaPRURHTaNqAN0N663u42tmVlI11VmnkgwwfBw4D1gs6dFaB1ZvmVzSLaul2V18zcxKqWbak1cB7wc+SNJTaw3wv/05qKQvSHpG0pOS7pJ0SMG2ayUtk/SspPML0i9I05ZJuqY/x6/GnpJIk0siZmalVHOF/BwwDrgJOCEizo6Iv+vncR8EXpVOJ/8ccC2ApNnA5SSN+BcAX5PULKkZ+CrJjMKzgfem+9ZM1iURM7OKqpnF9y2SRgIzIiIzEAeNiAcKHj4GvCu9fwlwR0R0Ai9IWgacnm5bFhErACTdke67dCDiKWbPyoYesW5mVlI11VlvAxaRjFZH0smS7hnAGK5g76zAhwOrC7atSdNKpZeK+UpJ8yTN27hxY5+C8qJUZmaVVVOddQNJaWAbQDrIcGalJ0l6SNKSIrdLCva5jqTn123dSUVeKsqkFxURt0bEnIiYM3ny5EqhFuW5s8zMKqumi282IrarlxfTiDi33HZJHwTeCpwTsWeGqjXA9ILdjgBeSu+XSq+JXPdU8C6JmJmVVE1JZImk9wHNko6V9G/Ab/pzUEkXAJ8ELo6ItoJN9wCXSxouaRZwLEnX4t8Bx0qaJWkYSeP7QFap7WfPolTOQ8zMSqomE/m/JL2lOoHbgR3An/fzuF8BxgIPSlok6WaAiHgKuJOkwfx+4OqIyEVEFvgo8HPgaeDOdN+ayeeDJkFvS2BmZkNJNb2z2oDr0hsAko4EVvX1oBFxTJltNwI3Fkm/F7i3r8fsrVyEq7LMzCooWxKR9DpJ75I0JX18kqTbSSZiPKglJRFnImZm5ZTMRCR9Afgm8E7gZ5L+nmSQ4FyStoqDWt4lETOzispVZ70FOCUiOiQdStIb6qSIeH5wQquv2+e+SFtXrt5hmJk1tHLVWe3ds/VGxFbg2aGSgQDsdgZiZlZRuZLI0T1Gps8sfBwRF9cuLDMzOxCUy0Qu6fH4i7UMpNG8+7Qj+PWyTfUOw8ysoZXMRCLikcEMpNHkw2NEzMwq8WIZJeQj8FIiZmbl+TJZQj6CZpdEzMzKqjoTkTS6loE0mpwHG5qZVVTNeiK/J2kpyZxVSHqNpK/VPLI6i4AmDzY0MyurmpLIl4Dzgc0AEfEE8IZaBtUIcukEjGZmVlpV1VkRsbpH0kE/Ei8frs4yM6ukmkWpVkv6PSDStTw+Rlq1dTDLh1c1NDOrpJqSyFXA1SRrmq8BTk4fH9TcxdfMrLJq1hPZBPzBIMTSUNzF18yssoqZiKSbiiRvB+ZFxI8HPqTGkMuHR6ybmVVQTYXNCJIqrOfT20nABOCPJX25hrHVVQReT8TMrIJqGtaPAd6UrnOOpK8DDwDnAYtrGFtdJb2z6h2FmVljq6YkcjhQOFp9NHBYROSAzppE1QBcnWVmVlk1JZHPA4skPQyIZKDhZ9NpUB6qYWx15eosM7PKqumd9R+S7gVOJ8lEPhURL6Wb/7qWwdVTLoJWZyJmZmVVOxKiA1gHbAGOkXTQT3viEetmZpVV08X3T4CPA0cAi4Azgd8Cb6ptaPXlEetmZpVVUxL5OPBaYFVEnA2cAmysaVQNIO8JGM3MKqomE+mIiA4AScMj4hng+NqGVX/5CDesm5lVUE3vrDWSDgHuBh6UtBV4qcJzDnju4mtmVlk1vbPent69QdIvgPHA/f05qKQvAG8DuoDlwIciYpukmSQzBD+b7vpYRFyVPuc04D+BkcC9wMcjIvoTRzkgIhOaAAAL+0lEQVQReO4sM7MKylZnSWqStKT7cUQ8EhH3RERXP4/7IPCqiDgJeA64tmDb8og4Ob1dVZD+deBK4Nj0dkE/Yygr51l8zcwqKnuZjIg88ISkGQN50Ih4oHsaFeAxkp5fJUmaBoyLiN+mpY/vAJcOZEw9uYuvmVll1bSJTAOekvQ4sLs7MSIuHqAYrgD+u+DxLEkLgR3A9RHxK/auZdJtTZpWlKQrSUotzJjRt/wv3MXXzKyiajKRT/flhSU9BLyiyKbruqeQl3QdkAVuS7etA2ZExOa0DeRuSSeSjJTvqWR7SETcCtwKMGfOnD61m3iNdTOzyqppWH9E0pHAsRHxkKRRQHMVzzu33HZJHwTeCpzT3UAeEZ2kkzpGxHxJy4HjSEoehVVeR1DjHmLJyobORczMyqnYdCzpT4EfALekSYeTdPftM0kXAJ8ELo6ItoL0yZKa0/tHkTSgr4iIdcBOSWcq6Xf7AaCmC2Ilgw2diZiZlVNN/6OrgbNI2iiIiOeBKf087leAsSTjThZJujlNfwPwpKQnSDKuqyJiS7rtI8A3gGUk3YLv62cMZeXdxdfMrKJq2kQ6I6Kre+CdpBbKtEdUIyKOKZH+Q+CHJbbNA17Vn+P2Rt5dfM3MKqrmMvmIpE8BIyWdB3wf+Eltw6q/fHjEuplZJdVkIteQTLi4GPgwyWjx62sZVCNwdZaZWWXVVGddAnwnIv691sE0EnfxNTOrrJqSyMXAc5K+K+ktaZvIQc9dfM3MKquYiUTEh4BjSNpC3gcsl/SNWgdWbx6xbmZWWVWliojISLqPpFfWSJIqrj+pZWD15uosM7PKqhlseIGk/yQZn/EukrEa02ocV92df+JUZh82rt5hmJk1tGpKIn8E3AF8OJ2WZEj48uWn1DsEM7OGV83cWZcXPpZ0FvC+iLi6ZlGZmdkBoao2EUknkzSqvwd4AfhRLYMyM7MDQ8lMRNJxwOXAe4HNJGt+KCLOHqTYzMyswZUriTwD/Ap4W0QsA5D0iUGJyszMDgjleme9E3gZ+IWkf5d0DsUXhzIzsyGqZCYSEXdFxGXAK4GHgU8AUyV9XdKbByk+MzNrYNWMWN8dEbdFxFtJVhRcRDIpo5mZDXG9WjEjIrZExC0R8aZaBWRmZgcOpcubH7QkbQRW9fHpk4BNAxjOQHFcveO4esdx9c7BGNeRETG5mh0P+kykPyTNi4g59Y6jJ8fVO46rdxxX7wz1uLwArJmZ9ZkzETMz6zNnIuXdWu8ASnBcveO4esdx9c6QjsttImZm1mcuiZiZWZ85EykiXYjrWUnLJNV8YKWk6ZJ+IelpSU9J+niafoOktZIWpbeLCp5zbRrfs5LOr1XsklZKWpwef16aNkHSg5KeT/8emqZL0k3psZ+UdGrB63ww3f95SR/sZ0zHF5yTRZJ2SPrzepwvSd+UtEHSkoK0ATs/kk5Lz/+y9LlVTT1UIq4vSHomPfZdkg5J02dKai84bzdXOn6p99jHuAbsc5M0S9LcNK7/ljSsH3H9d0FMKyUtqsP5KnVtqPt3bI+I8K3gBjQDy4GjgGHAE8DsGh9zGnBqen8s8BwwG7gB+Ksi+89O4xoOzErjba5F7MBKYFKPtM8D16T3rwE+l96/CLiPZI61M4G5afoEYEX699D0/qED+Hm9DBxZj/MFvAE4FVhSi/MDPA68Ln3OfcCF/YjrzUBLev9zBXHNLNyvx+sUPX6p99jHuAbscwPuBC5P798MfKSvcfXY/kXg7+pwvkpdG+r+Heu+uSSyv9OBZRGxIiK6SFZ1vKSWB4yIdRGxIL2/E3gaOLzMUy4B7oiIzoh4gWTp4tMHMfZLgG+n978NXFqQ/p1IPAYcImkacD7wYCQzHmwFHgQuGKBYzgGWR0S5AaU1O18R8UtgS5Hj9fv8pNvGRcRvI/lv/07Ba/U6roh4ICKy6cPHSKYxKqnC8Uu9x17HVUavPrf0F/SbgB8MZFzp674H+F6516jR+Sp1baj7d6ybM5H9HQ6sLni8hvIX9AElaSZwCjA3TfpoWiz9ZkERuFSMtYg9gAckzZd0ZZo2NSLWQfIlB6bUIa5ul7PvP3e9zxcM3Pk5PL0/0PEBXEHyq7PbLEkLJT0i6fUF8ZY6fqn32FcD8blNBLYVZJQDdb5eD6yPiOcL0gb9fPW4NjTMd8yZyP6K1QcOShc2SWOAHwJ/HhE7gK8DRwMnA+tIitTlYqxF7GdFxKnAhcDVkt5QZt/BjIu0vvti4PtpUiOcr3J6G0etztt1QBa4LU1aB8yIiFOAvwBulzSuVscvYqA+t1rF+172/aEy6OeryLWh5K4lYqjZOXMmsr81wPSCx0cAL9X6oJJaSb4kt0XEjwAiYn1E5CIiD/w7STG+XIwDHntEvJT+3QDclcawPi0GdxfhNwx2XKkLgQURsT6Nse7nKzVQ52cN+1Y59Tu+tEH1rcAfpNUXpNVFm9P780naG46rcPxS77HXBvBz20RSfdPSI73P0td6B8nKrt3xDur5KnZtKPN6g/8d600DylC4kaz2uIKkIa+70e7EGh9TJHWRX+6RPq3g/idI6ocBTmTfBscVJI2NAxo7MBoYW3D/NyRtGV9g30a9z6f338K+jXqPp+kTgBdIGvQOTe9PGIDzdgfwoXqfL3o0tA7k+QF+l+7b3eh5UT/iugBYCkzusd9koDm9fxSwttLxS73HPsY1YJ8bSam0sGH9z/oaV8E5e6Re54vS14aG+I5FhDOREh/cRSS9IJYD1w3C8f4PSRHySZL1WhalMXwXWJym39Pjn+26NL5nKehNMZCxp/8gT6S3p7pfj6Tu+X+A59O/3V9GAV9Nj70YmFPwWleQNIwuo+DC34/YRgGbgfEFaYN+vkiqOdYBGZJfdX88kOcHmAMsSZ/zFdIBwn2MaxlJvXj3d+zmdN93pp/vE8ACkiWxyx6/1HvsY1wD9rml39nH0/f6fWB4X+NK0/8TuKrHvoN5vkpdG+r+Heu+ecS6mZn1mdtEzMysz5yJmJlZnzkTMTOzPnMmYmZmfeZMxMzM+syZiB20JOW072y/ZWfplXSVpA8MwHFXSprUi/0fVjpDcvp4jqSH+xtH+lp/JOkrA/FaZsW0VN7F7IDVHhEnV7tzRNxcea+amSLpwoi4r/Kug0dSc0Tk6h2HNS6XRGzISUsKn5P0eHo7Jk2/QdJfpfc/JmlpOingHWnaBEl3p2mPSTopTZ8o6YF0Qr5bKJiPSNL702MsknSLpOYSYX0BuL5IrPuUJCT9VNLvp/d3pe9jvqSHJJ2elmpWSLq44GWmS7pfyfobf18ptvR1PyNpLskU4WYlOROxg9nIHtVZlxVs2xERp5OM0P1ykedeA5wSEScBV6VpnwYWpmmfIpmOAuDvgUcjmZDvHmAGgKQTgMtIJrE8GcgBf1Ai1t8CnZLO7sX7Gw08HBGnATuBfwTOA94OfKZgv9PT454MvDutLisX22iS6T/OiIhHexGPDUGuzrKDWbnqrO8V/P1Ske1PArdJuhu4O037PyRTXhAR/5uWQMaTLGj0jjT9Z5K2pvufA5wG/C5ZkoKRlJ947x9JSiOfrOK9AXQB96f3FwOdEZGRtJhkHqhuD0Y6YaCkH6XvI1smthzJhH9mFTkTsaEqStzv9haSzOFi4G8lnUj5abOLvYaAb0fEtVUFlGRM/0AyGV63LPvWGIwouJ+JvfMW5YHO9HXyBTPZFoutewrwUrF1uB3EquXqLBuqLiv4+9vCDZKagOkR8Qvgb4BDgDHAL0mrfNJ2iU2RrO1QmH4hySypkEyM9y5JU9JtEyQdWSGuG9NjdlsJnCypSdJ09k6T3hvnpcceSbJq3a/7GJvZflwSsYPZSEmLCh7fHxHd3XyHpw3HTSSLDhVqBv4rraoS8KWI2CbpBuBbkp4E2oAPpvt/GviepAXAI8CLABGxVNL1JCtDNpHMEHs1UHIp34i4V9LGgqRfk0zbvZhkptUFvToDiUdJZso9Brg9IuYB9DY2s2I8i68NOZJWkkyRvanesZgd6FydZWZmfeaSiJmZ9ZlLImZm1mfORMzMrM+ciZiZWZ85EzEzsz5zJmJmZn3mTMTMzPrs/wO+BlBPpoD4KQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "env = gym.make('Taxi-v2')\n",
    "#env = gym.make('CliffWalking-v0')\n",
    "nA = env.action_space.n\n",
    "agent = Agent(nA=nA, alpha=0.2, gamma=1, eps_start=0.0003, eps_decay=0.9, eps_min=0.0001, sars=0) #sars<- 0: sarsamax, 1: ex[sars]\n",
    "avg_rewards, best_avg_reward = interact(env, agent)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}