Digit Recognizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "np.random.seed(2)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "\n",
    "from keras.utils.np_utils import to_categorical # convert to one-hot-encoding\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "\n",
    "sns.set(style='white', context='notebook', palette='deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"d:\\BaiduNetdiskDownload/MNIST/train.csv\")\n",
    "test = pd.read_csv(\"d:\\BaiduNetdiskDownload/MNIST/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    4684\n",
       "7    4401\n",
       "3    4351\n",
       "9    4188\n",
       "2    4177\n",
       "6    4137\n",
       "0    4132\n",
       "4    4072\n",
       "8    4063\n",
       "5    3795\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEPCAYAAACHuClZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE+5JREFUeJzt3XuwXWV5x/Fvcs6JpmBAJmCCAYRBHtQWgnJpyy0qyKAoYsUIEQgKyCAOU1FQCeUyxbaMBsUSZeQSbBBoQawjibeoEIrghdso8pRWRSPJoKJyM5CTpH+sdciGJrBz3rPWPif5fmYynP3stfM+JCfnt9/17vWucWvWrEGSpBLje92AJGnsM0wkScUME0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVKx/l430JSIeBGwF7AMWNXjdiRprOgDpgI/zMynun3RRhsmVEGypNdNSNIYtT9wa7cHb8xhsgzg6quvZsqUKb3uRZLGhOXLlzNr1iyof4Z2a2MOk1UAU6ZMYdq0ab3uRZLGmg1aHnABXpJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUMk5atHly5UY4ladO2MV+0OCqN7x/gxxee0MpYrzvjslbGkSRnJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIGlUGBwc3yrE2dm70KGlU6e/v51Of+lQrY51++umtjLMpcGainln1dHtb5Lc5lrQpcmainumbMMDCY49vZaw3f/HKVsaRNlXOTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUME0kapVauWj1mxvKjwZuopwdXMqF/YKMZR9oYDfSN50M33tzKWHOPOLDo9YbJJmpC/wCzrzyt8XHmH/+ZxsfQyFk9uIrx/X0bzThqzyYVJk+vXMWEgea/gdsaRxpp4/v7uGfe9xofZ/dTZjQ+htq1SYXJhIE+jj7j6sbH+dKFsxofQ5JGk8bDJCI+CUzOzNkRMR24DJgE3AKcnJmDEbE9sADYBkhgVmY+HhFbAlcDOwG/Bd6Vmcub7lmblsGVq+hvYSbZ1jhSLzQaJhHxRuA44Ka6tAA4ITNvj4jLgROBzwHzgHmZeW1EnA2cDZwJ/COwJDPfEhHHAJ8BZjbZszY9/QN9fOKs6xsf5+MXvLPxMTRyVq9ayfi+5j880tY4TWssTCJiK+AC4BPA7hGxAzAxM2+vD5kPnBcRlwEHAG/vqN9MFSZvqZ8DuAa4JCIGMtMtYCU1anzfALd87dzGxzngsObHaEOT15lcCpwF/KF+vC2wrOP5ZcA0YDLwaGYOPqf+rNfUzz8KbN1gz5KkYWgkTCLiBODXmbn4OWOt6Xg8Dli9jjp1feiYTuM6npMkjRJNneaaCUyNiLuBrYDNqQJjascxU4CHgIeBLSKiLzNX1cc8VB/zm/q4pRHRD7wE+H1DPUuShqmRmUlmHpyZf5mZ04F/AL6amccDKyJi3/qwY4BF9frHEtYurB8LLKq/Xlg/pn5+ieslkjT6tH2dySzgCxExCbgTuLiunwJcFRFzgF8BR9X1s4H5EfFT4I/16yVJo0zjYZKZ86k+oUVm3gPsvY5jHgRmrKP+CPC2RhuUJBVz12BJUjHDRJJUzDCRJBUzTCRJxQwTaRQYXNneJ97bHEubjk1qC3pptOofGGDux97fylgf+qdLWxlHmxZnJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGL9Tf7mEXE+8E5gDXB5Zs6NiIOAucBE4LrMnFMfOx24DJgE3AKcnJmDEbE9sADYBkhgVmY+3mTfkqQN09jMJCIOBN4A7AbsCXwwInYHrgAOB14F7BURh9YvWQCcmpm7AOOAE+v6PGBeZu4K/Ag4u6meJUnD01iYZObNwOszc5BqVtEPbAk8kJm/qOsLgCMjYgdgYmbeXr98fl0fAA4Aru+sN9WzJGl4Gl0zycyVEXEecB+wGNgWWNZxyDJg2vPUJwOP1sHTWZckjSKNL8Bn5jnA1sB2wC5U6ydDxgGr6z66qVPXJUmjSJNrJrvWi+pk5pPAl4EZwNSOw6YADwFL11N/GNgiIvrq+tS6LkkaRZqcmewEfCEiXhQRE6gW3S8FIiJ2rgPiaGBRZj4IrIiIfevXHlPXVwJLgJl1/VhgUYM9S5KGockF+IXATcBdwI+B2zLzWmA2cAPVOsr9rF1cnwVcFBH3A5sDF9f1U4CTIuI+YH9gTlM9S5KGp9HrTDLzXODc59QWA7uv49h7gL3XUX+Q6vSYJGmU8gp4SVIxw0SSVMwwkSQVM0wkScUME0lSMcNEklSsqzCJiJevo/bqkW9HkjQWPe91JhGxVf3lwoiYQbVnFsAA1fYouzbXmiRprHihixavAQ6uv/59R32QtVeuS5I2cc8bJpl5CEBEXJGZ722nJUnSWNPVdiqZ+d76BlZbsfZUF5l5Z1ONSZLGjq7CpL7B1UeotoQfur/IGqqdgSVJm7huN3o8Ftg5M72XiCTp/+n2OpNfGySSpPXpdmayOCIuBP4T+PNQ0TUTSRJ0Hyaz6/8e2VFzzUSSBHT/aa4dm25EkjR2dftprg+tq56Zc0e2HUnSWNTtaa6/6vh6AnAgsHjk25EkjUXdnuY6vvNxRGwLXN5IR5KkMWdYW9DXHxN+xci2Ikkaq4azZjIO2JPqanhJkoa1ZrIG+BXV9iqSJG3Ymkm92eNAZv5Po11JksaUbk9z7Ux19fu2wPiI+B1wWGb+rMnmJEljQ7cL8P8KXJiZL83MLYB/BC5pri1J0ljSbZi8LDOvGnqQmVcCWzfTkiRprOk2TPo77gdPRExm7X1NJEmbuG4/zfVZ4PaIuI4qRN4NXNRYV5KkMaXbmclCqhCZALwaeDlwY1NNSZLGlm7DZD5wSWaeCbwHOAu4oqmmJEljS7dhMjkzLwbIzBWZ+WlganNtSZLGkg1ZgN926EFEvIxqWxVJkrpegJ8L3B0RX6daOzkIt1ORJNW6mplk5hVUAXIX8CPgkMz8UpONSZLGjm5nJmTmvcC9G/KbR8Q5wLvqhzdl5hkRcRDVTGcicF1mzqmPnQ5cBkwCbgFOzszBiNgeWABsAyQwKzMf35A+JEnNGtb9TLpRh8abgD2A6cDrIuIoqk+BHQ68CtgrIg6tX7IAODUzd6Fajzmxrs8D5mXmrlSzorOb6lmSNDyNhQmwDDg9M5/OzJXAz4BdgAcy8xeZOUgVIEfWuxFPzMzb69fOr+sDwAHA9Z31BnuWJA1D16e5NlRm/nTo64h4JdXprs9ShcyQZcA0qt2I11WfDDxaB09nXZI0ijQ5MwEgIl4DfIvq018/59l7eo0DVtd9dFOnrkuSRpFGwyQi9gUWAx+tdx1eyrMvdpwCPPQ89YeBLSKir65PreuSpFGkyQX47YCvAEdn5rV1+Y7qqdi5DoijgUWZ+SCwog4fgGPq+kpgCTCzrh8LLGqqZ0nS8DS2ZgJ8GHgxMDcihmqfB2YDN9TPLWTt4vos4AsRMQm4E7i4rp8CXBURc6juPX9Ugz1LkoahyQX404DT1vP07us4/h5g73XUHwRmjGhzkqQR1fgCvCRp42eYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYv1NDxARk4DbgMMy85cRcRAwF5gIXJeZc+rjpgOXAZOAW4CTM3MwIrYHFgDbAAnMyszHm+5bktS9RmcmEbEPcCuwS/14InAFcDjwKmCviDi0PnwBcGpm7gKMA06s6/OAeZm5K/Aj4Owme5YkbbimT3OdCHwAeKh+vDfwQGb+IjMHqQLkyIjYAZiYmbfXx82v6wPAAcD1nfWGe5YkbaBGT3Nl5gkAETFU2hZY1nHIMmDa89QnA4/WwdNZlySNIm0vwI8H1nQ8Hges3oA6dV2SNIq0HSZLgakdj6dQnQJbX/1hYIuI6KvrU1l7ykySNEq0HSZ3ABERO9cBcTSwKDMfBFZExL71ccfU9ZXAEmBmXT8WWNRyz5KkF9BqmGTmCmA2cANwH3A/axfXZwEXRcT9wObAxXX9FOCkiLgP2B+Y02bPkqQX1vh1JgCZ+YqOrxcDu6/jmHuoPu313PqDwIwG25MkFfIKeElSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUME0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUME0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScX6e91ANyLiaGAOMAB8OjMv6XFLkqQOo35mEhEvBy4A9gOmAydFxKt725UkqdNYmJkcBHwnMx8BiIjrgXcC57/A6/oAli9f/qziU0/+sYEWn23p0qXP+/xvH1vReA/d9LHij0/2vIdHnhodfxaPP/GHnvfw2BN/bryHbvp4+NHf9byHxx57rPEeuunjd4883vMennyk+b+Pzj46fmb2bcjrx61Zs2aEWxpZEfExYLPMnFM/PgHYOzNPeoHX7QcsaaFFSdoY7Z+Zt3Z78FiYmYwHOhNvHLC6i9f9ENgfWAasaqAvSdoY9QFTqX6Gdm0shMlSqlAYMgV46IVelJlPAV2nqiTpGf+7oS8YC2HybeDciNgaeAL4O+B5T3FJkto16j/NlZm/Ac4CvgvcDXwpM3/Q264kSZ1G/QK8JGn0G/UzE0nS6GeYSJKKGSaSpGKGiSSp2Fj4aHDPjJYNJiNiEnAbcFhm/rIH458DvKt+eFNmntF2D3Uf51NtpbMGuDwz5/aij7qXTwKTM3N2j8b/LrANsLIuvT8z72i5h7cC5wCbAd/MzNPaHL/u4QTg1I7SjsC/Zeap63lJU328B/hY/XBRZn64zfE7+vgocDzwFHBdZl7Q1tjOTNZjtGwwGRH7UF18uUvbY9fjHwS8CdiD6s/hdRFxRA/6OBB4A7AbsCfwwYiItvuoe3kjcFwvxq7HH0f1/bB7Zk6vf7UdJDsBnwfeTvV38tqIOLTNHgAy87KhPwNgFvAwcG6bPUTEXwAXAwcCuwP71/9uWlWPeTSwF9W/130i4h1tjW+YrN8zG0xm5hPA0AaTbTsR+ABdXPXfkGXA6Zn5dGauBH4GbN92E5l5M/D6zBykekfeT3URa6siYiuqNxmfaHvszjbq/34zIu6JiFbfhdeOoHrnu7T+vpgJtBpo6/A54OOZ2c7OiGv1Uf0s3YzqLMYA0M6unc+2B/CNzHw0M1cBX6cK+1YYJuu3LdUP0iHLgGltN5GZJ2RmzzaszMyfZubtABHxSqrTXQt71MvKiDgPuA9YDPymB21cSnURbfPbDK/fS6n+/48A3gicHBEHt9zDzkBfRHw1Iu4GTqGHfyb1u/KJmfkfbY+dmY8BZwP3U23/9Euq09JtuxM4JCK2iogXA2+j2n6qFYbJ+g13g8mNUkS8BvgW8JHMfKBXfWTmOcDWwHZUs7bW1Ofnf52Zi9sc97ky8/uZeWxm/ql+F3458OaW2+inmr2/D/gbYB96eOoPeD/QkzW0iNgNeC+wA9Wb0FVA62sm9fflfOB7VLOSW4Gn2xrfMFm/pVQ7Zw7paoPJjVFE7Ev1TvijmXlVj3rYNSKmA2Tmk8CXqc7Vt2km8Kb6nfj5wNsi4qKWeyAi9qvXbYaMY+1CfFuWA9/OzN9m5p+BG4G9W+4BgIiYQLVe8dVejA8cAizOzIfrDWbnAzPabiIiXgLckJm7ZeYMqkX4Dd6wcbj8NNf6ucEkEBHbAV8BZmbmd3rYyk7AefV9atYAhwNXtNlAZj5zKikiZgMzMvPv2+yhtiVwfkT8LdX5+eOAk1vu4WvAVRGxJfAYcCjV90kv7Ab8d7222Qv3ABdGxGbAk8Bb2cDt20fIjsAXI2JPqvWb99W/WuHMZD3cYPIZHwZeDMyNiLvrX23/4CIzFwI3AXcBPwZuy8xr2+5jNMjMr/HsP4srMvP7LfdwB3Ah1amU+4AHgSvb7KHDTlRnEnoiM78JXEP1d3EvVcD/cw/6uBe4oe7hB1SXM/xXW+O70aMkqZgzE0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRBohETEjIn7yAsesiYjJG/j7zo+InuxCK3XLMJEkFfMKeGmERcQuwCXAS6i25LmbageBFfUhF0TEXlRv5ubUFyESEe+j2jBxPPB74NTMvL/t/qXhcGYijbwTgasy86+pdtfdEXhLx/M/z8zXAu+h2pJk6/p+LccB+2fmHlRXl9/Yct/SsDkzkUbemcDBEXEG1U2stgU273j+8wCZ+ZOIuI9q1939qILnto57fr20vn+KNOoZJtLIu4bq39a/U+2htT3Vzr5DVnV8PZ5qx98+qtvNngkQEeOpQqiX902RuuZpLmnkHQKcn5nX1Y/3oQqLIbMBIuK1VLORO4BvAEdFxNBtD06m2vZfGhOcmUgj7+PAjRHxBPAn4Gaq0BiyU0TcRbWV/rsz8xGqW/D+C/CtiFgNPAq8IzPX9OhW99IGcddgSVIxT3NJkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSr2fy5dZTfrIvsDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "Y_train = train[\"label\"]\n",
    "\n",
    "# Drop 'label' column\n",
    "X_train = train.drop(labels = [\"label\"],axis = 1) \n",
    "\n",
    "# free some space\n",
    "del train \n",
    "\n",
    "g = sns.countplot(Y_train)\n",
    "\n",
    "Y_train.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       784\n",
       "unique        1\n",
       "top       False\n",
       "freq        784\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.isnull().any().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       784\n",
       "unique        1\n",
       "top       False\n",
       "freq        784\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().any().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize the data\n",
    "X_train = X_train / 255.0\n",
    "test = test / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\n",
    "X_train = X_train.values.reshape(-1,28,28,1)\n",
    "test = test.values.reshape(-1,28,28,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\n",
    "Y_train = to_categorical(Y_train, num_classes = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the random seed\n",
    "random_seed = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the train and the validation set for the fitting\n",
    "X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQMAAAEBCAYAAAB8GcDAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADClJREFUeJzt3VuIHfUdwPHvurH1AqF9UHLz8qD5VUTS4iVVE2sxFVoECfFC86BS4wUv+OAFwXhJoEgfDIFoULSRBVEERR/USq2bQtUQpbha1P6QGqExS30QFAOabLJ9mP9pjnZ3z8mey+wm3w+Ie86cs/Nj3P06Mzu7MzA+Po4kHVH3AJJmBmMgCTAGkgpjIAkwBpIKYyAJMAaSCmMgCTAGkgpjIAkwBpKKOf1eYUT8EDgbGAX29Xv90mFgEJgPvJOZ37b7po5iEBGrgbXAkcDGzHykjbedDfytk/VKasty4I12XzztGETEQuD3wJnAt8BbEbE1Mz9s8dZRgJ2f7WZsn78xKXXbnMEBFi08Fsr3Wtvv62CdK4DhzPwCICKeAy4D1rd43z6AsX3jjI0ZA6mHDuowvJMTiAv4bnlGgUUdfD5JNeokBkcAzf9rHwD2dzaOpLp0EoOdVGcsG+YBuzobR1JdOjln8BfggYg4DtgNrAKu78pUkvpu2nsGmfkZcA+wFRgBns7Mt7s1mKT+6ug6g8x8Gni6S7NIqpGXI0sCjIGkwhhIAoyBpMIYSAKMgaTCGEgCjIGkwhhIAoyBpMIYSAKMgaTCGEgCjIGkwhhIAoyBpMIYSAKMgaTCGEgCjIGkwhhIAmq4Jbt6Z/vx50y6bMnIQ1O+d88TU98ic+59r01rJs0e7hlIAoyBpMIYSAKMgaTCGEgCjIGkwhhIArzOQMUP1tw35fKvWrzf6xBmv45iEBFbgeOBveWpGzJze8dTSeq7accgIgaAxcBJmTnWvZEk1aGTcwZR/v3niHgvIm7pxkCS6tFJDH4MvA6sBC4CboyIX3VlKkl9N+3DhMzcBmxrPI6IPwK/ATyTJM1C094ziIhlEXFR01MDHDiRKGmW6eSnCT8C1kfEecCRwNXAjV2ZSlLfdXKY8FJELAXeBQaBR8qhgw5Bra5DwOsMZr2OrjPIzHuBe7s0i6QaeTmyJMAYSCqMgSTAGEgqjIEkwF9hPqQs/fztSZd908c5NDu5ZyAJMAaSCmMgCTAGkgpjIAkwBpIKYyAJ8DoDdclUt4OHqa+B0MzgnoEkwBhIKoyBJMAYSCqMgSTAGEgqjIEkwOsMDht7nlg/5fKWfwq9hSUjD039ggXLO/r86j33DCQBxkBSYQwkAcZAUmEMJAHGQFJhDCQBXmdw2Pjo4S+nXL5kTW/Xf/WCcyddNrRrW29Xrra0FYOImAu8BVySmZ9GxApgA3A08Gxmru3hjJL6oOVhQkQsBd4AFpfHRwNbgEuB04CzI+LXvRxSUu+1c87gOuBmYFd5fA7wcWbuyMwx4Cng8h7NJ6lPWh4mZOYagIhoPLUAGG16ySiwqOuTSeqr6fw04QhgvOnxALC/O+NIqst0YrATmN/0eB4HDiEkzVLT+dHidiAi4hRgB7Ca6oSipFnsoGOQmd9ExDXA88BRwCvAc12eS122ec7glMsf6/H6bxrbN+myoR6vW+1pOwaZeXLTx68DS3oxkKR6eDmyJMAYSCqMgSTAGEgqjIEkwF9hPmy0+jXhXv9oUTOfewaSAGMgqTAGkgBjIKkwBpIAYyCpMAaSAGMgqTAGkgBjIKkwBpIAYyCpMAaSAGMgqTAGkgBjIKkwBpIAYyCpMAaSAGMgqTAGkgBjIKkwBpIAYyCpaPsmKhExF3gLuCQzP42IJ4FlwO7yknWZ+UIPZpTUB23FICKWAo8Di5uePgu4IDNHezGYpP5q9zDhOuBmYBdARBwDnAhsiYj3I2JdRHjIIc1ibX0DZ+aazPxb01PzgGHgd8DPgeXAtd0fT1K/TOvGq5n5CbCy8TgiNgFXUR1KSJqFprVrHxFnRMSqpqcGgL3dGUlSHaZ7S/YBYGNEDANfA9cDQ12bSlLfTWvPIDPfBx4E3gQ+BEYy85luDiapvw5qzyAzT276eDOwudsDSaqHPw6UBBgDSYUxkAQYA0mFMZAEGANJhTGQBBgDSYUxkAQYA0mFMZAEGANJhTGQBBgDSYUxkAQYA0mFMZAEGANJhTGQBBgDSYUxkARM/74JmmWuXnBu3SNohnPPQBJgDCQVxkASYAwkFcZAEmAMJBXGQBLQ5nUGEXE/cEV5+HJm3hURK4ANwNHAs5m5tkczqgtuGttX6/qXjDw06bKvnlg/5Xvn3vdat8fRBFruGZRv+ouBnwE/Bc6MiN8CW4BLgdOAsyPi170cVFJvtXOYMArcnpl7MnMv8BGwGPg4M3dk5hjwFHB5D+eU1GMtDxMy84PGxxFxKtXhwiaqSDSMAou6Pp2kvmn7BGJEnA68BtwJfAKMNy0eAPZ3dzRJ/dRWDCLifOB14O7MHAJ2AvObXjIP2NX98ST1S8vDhIg4AXgRuDIzh8vT26tFcQqwA1hNdUJR0izVzo8W7wCOAjZEROO5R4FrgOfLsleA53own6Q+aecE4m3AbZMsXtLdcSTVxSsQJQHGQFJhDCQBxkBSYQwkAcZAUuGfSj9MbJ4zOOXyx/o0x0RuffTrGteuBvcMJAHGQFJhDCQBxkBSYQwkAcZAUmEMJAFeZ3DYGNq1bcrlm1r8ufIfrLmvo/W/99PbJ1029PnbHX1udYd7BpIAYyCpMAaSAGMgqTAGkgBjIKkwBpIArzNQ0fK2594W/ZDnnoEkwBhIKoyBJMAYSCqMgSTAGEgqjIEkoM3rDCLifuCK8vDlzLwrIp4ElgG7y/PrMvOFHswoqQ9axiAiVgAXAz8DxoFXI2IlcBZwQWaO9nZESf3Qzp7BKHB7Zu4BiIiPgBPLP1siYiHwAtWewf6eTSqpp1rGIDM/aHwcEadSHS4sBy4EbgK+BF4CrgUe78mUknqu7d9NiIjTgZeBOzMzgZVNyzYBV2EMpFmrrZ8mRMT5wOvA3Zk5FBFnRMSqppcMAHt7MaCk/mjnBOIJwIvAlZk5XJ4eADZGxDDwNXA9MNSzKSX1XDuHCXcARwEbIqLx3KPAg8CbwJHA85n5TE8mlNQX7ZxAvA24bZLFm7s7jqS6eAWiJMAYSCqMgSTAGEgqjIEkwBhIKoyBJMAYSCqMgSTAGEgqjIEkwBhIKoyBJKCeuzAPAswZHKhh1dKhr+l7a/Cg3tf9UVqaD7Bo4bE1rFo6rMwH/tXui+uIwTtUf1B1FNhXw/qlQ90gVQjeOZg3DYyPj/dmHEmziicQJQHGQFJhDCQBxkBSYQwkAcZAUmEMJAH1XHT0PxGxGlhLdVemjZn5SJ3zNIuIrcDxHLiH5A2Zub3GkYiIucBbwCWZ+WlErAA2AEcDz2bm2hky15PAMmB3ecm6zHyhhrnup7prOMDLmXnXDNpmE81W63ar7aKjiFgIvAGcCXxL9cX028z8sJaBmkTEALATOCkzx+qeByAillLd5fonwGLgP0ACvwD+TXWH7I2Z+ac65yox+AdwcWaO9nOW7821AlgH/BIYB14FngD+QP3bbKLZHgbWU+N2q/MwYQUwnJlfZOZu4Dngshrnada4qeSfI+K9iLil1mkq1wE3A7vK43OAjzNzRwnWU8Dldc8VEccAJwJbIuL9iFgXEXV8nY0Ct2fmnszcC3xEFdGZsM0mmu1Eat5udR4mLKDaKA2jVF/gM8GPqW5BfyvVIcxfIyIz87W6BsrMNQBNN7+daPst6vNYE801DxgGbgK+BF4CrqXae+jnXB80Po6IU6l2yTcxM7bZRLMtBy6kxu1WZwyOoNpFahgA9tc0y3dk5jZgW+NxRPwR+A1QWwwmMCO3X2Z+AqxsPI6ITcBV9DkGTes/nepw4E5gjGrvoKHWbdY8W2YmNW+3Og8TdlJ+nbmYx4Fd4FpFxLKIuKjpqQEOnEicKWbk9ouIMyJiVdNTtW27iDifag/v7swcYgZts+/PNhO2W517Bn8BHoiI46jOnq4Crq9xnmY/AtZHxHlUhwlXAzfWO9L/2Q5ERJwC7ABWA1vqHQmovog3RsQw8DXVf9Ohfg8REScALwJXZuZweXpGbLNJZqt9u9W2Z5CZnwH3AFuBEeDpzHy7rnmaZeZLVLtv7wJ/B7aUQ4cZIzO/Aa4Bngc+BP5JdRK2Vpn5PvAg8CbVXCOZ+UwNo9wBHAVsiIiRiBih2l7XUP82m2i286h5u/n3DCQBXoEoqTAGkgBjIKkwBpIAYyCpMAaSAGMgqTAGkgD4L7zd+ZPnK68hAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Some examples\n",
    "g = plt.imshow(X_train[5][:,:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From D:\\Anaconda\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "# Set the CNN model \n",
    "# my CNN architechture is In -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*2 -> Flatten -> Dense -> Dropout -> Out\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n",
    "                 activation ='relu', input_shape = (28,28,1)))\n",
    "model.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n",
    "                 activation ='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2,2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "\n",
    "model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n",
    "                 activation ='relu'))\n",
    "model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n",
    "                 activation ='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(256, activation = \"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(10, activation = \"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the optimizer\n",
    "optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n",
    "# Compile the model\n",
    "model.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a learning rate annealer\n",
    "learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n",
    "                                            patience=3, \n",
    "                                            verbose=1, \n",
    "                                            factor=0.5, \n",
    "                                            min_lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 1 # Turn epochs to 30 to get 0.9967 accuracy\n",
    "batch_size = 86"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# With data augmentation to prevent overfitting (accuracy 0.99286)\n",
    "\n",
    "datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        zoom_range = 0.1, # Randomly zoom image \n",
    "        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
    "        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n",
    "        horizontal_flip=False,  # randomly flip images\n",
    "        vertical_flip=False)  # randomly flip images\n",
    "\n",
    "\n",
    "datagen.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/1\n",
      " - 95s - loss: 0.4054 - acc: 0.8701 - val_loss: 0.0802 - val_acc: 0.9743\n"
     ]
    }
   ],
   "source": [
    "# Fit the model\n",
    "history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n",
    "                              epochs = epochs, validation_data = (X_val,Y_val),\n",
    "                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n",
    "                              , callbacks=[learning_rate_reduction])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEBCAYAAAB13qL/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX9//HXJCEBGiGKSFhbRTyILCkCkUURErWILLKJINS64FKW/qxYkBQBA18VFTAirXWtAkURCsFIVTb9UiWIBVTkqMhiIAQVU0NiIJD5/TGTfIeQQDJLMnjfz8fDx2PunHPv/ZwJznvu7nK73YiIiHNF1HQBIiJSsxQEIiIOpyAQEXE4BYGIiMMpCEREHE5BICLicAoCERGHUxCIiDicgkBExOEUBCIiDqcgEBFxuKiaLqA8xpgYoDOQDZyo4XJERM4WkUBjYLO19mhlZwrLIMATAu/XdBEiImepK4H/rWzncA2CbICFCxcSHx9f07WIiJwVDh48yMiRI8H7HVpZAQWBMWYEkALUAuZaa+dX0K8v8LS19sJKLvoEQHx8PM2aNQukRBERJ6rSLnW/DxYbY5oCM4EeQAIwxhjTppx+jYDHAZe/6xIRkdAJ5KyhZGCttfawtTYfWAoMKaffc8D0ANYjIiIhFMiuoSacvB8qG+ji28EYMx74GPgwgPWISDUoLi5m165d/PjjjzVdipxBdHQ0TZs25bzzzgvK8gIJggjA9zmXLqC4ZMIY0xYYDCQB2tEvEuays7NxuVz8+te/JiJClxiFq+LiYgoKCti1axdut5sGDRoEvMxA/tpZeM5XLREPHPCZHupt/wjIAJoYY3RKqEiY+u6772jevLlCIMxFREQQGxtLy5Yt+frrr/nhhx8CX2YA874LJBljGhpj6uL59b+6pNFa+5C19hJrbQJwPXDAWntlYOWKSKgcP36c6Ojomi5DKqlu3bq4XC6WL19OcXHxmWc4Db+DwFq7H5gCrAO2AoustZnGmAxjTKeAqhKRGuFy6eS+s0VERAQul4u8vDwKCwsDWlZA1xFYaxcBi8q8d305/fYAvwpkXSLiHNOnT+fjjz+mqKiIffv20bJlSwBGjx7N4MGDK7WMefPm0bZtW5KSkirsM2DAAFasWBFQrZs2beLpp5/mlVdeCWg5/nK5XAFvEYTrlcUi4mAPPfQQAFlZWYwePdqvL+sJEyacsU+gIfBzoSAQkbNKWloaW7duJTs7m1tuuYWLL76YOXPmUFhYyI8//sjkyZNJTk5m0qRJdOnShS5dujB27FhatWrF559/ToMGDZg3bx5xcXEYY7DWkpaWRk5ODnv37mX//v0MHTqUe+65h6KiIh566CG2bNlCo0aNcLlc3HvvvSQmJpZb2+7du5k6dSq5ubnUrVuXKVOm0L59e9LT03nuueeIjIykWbNmzJ49mx9++IH777+fgoICIiIiSElJISEhoZo/TQ8FgYicYu1H+3gnc19Iln1Nlxb07tQioGUcO3aMjIwMAMaPH09qaiotW7bkgw8+YNasWSQnJ5/Uf+fOncyaNYs2bdowbtw40tPTGTVq1El9rLUsXLiQvLw8kpOTGTlyJCtWrOCnn35i9erVHDhwgH79+p22rokTJzJmzBiuvfZatm7dyoQJE/jXv/7F3Llzee2112jQoAGPPvooX3/9NWvWrOHqq6/mjjvu4L333mPLli0KAhGRymrfvn3p69mzZ7Nu3TpWr17Ntm3byM/PP6V/gwYNaNPGcwecVq1a8d///veUPomJiURHR9OgQQPi4uLIy8tj48aNDBs2DJfLRdOmTenatWuFNeXn57Nv3z6uvfZaABISEqhfvz5ff/01vXr14uabbyY5OZnrrruOSy+9lIKCAsaNG8fnn39Oz549ueWWWwL9WPymIBCRU/TuFPiv9lCqXbt26esRI0aQmJhIYmIiXbt25f777z+lf0xMTOlrl8uF2+2uVJ/IyMhKH4gtb5lut5sTJ06QkpLCzp072bBhAxMnTmTs2LEMGDCAN998k/Xr15ORkcHy5ct58cUXK7WuYNOVIyJy1srNzWXPnj1MmDCBq666ijVr1nDiRPCeZdWtWzcyMjJwu93k5OSQmZlZ4Sm2sbGxNGvWjLfffhuArVu38t1339GqVSuuvfZazj33XO666y4GDBjA559/zmOPPcbKlSu58cYbmTp1Kjt27Aha3VWlLQIROWvFxcUxZMgQ+vbtS1RUFFdccQWFhYUUFBQEZfnDhg1j586d9OvXj4YNG9KkSZOTtkbKmj17NtOmTSMtLY1atWqRlpZGdHQ048eP57bbbiMmJoYGDRrwyCOPcOzYMf74xz+ybNkyIiMjefTRR4NSsz9c5W3O1DRjzK+A3WvWrNHzCESqyZYtW7j88struoywsn79etxuN7169SIvL4+BAwfyxhtvEBcXV9OlAZ6/2caNG7ntttuIjY0lKyur5LqJC73Xb1WKtghERCrQsmVLHnjgAebOnQt4zlAKlxAIJgWBiEgFmjdvzuLFi2u6jJDTwWIREYdTEIiIOJyCQETE4RQEIiIOpyAQEXE4BYGInPUmTZrEsmXLyMnJ4c477yy3jzHmtMv45ptvePDBBwH45JNPmDJlSsB1paWlkZaWFvByQk2nj4rIz0ajRo3429/+5te8Bw4c4JtvvgGgXbt2tGvXLpilhTUFgYic4tDa9eSsWRuSZTdK6s0Fva8+bZ+xY8fSr18/rrvuOgAGDRpEamoqR44cKffZAyVKHmSzdu1asrKymDhxIgUFBXTo0KG0T05ODg8++CB5eXkcOnSIG2+8kQkTJpCamkpWVhbTp0/nN7/5TelTxyp6xsCkSZOIjY3ls88+Iycnh9///venfXraunXrmDt3LsXFxTRv3pwZM2Zw/vnn8+ijj7Jx40YiIiJITk5m7NixfPDBB8yePRuA+vXr88QTT3Deeef5/6GfgXYNiUjYKbkzJ8CePXs4evQobdq04dVXXyU1NZXly5eTmprKvHnzKlzGww8/zKBBg1ixYgUdO3YsfX/VqlXccMMNvPbaa6Snp/Pyyy9z+PBhUlJSaNu2benT0UpMnDiRUaNGkZ6ezuTJk5kwYQLHjh0D4ODBgyxatIgFCxbw2GOPVVjL999/z9SpU5k/fz7p6el07NiRGTNmsH//ft577z1WrlzJ4sWL+eqrrzh69CjPPPMM06ZNY9myZXTr1i3kN6QLaIvAGDMCSAFqAXOttfPLtN8ITAcigc3AGGvtsUDWKSKhd0Hvq8/4qz2UevbsyYwZMzhy5AirVq2if//+QOWePVAiMzOTJ554AoD+/fuTkpICwO23386HH37I888/z5dffklRURE//fRTucs43TMGALp3747L5eKSSy4hNze3wlq2b99O+/btS++ddtNNN/Hss8/SqFEjYmJiGD58OL169eL+++8nJiaGpKQkxo4dS3JyMklJSXTv3r2Kn2DV+L1FYIxpCswEegAJwBhjTBuf9l8ATwPXWGsvA2oDtwZUrYg4QnR0NL169WLt2rWsXr2aG264AfA8e2D79u20bduWu++++4zLKbmppsvlIiLC83X3yCOP8Morr9CkSRPuuecezj333HKfJeA7f9n3Sm51XfIMg4puTV2i7DMN3G43x48fJyoqitdff50JEyaQm5vL8OHD2b17N7feeiuvvPIKLVq0YPbs2SxYsOCMYw1EILuGkoG11trD1tp8YCkwpKTR+96vrLU5xpi6wAXADwFVKyKOMWDAAF588UXi4uJo2rRplZ890K1bN1auXAnA22+/zdGjRwHYuHEjt99+O3369GH37t3k5ORQXFxMZGQkx48fP2kZp3vGQFV06NCBbdu2kZWVBcCSJUtITExkx44d3HLLLXTu3Jk//elPtGzZkt27dzN06FDy8/O59dZbufXWW8N611ATINtnOhvo4tvBWltkjOkDvArsB94OYH0i4iCXX345eXl53HzzzUDVnz0wdepUJk6cyJIlS2jbti2/+MUvALjrrrt44IEHqF27NvHx8bRt25asrCwuvfRS8vLymDhxIkOGlP6mrfAZA1Vx/vnnM2PGDMaOHUtRURFNmjRh5syZXHDBBSQkJHDDDTdQp04dOnbsyFVXXUWdOnWYNGkSUVFR1K1bl9TUVD8/xcrx+3kExpgpQG1r7Z+903cCl1try91eM8bMwrOFMKISy/4Veh6BSLXS8wjOPsF6HkEgu4aygMY+0/HAgZIJY8x5xphrfdoXAu0REZGwEkgQvAskGWMaeo8BDAZW+7S7gFeNMSVPwB4K/G8A6xMRkRDwOwistfuBKcA6YCuwyFqbaYzJMMZ0stZ+D4wBVhljtgEG+FMwihaR0AjHR9dK+cqeiRSIgK4jsNYuAhaVee96n9f/BP4ZyDpEpHpERUVx7Nix0lMiJbwVFBQELQx0ZbGIAJ4zW/bu3RvUX5oSfMXFxRw5coQvvviCgwcP4na7S6+R8JfuNSQiADRu3JhPP/2Ujz/++IwXSEnNKi4u5uDBg+zfv59zzjmH2rVrB7Q8BYGIABAREUG7du348MMP2bRpU8C/MiW03G43sbGxDBw4UFsEIhI8LpeLrl270rFjRwoLC2u6HDmNiIgI6tatS2RkZMDLUhCIyCliYmJ00NhBtO0nIuJwCgIREYdTEIiIOJyCQETE4RQEIiIOpyAQEXE4BYGIiMMpCEREHE5BICLicAoCERGHUxCIiDicgkBExOEUBCIiDhfQ3UeNMSOAFKAWMNdaO79M+wBgOp4H2e8Gfmet/SGQdYqISHD5vUVgjGkKzAR6AAnAGGNMG5/2esACoK+1tgOwHZgWULUiIhJ0gewaSgbWWmsPW2vzgaXAEJ/2WsDvrbX7vdPbgRYBrE9EREIgkF1DTYBsn+lsoEvJhLX2e2A5gDGmDjAJSAtgfSIiEgKBbBFEAG6faRdQXLaTMaY+8CawzVr7cgDrExGREAgkCLKAxj7T8cAB3w7GmMbA+3h2C90RwLpERCREAtk19C4wzRjTEMgHBgNjShqNMZFAOvCatTY1oCpFRCRk/A4Ca+1+Y8wUYB0QDTxnrc00xmQAU4HmQEcgyhhTchD5I2uttgxERMJIQNcRWGsXAYvKvHe99+VH6II1EZGwpy9qERGHUxCIiDicgkBExOEUBCIiDqcgEBFxOAWBiIjDKQhERBxOQSAi4nAKAhERh1MQiIg4nIJARMThFAQiIg6nIBARcTgFgYiIwykIREQcTkEgIuJwCgIREYdTEIiIOJyCQETE4QJ6ZrExZgSQAtQC5lpr51fQ7+/AWmvtS4GsT0REgs/vLQJjTFNgJtADSADGGGPalOnTxBiTDgwJqEoREQmZQHYNJeP5lX/YWpsPLOXUL/yRwArgtQDWIyIiIRTIrqEmQLbPdDbQxbeDtXY2gDGmRwDrERGREApkiyACcPtMu4DiwMoREZHqFkgQZAGNfabjgQOBlSMiItUtkF1D7wLTjDENgXxgMDAmKFWJiEi18XuLwFq7H5gCrAO2AoustZnGmAxjTKdgFSgiIqEV0HUE1tpFwKIy711fTr9bA1mPiIiEjq4sFhFxOAWBiIjDKQhERBxOQSAi4nABHSwOoUiAgwcP1nQdIiJnDZ/vzMiqzBeuQdAYYOTIkTVdh4jI2agxsKuyncM1CDYDV+K5f9GJGq5FRORsEYknBDZXZSaX2+0+cy8REfnZ0sFiERGHUxCIiDicgkBExOEUBCIiDqcgEBFxOAWBiIjDKQhERBwuXC8oC2vGmBbAq8AFgAVGWmuPlOkTDTwPdAJ+AkZYa3f6tEcB7wN/tda+VE2l+y2QMRtjYoEXgNZ4nm0901r7j+qsv7KMMSOAFKAWMNdaO79MewLwHFAPeA+421p7vDKfT7gKYMzdgTlANPA9cJu1dm+1Fu8nf8fs0/5r4ENrbUz1VR062iLwzzPAM9ba1sBHwJ/L6TMeyLfWXgr8AXipTPtU4JJQFhlkgYx5ErDPWtseSAKeNMY0Cn3JVWOMaQrMBHoACcAYY0ybMt1eBcZaay/BE2p3et+vzOcTdgIc80LgDmttgvf1U9VTdWACHDPGmLpAGp4A/FlQEFSRMaYWcBWw1PvWS8DQcrr2xfM/B9ba94CG3l+NGGO6AR2A9FDXGwxBGPMGvF8S1tpDwGEgPrRV+yUZWGutPWytzccz3iEljcaYXwJ1rLUfet96CRhahc8nHPk75hggxVq73fv+dqBF9ZUdEL/G7DP/E8Dcaqq1WigIqu584EefzcRsoFk5/Zp42/DtZ4yph2dzekxIqwyugMZsrX3HWrsPwBgzDIgBPgthvf4qt/5KtFf28wlHfo3ZWnvUWvsqgDEmApgG/DO0pQaNv39njDH9gbrW2qX8jOgYwWkYY4bi+dL29SVQ9gZNxeXMHlGmn8vbbz4wy1qbY4wJVqlBE6Ix+y57HvAb3/2tYeS09Z+mvez7UP7nE478HTNQelzoZTzfJbNCV2ZQ+TVmY0w8nuMKySGvsJopCE7DWvs68Lrve97dAN8bYyKttSfw3OnvQDmzZ3HyrWDj8fyySALaGWOm49mU7m2MKbLWLgzRMKokBGM+4F3GOGAicK219pMQlR+oLDx3vS1RWr9Pe+Ny2g8B9Svx+YQjf8eM9ySAlXgOFA+w1haFttSg8XfMNwANgPdKfsQZY7YCV1pr80JZcKhp11AVef+xvw/c5H1rNPBWOV0zvG0YY3oAhdbavdbaJtbaBO8BtpXA1HAJgYoEOOZ9xpiBwP8DuodxCAC8CyQZYxp6DwgOBlaXNHrPiCn0ni0DMAp4qwqfTzjya8ze168CXwE3WWuPVmPNgfL37/yctbalz/+/eF+f1SEACgJ/3YvnTIMdeH5ZpAAYY+42xszw9kkDYowxn+E5UDqqRioNnkDGPB2oA6QbY7Z6/+tUveWfmbV2PzAFWAdsBRZZazONMRk+9Y4E5hhjdgKx/N+ZMuV+PuHO3zF7T58cAHQHPvb+TTNqYAhVFuDf+WcpLJ9H4D0joTN6MI2ISFWUPpimKltp4XqMoDOeTW0REam6K4H/rWzncA2CbICFCxcSHx+Op5uLiISfgwcPljzrPftMfX2FaxCcAIiPj6dZs7PldGwRkbBRpV3qOlgsIuJwCgIREYdTEIiIOFy4HiMQEeDYsWPs2rWLgoKCmi5FwkhUVBTnn38+jRs3JiIi8N/zCgKRMLZr1y7i4uIwxgTlf3g5+7ndbo4dO8bevXvZsWMHbdu2DXiZ+pclEsYKCgpo1KiRQkBKuVwuYmJiuPjiiyksLGTbtm0BL1P/ukTCnEJAyhMREYHL5WL9+vUUFQV2vz/9CxMROYu5XC6OHg3snn86RiAilTJ9+nQ+/vhjioqK2LdvHy1btgRg9OjRDB48uFLLmDdvHm3btiUpKanCPgMGDGDFihVBqVkqR0EgIpXy0EMPAZCVlcXo0aP9+rKeMGHCGfsoBKqfgkBEApaWlsbWrVvJzs7mlltu4eKLL2bOnDkUFhby448/MnnyZJKTk5k0aRJdunShS5cujB07llatWvH555/ToEED5s2bV3qGlLWWtLQ0cnJy2Lt3L/v372fo0KHcc889FBUV8dBDD7FlyxYaNWqEy+Xi3nvvJTExsbSe48ePM23aNL788ku+++47jDE8+eST1K5dm5deeonFixcTGRlJr169mDhxIvv372fy5MkcPnyY2rVrk5qaSmxsLKNHj2bt2rWlYwQYN24cV1xxBW3btuXbb79l6dKlTJ8+vVLruueee0hKSmLNmjXExsaSlZXFmDFjyMio2Tt4KwhEziJrP9rHO5n7QrLsa7q0oHcn/58/f+zYsdIvtPHjx5OamkrLli354IMPmDVrFsnJJz/hcefOncyaNYs2bdowbtw40tPTGTXq5Md2WGtZuHAheXl5JCcnM3LkSFasWMFPP/3E6tWrOXDgAP369Tullv/85z/UqlWLJUuWUFxczG9/+1s2bNhA48aNWbRoEW+88QZ16tThjjvu4NNPP+Wpp57iuuuuY+TIkWzYsIEFCxYwceLECsf6ww8/cOedd5KYmMjmzZsrva49e/Zw9dVXs3r1aoYMGcI///lPBg4c6PdnHiwKAhEJivbt25e+nj17NuvWrWP16tVs27aN/Pz8U/o3aNCANm3aANCqVSv++9//ntInMTGR6OhoGjRoQFxcHHl5eWzcuJFhw4bhcrlo2rQpXbt2PWW+zp07ExcXx8KFC/n666/Zs2cPBQUFbN68mV69enHOOecA8NJLLwGwefNmnnzySQB69uxJz549ycrKOu14O3To4Ne6Bg8eTFpaGkOGDGHVqlW8/PLLp11PdVAQiJxFencK7Fd7KNWuXbv09YgRI0hMTCQxMZGuXbty//33n9I/Jiam9LXL5aK8h2SV1ycyMpLi4uJT+vpas2YNTz31FKNHj2bQoEH88MMPuN1uoqKicLlcpf1ycnKoU6cOUVH/91XodrvZtWsXderUOamm48ePn9SvZLxVXVfnzp05dOgQb7/9Ns2aNaNRo0anHUt10OmjIhJUubm57NmzhwkTJnDVVVexZs0aTpwI3oMGu3XrRkZGBm63m5ycHDIzM0/6wgX44IMP6NOnD4MHD6ZevXps2rSJEydO0KlTJzZs2EB+fj7Hjx/nj3/8I59++imdOnXizTffBODf//43f/7zn6lXrx65ubkcPnyYY8eO8f775T8rq6rrcrlcDBw4kNTUVAYNGhS0zyUQ2iIQkaCKi4tjyJAh9O3bl6ioKK644goKCwuDdr+kYcOGsXPnTvr160fDhg1p0qTJSVsjAEOHDuX+++/nzTffpFatWnTs2JGsrCyGDh3KLbfcwvDhwykuLuaaa66hW7duXHjhhaSkpLBo0SLq1KlDamoq55xzDnfccQdDhgwhPj6edu3alVtPVdcF0LdvX1544YVTjpvUlHB9ZvGvgN1r1qzRg2nE0bZs2cLll19e02WElfXr1+N2u+nVqxd5eXkMHDiQN954g7i4uJourVKKi4tZvHgxu3fvJiUlJaBlbdmyhY0bN3LbbbeVnoXkvUbjQmvtnsouR1sEInJWadmyJQ888ABz584FPGconS0hADB27Fiys7N5/vnna7qUUgoCETmrNG/enMWLF9d0GX575plnarqEU1QqCIwxI4AUoBYw11o7v0x7H+BR7+QnwF3W2iPGmJ7AMuAbb9t/rLW/C0rlIiISFGc8a8gY0xSYCfQAEoAxxpg2Pu1xwMvAcGtte2AbMMvb3Al43Fqb4P1PISAiEmYqc/poMrDWWnvYWpsPLAWG+LS3AvZaa3d4p1cBJZfKdQauNcZsN8asNMY0D1bhIiISHJUJgiZAts90NuB7Ks+XQHNjTAfv9DAg3vs6F0jzbilkAP8IrFwREQm2ygRBBOB7jqkLKL2sz1qbC4wGnjXGbAYOAMe8bXdba5d5X/8FuMwYUz9ItYtINbr55ptLL7oqUVBQQGJiIocPH65wvlGjRrFp0yY++eQTpkyZckp7VlYWvXv3Pu26t2/fzuzZswHPlbzz5s3zYwRSkcocLM4CrvSZjsfzZQ+AMSYSyLLWJnqnOwO7jDERwGTgEWut72WFxwOuWkSq3eDBg0lPT6dv376l77399tskJiZy3nnnnXH+du3aVXhR1pl89dVXfP/99wAkJSWd9nkGUnWVCYJ3gWnGmIZAPjAYGOPT7gbeNsYk4gmI+4Al1tpiY8yNeHYdvWaMGQ1s8h5nEJGzTJ8+fXjsscfIzc0tPW9/5cqV/Pa3vwXgrbfe4sUXX6SwsJBjx44xa9YsOnbsWDr/pk2bePrpp3nllVfYsWNH6dZB69atS/t88cUXPPzwwxQUFHD48GHGjBlD3759eeqppygoKGDBggU0atSIzMxMHnnkEbZu3crMmTM5evQo5557LjNmzOCXv/wlo0aNol27dmzZsoXDhw+TkpJCz549TxpPeeu6+eabyc3NZcqUKXz99ddER0czadIkunbtSnp6OgsWLMDlctGuXTsefvhh/vKXvwCeW1MD9O7dm7///e9kZmayfPlycnNz6dWrFzfccEOl15WVlcWHH37IE088AXhufx0TE8OYMWMIlTMGgbV2vzFmCrAOiAaes9ZmGmMygKnW2o+MMXcBq4EYPMEx2zv7b4G/GWMeAg7h2YUkIn46tHY9OWvWhmTZjZJ6c0Hvqyts/8UvfkFSUhKrV69m+PDh5OTksHv3bnr06EFxcTH/+Mc/+Mtf/sJ5553H0qVLefbZZ0u/KMv605/+xKRJk+jevTvz589n06ZNALz++uvce++9dO3alW+++Yb+/ftz8803M378eDIzM7nnnntYtmwZ4Lnt9X333cfcuXNp3749b731Fvfddx9vvPEGAEVFRSxZsoS1a9cyb968U4KgonXNmzePFi1aMH/+fKy1TJ06lYsuuoj/+Z//YdmyZcTHxzNx4kQ2bNhw2s8zJyeHjIwMoqKimDlzZqXX9cILLzBnzhyOHDlCbGwsq1at4u9//3sl/4r+qdR1BNbaRcCiMu9d7/P6TeDNcub7DOgWYI0iEiYGDRrEvHnzGD58OOnp6fTv35/IyEgA5s+fz9q1a9m9ezeZmZlERJR/CPLw4cMcOnSI7t27ly6z5Mt70qRJvP/++/z1r3/liy++OO39ifbs2UO9evVKb3/dp08fpk6dSl5eHgBXXunZo92qVStyc3NPmb+idW3evJnHH38cAGMMS5YsYfXq1XTs2JH4eM95MCXHKz7//PMK62vTpk3p3Uqrsi7w3Ar7nXfeoXnz5jRv3jzkdyjVlcUiZ5ELel992l/toda5c2e+/fZbsrOzWblyJU8//TQA+fn5DBkyhP79+9O5c2eMMSxcuLDcZZS95XRJkAD84Q9/oF69evTq1Yvrr7+eVatWVVhLebeidrvdpXc6LbmFddk7k55pXWVvH71r165T3is5OO5yuU6qo6ioqPS1743wqrKuCy+8kMGDB7NgwQKaNWtWLXco1W2oRaRKBg4cyIIFC6hfvz4tWniejbBnzx5cLhd33303iYmJvPPOOxXeevrcc8+lSZMmrF+/HuCkL/uNGzcyfvx4kpOTee9UZjl5AAAJDklEQVS99wA4ceIEkZGRHD9+8nkmF110Ebm5uWzfvh2AjIwMmjRpUun7DlW0Lt9bUu/atYs777yTdu3asXXrVr799lsAZs2axZo1azj33HP56quvAM+ZTSXtgazL5XLRqVMnDh48yKZNm6rlDqXaIhCRKhk0aBC9e/dm5syZpe+1bt2aSy+9lD59+uByuejRowdbtmypcBmzZ89m8uTJzJ07l4SEhNL3x40bx4gRI4iJiaF169Y0bdqUrKws2rdvz9NPP83jjz/ORRddBEB0dDRz5szh4Ycf5qeffqJ+/frMmTOn0uOoaF3jx48nJSWF/v37ExUVxWOPPUajRo2YMmUKt99+O8XFxSQkJDBo0CB+/PFH/vWvf3H99ddz2WWXlT5xLZB1lWwhXHPNNeTm5hIdHV3pMflLt6EWCWO6DbXzuN1uioqK+N3vfseDDz7IZZddVmHfYN2GWruGRETCyLfffkv37t3p0KHDaUMgmLRrSEQkjFxwwQVs3ry5WtepLQIREYdTEIiEufJOkxQJ5r8LBYFIGKtbty45OTkKAynldrs5evQoX375JYWFhUFZpo4RiISxli1b8sUXX7B///4KL4wS53G73eTm5nLo0CHcbje1atUKaHkKApEwFh0dzaWXXsry5cvJysoiNjZWgSCAJwzy8vIwxgR8rYGCQCTMRUZG0q9fP/79739z6NChCq/YFWeJjIykdevWXHHFFQH/OFAQiJwFYmJi6NWrV02XIT9TOlgsIuJwCgIREYdTEIiIOJyCQETE4RQEIiIOpyAQEXE4BYGIiMMpCEREHE5BICLicAoCERGHUxCIiDicgkBExOEUBCIiDqcgEBFxOAWBiIjDKQhERBxOQSAi4nAKAhERh1MQiIg4nIJARMThFAQiIg4XVZlOxpgRQApQC5hrrZ1fpr0P8Kh38hPgLmvtEWNMHLAQuAj4FhhmrT0YrOJFRCRwZ9wiMMY0BWYCPYAEYIwxpo1PexzwMjDcWtse2AbM8janAu9bay8F/gbMC275IiISqMrsGkoG1lprD1tr84GlwBCf9lbAXmvtDu/0KmCg93VfPFsEAIuBPsaYWoGXLSIiwVKZIGgCZPtMZwPNfKa/BJobYzp4p4cB8WXntdYeB34EGgZSsIiIBFdlgiACcPtMu4DikglrbS4wGnjWGLMZOAAc8+lLRfOKiEjNq0wQZAGNfabj8XzZA2CMiQSyrLWJ1trOwH+AXd7m/d7+GGOigHOA74NQt4iIBEllguBdIMkY09AYUxcYDKz2aXcDbxtjmhpjXMB9wBJvWwaerQWAm/AcOC4KTukiIhIMZwwCa+1+YAqwDtgKLLLWZhpjMowxnay1xcBdeMLBAj8As72z/xm4whjzGXAv8PsQjEFERALgcrvdZ+5VzYwxvwJ2r1mzhmbNmp2pu4iIAFlZWSQlJQFcaK3dU9n5dGWxiIjDKQhERBxOQSAi4nAKAhERh1MQiIg4nIJARMThFAQiIg6nIBARcTgFgYiIwykIREQcTkEgIuJwCgIREYdTEIiIOJyCQETE4RQEIiIOpyAQEXE4BYGIiMMpCEREHE5BICLicAoCERGHUxCIiDicgkBExOEUBCIiDqcgEBFxuKiaLqACkQAHDx6s6TpERM4aPt+ZkVWZL1yDoDHAyJEja7oOEZGzUWNgV2U7h2sQbAauBLKBEzVci4jI2SISTwhsrspMLrfbHZpyRETkrKCDxSIiDqcgEBFxOAWBiIjDKQhERBxOQSAi4nAKAhERh1MQiIg4XLheUBbWjDEtgFeBCwALjLTWHinTJxp4HugE/ASMsNbu9GmPAt4H/mqtfamaSvdbIGM2xsQCLwCtARcw01r7j+qsv7KMMSOAFKAWMNdaO79MewLwHFAPeA+421p7vDKfT7gKYMzdgTlANPA9cJu1dm+1Fu8nf8fs0/5r4ENrbUz1VR062iLwzzPAM9ba1sBHwJ/L6TMeyLfWXgr8AXipTPtU4JJQFhlkgYx5ErDPWtseSAKeNMY0Cn3JVWOMaQrMBHoACcAYY0ybMt1eBcZaay/BE2p3et+vzOcTdgIc80LgDmttgvf1U9VTdWACHDPGmLpAGp4A/FlQEFSRMaYWcBWw1PvWS8DQcrr2xfM/B9ba94CG3l+NGGO6AR2A9FDXGwxBGPMGvF8S1tpDwGEgPrRV+yUZWGutPWytzccz3iEljcaYXwJ1rLUfet96CRhahc8nHPk75hggxVq73fv+dqBF9ZUdEL/G7DP/E8Dcaqq1WigIqu584EefzcRsoFk5/Zp42/DtZ4yph2dzekxIqwyugMZsrX3HWrsPwBgzDIgBPgthvf4qt/5KtFf28wlHfo3ZWnvUWvsqgDEmApgG/DO0pQaNv39njDH9gbrW2qX8jOgYwWkYY4bi+dL29SVQ9gZNxeXMHlGmn8vbbz4wy1qbY4wJVqlBE6Ix+y57HvAb3/2tYeS09Z+mvez7UP7nE478HTNQelzoZTzfJbNCV2ZQ+TVmY0w8nuMKySGvsJopCE7DWvs68Lrve97dAN8bYyKttSfw3OnvQDmzZ3HyrWDj8fyySALaGWOm49mU7m2MKbLWLgzRMKokBGM+4F3GOGAicK219pMQlR+oLDx3vS1RWr9Pe+Ny2g8B9Svx+YQjf8eM9ySAlXgOFA+w1haFttSg8XfMNwANgPdKfsQZY7YCV1pr80JZcKhp11AVef+xvw/c5H1rNPBWOV0zvG0YY3oAhdbavdbaJtbaBO8BtpXA1HAJgYoEOOZ9xpiBwP8DuodxCAC8CyQZYxp6DwgOBlaXNHrPiCn0ni0DMAp4qwqfTzjya8ze168CXwE3WWuPVmPNgfL37/yctbalz/+/eF+f1SEACgJ/3YvnTIMdeH5ZpAAYY+42xszw9kkDYowxn+E5UDqqRioNnkDGPB2oA6QbY7Z6/+tUveWfmbV2PzAFWAdsBRZZazONMRk+9Y4E5hhjdgKx/N+ZMuV+PuHO3zF7T58cAHQHPvb+TTNqYAhVFuDf+WdJzyMQEXE4bRGIiDicgkBExOEUBCIiDqcgEBFxOAWBiIjDKQhERBxOQSAi4nAKAhERh/v/ZpIXJl+SBtoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the loss and accuracy curves for training and validation \n",
    "fig, ax = plt.subplots(2,1)\n",
    "ax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\n",
    "ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\n",
    "legend = ax[0].legend(loc='best', shadow=True)\n",
    "\n",
    "ax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\n",
    "ax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\n",
    "legend = ax[1].legend(loc='best', shadow=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEmCAYAAADiNhJgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXd4FPXWxz+bUEOR3gRFBI4VAZWOFAlNUUG5KkVQQLGDXNu1oNiwN7x6XwRRUOyNJk1FEUFRBAU5NkR6bxJKSPL+MRMImGRLZpKdcD4885CdnT2/s7OzZ8/8yvmGMjIyMAzDMKIjoaAdMAzDCCIWPA3DMGLAgqdhGEYMWPA0DMOIAQuehmEYMWDB0zAMIwaKFLQDRvSISCJwM9AL5zMsBkwC7lXVfXmw+T5wMvCcqo6K8vVnAXeo6iWxtO81InIM8IGqts/h+R+Atqq6PX89MwoLFjyDyYtAeeBcVd0hIqWA14GXgb4x2jwW6ASUUtW0aF+sqguBuAicLuWBJjk9qaoN89EXoxASsknywUJEagNLgeqqujPL/mpAS1V9z826XgAaAhnANOA/qnpARPYCI4GOQHXgMWACsAAQ4EfgYuA3oLKqbnbtZwCVgb3AK0A9IB34DrgGOAcYpaqnRdu+qr6YzfvcCzwFdABKA/cBPYHTgbVAN1XdLSJXue0XAyoAI1X1RRH5zPXpR+BMIAX4CDgD6A18676f63F+NFq7j78HeqvqZ5F/KsbRiPV5Bo8zgaVZAyeAqq5X1ffch88BW3ACzVk4AePf7nPFgc2q2gInU3waSAW6AntUtaGq/p5L+92BMm7mdra7r84Rx0TVvoiUyKad4sB6VW0CvIqTVQ8BTgGOAS4UkdLAIKCrqjYCLsX5MQC4Msv7ScPt2lBVcbPkTB503/+twHicHwALnEZYLHgGj3TCf25dcIJAhtsH+pK7L5OP3P+/xwlSpaJofy5wqoh8DtwBPKOqv/nUfuaPwe/Aj6q6RlXTgRVABVX9GzgfOE9EHgDuwslSc+LLI3e4gbU3cDsQAh7J5fWGcRALnsFjAXCyiJTJulNEjhWRKSJSEudzzdofkwAUzfJ4D4CqZh4TyqGtkGu7WOYOVV0B1MUJMmWBWSLS7YjXedV+1sGv1COfFJGawA/A8ThB/e4c7GTydw77j3d9OhGnr9QwwmLBM2Co6lqcwaGxIlIWwP3/v8AWVd0DTAduEJGQiBQHrgZmRtnUJpxbbnBG9XHbuhanz3OGqt7uttX4iNd60X4knOX6+SAwAycLzZw5cABIFJGcAjPuseVwzmd/YCIwxgc/jUKIBc9gch2wDJjnTrlZ4D4e6D5/E1AFZ7DkR0CBh6Js4ybgBRH5Hmf60jp3/2tAIrBMRL7D6X98LpvX5rX9SJgBrHbt/wwchxNM67r+fgMsFZGKudgYDUxW1Rk4g1J1ROQ6H3w1Chk22m4YhhEDlnkahmHEgAVPwzCMGLDgaRiGEQNxtzzTHZ09G6fDP+plgoZh+EYizqqwb2OtoZAdIlIBZ9pbJOxU1a1etZ0X4i544gTOf0xmNgwjbmiNM682z4hIhTSKbEnkQKQv2SYideMhgMZj8FwHsLFOL9KKRfpjFDmLnuvpuU2j4PBztkgolOsU0aOODevXc+UVveHQtDUvKJvIATYUP4sDoexW6R6iSMZequ5bWB4nS7XgmQ1pAGnFypJWrJznxo89tqbnNo2Cw4JngeB5d9qBhCTSEkrmflB6fA3RxGPwNAzjaCMEhPuxirPfMguehmEUPKEEZwt3TBxhwdMwjIInIdHZciXc8/lLfIXyXKhctgT64r+oX+MY6lQrw8wHujJjRBeeGdT8YLb/+JVN+XJkN6bd15mz6laKua309HRuvG4wbVo1p+O5bfn9tyMrrsWOn7b9th9U26mpqQzofwUd2p1D6xZNmTzpY89sB/Wc+H0dRk/IuW3PbYuz+/ZABM8iiSGeu6YFe/Y70xlG9mvCiInf0/HeaYSA888+js6Na1KvRlnOuXMSfZ78jKcHNo+5vY8/+pC9e/cyZ+7XPPDQSO64bZhH78Rf237bD6rtiW9MoELFCsz67As+nDSVW4bc6JntoJ4Tv6/DqAmFDt2657hZ8IyaR65owpgZy1m3LQWARnUq8uWy9QDMWLSGdqfX4OSa5Zj1wxoyMmDLrn2kpWdQtVyY0bscmPfVXJI7dQagabNmfPfdwjCviA/bftsPqu0eF/fk3vseOPi4SBHvequCek78vg6jJlzWeTD7jB98DZ4i0ktElonIryJyfSw2+rSty6Yde5m1eO3BfaEs6fvfe1M5JqkYS/7cSnLDmhRJDFG7SmlOrlWOpOKxfUl27dzJMcccc/BxYmIiBw5EPIm3wGz7bT+otkuXLk2ZMmXYtWsXvS/ryfAsgTSvBPWc+H0dRk3YrDOCAaV8xrcBIxE5FqeG45k4FcHnichnqrosGjt929UjA2jXoDoNaldg9I2tqXzMocm0pUsUZXvKfmYvWUvjupWYOrwzP67cxqI/trB1V2wryMqULcuuXbsOPk5PT/csW/HTtt/2g2obYPWqVVzaswdXD76WSy/vFf4FERLUc+L3+Y6aSDLLoyjz7AB8qqpbVXU38C4xSNN2Gj6NzsOn0eW+T1jy51YGPf8lMxatpvUp1QDo2OhY5v28gbrVy7Jpx1463juNpz78kfT0DHak7I/J8eYtWjJ92lQAFsyfz2mnnR6Tnfy27bf9oNresGED3bp24sGHR9Kv/1We2YXgnhO/r8OosczzMGpw+DKudeSiox0Nd772LaMGt6RYkQR09XY+mP8nRRMTSG54LP3a12Nvahq3vDw/ZvsXXtSdT2fNpG3rFmRkZPB/L7/ihdu+2/bbflBtP/7ow2zbvo2RDz/IyIcfBODDSVMpWTK2PvGsBPWc+H0dRk0AM0/fKsmLyF1ACVW9x308CDhTVQeHeV1tYMW6kwb7sjxzy8QrPbdpFBy2PDP/WLNmNV07ngtwgqr+6YXNzO/7mkrdSEvMTfgUEtP+5tjNkzxtPy/4mXmuxqm+kkk1YG0OxxqGcTSTkAiJYSbBZ8TXJHk/g+cs4D4RqQzsBi7GUVE0DMM4nAAuz/TNG1VdA9wFfIajrf2Gqn7jV3uGYQSYAM7z9HVugqq+AbzhZxuGYRQCMlcYhTsmjrDCIIZhFDwBHG234GkYRsETwD5PC56GYcQBkfRpWuZpGIZxOJZ5GoZhxIDJcHjHoud6+iLWVv7sGzy3mcmm+c/5ZhugSGJ8/fLGA0FeBWSro7IQSSX5sJXm85e4DZ6GYRxF2G27YRhGDFjwNAzDiAHr8zQMw4iFSOp1WuZpGIZxOLbCKP9IT0/n5huuY8mSxRQvXpwX//cyJ9atG5OtyuVLM++N2znv2lHcPbgrVSuWBeD4GhX45sc/ueKOV3jnmWuoeEwSqQfS2bNvPxfd8GJMbaWmpnLNgP6s/GsliQmJPP/i/xA5KSZb2eHleTHbBWs7NTWVwYMGsHLln+zbt4/b77yL87td4IltP/2OCevzzD+ySqcumD+fO24bxjvvfxS1nSJFEhh19+Xs2ZcKwBV3OBW1y5UpySejb+a2J94D4MRalWh88UN59nv6J1M5kHaA2Z/P5dNZMxkx/G5ef/PdPNvNxKvzYrYL3namZPKYca+xZcsWmjdp7Fnw9NPvmAhg5hlfoTwKvJJOHTm0O6Pfncu6TTsO23/Ptefx4ptzWL95J1UqlKFcmSTee3Yws8cOpUvr02L2u269+hw4cID09HR27dpJ0aJFY7aVHUGVqzXb/ySoksmxEAqFItriCd8zTxEpC8wDzveydH5O0qnRXGB9ujVl07a/mfX1z9x6VceD+yuXL03bJsKtbtZZrGgiz46fzag3PqdC2SQ+HXcLC3/6k03b/o7a79KlSvPXypU0bnAKW7ds5u33P47aRm54cV7MdnzYLl3akaXIT8nkglLQdBLP3INjnMVO33XbmwJzgfpe2/ZCOrXfRc05t9lJTB99Mw3kWMY80JeqFcvQvUMj3pq2kPR0ZwXI+i07Gf3OXNLS0tm07W8WL19N/dpVY/J71PPPcG6Hjvzw03LmfbOIwQOvZO/evTHZyo6gytWa7exZvWoVnZPbc3nvPoGRTI6FUEIooi2e8Pu2fRBwPT5oF3khnZo84Bk6DnyWToOeZYmuYcA949mwZRftmwozvlp68Lj2TU9iwmOOZG2pksU4pW51lq9YH5Pf5cuVp6z7i1++QgVSU1NJS0uLyVZ2BFWu1mz/k6BKJsdCiAhu22Oc6CkiT4jIOPfvhiKyUER+EZGXRaSIu/84EflCRJaLyEcikrsaHf5Xkh/oOua5bT+lU+vVrsqK1VsOPp7x1TKSm5/MnFeHkZ6RwfDnJ7Fl++6YbF9/0xCuu2YAHdu3ITV1P8NHPEipUqW8cj2wcrVm+58EVTI5FiLp04ylz1NEzgX6AVPcXROAgao6X0TG4CR4LwL/Bf6rqm+KyD3APcDtufrjZ3GCTETkT6BtJH2emVKkU2fMtsIgR2CFQQoXQSsM4qf08KZTbyCteO5S44n7tlN56ShwVHlXH/H0dlXdfoTtCsBU4C3gDGA48Kmqnug+3xq4H+gEbAEqqOoBEakFzFHVOrn5Y99GwzAKnlCEm8OXwIojtiHZWP0fjgjlNvdxDWBdlufXATWBSsBOVT1wxP5cCew8T8MwChGRTEU69Hy2mWfWByIyEFilqrNFpL+7OwHImu6HgPRs9uPuzxULnoZhFDhR9nmujqDb4FKguoj8AFQASuMEyOpZjqmGM5i9EThGRBJVNc09Juwgd77ctqtqbS/neBqGUbjwepK8qiar6mmq2hC4F/hYVa8E9opIS/ewvsA0VU3F6Qq41N1/BTAtXBuWeRqGUeDk4yT53sBod/HO90DmKO91wKsicjfwF3B5OEMWPA3DKHBCofCT4GOdQaCq44Bx7t+LgSbZHLMSaBuNXQuehmEUOH7N8/QTC56GYRQ4FjwNwzBi4fB5nDkfE0ccdcFz27ejfLNd46o3fLMNsHasd4UhjPBkFobxizhLpAoUyzwNwzBiIbpJ8nGBBU/DMAqcUASV5C3zNAzDOIIQEQTPOOv0tOBpGEbBYwNGhmEY0RMKEcFte/74EimBLUmXnp7OjdcNpk2r5nQ8ty2///ab5218s2ABHc9tG/PrE0Ihnh/YlGl3JzP5Px2oXaU0DY4vz8zhnZhyVwdG9j3z4AVx1yUNmDm8IzPu7UjjOhUL1O+c8POcB9U2QPMmjemc3I7Oye24ZpB3Fd9TU1MZ0P8KOrQ7h9YtmjJ5krd6V+DftRItCQkJEW3xRGAzT7+lU5984jEmThhPUh6qvHdudCwAXR6cScuTqvBgr8ZUL1+SO8d/xze/beY/Fzfgkua1Wb5mB2edWInk+2dQq1IpXh9yDufcHbYugW9+50RQJXz9tJ2pP/XJzM88sZcVP6WHwd9rJWoCeNseX6E8CvyWTq1T50TefOf9PNmY+v1qhoz9BoBalUqxacdeapRP4pvfNgPwza+baFa/Mj+u3MYljztfvloVneMK0u+cCKqEr5+2f1yymD0pKXTr2okunc7lmwXzPbPtp/Qw+HutREsQpYf9Vs8cLiJL3e0xL23nJJ3qFd17XOyJpnpaegYvXN2MR/uexcff/sXKTX/TQqoA0KnRsSQVL3LwuLsuacDEW9rw7td/Frjf2eHnOQ+q7ZJJSdw8dBgfT/mE555/kav69fHMdunSpSlTpowv0sPg77USLRY8syAiHYCOQCOgIXCmiHT3yn68SafmxvX/N5+zb5vEM1c1Zdi4bxna7RTevKUNm3fuY+uufQePe+jdJZx68wfc2PVkalcJK96X7wRVwtdP2/Xq1eeyXn0IhULUq1+fChUrsn7duvAvjBC/pIfjj0gC51ESPHF0QIap6n632OjPwHFeGY836dTs+FeL2gw5/xQA9uw7QHpGBh0b1uDGlxdw2VNzKF+6GJ/9tJ7WJ1flsSvOAmBvahqpaRm+Lw2MhaBK+Ppp+7VxY7nztmEArFu7ll07d1KtevUwr4oMP6WH440gZp6+pWqqelD4XETqAf8CWub8iuiIN+nU7Ji8cBWjBjVj8n86ULRIAv95/TvS0+GtYW3Zs/8AX/68gVlL1pIQCnFhk+OYdncyiQkhxsz+hb82xyZt7CdBlfD103a/Kwdw9cAr6dCuNaFQiBf/b4xnWa2f0sNxRwAHjHyXHhaRU3E0k4er6qsRHF8bH6WH/cQKgxQuglwYJGjSw6nt7oGkCrkfnLKVop894Gn7ecHXTkJXK+Q9YIiqvulnW4ZhBJdIJsnHW+bpW/B0heM/BC5V1U/9ascwjOCTkBCCMDIcYZ/PZ/zMPP8NlACeEpHMfS+p6ks+tmkYRgCJaDA9vmKnrwNGNwM3+2XfMIzCQyRVleItesbnxEjDMI4uIoidGfEVOy14GoZR8CQkhJcezkgIkZ5P/kSCBU/DMAqcCArJx9tduwVPwzAKnohWEB0tK4wMwzAixTJPwzCMGLDM00MyMjLwY+mon8UF/F4+WaXva77Z3jj+Ct9s+7nMMSHOJk5Hg58ro+MszkRA+OCZEWepZ9wGT8Mwjh4SEkLhfwjj7IfSgqdhGAVOJH2e8ZZNW/A0DKPAcYJnsNQzLXgahlHgBDHzDKwAnJ+yrEGQNc6kUtkSLBt1MfVqlKVO1TJMH96ZT4Z34qmrmh52sdWpWob5j3XLU1tBlfDNj89z48aN1D/xOHT58kDZjhfpYaskn4/4KcsaBFljgCKJIZ4d2Iy9+9MAeLjvWTzw9iLm/ryBpwc05bwzazF54Soua1WHwV1OomKZEnlqL6gSvn5/nqmpqdx0/WBKlvC+wruftuNJetgyz3zET1nWIMgaAzzU+yzGzvqFddtSAGh4QkXm/rwBgJk/rKXd6Y6Wzvbd++g6Ykae2wuqhK/fn+edt/+bAYOuoXqNGp7a9dt2PEkPmwDcEYjICBFZ5koP3+KlbT9lWYMga9zrnBPZvGsvs5esPbgv6y/z33tTKZtUDIBPFq0hZV/e/Q+qhK+ffo9/bRyVK1cmuWMnT+zll22IN+nhyLZ4ws9K8m2A9kADoCiwTESmqKp61cbqVau4tGcPrh58raeyrEGQNe7bti4ZZND2tOqcfnwF/u/aVlQue+i2vHSJouzYvd/TNv2W8D3xxLr/kPCtWatWnm376ff4V18hFArx2aezWbL4BwYN6Mfb731EtWrV4tp2vBFJn2a89Xn6lnmq6hygnaoeAKrgBGrPJCH9lGUNgqxxlxHT6TpiBuc9MIMfV27l6hfnMvOHNbQ6uSoAyQ1rMG/5Rk/bDKqEr59+z5g9h+mzPueTmZ/R4IyGjB7zqmfBzU/b8UbmJPlwWzzhazqlqqkicj+OJMc7wBqvbPspyxoEWePsuGvCQp67ujnFEhPRtTv4cMFKT+0HVcI3qJ/n0UQQB4x8lx4GEJEkYBLwlqr+X5hjawMrpkyf5Yv0cLyl/tFga9v/iZ/ZiN/Sw37ix3nxU3q44uWPklimUq7Hpu3azJaJt0fcvoiMAC4BMoAxqvqUiHQAngJK4sSju91jGwIvA2WBL4DB7l1zjvh22y4iJ7kOoaopwPs4/Z+GYRiH4fU8zyPGXM4CbhSRM4CxwIXAycDZItLFfckE4AZVrY8zrD8oXBt+jrbXAUaLSHERKYbj8Fwf2zMMI8B4OdKew5hLOeBXVV3h7p8A9BSR44GSqpo5P24c0DNcG36qZ04VkSbAIiANeE9V3/SrPcMwgkuUo+01s8iZZ7JdVbdn3ZHNmEsNYF2WQ9YBNXPZnyt+DxjdB9znZxuGYQSfKAeMvszm6fvJJtao6nAReRRnzKU+Tv/nQZNAOs4deHb7cyW+Ji8ahnFUEmVVpdbA6iOePizrFJGTgBKq+oOqpojI+ziDR2lZDqsGrHVtVc9mf65Y8DQMo8CJMvNcHcFoex3gfhFphZNVXgj8D3hcROoCK4BewFhVXSkie0Wkpap+BfQFpoXzObBr2w3DKDwkJoQi2iJFVacCU3DGXL4D5rljLv2B94BlwHLgXfclvYGnRWQ5UBp4LlwblnkahlHw+CAAl92Yi6rOBs7I5tjFQJNo7FvwNAyjwEkgvERRvN0m5xg8RSTXtFVVb/LeHcMwjkaCWBgkt8xzS755kQ1+VY72czmq3x+un0soy3d6xDfb26bf6Zvt/Fhe7Bd+Li09kBZ2pk3UpKX5+d0J3tr2HIOnqt6f+beIlATqAktxhv9T8sE3wzCOEkLuv3DHxBNhuxFEpCnwO87IVQ1glYi08NsxwzCOHhJCkW3xRCR9sE8AHYAtqroaZw7Us756ZRjG0UUkRUHi7L49kuCZpKrLMh+486dslN4wDM8IogxHJMEzVUTK4679lGxW5BcEfsrJ+ilrnIlfkq9enpfK5ZL4deL11K9V4eC+S9ufwufPHxq4urLrGcz9b3/mPH8FXZrVjQu/jyQtLY1rBl1F+zatSG7fhj9+/90z2xA8yeRvv1lAl+T2APz++28ktzuHju3bMOTG60hP936gKRISQqGItngikuD5IDAHqCUiE4F57r4CJauc7AMPjeQOV8LBCzJljWd99gUfTprKLUNu9Mw2OJKv110z8KDcrpd4dV6KJCYwamhn9uw/VA+2wYlV6NfljIPd9lXLl+K67mfR/ubxdLvjLUYMaEOxookF6nd2TJk8CYBP58zlnuH3c/ut3tnOKpn8yczP+N/osZ7Z9uOcPP3k49xw7dXs3ef4fedtw7j3vhHM+HQOGRkZTJ7knSRzNCSEIpDhCFrwVNXJQA/gXuAroJWqvue3Y+HwU07WT1lj8Ffy1avzMnJwe0ZPWsS6zY5wWoWyJXlgYDtu/e+sg8ecdVJ1vl66mv2paezcvY8/1m7j9DpVCtTv7Ljgwot44UVHwOCvlSupUjU2H7MjaJLJderU4fW33j34+IdF39PqnDYAJHfqzOefzs5zG7FQWG/bwVG/TARS3S1iROQJERkXpV9h8VNO1k9ZY/BX8tWL89Kn0+ls2p7CrIUrHBsJCbz0767c9uIsdqUcUuQsU6o4O3fvO9R2yn7KlipeYH7nRpEiRRh0VX+GDb2J7j0u8cxu0CSTL+x+MUWLHLr2MjIyDs5PLlO6DDt37MiT/VgJhcLfugcueIrIlcBnwNk4paC+FJGLIzEuIucC/fLkYQ74LQ+8etUqOie35/LefTyVNfYbL85Lv84NOPfME5j+ZC8a1K3KwpcHcuoJlXnu5k6Mv/tCTjq+Eo9f14Fdu/dRuuShYFkmqRg7/o6tKyI/5J5Hjx3H4qXK9ddeze7d3gi51qtXn8t69fmHZLIX5Mc5SUg4FAJ2/b2LY8qV89R+pIQi3OKJSDLPW4BGqtpPVfsATYHh4V4kIhWAh4CH8+Zi9vgpJ+unrLHfeHFekoe+TsdbXqfTsDdY8tsGGl31f5za9yU6DXuDvg9+xPKVm7n1v7NYuHwdLU+vSfGiiZQtVRw5rhJLV2wqML9z4o0J43n8UWcFVVJSEgkJCSQmxtY3eyRBlUzOpMEZDflyzucAzJz+CS1atvK8jUjwWsMoP4jkZ2y/qh78KVXVv0Qkklv3/wF3AbVidS43/JST9VPW2G/yU2Z3w7bd/PeDhcx+pi+hBLhv7Bz2paaFf2E2+On3hd17cM3Aq0hu34bU1FQee+JpSpQo4YntoEsmP/zoE9x47TXsv/cuRE7iIg+7NKIhkknw8TZJPkfpYRFp7P55A5CCEwzTcOrhlcitMIiIDAROUdVbRKQ/0FZV+0fiUKYU6dQZs32RHg7y2nY/sbXt2dn2zTQQvLXta9esoVuXDuCD9HCDG0dRvFzuA3n7tm9kyfM3eNp+XsjtJ/LIEfXzsvydAeRWVelSoLqI/ABUAEqLyNOqOjQ2Nw3DKMwUtsIgJ8RqVFWTM//Oknla4DQMI1sKW0k6AESkEs569tI4A16JQF1V7e2zb4ZhHCUkhMLLbMTbJPlIerbfBvYApwIzgWSyl/7MFlUdhyMibxiGkS2RTEWKr9AZ2VSl41X1PGAqMApoCZzkq1eGYRxVFNa17evd/38FTlPVNTgrjgzDMDwhiMszI7lt3ygitwJf4+gg7wSS/HXLMIyjiSAOGEWSeV4D7FPVucBCYARwu69eGYZxdBFJ1hlfsTN85qmqG3EF4FX1dixwGobhMZH0acZbn2du0sO7cAsgZ4eqlvXFI8MwjjoK1SR54LR88yIfibd+k3jBzyWU9Yb4V2D3l6cv8M22n8sn/aZIYqTVJiMnMdG/8xEi/Hcz3j6N3FYYrcxPRwzDOHpJIPwAjPc/B3nDhNwMwyhwEhIiWGEUZ3cCFjwNwyhwgliSLqLgKSIlgbrAT0BJVU3x1SvDMI4qCuU8TxFpBvwOTAGOBVaJSAu/HQuHn1K1ftrOxC/pYT9te3FeEkLwRO+GvD+0Fe8OacnxlZKoWLoYY65uwrtDWvL+0FYcX8lZg9H2lCp8NKw1Hw1rzUP/ahCz335KDwf1WskPv6MhM/MMt8UTkWSejwMdgNdVdbWI9AWexdE0KjCyyrIumD+fO24bxjvvezOq66dtcKSHJ04YT1KpUp7ZzA/bXpyX5NOrAdDj6bk0q1eRe3ucxo6UVD74djWTF62leb1KnFi1DJt37efui06l57NfsW33fgZ3qEuF0sXY+vf+MC38k6zSw1/M+Zzbbx3GO+9/GLWd7AjqteK339ESxKlKkQxgJanqsswHqjqVOOgr9VOq1k/b4K/0cLzLGk9fsp7bJy4GoGaFJDbt2sdZdSpQvXxJ3rihBd3PrsnXv27mrDrlWb52J/f0OJX3hrRi8659MQVO8Fd6OKjXit9+R0sogqIggbttB1JFpDzuhHkRkUiNi8hnIrJURH5wt6axOnokfkrV+i2D66f0cLzLGgOkpWfwVN9GjLjkdKYuWkvNiknsSEml16h5rNmawnXJ9ShfqjjN61fikQ+X0fe/XzOgbR1OqBJ79uWX9HBQrxW//Y6WhAi3eCKSDPJBYA5QTUQmAh2Bq8O9SERCQH2cknaefyp+yrLmh+RrEPHyvNwyfhGPlFnGx7dVkl2+AAAfDklEQVSew86UVGb86GgMzvppA7d1O5nv/tjK4pXb2bTL0YVf8NsWTj32GFZsjF0yePTYcTywfiRtWjXj+8VLKeXBrXBQr5V489uZJB/+mHgibDBX1clADxy54a+AVqp6pL5RdmRmqDNEZLGI3BC7m//ET1nW/JB8DSJenJceZ9fk+o71ANiTmkZ6egbzf9tC+1OqAtC0bkV+WbeTJau2I9XLUL5UMRITQjQ+oQK/rt+Vm+kc8VN6OKjXSrz5HcR6npHIcFQAtgJvZd2nqlvDvLQ8MBu4Eaf+5+cioqo6Mw/+HsRXqdp8lO8NEl6cl2mL1/Fkn0a8O6QlRRITuP+9n1i6egeP9W5I39YnsGtPKjeO+44de1J5dNLPTLi+OQCTv1+DrostePopPRzUayXe/E5IgHArShPi7L49R+nhTEQknX8WCFmnqlHpAovIUOC4cEJwfksPG/lPUNe2x9sARUGzZs1qunY8F3yQHr7w/rGUrlg112P/3rKBj4Zf5Wn7eSGSknQH472IFAN6ceiWPEdEpBVQXFVnu7tCQGqMfhqGUYgprFOVDqKq+11Bt+RwxwLlgMdFpISIlAH6AR9E76JhGIWdQjlJ3u3zzCQEnIXTn5krqjrZnZq0CEeu+AVV/TpWRw3DKLyE3H/hjokGERkO/Mt9OEVVbxORDsBTQEngLVW92z22IfAyUBb4AhgcbpZQJHMTNuP0eWZ6vhG4KRLnVfUe4J5IjjUM4+glFEFmGc1tuxskOwKNcOLXJyJyOfAo0AZYBUwRkS6qOg2YAAxU1fkiMgYYBLyYWxuRBM+zVfW7yN02DMOIjgQiqKp06M+a2azV2a6q27M8XgcMU9X9ACLyM868819VdYW7bwLQU0SW4RQ8mu++dhxwPx4EzwnAyREcZxiGERNRVlX6Mpun7wfuy3ygqksz/xaReji378/jBNVM1gE1gRo57M+VSILnEhHpBcwF/s7iXLh5noZhGBERZT3P1sDqI57eTjaIyKk4FeFuBQ7gZJ+ZhIB0nKQ2I5v9uRJJ8LwQ6HnEvgycQSDDMIw840ySD1dJ/uCfqyOZ5ykiLYH3gCGq+qaItAGqZzmkGrAWJxBntz9XclPPLK6q+1TVm6UYhmEYOeB1JXkRqQV8CFyqqp+6uxc4T0ldYAXOnPWxqrpSRPaKSEtV/QroC0wL10ZumefXQOPI3TUMw4gNHybJ/xsoATyVZXDpJaA/TjZaApgKvOs+1xsYLSJlge+B58I1kFvwLNApqRkZGYRbOhoLtuQu//n1mQt9s13xcv/WZG+ZeKVvto3DSSBEQpiQE+75rKjqzcDNOTx9RjbHLwaaRNwAuQfPEiLSiByCqKp+H01DhmEYORHE5Zm5Bc86OOltdi5nuM8bhmHkmRARTJLPF08iJ7fguUxVG+WbJ4ZhHLVEUq8zcPU8DcMw/Kaw3bZ/kW9eGIZxVBPEzDPHknTuaFXcElQtbr/1soPqu5e2K5ctgb74L+rXOIY61cow84GuzBjRhWcGNT+YvTx+ZVO+HNmNafd15qy6leLC75w4GnTbMzPPcFs8EWeF7SMnqxb3PcPv5/Zbh3lmO6um9QMPjeSO24Jh22/7QbBdJDHEc9e0YM9+p5rYyH5NGDHxezreO40QcP7Zx9G5cU3q1SjLOXdOos+Tn/H0wOYF7ndOPPnEY1x3zUD27t3rqV2//Y6WhFCIxDBbYDLPeCeoWtx+62UH1XevbD9yRRPGzFjOum0pADSqU5Evl60HYMaiNbQ7vQYn1yzHrB/WkJEBW3btIy09g6rlShao3zlx1Oi2R7jFE74GTxHpJiILReRnEXnWa/tB1OL2Wy87qL57YbtP27ps2rGXWYsPLUvOWkD3772pHJNUjCV/biW5YU2KJIaoXaU0J9cqR1Lx2MZOTbfdGwqlemasiEgdnOVQTYENwKdZCo96RtC0uP3Wyw6q717Y7tuuHhlAuwbVaVC7AqNvbE3lYw6VZihdoijbU/Yze8laGtetxNThnflx5TYW/bGFra4+fEH4XRDEm9+RZJbxFTr9zTy745S5X62qqcClOAvzPSGoWtx+62UH1XcvbHcaPo3Ow6fR5b5PWPLnVgY9/yUzFq2m9SnVAOjY6Fjm/byButXLsmnHXjreO42nPvyR9PQMdqTsLzC/C4J48zuIA0Z+/tTUBfaLyMfAccBkPJTkCKoWt9962UH13S/bd772LaMGt6RYkQR09XY+mP8nRRMTSG54LP3a12Nvahq3vDw/vKF89ttv4s/v8MWQ4y33DKvbHisiMhpoAbTFKaL8MfC6q76Z2+tqAyumTJ/li267FQYpXFhhkPzDT932oc+9Qfkq1XI9dtvG9Tx9Uy9P288Lft62rwdmqeomVd2DIzscVdUSwzCODjJlOMJt8YSft+2TgVdFpBywC+iCU5zUMAzjMGzAKAuqugB4DEf7aBmwEijojhXDMOKQxBBhJ8knxln09HVugqqOBcb62YZhGIWASG7Lj6LbdsMwjIgI4m27BU/DMAqcwlaSzjAMI1/wWsMoP7DgaRhGgWOZp2EYRgyE3H/hjokn4jZ4xuOkWCP+8HMVUPnkB32zDbBt5t2+2g8SlnkahmHEQCiCPk/LPA3DMI7AMk/DMIwYSCACATjLPA3DMA4nIeRs4Y6JJyx4GoZR4ARxtD2wAnBBkcHNCT/kZCG45yUotiuXS+LXt26ifq2KB/ddeu6pfD6q/2HHhULw4cjLGNitccxtBfk6jJpIqsjHV+wMbvAMggxuTvglJwvBPS9BsF0kMYFRt3Rlz77Ug/sanFiVfl0b/mMw474B7ahQNjZFzkyCfB1GSyjCf/FEYINnEGRwc8IvOVkI7nkJgu2R13Zg9KTvWbflbwAqlC3JA1e359ZRMw47rvs5J5GensH0Bb/Hhd854ed1GC2ZfZ7htnjCt+ApIgNF5Ics2w4RGeWV/XiXwc0Nv+RkIbjnJd5t9+nUgE3bU5j17R+ujRAv3Xo+t70wg11ZxONOqV2ZS889jRGvfB4XfueGn9dhtDh35UHKO30cMFLVl4GXAUTkVJwq8vd5ZT/eZXALiqCel3i33a9LQzLIoP2ZJ9CgblUWjrmGP9dt57mhXSlRLJGTjq/M49cns/9AGjUql+GTp/pyfLVj2J+axsr125npBt389jsoBHGeZ37dtr8I/EdVN3tlMN5lcAuKoJ6XeLedPOQ1Og4ZT6eh41ny2wYa9X+JU/u8QKeh4+k74gOWr9zErS/M5K7/fco5171Cp6HjGf/JEp57Z0FMgdMrv4NCKMItnvD9Z0xEOgAlVfUdL+0GUQY3PwjqeQmqbT8Jqt+xkOBKbYQ7Jp7wTXo4ExF5B3hfVSdGeHxtYMXUGbN9kR42jEixwiCH46f08BOvfEDlqjVyPXbThrX8+8runrafF3y9bReRYkAbHM12wzCMbAniVCW/b9sbAL+o6m6f2zEMI8AEccDI7+BZB1jtcxuGYQQcE4A7AlV9G3jbzzYMwygkxFt0DEPhnDRmGEag8KswiIiUBeYB56vqn+7sn6eAksBbqnq3e1xDnHnpZYEvgMGqmuuKhMAuzzQMo/AQrihIJH2iRyIiTYG5QH33cUlgLHAhcDJwtoh0cQ+fANygqvVxcuBB4exb5mkYRoETZZ9nTRE58untqrr9iH2DgOuB8e7jJsCvqroCQEQmAD1FZBnOXPT57nHjgPtxFvfkiAVPwzAKnkgEHw89/2U2z97PEcu/VXUgQJZAWwNYl+WQdUDNXPbnigVPwzAKnCinKrXmn7N4jsw6syMByLoqKASk57I/Vyx4GoZR4ER52746xhVGq4HqWR5XA9bmsj9XLHgavpOW7t8S4EQfizxunXGXb7YB6g35yDfbvzx9gec2fV3KnT8TPRcAIiJ1gRVAL2Csqq4Ukb0i0lJVvwL6AtPCGbPRdsMwCpz8WJ6pqnuB/sB7wDJgOfCu+3Rv4GkRWQ6UBp4LZ88yT8MwChw/l2eqau0sf88GzsjmmMU4o/ERY8HTMIwCx5ZnGoZhxEIAo2dg+zyDLssaROlh8PecPPHYI7Q/pwWtmp3Fq6+M8dy+H76npqYyoP8VdGh3Dq1bNGXypNiqLyaE4IneDXl/aCveHdKS4yslUbF0McZc3YR3h7Tk/aGtOL5SEqccW5a3b255cPv16fNpe3KVqNtLS0vjmkFX0b5NK5Lbt+GP3/MmVpdXrCRdPpJVlnXB/Pnccdsw3nnfu9HLJ594jIkTxpNUqpRnNvPDtp/nxU+/v5jzOQu+/ppZn88lJSWFZ59+wlP7fvk+8Y0JVKhYgTHjXmPLli00b9KY87tFP9KdfHo1AHo8PZdm9Spyb4/T2JGSygffrmbyorU0r1eJE6uW4dOlG/jXs18BcF6jGmzYsZfPf94YdXtTJk8C4NM5c/lizufcfusw3nn/w6jteEUQS9IFNvMMsixrUKWH/fR79szpnHraaVzeswc9e1xAl67ne2rfL997XNyTe+974ODjWAXapi9Zz+0TFwNQs0ISm3bt46w6FaheviRv3NCC7mfX5OtfD0mAlSyWyC1dT+Led36Mqb0LLryIF178PwD+WrmSKlWjz169xst17fmB35Xk+4jIUnfzNJUIsixrUKWH/fR7y5bNfP/dd4yf+DbPPv8iA/r18XReoV++ly5dmjJlyrBr1y56X9aT4VkCabSkpWfwVN9GjLjkdKYuWkvNiknsSEml16h5rNmawnXJ9Q4ee1nz45myaA3bdu/PxWLuFClShEFX9WfY0Jvo3uOSmO14QRBv2/3UbU/CmSvVBmdqQGu3HJQnHE2yrNEQ1PNSoUJFOiR3pFixYtQXoXiJEmzatKmg3YqI1atW0Tm5PZf37sOll/fKk61bxi+izYjZPNqrITtTUpnxo7PketZPG2hwXLmDx3U/uyYT563MU1sAo8eOY/FS5fprr2b37oITfPCjqpLf+Jl5Jrr2SwFF3W2PV8aPJlnWaAjqeWneohUzZ0wnIyODdWvXkrJ7NxUrVixot8KyYcMGunXtxIMPj6Rf/6tittPj7Jpc39HJLPekppGensH837bQ/pSqADStW5Ff1u0EoEyJIhQrksC67Xtjbu+NCeN5/NFHAEhKSiIhIYHExMSY7eUVkx7OgqruEpF7cGbxpwBzcIqSesLRJMsaDUE9L13OO5+v5n5Bm5ZNSU9P56lnRxXolzlSHn/0YbZt38bIhx9k5MOO2uaHk6ZSsmTJqOxMW7yOJ/s04t0hLSmSmMD97/3E0tU7eKx3Q/q2PoFde1K5cdx3ANSpUprVW1Ly5PeF3XtwzcCrSG7fhtTUVB574mlKlCiRJ5t5IoBTlXyTHhaRBsCrQCdgB06x0W9U9fEwr6uNSQ8XKoK6tt1vWe76Q/0TlfVjbfuaNas5r1MH8EF6eMxbU6ha/dhcj92wbg0DLj3P0/bzgp+37Z2A2aq6UVX34RQYbetje4ZhBJQg9nn6OZKwGHhMRErh3LZ3A771sT3DMAJKAO/a/cs8VXUGMBH4DliCM2A00q/2DMMIMAEcMfJbevhR4FE/2zAMI/iEQiESwtyXh5XpyGfifwKgYRiFniDetlvwNAyj4Alg9LTgaRhGgRPJ8st4W55pwdMwjAIniFWVLHgahlHgBPCu3YKnYRgFT4gIMs988SRy4jZ4ZmRk+LI8Lt6mO8QLfi5F9HMJpZ/sP5Duq/1fn7nQN9s1B77puc2Evdso77nVTIKXe8Zt8DQM4+jB+jwNwzBiIHh5pwVPwzDiAFthZBiGEQsBTD0DKwDnleRrdvgp3+u3NLCf9u2cH87CbxZwXqf2AGzauJHLe3anS3JbOrVvzYo/vJPyzatkckIoxLNXNWHKXecy6c721K5cmtOOK8cn93Rg8n/O5dmrmhzsT7y+szD7vo7MvDeZro1zr6/pJQGsCxLczNMrydfs8FO+12/JZD/t2zk/xLNPPc5bE18nKSkJgHvvvoN/XdaL7hf35Is5n/GLLueEOifm2XcvJJM7NaoBwHkPzablSVV44PKGpGfAEx8tZdaSdbx0TTM6nlGDr3/ZxKDk+px92xSSiify+YjOTP1+TZ7fQyQEccAosJmnV5Kv2eGnfK/fksl+2rdzfojadU5k/MR3Dj5e8PU81qxZzYXndeSdNyfS6py2eW4DvJFMnvb9Gm4Z55TSrVkxiU079/Hjym2UL1UMgNIlipKalk7KvgOs2pxCUvFEkooXId3nSvpZMfXMIxCRO0RERWSJiNzlpW0vJV+PxE/5Xr8lk/20b+f8EBde1IMiWaSM/1r5J+XKleejKTOoWasWzzz5WJ7sZ+KVZHJaegajBjZlZJ8z+fjbVfyxYRcP927M1490pXLZEny1fCMAa7emMO/hrnx6fydGz/wlz+1GTADv2/2UHu4A9ALOBhoBTUWkh5dteCn5mhU/5Xv9lgb2276d8+ypUKEiXc/rBkDnrufzw/ffeWrfC254eQFN75jC01eezcO9G3P+w7NpfudU3pq3ghGXNaLD6dWpWq4EjW+dRMNhH9O1cU0anVAhX3wLYOz0NfNsBExX1Z2qmgZ8AlzklXGvJF+zw0/5Xr+lgf20b+c8Z5q1aMmM6dMAmPfVl5x0yimetxErPVvU5ubzTgYgZd8B0jMy2LZ7P7v2Otn3+m17KJdUjO0pqezZn8a+1HT2paazI2U/xyQVyxcfTcPocL4HnhaRR3A0jC7Aw2DtleRrdvgp3+u3NLCf9u2c58yDIx/npuuuZuzolyhb9hheHjfB8zZiZcrCVTw3sCmT7mxPkcQE7n5jEVv/3sfoa5uTlpbB/gPpDB33Las276bNKVWZfk8y6RkZLPh1E58vXZ8vPgaxJJ1v0sMAInIL0B/YCswCmqnq+WFeUxtYMWX6LF+kh+Ntom284Od1ENRzvi81zVf7xYv6p0vv29r2Rc+AD9LD70+ZSY0auU+NWrt2DT3OS/a0/bzgZ59nGeA9VW2gqm2BfYB3k98Mwyg0ZFZVynUraCePwM/b9hOA10TkLKAUMMDdDMMwDiOIt+1+Sg8vAd7DkR3+BnhGVb/yqz3DMIKLDRgdgao+AHg3GdAwjEJJAJe2B3d5pmEYhYgARk8LnoZhFDhO7AzX5xlfWPA0DKPACWJhEAuehmEUOH7ctYtIL+BuoCjOgPULsfiWE4GtqmQYRiEiFCIUZosm9RSRY4GHgFZAQ+BqEfF0zWw8Zp6JABvW+7MsLKirXfzGVhj9k/2p/qpnFivqX+6SsHeb9zb37cj80/OlURs3rA8bGzduOBgTaorIkU9vV9XtWR53AD5V1a0AIvIucAkwwgt/IT6DZ3WAq/r1KWg/DCOw+CcRDDjfUa9WC+4Etl15Re9IXd4LfJnN/vuB+7I8rgGsy/J4HdAkFgdzIh6D57dAa5w36+/iYsMwoiERJ3B+65VBVd0qInWBsnk0tf2IxwlA1tupEODprUTcBU9V3QfMLWg/DMPIFs/rU7i31ls9NrsaJwnLpBqw1ssG4i54GoZheMAs4D4RqQzsBi4GrvayARttNwyj0KGqa4C7gM+AH4A3VPUbL9vwtZ6nYRhGYcUyT8MwjBiw4GkYhhEDFjwNwzBiwIKnYRhGDFjwNOICESlV0D7EgoicHFTfjbwRyHmeIlIHZ0XCUiDd1YWPe0SkAc5k3RXAX+6CAK9s18NZlfeTqqZ4Zde1fQZwkqq+5aXdLPYvANqJyN2quttj2w2AmsBvwG+q6tkqExHpALwAnOfa9wzX7+rAcmCVx36fDFTEWcW3zuvr5WghcJmniPQApgCPA+OAISJyjIf2G4lINa/sZbF7ITABuBanOEE3D21fBLwLPIKjG+WV3ZCIFAEGAneKSPesz3nUxvk4a5Lf8CFwdgPeAG4DzsfDghYi0hF4AigNNPPKrmv7Ipxr5Xrgf7j1Hjy0/TrOdXgP8LiI+LwUvnASqODp3h71AXqpajLwEXA88B8RydPa2CzBYBjwtohUzZOzh9uuBgwB/qWq3YFlQC+PbV+qqucClUTkAhGpLSIl82JbVTNU9QDOJONtQFcR6Z/5XB5dz8yungHaAVVFZJyIvCYiffP6gygixYB/AVe50tdvAieLSK28nhc3cD6D86NyC1DF3Z/nHxT3Gu8P9FPVC4ADQFsROdGV886L7SrAzUB/Ve2N80PbHnhWRCrkzfOjj0AFT5yF/pWB49zH7wKTgBLAFSISc2aRJRikA6cB4z0MoKlAMXcDGA1UzOuXIYvtUsAu98txIk5W8TxOtljCgza2ADuAeUBrEblHRP4NeQ4YO3GuweE4X+qXgD+ARsB1efk8cQpBnAiUd39Y3wYeBZ7D+bFNisWoG5SbAYNUdSGwB7hSRKp78YOCc41XAZq5n+fpwGU4We5/8/ijkorzXSkLoKqTcJYx7gBuyuP5PuoIVPB0+2bGAF1E5Ay3H+hTYCFwDoeCU15YCTTH6Q+a6FEA3YFz+7VfRBJwfgCq4FZ9cbOKWH3fAdzuLkc7GbhOVbu47dV328krC4DFqvoKsAn4N27Vs7wEDFX9E7gCp8/wXVWdDzyIU3KsMXn4PN3+5DdwMqsHgf+552UMznmpGaPd/cAjqvqVG2w+w/lRqQWQ1wDkXuP3AzfidE+NUdVuwL0439dT82B+BzADuExELhGRkUA54HOc8+3FD+1RQ6CCp8sUnPJTl7kBNE1VxwMVgLoe2H8T+FNV+wFr8CCAure+bwO/ugG/KJCmqn+LSD+cL3dMmZCqHlDVT92/56jqm+7fk12bXgTPdJwCtD2BzsBYoK6IXOqB7Xk48tRL4eD7+QDn8zw+j7Y/whkYaYyTPWeel1LAsbEaVdVU9/80Vd2FE5Qeydzn/kDGjKpOB5oC3+F0maCqP7pPx9w/6V57/8MZsLwYKKOqfVX1PZxs1PO+/sJMINe2i0gt4AagEs4vaQZOEYBkVd3ocVuvAA2ALl7Zdkc7bwM+xvG7v6r+5IHdosBJQEmcoDkc6KaqeSrL72ZTY3C+0Ner6qcicgUw281480SWYHMOsB8nMx8BnKuqm/Nouy7wLPAXh7p47gPOU9VVebQdysy8ReQdYI2qDsmLzSPsX4tzN/E+ToZ4N3Cxqq70wHYCzkBUGbeN4Tjne0tebR8tBDHzxL3oHwdmA31xRq77eRk4M7/QqnolTvHXPA0yHMEBoB+OxkpfLwKnSyLQEngKuA4YmNfACU42lWkzM8vFGR3Pc+B07afjZINn4nyug3E+zzwFTtf2bzjn4g+cW+GeOOc8T4HTtZ2RJfCPAop7PPDyEU6dy7uBQcAALwKnSwZwBvAaMADnB9wCZxQEMvPMittXmJF5K+Wx7QQv59dlsZuEc+t7r6r+4rHtIkBxIMG9pfSUrNmWD7YTcX6kElR1pw/2iwEhL+fXZrFdAiiiqn97bDfEoXPiqW3Xfimcc+K57cJO4INnUBGRYu7gg2EYAcSCp2EYRgwEss/TMAyjoLHgaRiGEQMWPA3DMGLAgqdhGEYMBLIknXE4IlIbR0/7xyy7Q8Czqjo2j7Yn4yydHCciPwBtVXV7DsceA3ygqu2jbOMS4Aa3gEfW/W2BUap6WpjXZwCVo5kXKiLjcMr3PRGNr4aRiQXPwsMeVW2Y+UBEjgV+EpGFqrrEiway2s+B8kATL9oyjHjHgmchRVXXiMivQH0RaYyziqQUsENV24nIAJyVNwk4675vUNXlIlIDeBWogVMk5eDa+KwZnojcibNK6gDwK04ZtVeAkm6GeiZOAY5ncdaXJwLPZWbCIjIC6O22/Wu49yMi9XEKD5fBWVb4A04Zvr3uIQ+JyNnu+7nbXcNOTu8zqpNpGNlgfZ6FFBFpjlMoZYG761ScW+52ItIGJ/C1VtVGwGPAB+5xLwDzVfVU4CactfJH2r4AJ1g2d2+pV+DUGriSQxlwCKdk4B2qeibQBvi3iDRzC0NfDDQEWgCRlFkbBLyqqs3c93UCTjWmTP5Q1cY49V5fFZHKYd6nYeQJyzwLD5kZHzif62agt6quEhGAJVmWPJ6HE4Dmuc+BU/eyAtABp+QcqvqbiGSuZc9KB+AdVd3mHncLHOx7zaQ+Tj3NsVnaKIlTq/MU4P3M5aMiMhYnUOfG7UCyiNzm2q6BU8U9k5dcX34SkWU4ZQVb5fI+DSNPWPAsPOwJ0yeZde1yIjBeVW+Hg0VQauBUi8/AyRozOZCNrQPucbivL4dT9ScriThdBFn7YavilG97PII2jmQizvX6Nk5ZwuOOsJFVxyoBp/Bvbu/TMPKE3bYfnUwHLheRTG2cwTgVqgA+Aa4GEJHjcCQyjmQW0EMOSZ/chyNHcQBIdItZKLBHRPq4tmoBP+H0hU4DeopIOTeg9Y3A507ACD0kQteUwzWJ+rvtNOZQd0Vu79Mw8oRlnkchqjpDRB4FZopIOo4cRg+3xNr1wCsi8jOwGrcY7xGvnyoipwBfubfDS3H6JFOAb9zHrYELcfRxbsMpAH2Pqn4FICKn4ygAbAMW49TwzI3/AB+IyG6c7HUOhxe/riMii3Ay4stUdSuQ2/uM5pQZxj+wwiCGYRgxYLfthmEYMWDB0zAMIwYseBqGYcSABU/DMIwYsOBpGIYRAxY8DcMwYsCCp2EYRgz8P8ppf9ihjOYhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Look at confusion matrix \n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "# Predict the values from the validation dataset\n",
    "Y_pred = model.predict(X_val)\n",
    "# Convert predictions classes to one hot vectors \n",
    "Y_pred_classes = np.argmax(Y_pred,axis = 1) \n",
    "# Convert validation observations to one hot vectors\n",
    "Y_true = np.argmax(Y_val,axis = 1) \n",
    "# compute the confusion matrix\n",
    "confusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n",
    "# plot the confusion matrix\n",
    "plot_confusion_matrix(confusion_mtx, classes = range(10)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display some error results \n",
    "\n",
    "# Errors are difference between predicted labels and true labels\n",
    "errors = (Y_pred_classes - Y_true != 0)\n",
    "\n",
    "Y_pred_classes_errors = Y_pred_classes[errors]\n",
    "Y_pred_errors = Y_pred[errors]\n",
    "Y_true_errors = Y_true[errors]\n",
    "X_val_errors = X_val[errors]\n",
    "\n",
    "def display_errors(errors_index,img_errors,pred_errors, obs_errors):\n",
    "    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n",
    "    n = 0\n",
    "    nrows = 2\n",
    "    ncols = 3\n",
    "    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n",
    "    for row in range(nrows):\n",
    "        for col in range(ncols):\n",
    "            error = errors_index[n]\n",
    "            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n",
    "            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n",
    "            n += 1\n",
    "\n",
    "# Probabilities of the wrong predicted numbers\n",
    "Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n",
    "\n",
    "# Predicted probabilities of the true values in the error set\n",
    "true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n",
    "\n",
    "# Difference between the probability of the predicted label and the true label\n",
    "delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n",
    "\n",
    "# Sorted list of the delta prob errors\n",
    "sorted_dela_errors = np.argsort(delta_pred_true_errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEUCAYAAAAhqy2HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcHEXdx/HPzOy9uW9yYIDAj/sK4ZBLTgVUFBAEBAEFfVQU5BLhEURRRG7EBxAiKqL4AKLcR4IghwEeMIBgyREgFwkhIdcm2fP5o3u7p4bM7uzs7O5s7/f9eiWvqqme7pr5zdbUVHdXpdra2hARkeRJ93UFRESkZ6iBFxFJKDXwIiIJpQZeRCSh1MCLiCSUGngRkYSq6OsKmNlk4E3g5ayHU8DVzrnp3dz3vcAdzrlbzOyfwCeccx/m2XYo8Gfn3L5dPMYRwLecc5/IefwTwC+cc1t38vw2YLRzbkkXjnkL8Ipz7rJOtjsV+BawBngN+KZzbmmhx+kOxbVn4mpmxwPfzXpoKDARmOicW1TosYqluPavuPZ5Ax9a45zbvj1jZhOAV8zseefcS6U4QPb+8xgO7FyKY5UDM9sHOAfY1Tk3z8yOA24EjujFaiiuJeac+y3wWwAzqwSeAC7pjcY9i+JaYj0V13Jp4D3Ouflm9jqwmZntCHwFqAeWO+f2MbOvAN8gGGL6gOAb+d9mNh74DTAeeAcY077P7G9eMzsX+DLQDLwOnAD8GqgNew5Tgc2Aq4GRQAa4pr2HYmYXAceGx369s9djZpsB1wGDgQ2AfwJHOefWhptcbGbTwtdzvnPu3vB5632dHRxnJ+Cm8I9jKvCoc25eWHwXcJOZVTnnGjurc09QXEsS12znAIudczd0VteepLiWb1zLcgzezHYDpgCzwoe2Ivi5to+Z7U0Q7D2dczsAlwJ/Dre7DviHc24r4NvA5uvZ92cJPiC7hT/H5hAMY5xI3DNJAXcA33POTQX2Bs40s13N7FDgcGB74OMEP6U6czLwG+fcruHr2gg4JKv8LefcjsCXgN+Y2ehOXud6Oeeez/qwzAL2NbOPhfkTgSqCP4A+obiWJK7tr3cUcAZwegH17FGKa/nGtVx68O3fxBDUaQlwrHNurpkBvOScWxGWH0Lwpj8dlgEMN7MRwP7AmQDOuTfMbOZ6jrU/8L/OuWXhdt+FaGyx3WbAJsD0rGPUAjsAWwJ3OedWhs+bTvDh7Mg5wAFmdna47/HAoKzy68O6vGJmrwK7AXt08Do75Zz7u5n9EPizmbUC04GlQG/23hVXShvXLKcAf3HOvdXF55WC4kr/iGu5NPBrcr/JcqzKSmeA3znnzgEwszRBAJYBbQTf5u2a17Ov5nA7wucPA4blbJMh+HmZPc44FlgO/LyAY+T6A8F7/SfgPmDDnH20ZKXTQBMdv85Omdlg4HHn3M1hfgLwI4JGvrcorrGSxDXLUXTeUPUUxTVW1nEtyyGaTjwEHG1mG4T5rwMzwvSDBN+AmNmGwD7ref6jwGFmNiTMX0hw9roZyJhZCnDAGjP7UrivScArBGN9DwBfMLNhYRCPK6DOnwQucs7dHuZ3IfhAtDshPM6OxD91O3qdhRgP/C3rdZ4H/ME5V66zyymuBTKz4eH+nu7K8/qI4lqgnohrufTgC+ace9jMfgY8Eg49rAAOc861mdk3gV+b2WvAPIKTI7nPv9/MtgSeCn9K/YtgzK0BeDbM7wkcClwd/kyrBP7bOfcUgJltAzxP8O08GxjdSbW/TzBUspqgV/E4QSDbbWxmLxL0VL7ogksZO3qd6z1I9kkb55wzs0uAWeEH+0mCscuypLgWFtfwoSnAQudcUyf163OKa9/GNaXpgkVEkqk/DtGIiEgB1MCLiCSUGngRkYTqdydZAczsGmCvMLslwc0Pa8L8bs65Net9YvHHm0wwl8SgTrZ7GzjCOfd8F/Z9ITDKOVfQCVAL7qB7EpjgujAfRn8wEONqZkcCFxBcFTIP+IZz7p1Cj9MfKK59F9d+2cA756LrRMMgHduVIPVXFtzldj3BHamJM9DiamabAjcAeznnXjazvQjuyJzWtzUrLcW17+LaLxv4zpjZOuAvwHYEc1A8R9YMcObPc/EZ4HyCRrMBONM590wH+x5LELyxwDiCOTSOdM4tDjf5ppltB1QDl7t4PoyuHmc8cD9wsHNuQXip460El3A9WMTb0u8lLa7h65jtnHsZwDn3hJlNNrPJzrm3u/wG9VOKa89J6hh8FXCPc8466imE37Q/IWhEdyC46eIuM6vvYN9fBJ5xzu0GbEwQ/OybJ9aE81QcAPzUzLYq5jjOuQXh9ewLwocuAp51zj3UyWtPsqTF9UVgazPbPqz3ZwjmCtog3/MTSnHtIYnswYf+XsA2BxC86TOybkZoJbjhYPb6nuCcu9rM9jSz7wKbAlsTT7IEQW+BsNf9MLAfwThcvuN0yswOIbib7pOFbJ9wiYmrc+5NMzsJuN7Mqgl6sbPp3fmCyoXi2gOS3MCvysmnAMwse/w6A8xwzh3V/oAFtzkvII/wbrWdCSbveozgrrnO5qmo6OA4ny/gtZxEMPn/C1kfuMfM7MQkj2XmkZi4hn/8b7hg1sL2/GkEJyEHGsW1ByR1iCbX+8BOYfqYrMdnAAea2eYAZnYw8BLBTHT5fBK4yjn3O2AxQa9iffNUbEgwE96MIo8Tcc4d7pzbIvwJ2H5b8z4DsHHP1a/jSjDu+1TYeEDQCDzpemnVrTKmuJZIknvw2b4NXGdmHwKPAAsBnHOvmtkpwB8tmLSoGfiscy63N5HtIuAyM/sRwbf9k/g/3WrM7AWCccVTnXP/Ach3HMs/T4V3krXYF55w/T6uZnYy8ICZZQiWVTyhmDciYRTXEtFcNCIiCTVQhmhERAYcNfAiIgmlBl5EJKG6dZLVzI4huNurkuBM9XUFPKea4JbdhfiXKEnvyhBc6/ucc25dd3emuJaVksVWcS0rXY5r0Q28BWt8XkywLNY6gsVmH3POvdrJU6dR2E0N0jv2JLiyoLsU1/JTitgqruWn4Lh2pwe/PzCz/dpOM7sDOILgsqSOLASYN381zS26gqevVGRSTJxQD2E8SkBxLRMljq3iWiaKiWt3GvjxOQdaSHDHWGdaAJpb2mhu1gemDJTqZ7fiWn5KEVvFtfwUHNfunGRNEyw62y5FMF+DiIiUge408PPwZ0cbRwdzQoiISO/qzhDNo8CFZjYaWA0cTjCtpoiIlIGie/DOufnAeQQztP0TuM0592ypKiYiIt3TrevgnXO3AbeVqC4iIlJCupNVRCSh1MCLiCSUGngRkYRSAy8iklBq4EVEEkoNvIhIQqmBFxFJKDXwIiIJpQZeRCSh1MCLiCSUGngRkYRSAy8iklBq4EVEEqpbs0lKYS4cv4+X/97z8bK1X9/pHK/slgVP90qdRCT51IMXEUkoNfAiIgmlBl5EJKE0Bt8Ljqz5wH+grTVKXjBqmVd0i5Yt7xGDq+ui9JNjzCvb5Kr4HEnFrod6ZY23/NTL73TZy1H6P8vml7KKUoRRdUOj9Ms7j/bKBn39U1G6cq+jvLKmh3/t5bf89j1Reu6K972yk8fvHqU/3lTllZ34/mNdrHHvUg9eRCSh1MCLiCSUhmiKtNPoTb387WMqvfzYX5wQpTObTM27n7F3/MzL77XnuVH6iUX/6kYNB550Ku6vXDxub6/sm6fFQzTpPT/llaWynvf+507xypYtqvO3JRWlJw4e5ZUtXbcqSjc0ri202tIF+47dxsvffcbHonTVsWflfV5bc6OXr9jvOC8/+9BZUXrkrUv852alD/uyH9cTL+uwun1OPXgRkYRSAy8iklBq4EVEEkpj8B3IHWe/d5t4NG7IjVd6ZenawUUdI10/1MtvUTE8Sj9R1B4HjvGDR3r5u+s2jtJb/v6TXtmGh1wcpZeuua/gY2wybAMv/8C4+JiTHrnWK2t+cHqUnnr2TK9Ml1QWrrrCvxTxziG7Rul97j/BK0uPjsfg1/3ifK/srpvi5u2hygav7MZTR3j5ukt+GaWXNPjnYX78j0xWrpX+RD14EZGEKqgHb2ZDgKeBTzvn3jaz/YErgFrgdufc+R3uQEREel2nDbyZ7QL8CtgszNcC04G9gbnAfWZ2kHPugZ6saG+5dVR8V+Pn/3qEV5bZcOu8z2u66xde/uifvBml/2fiaq9s9N03Ruk7t7vQK/v1sllIfmPqh0Xpl/b1h2jqr/pRlH5gqn8H6tI1K/PuM3sY5g/VE72yrR461cu3rYzvSm554zmvrPLgr0bp4d9/Nu/x5KOyLzl99Xs7eWVVJ8SXDrc1rfPKrt85Hno7/b2/e2WDqmqj9BOjtvTKKj73VS/fmvX5qL/mRq/sO4d8LSs3nP6kkCGak4FvAu030e8MvO6cm+OcawZuBb7QQ/UTEZEiddqDd859FcAsmr9jPLAwa5OFwERERKSsFHOSNY1/c1eK/nZqWURkACjmMsl5QPa1Y+OIh2/6hcM2mBalf/vrQ7yyzOYfj9LZ460Aay8+PUrv96cVXtlLy9728k0tzVH6tv86IG9dHq7yb31ubGnKu63ARfU7RulBv7zAK3t6m3is9rCl+VfG+u54fxqDC0+NpyPIHZv9516XevkTmufF+6n0Z6U84cVd8h5TfCPrhnj5124+JkpX7n64V9a6NG5e/rSvPz5+etZsjkeN99//m34cj7tX7OPPJtl4k3+O5o3p8XmyrZ6/3CsbslX/vdiwmAZ+FmBmNgWYAxxDcNJVRETKSJe/mpxza4ETgDuBV4F/A3eUtloiItJdBffgnXOTs9IzgO16okI94Yc5i16feffRUTozwf+Znf1z8KD9L/bKHl/0SlHHTw2q94+xKl7kY07LitzNJUv2Qh0Ax146OUo3/v7nXtm+HQzL3D7yE1H6kNv9RT14Px52+c5+V3hFv3o//2WrqZyFQ0j135/yvSF7ts/XP+9fl5E7LJPtvaPjReofqxzmlX34te2jdNWp3/XKWl0cu6/ufqFX9vsF//Dyc6flxDIh9IkUEUkoNfAiIgmlBl5EJKESO5vkzqM3i9LZY+7gj7s3v/K4V3buSY9G6WLH3AE2Gz4hPt6nT/IL18WXZC1pyn8LvcC9g/xTPdkr8bS8NMMrW3l1dZwZ688CmdnxwCj9j10v8cqOXBuvnLWkYXnBdWv7yAPx7SC7V47zimbhCt5vUg2ric9F1V36PwU/b/xD10fpG3LK2tbGf0tzDznbK5s6560ovXKdP5tkruE/PjZv2Tt/q4nSGx+dd7OypB68iEhCqYEXEUmoxA7R/GFM/NJyL4XMdumJj3n5axeUZpmNCzLxEFHuoh4ffj2+nOvVpe+W5HhJdU1Vs5f/7YJ/R+mK7f07hFs33TlKr/uZP4P19894KUpf28HllKXyVNN7PX6M/mZN1sLXzS886JVV7Pip3M0jbY1rovSy4/zZPc96K55R9PcLih9S7cj9rfHf77foX4upqwcvIpJQauBFRBJKDbyISEIldgy+qTHT+UbA2X/6nJc/7bp47PSNmYO8sofScf4vWbMKAry9erGX37wy/+WP818blrdMfHcu9FdNenyf/0TpvYdu5pU9vTJeRWvhqqU9WzHgkI9pIe2uWJO1GtOnvvIXr+yaioejdENjpVd2XmZVlH580es5e83NF2bvsf7qbOlN4llKW5f7f8tNqTjdsrDwy2jLgXrwIiIJpQZeRCSh1MCLiCRUYsfgd507J0rP+YF/7WzN+fEqPRUbbe+VVVwW3xa9bc4+s/NnrPbH4lrfecnLpydukbduGx8cXw/MzXk3k/XInkrgzobnOtiy5w09YkqfHr8/e3Lxq15+xzzb9ZQfN/vn19K1g6N0421Xe2XziP9e73zEn4IiWBKjfKkHLyKSUGrgRUQSKrFDNB+ujS+tGn6zP3yy171nROkjGeuVfemo+PLGin0/kXf/6a39hZsrttyz4LplPja2842k/KVSOXn1l/qL7X+efwg1SfSJFBFJKDXwIiIJpQZeRCShEjsG35EnFsUr+DzBv7yyb12Tlbkm/2V494yY5eUPePlHebdd+bWvevnjnq/Ps6X0K205azplregk5S01Ovdyx9hzV+Ws/lSz/u36A/XgRUQSSg28iEhCDcghmlIYmVnXYXn2ijU2w1/dZ+kaLbQtUq5GD8oZommui5KzK5t6uTbdox68iEhCFdSDN7MLgCPD7H3OubPNbH/gCqAWuN05d37eHYiISK/rtAcfNuQHAjsA2wNTzexoYDpwKLAFMM3MDurJioqISNcU0oNfCJzhnGsEMLPXgM2A151zc8LHbgW+ADzQUxUtN5vu0vGKQa+d/EiU1pi7SN/aYsQkL5+aYHm3/dgR1f4Df4yTD65+k/6k0wbeORddKG5mmxIM1VxL0PC3WwhMLHntRESkaAWfZDWzrYBHgLOAt4DsuzxSgO7yEBEpI4WeZN0duBM4zTn3RzPbG9gga5NxwIIeqF9ZOXn87lG6/pJve2Ut777i5U9sfL9X6iQinduoapSXz4zdKO+2qWGDvfxTa/8dpeev/qC0FethnTbwZjYJuBs4yjk3M3x4VlBkU4A5wDEEJ11FRKRMFNKDP5NgNoYrzKITE9cDJxD06muA+4E7eqB+IiJSpEJOsn4H+E6e4u1KWx0RESkVTVXQBd/KxLcwp4f7s9Gd9slrvfwrS9/plTpJH9KKTgPCa0vn9nUViqZPpIhIQqmBFxFJKA3RdGDSkNFefpNbj43SzbNneGX/u8xf2FuS77lLlnn53Y/VrSDlauv0kL6uQp9QD15EJKHUwIuIJJQaeBGRhNIYfAf+Otifga5iyrQofcsOP/TKPmhY0St1kvIx7ZxhfV0FKdArrQPz71M9eBGRhFIDLyKSUBqi6cBDrf5P8I0vPydKn7fq1d6ujpSZv/7cX5z5iC/1UUVE8lAPXkQkodTAi4gklBp4EZGE0hh8B763cKafv6KPKiJl6d6qnDH4PqqHdO6BRf/08s0vPOjlK3b8VJR+8OqmXqlTb1APXkQkofq8ga+oSLHJRoOYNKHO+zd4cPd/XGwwtpbBg4L9TJpQR7qDV5tOwfgNart8jPr6Cias53m1NRkmTazr9PlTNh5MOp3qdLtsY0bXMGxoZafbVVWmmbBBLZMm1DFxQh3VVb0X7oEe1w8GL6A11dKlYxYS18GDKrz382OT6tlko0FkMl37DBWr38e1LsP4cR993rMvvsTnjv9Gp8/vyb/XutoMkybUseHEesaNqfnIcgPFKIshmrY2mDs//rmbyaTYcGI969Y10NhYmhn6sve/PulMiprqTEmOVQ5S4R/A4vfX0rCmhfq6CsaOqeHdeR2/D6WU9Lj+ccEsPz9h7yg9ZePB/N+SN2ltbSvpMVeuamblquYoP3F8Hcs+bKSlpbTH6Uh/jGtbW/D+tP8/6DM/9cprazKMGlVNTVYMe1M6nWLMmBrmz2+gqbmNkSOqGDWimvc/WNet/ZZFA5+rpaWNpqZWKivTVFelGTK4klQ6RWtrGwsWrmHw4EqGDqkkFW77/gfraGpqJZNJMXZ0DZmKFM3NbV6vZsrGg3nr7VW0trYxfFgVgwdV0AY0NbWyePFaxowOvjEnTahj7vwGKivTjB5ZTTqTIgV8uKKRlSuDP6wRw6sYPKgyqmdnKitTjB5ZQzqdIpNJsa6xhUWL1xJ+5hg5oorq6gwp4INl62hoCHp++V5nPtVVacaMrmHu/Abqaitoam6lYU2wr9UNzTQ19+10topr9+OabfiwKppbWlmxsm/HjBXXEvy91mVYt66VpubgIMtXNDFpYn0yG/ia6jSVlWnWrW2htjZDVVWGt99dRVsb1NRkGDKogvkLGmhrg9raDBuMDXqmo0dVs3ZdC0vfa6SyIsWkifUf2XddXYbBgyqZt2A1ra0wakQ1Q4dWsfj9tWw4sT76Ixo3tobFi9eyrrGVdAomTqijsbGVikya+voK3p23mra24GdlZ4YMrmLFqiZWhT2vSRPqqKurYPXqIN/U1Mb7SxqCIZXxdbwzdzVVVem8rzOfdY2tUf0rK9O0NLcxZlQ1VdUZWlva+GBp9z4s3aW4dj+u7dLpFMOGVjF3/uqC3/+eorh2P64VmTTNWR2w5uY2MukUqRTRF0sx+qKBzwBUhN/WFZngRWyYNa7Z2gpLPlgLqeDnX/u3PQRjkJWVaSZNiLfPZFJUVaWoq63gw+UNVFSkaAPWrm0hk0lRUREeqwIG1VfQsKaZdDpFOh1803v1qUhRWZGiqjLN2DE10TFS6RR1tRkqK9OsWdMS1Wd1QxNDBldFx8iuUyrc3/IVjdTUZBgxvIrKyjQVFWkqKuJ6rW5ooqIiRWtb0MOor89QU53J+zrTqeAPPPeY/vGhrq6C9xavofHDRmprM4zfoDbrAxU9t1TjF4prL8S13dAhlaxZ0xy9tmwljq3i2kt/r6nUR7epqEhFDXwxcU21defroQhmtgfw9149qHRkT+fck93dieJalrodW8W1LBUc1764iuY5YE9gCrBRmG4I0+v7dxYwMyt/PLAA2DnM/zfwVpi+D7ghTO8OrAifv1F47B2BE4HXgW3Cx38N3ALsBqwLH9sMmA+cHuY/DiwluNT5xPB42wEbA/cDs9ZT76OB/4TplcDBYfrAMH9OVr0uCNOfBpYA23fyOu8EftLBe7ZR+LzlwGfC/FHAB8DmYX5K+N4/lydOXaW49k5cNwrruAbYNE95KWOruPZOXHcK97VPmP8f4I6cbboc114fonHOrQOibx8zA2hzzr29vu3NbAmwJqv8bTMbQhDoVoIPxWedc2+b2Ynh4w8A84AXgCVhGcBc59yLZjYK+EO4v38BJxN8aGcRfAD2BA4BrgZOAiqB85xzd4Z1GgvcBSwDZgP1ufU3s8lAU3js7wHXAasJGt2/AcOz6jUM+DPBSf4jnXOzgdkdvM5VwLL1HHMn4Cbn3Pbh+/RZ4OdAPcEfw+eccy7rKW+u7z0vhuLaa3HFzKYBC5xzb6zvvQ2VJLaKa6/+vX45fA1VBPE73jm3NOct7lJce32IRkREekef3+gkIiI9Qw28iEhCleV18J0xs2uAvcLslsAcgpNOALs559as94nFH28y8IpzblAn270NHOGce74L+74QGOWc+1aB208jGBOd4JxbUuhx+oOBGFczO5LgpF0zwTj0N5xz7xR6nP5Ace27uPbLBt459+32dBikY7sSpP4qPNl0PcFJmMQZaHE1s00JriLZyzn3spntRXDlxLSOn9m/KK59F9d+2cB3xszWAX8huDTqWILLika393jNrK09b2afAc4naDQbgDOdc890sO+xBMEbC4wD3iE4k7443OSbZrYdUA1c7pybHj6vq8cZT3CFwMHOuQVmlgZuBb4PPJjveUmWtLiGr2O2c+5lAOfcE2Y22cwm57tKJYkU156T1DH4KuAe55x11FMIv2l/QtCI7gCcAtxlZh+9Zzr2ReAZ59xuBNfVNgDHZZWvcc7tCBwA/NTMtirmOM65Bc657Z1zC8KHLgKedc491MlrT7KkxfVFYGsza7/88TPASGCDTt6HpFFce0gie/ChQu6+O4DgTZ8RXt8KwTWsUwiul/0I59zVZranmX2X4EaTrQmux213Q7jdAjN7GNiPYBwu33E6ZWaHALsAnyxk+4RLTFydc2+a2UnA9WZWTdCLnQ00FvL8hFFce0CSG/hVOfkUgJllj19ngBnOuaPaHzCzSQR3pK2Xmf2M4G616cBjBDdVpLI2yZ4EPA00EbzP+Y7z+QJey0nAROCFrA/cY2Z2YpLHMvNITFzDP/43nHO7ZuVPIzgJOdAorj0gqUM0ud4nuBUY4Jisx2cAB5rZ5gBmdjDwEtDRlHOfBK5yzv0OWEzQq8ie/OeEcF8bAvuHxyjmOBHn3OHOuS3Cn4Dbhw/vMwAb91z9Oq4E475PhY0HBI3Ak+u5e3GgUVxLJMk9+GzfBq4zsw+BR4CFAM65V83sFOCPZpYi+Gn2Wedcbm8i20XAZWb2I4Jv+yfxf7rVmNkLBOOKpzrn/gOQ7zhZPXJP7knWYl94wvX7uJrZycADZpYBXiNscAY4xbVENFWBiEhCDZQhGhGRAUcNvIhIQqmBFxFJqG6dZDWzYwju9qokOFN9XUlqJSIi3Vb0SVYzm0BwRnoqwWISTwNHO+de7eR51QRzMizEvwZVeleG4GaO51ywqEO3KK5lpWSxVVzLSpfj2p0e/P7AzPZrO83sDoIlsi7q5HnT0BqP5WRPslbs6QbFtfyUIraKa/kpOK7daeDHE16fGlpIcMdYZxYCzJu/muYWXaLZVyoyKSZOqAc/ht2huJaJEsdWcS0TxcS1Ow18mmBNwnYpgvkaOtMC0NzSRnOzPjBloFQ/uxXX8lOK2Cqu5afguHbnKpp5+LOjjaODOSFERKR3dacH/yhwoZmNJlh9/HCCaTVFRKQMFN2Dd87NB84jmKHtn8BtzrlnS1UxERHpnm5dB++cuw24rUR1KTvvTvUnFhpzz01R+sqpP/DKzl34WK/USUSkULqTVUQkodTAi4gklBp4EZGEGigLfhRl+Nf9+7baWpqj9JEjFnll55bqdiERkRJRD15EJKHUwIuIJJSGaHIMramPM/WD82536YcjeqE2IiLFUw9eRCSh1MCLiCSUGngRkYTSGHyO40dOjdIVe34h73YPr3qjN6oj/cjUUVOi9OTK4V7ZVqlBUfqiBZrWore9t+8ULz/sdzcXtZ/G3//cy5961QdR+jcLnilqnz1JPXgRkYRSAy8iklAaoumCxmvOi9LvNSzrw5pIT9lltD+D6OaV/uWwl0+Lf5IveLHeK9v4thPizJJ5XllbY7xG8kVHaoimJ5w//hNe/sxDl0fpmvOv8sraWv3F55ofnB5nVq7Me4yqY8/y8tdNmxWlZx7qD9vOXfF+h/XtDerBi4gklBp4EZGEUgMvIpJQGoPvgobnFkfpNU3rOthSytnWIz7m5WfuXBml6y86wytLj9vEy7etWhqlN1nln4fJTIjH71f85Bde2ZqFmeIqK566qhov/9iwbaP0NjNO9cpSg+LzJ0333OCVHfSDf3r5Zz8hom8SAAANg0lEQVR4PUo3tjTlPf77j/3Lyw/65Y1RekzVUK9sLhqDFxGRHqIGXkQkoTREIwPCqLr45/PTp/uXQladdF7u5pHmt17w8q2P3BOlb7zR3/Yny5+L0isb13hlTVmLxUjXVFdURemnRm/plW3+j5/FmZyhlTVnfyNKb3uvP1wyb+WSouqy4q1KLz8oK12brqLcqAcvIpJQauBFRBJKDbyISEJpDF763OShY73828sX5dmycEeN38XL/+q/4jH43DH31mXvRem1F1/ole364AovP2dFXLdix9XPybml/rDmhii9y+Jni9pnkr0waYsovfETl+fdrvnem7z8iD+8VvK6LP2gzsuPz0rfuW2jVzb20ZIfvsvUgxcRSaiCevBmNgR4Gvi0c+5tM9sfuAKoBW53zp3fg3UUEZEidNrAm9kuwK+AzcJ8LTAd2BuYC9xnZgc55x7oyYr2lh8fnf8uNukZo6qGePm3KW6IJnvBjel/PsEry0yML6+bvYM/I+CVFako/dclc7yyhsa1RdVlk2EbePmnd4pnnhxyw9leWeP1P44z/noSA1Lu8NomT10WZ9pyZoF8I740dYtzZ/ZovQAmT1vu5VPpeBCk/r8+62/86BU9Xp/OFDJEczLwTWBBmN8ZeN05N8c51wzcCuRf+khERPpEpz1459xXAcyim0PGAwuzNlkITCx5zUREpFuKOcmaBtqy8imgNc+2IiLSR4q5THIekD3AOI54+KbfS40clrfsiFcr85ZJ8Z5///XON1qPnUZv6uWn18Y3jretXe2V3b/ND6L0Ucv96QeKvdwxd5x9UCae6XBkxSCvrP5r+0XpVI2/ElT1aT+N0t+57b+9sqvnP1FU3fqbykzcFP3qFP+9yx53b37hQa9o6vG/i9LFTj/QFTXHHeTls1eGWvvbe3v8+F1VTAM/CzAzmwLMAY4hOOkqIiJlpMtDNM65tcAJwJ3Aq8C/gTtKWy0REemugnvwzrnJWekZwHY9UaHelkn7CzGkqqvjdNr//lvbqkso+9o2IydH6Znf3sgra50XL8iyySEXe2VL18QLKXdnZsczx+8dpZ9t8YcEqlLxZ2mXtL9Yd3qr3aN0y3tvemXzjo4vp5u+5K2i69afXT1qryhd+RX/tprsO43P/fpTXplb5i9u3tMqdz/cy2cP0bz+dP7h3b6iO1lFRBJKDbyISEKpgRcRSagBP5vkUeOmefmKL3wnSucu1PtOQ3GL6OaO808eMiZKr8tZhaY3LvXqT0bW+dMYPH3l/lG67S1/WoGX74gvU1yUsyB2ocYOGu7ltx20oZcfnNUnemKRvwDzXSPi8flPPXOaV5Z92eYXP32dV3bPwleKqmt/lvs3cezx+aeEWHdlfD7l2vkv9Vid8qmtrO58ozKlHryISEKpgRcRSagBP0TTkcbH/DselzQsz7Nlx7KHZABeefm2KN3y+iyvbM8jfxWlX1ziX043EN1Ws62Xr9zvuDizn7/ttmvihTx+f/M+XtnvKuIhm+dX+EM72XH9wtCtvbJfLfLj8++sxbu3H7WxV3bAxfGUTK2L/WM8c2h8q8g9S/+Pge6mkXt5+apv/yDPlnDzfaN6ujodenjIDgVt92xmUOcb9TL14EVEEkoNvIhIQqmBFxFJKI3B95Apw+LlePer3zjvdi1/vcvLa9zdd211g5ffY96rUbr5r3/0ykZd9o8ovdso88qeeC++pHHvsf44++NZY/C/mP/3DuszvjqeguCxJ37ilaVqB0fpl3b2b7c/YKkW08524Kb+FAPZ04I03nqpV3bmez2/UlNHtv1yVibl94lbnvtrlP7h8ucoN+rBi4gklBp4EZGEUgMvIpJQGoPvwOOPjM155N95t31wxJ5efo9bPh6lMzt8Ku/zrvxTfd4ygXsX+vcifPFzN0XpDCmvLHsa4H+vmp93n48vKn5qgIdPGRel00P867Mbp8e31O+yWGPuHRl6xme8fPa0u4/+bFVvV8czNGfFrcx+8fQY2atLAfztq/G4e/aU1OVCPXgRkYRSAy8iklAaounA/mfUevkZl8er8kw7y5/lsPLo0718KmsGuuyfn7m+e4r/HXvzFaOj9NwVxc1emWT3LCzsNv/Fqz8sav+jsqYiAHh6w0levvLYb0XpdVed65Vd8vuqoo45UOw/Lp52Ir3dvl5ZW9Ysqhen8w+v9YaTR+zk5TPb7pdnS7i9prxXeVMPXkQkodTAi4gklBp4EZGEGvBj8O80+1MAt8yNL6GrPP4cr+zjxxe+34UHfT1Kf2+JP15/8w3x+GPuCvL/4sdResvLZ3tlWu2p5/21fnMvv+GMS/NsCSv/7sfjpwv+0yN1SopBqfgcRaravxSx9YO5Ufr/lrzRa3Vqt8eYLaP0Dx84Je92jb+6yMv/cVH5TU+QTT14EZGEUgMvIpJQA36I5qnFr3n51sfvj9KZL22du3lezQ/f4uV3feu9KL1olX+Mq6/8IEoP/c2BXtnqme9G6Q/XrUZKrypT6eW/NDZeeH2bS7bM3dyz/MtfidLbvbS4tBUbwFIj48tRLxzvr8Z14YLHSn68W0f5xzhsZjykmho6JnfzyI3Xt3n57Luny5F68CIiCVVQD97MLgCODLP3OefONrP9gSuAWuB259z5eXcgIiK9rtMefNiQHwjsAGwPTDWzo4HpwKHAFsA0MzuoJysqIiJdU0gPfiFwhnOuEcDMXgM2A153zs0JH7sV+ALwQE9VtLcce008rvqHvf1ZBzOT4jH5g3c61SubveIdL9/RzHIzXorHGw/LKbv4zXi2wlWNr3daXynM4Oq6KP3qtv70A6Pv/mne5zXe4pdt8dSiKF2OsweWs7sXPh+lm3Le16qTzovSZ96wq1e26Ylx+sRlT3plk4fE4+X/WeZPcbDZ8AlR+okd/GlHhv3uwsIqDTS7Z+K69fHqUl3VaQPvnIvWOjOzTQmGaq4laPjbLQQmlrx2IiJStIJPsprZVsAjwFnAW0D26eQUkH9GLRER6XWFnmTdHbgTOM0590cz2xvYIGuTccCCHqhfr8uerfCIw1q8sjnr4jsX//Oh/3Owrc2/fKojZ617KUp/9u//65VdctXUKD3ja297Za8tnYsUJpPOePl3vzQlStdedK1X1po11DJ7N/9OxW+1LfXyGpYpjeN/6b+vvxt+XZSuOPS/vLLDZ8cL5nxugT9smcpanKOtwZ9BtG1J/DfadMdfvLKntvm+l9915teidHqkP4T38BH9d+S50wbezCYBdwNHOefaB6BmBUU2BZgDHENw0lVERMpEIT34M4Ea4Aoza3/seuAEgl59DXA/cEcP1E9ERIpUyEnW7wDfyVO8XWmrIyIipTLgpyroyIPv/bNH9rtgZTxVweBjftkjxxiIqivi2QrfnDrZK6v9UTzGm7tw8gu7xjN47rFEi2X3huxLJgG2+n48RcftP3zbK9vm+j3iTM7qaNlnvppnPu6VHfXnuPSh9172yk4ev7uX/3jWuHvLv/z9nNjg17U/0VQFIiIJpQZeRCShNEQjibHolPhO45rzrvQLs4Zlfr/DD72iG1L+JXvS+95dEd9Bvhs5s3QeXvphs9Nr8l/ueu5X/LtVl6/tv7O6qgcvIpJQauBFRBJKDbyISEJpDF4SI7NtvGB20/03eWV/OXdelP7KktKvECT9y7Er/HH1J7MW/X63tf+OuedSD15EJKHUwIuIJJSGaCQxBn/9tr6ugvQTLy5508vXb398H9WkZ6kHLyKSUGrgRUQSSg28iEhCqYEXEUkoNfAiIgmlBl5EJKHUwIuIJJQaeBGRhFIDLyKSUH1xJ2sGoCKT6oNDS7us9z9Tol0qrmWixLFVXMtEMXHtiwZ+A4CJE+r74NCyHhsAb3a6VWH7UVzLSyliq7iWn4Lj2hcN/HPAnsBCoKUPji+BDMEH5bkS7U9xLR+ljK3iWj66HNdUW1tbz1VHRET6jE6yiogklBp4EZGEUgMvIpJQauBFRBJKDbyISEKpgRcRSSg18CIiCdUni26b2THA+UAlcJVz7rq+qEc5MLMLgCPD7H3OubPNbH/gCqAWuN05d36fVbALFNeY4ppc/Sm2vd6DN7MJwMXAHsD2wClmtmVv16MchB+KA4EdCN6LqWZ2NDAdOBTYAphmZgf1XS0Lo7jGFNfk6m+x7Yshmv2Bmc65pc651cAdwBF9UI9ysBA4wznX6JxrAl4DNgNed87Ncc41A7cCX+jLShZIcY0prsnVr2LbF0M04wnepHYLgZ37oB59zjn3r/a0mW1K8LPvWj76/kzs5aoVQ3ENKa7J1d9i2xc9+DSQPQFOCmjtg3qUDTPbCngEOAt4i/75/iiuORTX5Oovse2LBn4e4RSkoXHAgj6oR1kws92BGcD3nHO/of++P/213j1CcU2u/hTbvhiieRS40MxGA6uBw4FT+qAefc7MJgF3A0c552aGD88KimwKMAc4huAETrlTXEOKa3L1t9j2egPvnJtvZucBjwFVwE3OuWd7ux5l4kygBrjCzNofux44AbgzLLuf4MRWWVNcPYprcvWr2Go+eBGRhNKdrCIiCaUGXkQkodTAi4gklBp4EZGEUgMvIpJQauBFRBJKDbyISEKpgRcRSaj/B3ZtPKrFppIUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Top 6 errors \n",
    "most_important_errors = sorted_dela_errors[-6:]\n",
    "\n",
    "# Show the top 6 errors\n",
    "display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict results\n",
    "results = model.predict(test)\n",
    "\n",
    "# select the indix with the maximum probability\n",
    "results = np.argmax(results,axis = 1)\n",
    "\n",
    "results = pd.Series(results,name=\"Label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n",
    "\n",
    "submission.to_csv(\"cnn_mnist.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}