anom_wtop_vs_gam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense, Input, Conv2D, Conv2DTranspose, Lambda, Flatten, Reshape, MaxPooling2D\n",
    "from keras.models import Model\n",
    "from keras.losses import mean_squared_error, kullback_leibler_divergence\n",
    "from keras.callbacks import EarlyStopping, TerminateOnNaN, ReduceLROnPlateau\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os, scipy\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from phenom_to_df import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get the data\n",
    "df = csv_to_df('gam_samples/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.fillna(0)\n",
    "variables = [entry for entry in df.columns if entry[0] == 'E'] + [entry for entry in df.columns if entry[0:2] == 'pt'] + [entry for entry in df.columns if entry[0:2] == 'et'] + [entry for entry in df.columns if entry[0:2] == 'ph']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_ID</th>\n",
       "      <th>process_ID</th>\n",
       "      <th>event_weight</th>\n",
       "      <th>MET</th>\n",
       "      <th>MET_Phi</th>\n",
       "      <th>obj1</th>\n",
       "      <th>E1</th>\n",
       "      <th>pt1</th>\n",
       "      <th>eta1</th>\n",
       "      <th>phi1</th>\n",
       "      <th>...</th>\n",
       "      <th>obj11</th>\n",
       "      <th>E11</th>\n",
       "      <th>pt11</th>\n",
       "      <th>eta11</th>\n",
       "      <th>phi11</th>\n",
       "      <th>obj12</th>\n",
       "      <th>E12</th>\n",
       "      <th>pt12</th>\n",
       "      <th>eta12</th>\n",
       "      <th>phi12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>18984</td>\n",
       "      <td>2gam</td>\n",
       "      <td>1</td>\n",
       "      <td>42260.8</td>\n",
       "      <td>-1.43297</td>\n",
       "      <td>j</td>\n",
       "      <td>287062</td>\n",
       "      <td>248208</td>\n",
       "      <td>-0.548137</td>\n",
       "      <td>0.264839</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>18984</td>\n",
       "      <td>2gam</td>\n",
       "      <td>1</td>\n",
       "      <td>6035.72</td>\n",
       "      <td>1.96804</td>\n",
       "      <td>g</td>\n",
       "      <td>85958.2</td>\n",
       "      <td>23583.2</td>\n",
       "      <td>-1.9671</td>\n",
       "      <td>0.951141</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>18984</td>\n",
       "      <td>2gam</td>\n",
       "      <td>1</td>\n",
       "      <td>6622.72</td>\n",
       "      <td>-1.28153</td>\n",
       "      <td>g</td>\n",
       "      <td>33021.6</td>\n",
       "      <td>29679.4</td>\n",
       "      <td>-0.47023</td>\n",
       "      <td>1.32244</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>18984</td>\n",
       "      <td>2gam</td>\n",
       "      <td>1</td>\n",
       "      <td>6343.79</td>\n",
       "      <td>0.247373</td>\n",
       "      <td>g</td>\n",
       "      <td>30126.4</td>\n",
       "      <td>23789.3</td>\n",
       "      <td>0.714609</td>\n",
       "      <td>-2.42662</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>18984</td>\n",
       "      <td>2gam</td>\n",
       "      <td>1</td>\n",
       "      <td>8157.26</td>\n",
       "      <td>1.55667</td>\n",
       "      <td>g</td>\n",
       "      <td>100611</td>\n",
       "      <td>26763.5</td>\n",
       "      <td>1.99919</td>\n",
       "      <td>-2.17258</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1220762</td>\n",
       "      <td>26615</td>\n",
       "      <td>Wgam</td>\n",
       "      <td>1</td>\n",
       "      <td>1421.49</td>\n",
       "      <td>-2.474</td>\n",
       "      <td>j</td>\n",
       "      <td>204644</td>\n",
       "      <td>67794.3</td>\n",
       "      <td>1.76918</td>\n",
       "      <td>-0.687769</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1220763</td>\n",
       "      <td>26615</td>\n",
       "      <td>Wgam</td>\n",
       "      <td>1</td>\n",
       "      <td>186690</td>\n",
       "      <td>0.211668</td>\n",
       "      <td>j</td>\n",
       "      <td>270526</td>\n",
       "      <td>142188</td>\n",
       "      <td>-1.25264</td>\n",
       "      <td>-2.68393</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1220764</td>\n",
       "      <td>26615</td>\n",
       "      <td>Wgam</td>\n",
       "      <td>1</td>\n",
       "      <td>5755.02</td>\n",
       "      <td>-0.0294825</td>\n",
       "      <td>j</td>\n",
       "      <td>127753</td>\n",
       "      <td>56116.6</td>\n",
       "      <td>1.46162</td>\n",
       "      <td>2.70033</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1220765</td>\n",
       "      <td>26615</td>\n",
       "      <td>Wgam</td>\n",
       "      <td>1</td>\n",
       "      <td>21209.7</td>\n",
       "      <td>1.45601</td>\n",
       "      <td>j</td>\n",
       "      <td>178814</td>\n",
       "      <td>69628</td>\n",
       "      <td>1.5946</td>\n",
       "      <td>-0.971769</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1220766</td>\n",
       "      <td>26615</td>\n",
       "      <td>Wgam</td>\n",
       "      <td>1</td>\n",
       "      <td>23873.7</td>\n",
       "      <td>-1.6074</td>\n",
       "      <td>g</td>\n",
       "      <td>43675.7</td>\n",
       "      <td>32792.8</td>\n",
       "      <td>0.793701</td>\n",
       "      <td>-1.71254</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1220767 rows × 65 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        event_ID process_ID event_weight      MET     MET_Phi obj1       E1  \\\n",
       "0          18984       2gam            1  42260.8    -1.43297    j   287062   \n",
       "1          18984       2gam            1  6035.72     1.96804    g  85958.2   \n",
       "2          18984       2gam            1  6622.72    -1.28153    g  33021.6   \n",
       "3          18984       2gam            1  6343.79    0.247373    g  30126.4   \n",
       "4          18984       2gam            1  8157.26     1.55667    g   100611   \n",
       "...          ...        ...          ...      ...         ...  ...      ...   \n",
       "1220762    26615       Wgam            1  1421.49      -2.474    j   204644   \n",
       "1220763    26615       Wgam            1   186690    0.211668    j   270526   \n",
       "1220764    26615       Wgam            1  5755.02  -0.0294825    j   127753   \n",
       "1220765    26615       Wgam            1  21209.7     1.45601    j   178814   \n",
       "1220766    26615       Wgam            1  23873.7     -1.6074    g  43675.7   \n",
       "\n",
       "             pt1       eta1       phi1  ... obj11 E11 pt11 eta11 phi11 obj12  \\\n",
       "0         248208  -0.548137   0.264839  ...     0   0    0     0     0     0   \n",
       "1        23583.2    -1.9671   0.951141  ...     0   0    0     0     0     0   \n",
       "2        29679.4   -0.47023    1.32244  ...     0   0    0     0     0     0   \n",
       "3        23789.3   0.714609   -2.42662  ...     0   0    0     0     0     0   \n",
       "4        26763.5    1.99919   -2.17258  ...     0   0    0     0     0     0   \n",
       "...          ...        ...        ...  ...   ...  ..  ...   ...   ...   ...   \n",
       "1220762  67794.3    1.76918  -0.687769  ...     0   0    0     0     0     0   \n",
       "1220763   142188   -1.25264   -2.68393  ...     0   0    0     0     0     0   \n",
       "1220764  56116.6    1.46162    2.70033  ...     0   0    0     0     0     0   \n",
       "1220765    69628     1.5946  -0.971769  ...     0   0    0     0     0     0   \n",
       "1220766  32792.8   0.793701   -1.71254  ...     0   0    0     0     0     0   \n",
       "\n",
       "        E12 pt12 eta12 phi12  \n",
       "0         0    0     0     0  \n",
       "1         0    0     0     0  \n",
       "2         0    0     0     0  \n",
       "3         0    0     0     0  \n",
       "4         0    0     0     0  \n",
       "...      ..  ...   ...   ...  \n",
       "1220762   0    0     0     0  \n",
       "1220763   0    0     0     0  \n",
       "1220764   0    0     0     0  \n",
       "1220765   0    0     0     0  \n",
       "1220766   0    0     0     0  \n",
       "\n",
       "[1220767 rows x 65 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/joe/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:4102: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    }
   ],
   "source": [
    "#Now we one_hot encode the data to make sure we have a class definition\n",
    "df = df[['process_ID']+variables]\n",
    "one_hot = pd.get_dummies(df['process_ID'])\n",
    "processes = one_hot.columns\n",
    "df.drop('process_ID', axis = 'columns', inplace = True)\n",
    "df = pd.concat([df, one_hot], sort = False, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We have created a df of our values and some kind of class label\n",
    "#We should normalize\n",
    "x = df[variables].values\n",
    "x_scaled = StandardScaler().fit_transform(x)\n",
    "df[variables] = x_scaled\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(df[variables].values,\n",
    "\t\t\t\t\t\t    df[processes].values, \n",
    "                                                    shuffle = True,\n",
    "                                                    random_state = 42,\n",
    "                                                    test_size = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reshape the data. This will have to be done explicitly until you think of a clever way of doing it\n",
    "x_train = x_train.reshape((x_train.shape[0], 4, 3, 4))\n",
    "x_test = x_test.reshape((x_test.shape[0], 4, 3, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sampling(args):\n",
    "    z_mean, z_log_var=args\n",
    "    epsilon=tf.random.uniform(shape=(K.shape(z_mean)[0], latent_dim))\n",
    "    return z_mean+K.exp(z_log_var/2)*epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#VAE Model\n",
    "original_dim = x_train.shape[0]\n",
    "input_shape = x_train.shape[1:]\n",
    "latent_dim = 4\n",
    "intermediate_dim = 50\n",
    "kernel_max_norm = 1000.\n",
    "act_fun = 'relu'\n",
    "epsilon_std = 1.\n",
    "filters = 32\n",
    "kernel_size = (3, 3)\n",
    "strides = (1, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Layers\n",
    "x_input = Input(shape = input_shape, name = 'Input')\n",
    "conv_1 = Conv2D(filters = filters, \n",
    "                activation = act_fun,\n",
    "                kernel_size = kernel_size,\n",
    "                strides = strides, \n",
    "                padding = 'same')(x_input)\n",
    "pool_1 = MaxPooling2D(pool_size = (1, 1),\n",
    "                      strides = 2,\n",
    "                      data_format = 'channels_first')(conv_1)\n",
    "conv_2 = Conv2D(filters = filters, \n",
    "                activation = act_fun, \n",
    "                kernel_size = kernel_size, \n",
    "                strides = strides, \n",
    "                padding = 'same')(pool_1)\n",
    "\n",
    "#Shape info needed to inform the dense layer in the decoder\n",
    "flatten = Flatten()(conv_2)\n",
    "#cnn = Model(x_input, flatten, name = 'cnn')\n",
    "#cnn.summary()\n",
    "\n",
    "shape = input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"encoder\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input (InputLayer)              (None, 4, 3, 4)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 4, 3, 32)     1184        Input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 4, 2, 16)     0           conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 4, 2, 32)     4640        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_encoder_1 (Dense)         (None, 50)           12850       flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_encoder_2 (Dense)         (None, 16)           816         dense_encoder_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "z_mean (Dense)                  (None, 4)            68          dense_encoder_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "z_var (Dense)                   (None, 4)            68          dense_encoder_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "sampling (Lambda)               (None, 4)            0           z_mean[0][0]                     \n",
      "                                                                 z_var[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 19,626\n",
      "Trainable params: 19,626\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Encoding\n",
    "dense_encoder_1 = Dense(intermediate_dim,\n",
    "                        activation = act_fun,\n",
    "                        name = 'dense_encoder_1')(flatten)\n",
    "dense_encoder_2 = Dense(16,\n",
    "                        activation = act_fun,\n",
    "                        name = 'dense_encoder_2')(dense_encoder_1)\n",
    "z_mean = Dense(latent_dim, name = 'z_mean')(dense_encoder_2)\n",
    "z_var = Dense(latent_dim, name = 'z_var')(dense_encoder_2)\n",
    "\n",
    "z = Lambda(sampling, output_shape = (latent_dim, ), name = 'sampling')([z_mean, z_var])\n",
    "\n",
    "encoder = Model(x_input, [z_mean, z_var, z], name = 'encoder')\n",
    "encoder.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"decoder\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "decoder_input (InputLayer)   (None, 4)                 0         \n",
      "_________________________________________________________________\n",
      "dense_decoder_1 (Dense)      (None, 48)                240       \n",
      "_________________________________________________________________\n",
      "reshape_1 (Reshape)          (None, 4, 3, 4)           0         \n",
      "_________________________________________________________________\n",
      "deconv_1 (Conv2DTranspose)   (None, 4, 3, 32)          1184      \n",
      "_________________________________________________________________\n",
      "deconv_2 (Conv2DTranspose)   (None, 4, 3, 16)          4624      \n",
      "_________________________________________________________________\n",
      "decoder_output (Conv2DTransp (None, 4, 3, 4)           580       \n",
      "=================================================================\n",
      "Total params: 6,628\n",
      "Trainable params: 6,628\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Decoding\n",
    "decoder_input = Input(shape = (latent_dim, ), name = 'decoder_input')\n",
    "dense_decoder_1 = Dense(shape[0]*shape[1]*shape[2], \n",
    "                        activation = act_fun, name = 'dense_decoder_1')(decoder_input)\n",
    "reshape = Reshape((shape[0], shape[1], shape[2]))(dense_decoder_1)\n",
    "\n",
    "deconv_1 = Conv2DTranspose(filters = filters,\n",
    "                           kernel_size = kernel_size, \n",
    "                           activation = act_fun, \n",
    "                           strides = strides, \n",
    "                           padding = 'same', \n",
    "                           name = 'deconv_1')(reshape)\n",
    "deconv_2 = Conv2DTranspose(filters = 16,\n",
    "                           kernel_size = kernel_size, \n",
    "                           activation = act_fun, \n",
    "                           strides = strides, \n",
    "                           padding = 'same', \n",
    "                           name = 'deconv_2')(deconv_1)\n",
    "\n",
    "output = Conv2DTranspose(filters = 4, \n",
    "                         kernel_size = kernel_size, \n",
    "                         activation = 'sigmoid', \n",
    "                         padding = 'same', \n",
    "                         strides = strides, \n",
    "                         name = 'decoder_output')(deconv_2)\n",
    "\n",
    "decoder = Model(decoder_input, output, name = 'decoder')\n",
    "decoder.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Build the VAE model\n",
    "outputs = decoder(encoder(x_input)[2])\n",
    "vae = Model(x_input, outputs, name = 'vae')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define some losses\n",
    "def kl_loss(y_true, y_pred):\n",
    "    kl_loss = 1 + z_var - K.square(z_mean) - K.exp(z_var)\n",
    "    kl_loss = K.sum(kl_loss, axis=-1)\n",
    "    kl_loss *= -0.5\n",
    "    kl_loss = K.mean(kl_loss)\n",
    "    return kl_loss\n",
    "\n",
    "def recon_loss(y_true, y_pred):\n",
    "    recon_loss = mean_squared_error(K.flatten(y_true), K.flatten(y_pred))\n",
    "    recon_loss *= shape[0]*shape[1]*shape[2]\n",
    "    recon_loss = K.mean(recon_loss)\n",
    "    return recon_loss\n",
    "\n",
    "def my_vae_loss(y_true, y_pred):\n",
    "    recon = recon_loss(y_true, y_pred) \n",
    "    kl = kl_loss(y_true, y_pred)\n",
    "    vae_loss = recon + kl\n",
    "    return vae_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"vae\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Input (InputLayer)           (None, 4, 3, 4)           0         \n",
      "_________________________________________________________________\n",
      "encoder (Model)              [(None, 4), (None, 4), (N 19626     \n",
      "_________________________________________________________________\n",
      "decoder (Model)              (None, 4, 3, 4)           6628      \n",
      "=================================================================\n",
      "Total params: 26,254\n",
      "Trainable params: 26,254\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "vae.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"vae\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Input (InputLayer)           (None, 4, 3, 4)           0         \n",
      "_________________________________________________________________\n",
      "encoder (Model)              [(None, 4), (None, 4), (N 19626     \n",
      "_________________________________________________________________\n",
      "decoder (Model)              (None, 4, 3, 4)           6628      \n",
      "=================================================================\n",
      "Total params: 26,254\n",
      "Trainable params: 26,254\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "vae.compile(optimizer = 'rmsprop', loss = my_vae_loss, metrics = [kl_loss, recon_loss])\n",
    "vae.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1098690 samples, validate on 122077 samples\n",
      "Epoch 1/100\n",
      " - 50s - loss: 45.4371 - kl_loss: 0.4045 - recon_loss: 45.0290 - val_loss: 36.0624 - val_kl_loss: 0.5234 - val_recon_loss: 35.3903\n",
      "Epoch 2/100\n",
      " - 60s - loss: 44.1258 - kl_loss: 0.5625 - recon_loss: 43.5563 - val_loss: 35.4694 - val_kl_loss: 0.6266 - val_recon_loss: 34.6936\n",
      "Epoch 3/100\n",
      " - 62s - loss: 43.7175 - kl_loss: 0.6192 - recon_loss: 43.0923 - val_loss: 35.2114 - val_kl_loss: 0.5994 - val_recon_loss: 34.4643\n",
      "Epoch 4/100\n",
      " - 54s - loss: 43.5488 - kl_loss: 0.6404 - recon_loss: 42.9057 - val_loss: 35.1420 - val_kl_loss: 0.6690 - val_recon_loss: 34.3250\n",
      "Epoch 5/100\n",
      " - 53s - loss: 43.4499 - kl_loss: 0.6542 - recon_loss: 42.7914 - val_loss: 35.0525 - val_kl_loss: 0.6401 - val_recon_loss: 34.2651\n",
      "Epoch 6/100\n",
      " - 53s - loss: 43.3737 - kl_loss: 0.6673 - recon_loss: 42.7009 - val_loss: 35.0054 - val_kl_loss: 0.6551 - val_recon_loss: 34.2040\n",
      "Epoch 7/100\n",
      " - 55s - loss: 43.3180 - kl_loss: 0.6854 - recon_loss: 42.6314 - val_loss: 34.9486 - val_kl_loss: 0.7046 - val_recon_loss: 34.0962\n",
      "Epoch 8/100\n",
      " - 52s - loss: 43.2715 - kl_loss: 0.6982 - recon_loss: 42.5697 - val_loss: 34.9004 - val_kl_loss: 0.6824 - val_recon_loss: 34.0712\n",
      "Epoch 9/100\n",
      " - 72s - loss: 43.2312 - kl_loss: 0.7059 - recon_loss: 42.5186 - val_loss: 34.8377 - val_kl_loss: 0.6927 - val_recon_loss: 33.9980\n",
      "Epoch 10/100\n",
      " - 61s - loss: 43.2008 - kl_loss: 0.7096 - recon_loss: 42.4853 - val_loss: 34.8735 - val_kl_loss: 0.7352 - val_recon_loss: 33.9909\n"
     ]
    }
   ],
   "source": [
    "#Fit the model\n",
    "epochs = 100\n",
    "batch_size = 1000\n",
    "\n",
    "history = vae.fit(x = x_train,\n",
    "                  y = x_train,\n",
    "                  validation_data = (x_test, x_test),\n",
    "                  epochs = epochs,\n",
    "                  batch_size = batch_size,\n",
    "                  verbose = 2, \n",
    "                 callbacks = [TerminateOnNaN(), ReduceLROnPlateau(), EarlyStopping()])\n",
    "\n",
    "vae.save('cnn-vae_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3hddZ3v8fd3X3NvekkLbdqmFxQqSi2VUfAMgjeoyOiogyjqdMAefM4II8ejOGeeozjjGTjjBRVmkNH24Ig6CnJEHh3U8TKHowJtKSKt2AK9xBabXpM0Tfbte/5Yayc76U6atNnZSdbn9Tzr2Wv91mV/s6Gf37rttc3dERGR6IhVuwAREZlYCn4RkYhR8IuIRIyCX0QkYhT8IiIRo+AXEYkYBb/IMMyszczczBKjWPbPzeyR092OyERQ8Mu0YGY7zSxjZnOGtG8JQ7etOpWJTD4KfplOngeuLk6Y2UuB2uqVIzI5KfhlOvkX4L0l0+8Dvlq6gJnNMLOvmlmHme0ys78xs1g4L25mnzazA2b2HPCmMut+xcz2mdnvzezvzCw+1iLNbL6ZPWhmh8xsh5m9v2TeBWa20cw6zewPZvbZsL3GzL5mZgfN7IiZPW5m88b63iKg4Jfp5VdAk5mdEwbyVcDXhizzRWAGsBS4mKCjWBvOez9wBfByYDXw9iHr3gPkgOXhMm8ArjuFOr8BtAPzw/f4n2b22nDe54HPu3sTsAz4Vtj+vrDuhcBs4Hrg+Cm8t4iCX6ad4l7/64HfAr8vzijpDD7m7l3uvhP4DPCecJE/A2539z3ufgj4+5J15wGXA3/l7sfcfT/wOeCdYynOzBYCrwY+6u697r4F+HJJDVlguZnNcfdud/9VSftsYLm75919k7t3juW9RYoU/DLd/AvwLuDPGXKaB5gDpIBdJW27gAXh+Hxgz5B5RYuBJLAvPNVyBPgSMHeM9c0HDrl71zA1XAu8CPhteDrnipK/62Hgm2a218z+l5klx/jeIoCCX6YZd99FcJF3DfCdIbMPEOw5Ly5pW8TAUcE+glMppfOK9gB9wBx3bw6HJnd/yRhL3AvMMrPGcjW4+3Z3v5qgQ7kNuM/M6t096+63uPsK4EKCU1LvReQUKPhlOroWuNTdj5U2unue4Jz5p8ys0cwWAzcxcB3gW8ANZtZqZjOBm0vW3Qf8EPiMmTWZWczMlpnZxWMpzN33AL8A/j68YPuysN57AczsGjNrcfcCcCRcLW9ml5jZS8PTVZ0EHVh+LO8tUqTgl2nH3Z91943DzP4gcAx4DngE+DqwPpz3zwSnU54ENnPiEcN7CU4VbQUOA/cBZ55CiVcDbQR7/w8AH3f3H4XzLgOeNrNuggu973T3XuCM8P06gW3AzznxwrXIqJh+iEVEJFq0xy8iEjEKfhGRiFHwi4hEjIJfRCRipsRjYufMmeNtbW3VLkNEZErZtGnTAXdvGdo+JYK/ra2NjRuHuztPRETKMbNd5dp1qkdEJGIU/CIiEaPgFxGJmClxjr+cbDZLe3s7vb291S5lQtTU1NDa2koyqQcyisjpmbLB397eTmNjI21tbZhZtcupKHfn4MGDtLe3s2TJkmqXIyJT3JQ91dPb28vs2bOnfegDmBmzZ8+OzNGNiFTWlA1+IBKhXxSlv1VEKmtKB//JHOnJcLgnU+0yREQmlWkb/O7O4Z4sew71sOdQD/nC+D5++uDBg6xcuZKVK1dyxhlnsGDBgv7pTGZ0nc3atWt55plnxrUuEZGTmbIXd0/GzGibXcf+rj72d/bSk8mxcFYddanx+ZNnz57Nli1bAPjEJz5BQ0MDH/7whwct4+64O7FY+f51w4YN41KLiMhYTNs9fgjCf15TDUtbGig4PLv/GPu7eqnkj8/s2LGDc889l+uvv55Vq1axb98+1q1bx+rVq3nJS17CJz/5yf5lX/3qV7NlyxZyuRzNzc3cfPPNnHfeebzqVa9i//79FatRRKJtWuzx3/K9p9m6t3PEZRzI5PLk8k48ZqSTcUa6XLpifhMff/NYf0c7sHXrVjZs2MBdd90FwK233sqsWbPI5XJccsklvP3tb2fFihWD1jl69CgXX3wxt956KzfddBPr16/n5ptvLrd5EZHTMq33+EsZkE7ESSdj5N05nsmP+3n/omXLlvGKV7yif/ob3/gGq1atYtWqVWzbto2tW7eesE5tbS2XX345AOeffz47d+6sSG0iItNij3+se+a92Ty7D/XQm80zpyHNGU01xGLjd7tkfX19//j27dv5/Oc/z2OPPUZzczPXXHNN2fvxU6lU/3g8HieXy41bPSIipSKzx1+qJhlneUsDcxrSHOjuY0dHN73ZfEXeq7Ozk8bGRpqamti3bx8PP/xwRd5HRGS0psUe/6mIxYz5zbU0pBO0Hz7Ojv3dnDmjhln1qXH9stSqVatYsWIF5557LkuXLuWiiy4at22LiJwKq+QdLuNl9erVPvSHWLZt28Y555wzLtvP5gvsOdRDd1+OGbVJFjTXkohPvoOh8fybRWT6M7NN7r56aPvkS7cqSMZjLJlTz5kzaujszbF9fzfdfTrHLiLTk4I/ZGa0NNawrKWemMHzHd28cLSy9/yLiFSDgn+IulSC5XMbaa5Lsb+rl2c7jpHJVebCr4hINSj4y4jHjIWz6lg0q46+bJ7t+7s5ooe9icg0oeAfQXNdirPmNZBOxNl9qIf2CjzsTURkoin4TyKViLO0pZ65jTUc6smwY383xzO68CsiU1dk7+Mfi5gZZ8yooSGdYM/hHnZ0HCOVPcZVb1kDwAsvvEA8HqelpQWAxx57bNA3cUeyfv161qxZwxlnnFGx+kVESin4x6ChJsFZcxtoP3ycTq/jgR//P1pn1vKpv/1k2ccyj8b69etZtWqVgl9EJoyCf4wS8RiLZ9dx6FiGfUd72f6HbvpyeRpKlrnnnnu48847yWQyXHjhhdxxxx0UCgXWrl3Lli1bcHfWrVvHvHnz2LJlC1dddRW1tbVjOlIQETlV0yP4f3AzvPDU+G7zjJfC5beWnWVmzG5IU59OsPtQD0d6siTTWQrubH36aR544AF+8YtfkEgkWLduHd/85jdZtmwZBw4c4KmngjqPHDlCc3MzX/ziF7njjjtYuXLl+NYvIjKM6RH8VVJ82FttKs6xvhzP7u/mBw//kMcff5zVq4NvSR8/fpyFCxfyxje+kWeeeYYbb7yRNWvW8IY3vKHK1YtIVE2P4B9mz3wixGJGU02SRDpFNl+go6uXq695H/9w66dOeNjbr3/9a37wgx/whS98gfvvv5+77767SlWLSJTpds5xUpOMc9bcRi655LXcd9+32bJ9N7lCgYMHD7J79246Ojpwd97xjndwyy23sHnzZgAaGxvp6uqqcvUiEiXTY49/kkgmYlz2xxfw0b/+71z91itwd+pqUtz9pS8Rj8e59tprcXfMjNtuuw2AtWvXct111+nirohMGD2WuUJ6Mjl2H+ohm3PmNqWZ25g+7ef8T/a/WUQml6o9ltnM4mb2hJk9NKT9i2bWXen3r5a6VHDPf3Ndkj909rJ1Xyc7Dxxjf2cv3b1ZPfpBRKpmIk713AhsA5qKDWa2GmiegPeuqngsxsJZdTTVJuk6nuVYJk9nbxYAw6hJxqhLJ6hPxalLxUnGY+P6618iIuVUNPjNrBV4E/Ap4KawLQ78A/Au4K2ns/3i+fLJbkZtkhm1SQBy+QI92Tw9fXl6MjkOH8twsDvY+0/EY/2dQF0qQW0y3v8j8FPhlJyITA2V3uO/HfgI0FjS9pfAg+6+b6TQNrN1wDqARYsWnTC/pqaGgwcPMnv27CkR/kWJeIymeIymmqAjcHd6s3l6MsFwLJPj6PHwqMCM2mSc2mSMXE8nltCFXxE5fRULfjO7Atjv7pvM7DVh23zgHcBrTra+u98N3A3Bxd2h81tbW2lvb6ejo2M8y54UYgUnky+QyRU4nCvQlyuw80iGLz56mMbaXaxaPJPzFzVz/uJZnH1mI8lJ+PvAIjJ5VXKP/yLgSjNbA9QQnON/GugDdoR76XVmtsPdl49148lkkiVLloxnvZNWJldgzt6j3Nh0hM27DvP484f43pN7AahJxjivtZnzF89k1aKZrFo8k1n1OjIQkeFNyO2c4R7/h939iiHt3e7eUH6tAeVu54y6vUeOs2nXYTbtOswTuw/z9N5OcuGdQkvm1LNq0cygM1jczFlzG4nHps7pMBEZH8PdzqkvcE1R85trmd9cy5vPmw/A8UyeX7cfYfPuI2zadZifPrOf+ze3A9CYTrByUTOrFs1k0aw6WhrT/cPMupQ6BZGImbJf4JKRuTu7DvYERwW7D7N512Ge+UMXQ/9zxwxmN6RpaUgP6hDmFKdL2ptqElPqQrpI1GmPP2LMjLY59bTNqedt57cCwbeJ93f20dHdx4Gu4LWjq2To7mP7H7ro6O4jmz9xhyAVjwWdwpAOoaUhVTJew5zGFHUp/a8lMlnpX2eE1KUStM1J0DanfsTl3J2jx7ODOoSOIR1F++Eetuw5zMFjmROOIgDqU/FBRxAtDQNHEXMa0jTVJmlIJ2isSdCQTlCfTpBK6O4kkYmg4JcTmBnNdSma61KcNa9xxGVz+QKHejKDjhwOdGdKOopennmhi0e6DtDZO/KP1KcTMRprgk6gIRyKHUNDTYKGdHJgOmxr7J9XnE5Sk9Q3oEVGouCX05KIx5jbWMPcxpqTLtubzXPwWIYDXX109+Xo6s3R3ZejuzcbTPfl6O5vC6b3HukNpvtydPVmy56CGioes2E6jpLpdJL6dJza8JvStclE+I3pODXJkm9Pp+LUJuM6GpFpRcEvE6YmGWdBcy0LmmtPeRt9ufxA51DaUQzqSEqng07l8LEMuw/19M/ryeTH9L6JmJV0EnFqUwMdRTA9uAMpdhiDxxODtlE6L6Ev4ckEUvDLlJJOxEk3xJndkD6t7eQLzrFMjuOZ4uMycoMenVFsP57NczyTG9R+PFucF3QiHV19A23htsb68NV4zEgnYqQTMVKJWPB3JmKkkwPjqXD+0HnDtZ+wTnLIciXtqXis/7lQMv0p+CWS4uFPZhafmTSe3INHbgx0KqUdRklnk83TG45n8nky4eM5+rIF+nL5YDxXCNvzdPflBs3rXz6XH9UpsJNJxo1kPOgskvGgM0gXxxMxknHrn1fanorHSIav/dPF8USMVMl6qSHrnbB8PEYyEdRRrCERNxIx03WbcaTgFxlnZhbuVcdprpuY98wXvL+DKO0QerMD4wPtBfqygzuW3myeTL5ANlcIXvPBvGzeyYQdSyZctiub42AuWKZ0nb5iW64w5iOe0Sh2AsVOIdk/Xq6tZDwRIxkbZjweTicGr5MIl4nHjGTciMeCtkTciMeMRKx0XjBd7KDKTRe3NVk6MAW/yDQQD69B1Kbi1S4FGOiIig8bLHYImZLX0k4mU9IJ5QpONh90OtlwuWyhZDwfTpcbD9fpyeQG1g/bc/kCmZK2XD44MptoMQtuiijXKZSbvuua81k4a3z3IBT8IjLu+jsiJkdHNBx3J1fw/k6gv6PIOblCgXwhmJ8PO57idC4/eP6g6XxxnULJvGA6m/eSbZ44PbDswPtV4um7Cn4RiSwzC0/vMOk7qfGke8hERCJGwS8iEjEKfhGRiFHwi4hEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxCj4RUQiRsEvIhIxCn4RkYhR8IuIRIyCX0QkYhT8IiIRo+AXEYkYBb+ISMQo+EVEIqbiwW9mcTN7wsweCqfvNbNnzOw3ZrbezJKVrkFERAZMxB7/jcC2kul7gbOBlwK1wHUTUIOIiIQqGvxm1gq8Cfhysc3dv+8h4DGgtZI1iIjIYJXe478d+AhQGDojPMXzHuDfyq1oZuvMbKOZbezo6KhslSIiEVKx4DezK4D97r5pmEX+EfgPd/+/5Wa6+93uvtrdV7e0tFSqTBGRyElUcNsXAVea2RqgBmgys6+5+zVm9nGgBfjPFXx/EREpo2J7/O7+MXdvdfc24J3AT8LQvw54I3C1u59wCkhERCqrGvfx3wXMA35pZlvM7H9UoQYRkciq5Kmefu7+M+Bn4fiEvKeIiJSnb+6KiESMgl9EJGIU/CIiEaPgFxGJGAW/iEjEKPhFRCJGwS8iEjEKfhGRiFHwi4hEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxCj4RUQiRsEvIhIxCn4RkYhR8IuIRIyCX0QkYhT8IiIRo+AXEYkYBb+ISMSMKvjNbJmZpcPx15jZDWbWXNnSRESkEka7x38/kDez5cBXgCXA1ytWlYiIVMxog7/g7jngrcDt7v4h4MzKlSUiIpUy2uDPmtnVwPuAh8K2ZGVKEhGRShpt8K8FXgV8yt2fN7MlwNcqV5aIiFRKYjQLuftW4AYAM5sJNLr7rZUsTEREKmO0d/X8zMyazGwW8CSwwcw+W9nSRESkEkZ7qmeGu3cCfwpscPfzgdeNZkUzi5vZE2b2UDi9xMweNbPtZvavZpY6tdJFRORUjDb4E2Z2JvBnDFzcHa0bgW0l07cBn3P3s4DDwLVj3J6IiJyG0Qb/J4GHgWfd/XEzWwpsP9lKZtYKvAn4cjhtwKXAfeEi9wBvGWvRIiJy6kZ7cffbwLdLpp8D3jaKVW8HPgI0htOzgSPhdwIA2oEFo65WRERO22gv7raa2QNmtt/M/mBm94d78yOtcwWw3903lTaXWdSHWX+dmW00s40dHR2jKVNEREZhtKd6NgAPAvMJ9tC/F7aN5CLgSjPbCXyT4BTP7UCzmRWPNFqBveVWdve73X21u69uaWkZZZkiInIyow3+Fnff4O65cPjfwIhp7O4fc/dWd28D3gn8xN3fDfwUeHu42PuA755a6SIicipGG/wHzOya8NbMuJldAxw8xff8KHCTme0gOOf/lVPcjoiInIJRXdwF/gK4A/gcwTn5XxA8xmFU3P1nwM/C8eeAC8ZSpIiIjJ9R7fG7+253v9LdW9x9rru/heDLXCIiMsWczi9w3TRuVYiIyIQ5neAvd2umiIhMcqcT/GXvvxcRkcltxIu7ZtZF+YA3oLYiFYmISEWNGPzu3jjSfBERmXpO51SPiIhMQQp+EZGIUfCLiESMgl9EJGIU/CIiEaPgFxGJGAW/iEjEKPhFRCJGwS8iEjEKfhGRiFHwi4hEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxCj4RUQiRsEvIhIxCn4RkYhR8IuIRIyCX0QkYhT8IiIRo+AXEYkYBb+ISMRULPjNrMbMHjOzJ83saTO7JWx/rZltNrMtZvaImS2vVA0iInKiSu7x9wGXuvt5wErgMjN7JfBPwLvdfSXwdeBvKliDiIgMkajUht3dge5wMhkOHg5NYfsMYG+lahARkRNVLPgBzCwObAKWA3e6+6Nmdh3wfTM7DnQCrxxm3XXAOoBFixZVskwRkUip6MVdd8+Hp3RagQvM7FzgQ8Aad28FNgCfHWbdu919tbuvbmlpqWSZIiKRMiF39bj7EeBnwOXAee7+aDjrX4ELJ6IGEREJVPKunhYzaw7Ha4HXAduAGWb2onCx14dtIiIyQSp5jv9M4J7wPH8M+Ja7P2Rm7wfuN7MCcBj4iwrWICIiQ1Tyrp5fAy8v0/4A8ECl3ldEREamb+6KiESMgl9EJGIU/CIiEaPgFxGJGAW/iEjEKPhFRCJGwS8iEjEKfhGRiFHwi4hEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxCj4RUQiRsEvIhIxCn4RkYhR8IuIRIyCX0QkYhT8IiIRo+AXEYkYBb+ISMQo+EVEIkbBLyISMQp+EZGIUfCLiESMgl9EJGIU/CIiEaPgFxGJmIoFv5nVmNljZvakmT1tZreE7WZmnzKz35nZNjO7oVI1iIjIiRIV3HYfcKm7d5tZEnjEzH4AnAMsBM5294KZza1YBf/xaTjWASv+BBb+EcTiFXsrEZGpomLB7+4OdIeTyXBw4APAu9y9EC63v1I10LkXnvgaPHoXNMyDs68IOoHFF0G8kn2eiMjkZUE+V2jjZnFgE7AcuNPdP2pmB4HPAm8FOoAb3H17mXXXAesAFi1adP6uXbtOrYi+Ltj+Q9j6Xdj+I8j2QN1sOPtNQSew5GKIJ09t2yIik5iZbXL31Se0VzL4S968GXgA+CDwK+Dj7v4ZM/tT4EPu/p9GWn/16tW+cePG0y8k0wM7fhx0Ar97GDJdUNMML14TdALLLoFE+vTfR0RkEqhq8IcFfBw4BlwHXObuO83MgCPuPmOkdcct+Etle+G5nwadwG+/D31HId0EL7os6ASWvxaSteP7niIiE2i44K/YiW4zawGy7n7EzGqB1wG3Af8HuBRYD1wM/K5SNYwoWQMvvjwYchl4/udhJ/AQPPUtSNbDi94QdgKvh3RDVcoUERlvFdvjN7OXAfcAcYLbRr/l7p8MT/vcCywiuPh7vbs/OdK2KrLHP5x8FnY+AtsehG3fC+4KStTA8tfBirfAi94INU0TU4uIyGmo+qme0zGhwV+qkIfdv4StDwYdQdc+iKdg2aXBkcCLL4famRNfl4jIKCj4T1ehAO2PB6eDtj0IR/dALAFLXwPnXBncKlo/u7o1ioiUUPCPJ3fYuznoBLZ+Fw7vBItD26thxZVw9puhcV61qxSRiFPwV4o7vPDUQCdwcDtgsPjC4EjgnDfDjAXVrlJEIkjBPxHcoeO3A53A/q1Be+sFsPhVMGsZzFoKs5dBwxkQ0zPyRKRyJvx2zkgyg7nnBMNrboYD28NrAt+DX/4jFLIDyyZqg05g1pKBzmDW0qBzaDxTnYKIVIz2+CdKIQ9H2+HQs3DoOTj4XPB66NngGkE+M7BsonagQ5i1ZPCRQuN8dQoiMira46+2WBxmLg6GZZcOnlfIQ+fv4WDYKRSHA9uD5wwN6hRqYGZJp1B6pNC0QJ2CiJyUgn8yiMWheVEwLLtk8LxCPnjKaP+RwrNw6PlgfMePId83sGw8DTPbSjqDpQNHCk0L9FhqEQEU/JNfLA7NC4Nh6WsGzysUoGtvyZFCSafw7E8g1zuwbDwFzYuDL5wla0uGuoHxxNC2uuDRFkPbEjWD11OHIjKlKPinslgMZrQGw9KLB88rFIJvGpceKRzeGTymOnscjh8OOobs8eBR1cXXUxFPhx1E3eBOIlGmrTik6iHVAOnG8uOp+mBaj8wWGXcK/ukqFgu+PzBjASz549Gt4w65vpKOoKRTyA2ZHjT0DFm2d2C8+4VwXu+QDmaUNxXE02En0ACpxpLx+mC6f3xoJ9IQdiAlnUiqIXjsttkpf6wi04GCXwaYhXvuNZV9H/cg/Pu6IVMcjoXTXSXj4dAXzs90B0csvZ3BdY/SZQq5Uf6N8ZJOpC7oWBKp4FRYcUikS8ZTwTKDxpPhMqXj5dY92XZSuhgvVaHgl4lnFu6Z1wPj8GgL9+DOpxM6jtLxY8F0aSeSORasl+sLX3uhrzN4THe+2FYczwbLlX4XYzzEEmHnkw5OjY35tWQ8WTP2dXR9JpIU/DL1mYVBlq78g/IKhSD8i51FaccxqKMoN15u+b5wui/oeHJDX3uh90iZ9r7gtNloT5kNJ5YY6AiGHs2ccISTKrPcaNtKttd/hDVMm34Pu+L0CYuMRSwGsfTk+IlO9+AU13AdRrnOYrjX7PGw8ynTQWWPQ+/RIUdHQzovz4/f32XxkpsDiq81wV1niXSZeaWv6fDutJoRXmtOXO9UjnwK+eBIsJANX3ODp/vHcyXLDJ3OldlGZvC8P7p+3HdoFPwiU5VZsEcdTwYXr6upkC9/FHRCW7nOpaRtUKcU3hQw9LWvc6CzyvUOzCv9ouNYxZKDO45EOujMRgrt0z3aGq2Xvl3BLyKTUCweXCynrno1FPKDj2CKtyuf8DpCp5IrGSwedKqxRPiaLDOdOLE9nhp+3km3EU4XtxGLV+QuNAW/iEwPsXjJTQMyEt1LJiISMQp+EZGIUfCLiESMgl9EJGIU/CIiEaPgFxGJGAW/iEjEKPhFRCJmSvzYupl1ALtOcfU5wIFxLGeq0+cxQJ/FYPo8BpsOn8did28Z2jglgv90mNnGcr8yH1X6PAbosxhMn8dg0/nz0KkeEZGIUfCLiERMFIL/7moXMMno8xigz2IwfR6DTdvPY9qf4xcRkcGisMcvIiIlFPwiIhEzrYPfzC4zs2fMbIeZ3VzteqrFzBaa2U/NbJuZPW1mN1a7psnAzOJm9oSZPVTtWqrNzJrN7D4z+234/8mrql1TtZjZh8J/J78xs2+YWU21axpv0zb4zSwO3AlcDqwArjazFdWtqmpywH9193OAVwL/JcKfRakbgW3VLmKS+Dzwb+5+NnAeEf1czGwBcAOw2t3PBeLAO6tb1fibtsEPXADscPfn3D0DfBP4kyrXVBXuvs/dN4fjXQT/qBdUt6rqMrNW4E3Al6tdS7WZWRPwx8BXANw94+5HqltVVSWAWjNLEPyI8N4q1zPupnPwLwD2lEy3E/GwAzCzNuDlwKPVraTqbgc+AhSqXcgksBToADaEp76+bGaR/OFad/898GlgN7APOOruP6xuVeNvOgd/uZ+mj/S9q2bWANwP/JW7d1a7nmoxsyuA/e6+qdq1TBIJYBXwT+7+cuAYEMlrYmY2k+DMwBJgPlBvZtdUt6rxN52Dvx1YWDLdyjQ8ZBstM0sShP697v6datdTZRcBV5rZToJTgJea2deqW1JVtQPt7l48CryPoCOIotcBz7t7h7tnge8AF1a5pnE3nYP/ceAsM1tiZimCCzQPVrmmqjAzIzh/u83dP1vteqrN3T/m7q3u3kbw/8VP3H3a7dWNlru/AOwxsxeHTa8FtlaxpGraDbzSzOrCfzevZRpe6E5Uu4BKcfecmf0l8DDBlfn17v50lcuqlouA9wBPmdmWsO2v3f37VaxJJpcPAveGO0nPAWurXE9VuPujZnYfsJngbrgnmIaPbtAjG0REImY6n+oREZEyFPwiIhGj4BcRiRgFv4hIxCj4RUQiRsEvAphZ3sy2lAzj9s1VM2szs9+M1/ZETte0vY9fZIyOu/vKahchMhG0xy8yAjPbaWa3mdlj4bA8bF9sZhTnU6MAAAFOSURBVP9uZr8OXxeF7fPM7AEzezIcil/3j5vZP4fPef+hmdVW7Y+SyFPwiwRqh5zquapkXqe7XwDcQfBUT8Lxr7r7y4B7gS+E7V8Afu7u5xE876b4bfGzgDvd/SXAEeBtFf57RIalb+6KAGbW7e4NZdp3Ape6+3Phg+5ecPfZZnYAONPds2H7PnefY2YdQKu795Vsow34kbufFU5/FEi6+99V/i8TOZH2+EVOzocZH26ZcvpKxvPo+ppUkYJf5OSuKnn9ZTj+CwZ+ku/dwCPh+L8DH4D+3/RtmqgiRUZLex0igdqSJ5dC8PuzxVs602b2KMGO0tVh2w3AejP7bwS/XlV8muWNwN1mdi3Bnv0HCH7JSWTS0Dl+kRGE5/hXu/uBatciMl50qkdEJGK0xy8iEjHa4xcRiRgFv4hIxCj4RUQiRsEvIhIxCn4RkYj5/2sYKEMrnbseAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['loss', 'kl_loss', 'recon_loss']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae.metrics_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxV9Z3/8dcnIYEkLFlB9oRVEAQ0ggpjVVzAKnZa26rVWm3HtlN/dp/qTMda25naWbpNnWmtYpex2qmdVqpRahc7bixBgbBIDWFJSALZWEP2z++Pe4KXeIEAudzce9/Px+M+vOec7zn3c6/hvu/5nuVr7o6IiEhPKbEuQERE+icFhIiIRKSAEBGRiBQQIiISkQJCREQiUkCIiEhECgiJG2Z2v5n9d/C80MzczAYE0y+a2cf6+PWOeo3T2M44MztoZqm9aPsRM3v5OMv7/H2KHIsCQs4oM9tuZleETd9oZk1m9q5Y1tUXjvXl7e473X2wu3fGoq5j6asAlMSlgJCYMbPbgIeAd7v7n2NdTyJTCMipUEBITJjZncC/A1e7+6vBvEvNrKpHu6P2OE5gopmtMrN9Zva0meWGbeeXZlYbLPs/MzsnbFmGmf27me0Ilr9sZhkRan5fUM+Mk3yvPbvDhpnZo2ZWY2a7zOzrPbqfzMz+I6jlTTNbeJxt32Fmm4O9sOVmNj5smZvZp8zsLeCtk6x5oJl9x8yqg8d3zGxgsCzfzJ4xs71m1mhmL5lZSrDsS8F7OmBmW45Xu/R/CgiJhU8CXwMWuntpH273w8AdwCigA/he2LLngMnAcOB14PGwZf8GnA9cDOQCfwd0hW/YzG4Hvglc4e4bTrPOnwT1TQLmAFcB4V1T84AKIB/4CvC/4WEXVtN7gL8H3gsUAC8BT/Ro9p5ge9NPssZ/AC4EZgOzgLnAl4NlnweqgtccEdTgZjYVuAu4wN2HAFcD20/ydaUfUUBILFwJrADK+ni7P3P3De5+CPhH4APdv8zdfam7H3D3VuB+YFbwSz6FUKh82t13uXunu78atOv2GeCLwKXuXn46BZrZCGAx8Bl3P+Tue4BvAzeGNdsDfMfd2939F8AW4N0RNvdx4BvuvtndO4B/BmaH70UEyxvd/fBJlvoh4AF33+PudcBXgVuDZe3ASGB8UONLHrqpWycwEJhuZmnuvt3dt57k60o/ooCQWPgEMAV4xMysD7dbGfZ8B5AG5JtZqpk9aGZbzWw/b/+qzQ8eg4DjfZF9EXjI3Y90f5nZD4Izkw6a2d+fRI3jg7pqgi6avcAPCe3ZdNvlR99FcwehvaJI2/pu2HYaAQNGh7WpjLBeb4wKXjdSDf8KlAO/M7MKM7sHIAjPzxAK4D1m9qSZRapb4oQCQmJhD7AQ+CvgP8PmHwIyuyeCX/8FJ7HdsWHPxxH6pVsP3AxcD1wBDAMKu18iWN4CTDzOdq8Cvmxm7+ue4e6fCM5MGuzu/3wSNVYCrUC+u2cHj6Hufk5Ym9E9gnMcUH2MbX08bDvZ7p7RfUynu9STqC1cNaEAekcNwZ7Y5919AnAd8LnuYw3u/nN3XxCs64S65SROKSAkJty9GrgcWGRm3w5m/wUYZGbvNrM0Qn3eA09is7eY2XQzywQeAJ4KTi0dQuhLuYFQAB35Qnf3LmAp8C0zGxXsbVzUfUA2sBFYBDxkZktOUMMAMxsU9kjr8b5rgN8B/25mQ80sxcwm9jjNdzhwt5mlmdn7gWlASYTX+gFwb/cB96DL7P0nqC+SgT1qTiF0LOPLZlZgZvnAfUD3NSjXmtmkIMT2E+pa6jSzqWZ2efDZtQCHg2USpxQQEjPuXkkoJG4ws2+4+z7gb4FHgF2E9iiqjrOJnn4G/BioJdRtdHcw/6eEukh2AZsIHf8I9wVCx0NWE+qm+SY9/m24+zrgWuBHZrb4ODX8F6Evxu7HYxHafBhID2ppAp4i1KffbSWhA+r1wD8BN7h7Q8+NuPuvg1qfDLrONhA6vnGyDvao+XLg60ApsJ7QZ/N6MI+gtt8H670G/Ke7v0gozB8M6q4lFHQn0/0m/YxpwCAREYlEexAiIhKRAkJERCJSQIiISEQKCBERiShhbuCVn5/vhYWFsS5DRCSurFmzpt7dI15vlDABUVhYSGlpX97WR0Qk8ZnZjmMtUxeTiIhEpIAQEZGIFBAiIhJRwhyDiKS9vZ2qqipaWlpiXcoZM2jQIMaMGUNaWtqJG4uIHEdCB0RVVRVDhgyhsLCQvr2rdP/k7jQ0NFBVVUVRUVGsyxGROJfQXUwtLS3k5eUlRTgAmBl5eXlJtcckItGT0AEBJE04dEu29ysi0ZPQXUwiIonA3Wlp76KpuY29ze3sbW5j7+H2I9M5mencPG9cn7+uAiKKGhoaWLhwIQC1tbWkpqZSUBC6YHHVqlWkp6efcBu3334799xzD1OnTo1qrSJyZrR2dAZf8u0Rv/D3HTW/nb2H22hqbqeto+uY2zxvXLYCIt7k5eWxdu1aAO6//34GDx7MF77whaPauDvuTkpK5N6+xx6LNN6MiMRaV5fTcKiNfcEXePcX/pEv+MPBF39zO03N7exrDrU73H7sQfbSU1PIzkwLHumMz8tkdmb2kenszDRyMtMYlpFOTlYa2RmheYPSUqPyHhUQMVBeXs573vMeFixYwMqVK3nmmWf46le/yuuvv87hw4f54Ac/yH333QfAggUL+P73v8+MGTPIz8/nE5/4BM899xyZmZk8/fTTDB8+/ASvJiKnyt1pPNTGtvpDVNQfYnv9IbaFPVqP8as+NcWCL/I0cjLTGZ09iHNGDSU7I42crPQj88PDICczjYy01H51HDGqAWFmi4DvAqnAI+7+YI/l3wYuCyYzgeHunh0su43QmMQAX3f3n5xOLV/97UY2Ve8/nU28w/RRQ/nKdeecuGEEmzZt4rHHHuMHP/gBAA8++CC5ubl0dHRw2WWXccMNNzB9+vSj1tm3bx/vete7ePDBB/nc5z7H0qVLueeee077fYgku4OtHWwPQmBb3SG21R9kW0Mz2+oOsr+l40i7ASnGuLxMJuRnsWBSPmNzM8nJSg998Yd94Q8eOKBffdGfqqgFhJmlAg8BVxIaV3i1mS1z903dbdz9s2Ht/x8wJ3ieC3wFKAYcWBOs2xStes+0iRMncsEFFxyZfuKJJ3j00Ufp6OigurqaTZs2vSMgMjIyWLw4NOTw+eefz0svvXRGaxaJZ60dnexsaD5qT6Ai+G/dgdaj2o7OzqAoP4vrZ4+mKD/ryGNMTgYDUhP+5M8jorkHMRcod/cKADN7Erie0EDtkdxEKBQArgZecPfGYN0XgEXAE6dazKn+0o+WrKysI8/feustvvvd77Jq1Sqys7O55ZZbIl7LEH5QOzU1lY6Ojne0EUlmnV1O9d7DwZ7AwVBXUEMz2+oPsqvpMF3+dtv8wekU5mVx6ZQCigqymJCfRVH+YMbnZUatTz/eRDMgRgOVYdNVwLxIDc1sPFAE/PE4646OsN6dwJ0A48b1/RH8M2X//v0MGTKEoUOHUlNTw/Lly1m0aFGsyxLpt/YcaKGi7p17AjsbmmnrfPu4wOCBAyjKz2LO2Bz+es6YIASyKMzPYliGbkdzItEMiEgdcB5hHsCNwFPu3n14v1fruvvDwMMAxcXFx9p2v3feeecxffp0ZsyYwYQJE5g/f36sSxLpN9yd7Q3NrNrWwMqKRlZua2TX3sNHlqenpjA+OC6wcNpwJuRnUZiXRVFBFgWDBybEsYBYMffofK+a2UXA/e5+dTB9L4C7fyNC2zeAT7n7q8H0TcCl7v7xYPqHwIvufswupuLiYu85YNDmzZuZNm1aH72j+JGs71sSQ1eXU153kJUVDazc1siqbY3sCY4R5GWlM29CLuePz2Xy8MEU5WcxKjuD1BSFwKkyszXuXhxpWTT3IFYDk82sCNhFaC/h5gjFTQVygNfCZi8H/tnMcoLpq4B7o1iriMRIZ5ezuWY/K7c1srKigdXbG2lqbgfgrKGDuGhiHnOLcplXlMfEgiztEZxBUQsId+8ws7sIfdmnAkvdfaOZPQCUuvuyoOlNwJMetivj7o1m9jVCIQPwQPcBaxGJb+2dXZTt2sfKikZWbWugdHsTB1pDJ1yMy81k4bQRzAsCYWxuhgIhhqJ6HYS7lwAlPebd12P6/mOsuxRY2gc1JNUfWLS6DEVOVUt7J2sr97Iq6C5as6PpyNXEEwuyuG72KOYV5TK3KJeRwzJiXK2ES+grqQcNGkRDQ0PS3PK7ezyIQYMGxboUSWKHWjt4fWcTq7Y1srKikbWVe2nr7MIMzj5rKB+8YCzzinK5oCiX/MEDY12uHEdCB8SYMWOoqqqirq4u1qWcMd0jyomcKfsOt7NmR+ORM4w27NpHR5eTmmLMGDWUj8wvZG5hLhcU5jIsU6eWxpOEDoi0tDSNrCbSxxoOtrJ6e2NwULmRzbX7cYe0VGP22Gw+/q4JzC3K4/zxOQwemNBfMQlP//dE5IQq6g6ybF01z5XVsmX3AQAGpaVw3rgcPr1wMvOK8pgzLltXICcYBYSIRFS99zC/XVfNsnXVbKzejxnMK8rl7xZNZV5RLjNHZ5M+IHnuS5SMFBAickTDwVZKympYtq6a1dtD98acNTabf7x2OteeO5IRQ3UCRDJRQIgkuf0t7SzfUMtv19fwSnk9nV3OlBGD+cJVU7hu1ijG52WdeCOSkBQQIkmopb2TP2zew7J1u/jTljraOroYm5vBxy+ZwJLZozj7rKGxLlH6AQWESJJo7+zi5bfqWbaumt9trOVQWycFQwbyoXnjWDJrFLPHZifF9ULSewoIkQTW2eWs2tYYOgNpQw17m9sZlpHGdbNGsWTWKOZNyNON7uSYFBAiCcbdWV+1j2XrqnlmfTW797eSkZbKldNHsGTWKC6ZUqCzj6RXFBAiCeIvuw+wbG01v11fzY6GZtJTU3jX1AKWzBrFwmnDyUzXP3c5OfqLEYljlY3NLFtXzW/XVfNm7QFSDC6emM+nLp3E1eecpVtbyGlRQIjEmT37W3hmfehahbWVewE4b1w29183nWvOHcnwIbpWQfqGAkIkDuxtbuP5DbUsW1fNiooGuhymjRzKlxadzbXnjmRsbmasS5QEpIAQ6Yf2t7RTur2RFRWNvLa1gY3V++hyKMzL5K7LJnHdrFFMHjEk1mVKglNAiPQDB1raKd3exIqKBlZUNFC2KxQI6akpzB6XzV2XTeKK6SOYOXqYrlWQM0YBIRIDB1s7WL29MQiE0BgKnV1OWqoxZ2wOd102iQsn5HHe+BzdIVViRgEhcgYcau2gdEcTr219ew+hOxBmjcnmby+dGAqEcTlkpCsQpH9QQIhEQXNbx5Euo9cqGiirCo2yNiDFmDU2m0+8awIXTcjnvPHZuj5B+i39ZYr0gea2DtbsaDrSZbSucu+RQDh3zDDuvGQCF00MjbKmQJB4ob9UkVNwuK0zLBAaWFe1l/bO0DjM544Zxt9cMoELJ+RRPD6HLA27KXFKf7kivdDS3snrO5p4LQiEtZVvB8LM0cP46IIJXDghl+LCXI3DLAlDf8kiEbR1dLG2ci8vl9ezYmsoENo6u0gxmDkmmzsWFB3ZQxgySLezkMQU1YAws0XAd4FU4BF3fzBCmw8A9wMOrHP3m4P5nUBZ0Gynuy+JZq2S3Lq6nDdrD/BKeT2vbK1n1bZGmts6STGYMXoYt88vDAVCoQJBkkfUAsLMUoGHgCuBKmC1mS1z901hbSYD9wLz3b3JzIaHbeKwu8+OVn0ilY3NvFxezyvl9by6tYHGQ20ATCzI4obzxzB/Uj4XTshjWIYCQZJTNPcg5gLl7l4BYGZPAtcDm8La/A3wkLs3Abj7nijWI0mu4WArr25t4NWt9bxcXk9l42EARgwdyKVTCpg/KZ+LJ+UxclhGjCsV6R+iGRCjgcqw6SpgXo82UwDM7BVC3VD3u/vzwbJBZlYKdAAPuvtver6Amd0J3Akwbty4vq1e4t6h1g5WbW/k1fJ6Xi5vYHPNfgCGDBrAhRPy+NiCCcyflMfEgsG6fYVIBNEMiEj/4jzC608GLgXGAC+Z2Qx33wuMc/dqM5sA/NHMytx961Ebc38YeBiguLi457YlybR3drGuci+vlDfwSnk9b1Q20d7ppKemcP74HL549VQunpjHzNHDGJCqEdVETiSaAVEFjA2bHgNUR2izwt3bgW1mtoVQYKx292oAd68wsxeBOcBWRALuzpbdB44EwsqKBg61dWIGM0aFTj2dPymP4vG5un2FyCmIZkCsBiabWRGwC7gRuLlHm98ANwE/NrN8Ql1OFWaWAzS7e2swfz7wL1GsVeJEVVMzr5Y38HJwYLn+YCsARflZ/PV5o5k/MZ+LJuaRnZke40pF4l/UAsLdO8zsLmA5oeMLS919o5k9AJS6+7Jg2VVmtgnoBL7o7g1mdjHwQzPrAlIIHYPYdIyXkgTWdKiN1yqCQCivZ3tDMwD5gweyYFIeF0/KZ/6kfEZn68CySF8z98Toui8uLvbS0tJYlyF9YM/+Fv6ntJLnN9aysXo/7jB44AAunJDLxRPzWTA5n8nDdWBZpC+Y2Rp3L460TFdSS7/Q1eW8VtHA4yt38LuNu+noci4ozOGzV0xh/qR8zh0zjDQdWBY5oxQQElNNh9r41etVPL5yJ9vqD5GTmcYdC4q4ae44ivKzYl2eSFJTQMgZ5+68vrOJx1fs5JmyGto6uigen8PdCyexeMZIjaAm0k8oIOSMOdDSzm/WVvP4ih28WXuAwQMHcOMFY7l53jjOPmtorMsTkR4UEBJ1G3bt4/GVO3l67S6a2zo5Z9RQvvHemSyZNUpjJYj0Y/rXKVFxuK2TZ9ZX8/jKnayt3MugtBSuO3cUt1w4nnPHDNMZSCJxQAEhfap8zwEeX7mTX62pYn9LB5OGD+Yr103nvXPGMCxTd0UViScKCDltbR1dLN9Yy+Mrd7CiopG0VGPRjJHcMm8cc4tytbcgEqcUEHLKKhub+fmqnfyytJL6g22Mzc3gS4vO5v3FY8gfPDDW5YnIaVJAyEnp6OziT1vqeHzlDv78lzoMWDhtBB+aN45LJheQkqK9BZFEoYCQXtm9v4UnV1Xy5Oqd1OxrYcTQgdx9+WRunDtWA+yIJCgFhBxTV5fzytZ6Hl+xkxc276azy/mryfl85bpzuGLacI2pIJLgFBDyDo2H2vhlaSU/X7WTHQ3N5GSm8bHg9heFuv2FSNJQQMhRnly1k/ue3khbZxdzC3P53JVTWDTjLAYO0O0vRJKNAkKOeHZ9Dff+uowFk/L58runM/WsIbEuSURiSAEhALz0Vh2f+cUbFI/P4eFbizVEp4igo4zC2sq9fPxna5hYMJhHbrtA4SAigAIi6ZXvOcBHHltF/uCB/PSOuQzL0O0wRCREAZHEdu09zK2PrmJASgo/++hchg8dFOuSRKQfUUAkqYaDrdz66EoOtnbw0zvmMj5Pp6+KyNEUEEnoYGsHt/94NbuaDvPobRcwfZQG6xGRd9JZTEmmtaOTO39aysbq/Tx86/nMLcqNdUki0k9pDyKJdHY5n35iLa9ubeBfbziXhdNGxLokEenHohoQZrbIzLaYWbmZ3XOMNh8ws01mttHMfh42/zYzeyt43BbNOpOBu/Pl35Tx/MZa/vHa6bz3vDGxLklE+rmodTGZWSrwEHAlUAWsNrNl7r4prM1k4F5gvrs3mdnwYH4u8BWgGHBgTbBuU7TqTXT/unwLT6yq5K7LJvHRBUWxLkdE4kA09yDmAuXuXuHubcCTwPU92vwN8FD3F7+77wnmXw284O6NwbIXgEVRrDWhPfJSBf/54lZunjeOz181JdbliEiciGZAjAYqw6argnnhpgBTzOwVM1thZotOYl3M7E4zKzWz0rq6uj4sPXE8taaKrz+7mWtmnsXXrp+h4T9FpNeiGRCRvom8x/QAYDJwKXAT8IiZZfdyXdz9YXcvdvfigoKC0yw38fx+026+9Kv1LJiUz7c/OJtUjfYmIichmgFRBYwNmx4DVEdo87S7t7v7NmALocDozbpyHCsrGvjUz19nxqih/PDW83W7bhE5adEMiNXAZDMrMrN04EZgWY82vwEuAzCzfEJdThXAcuAqM8sxsxzgqmCe9MLG6n187CeljMnJ4LHb55I1UJe7iMjJi9o3h7t3mNldhL7YU4Gl7r7RzB4ASt19GW8HwSagE/iiuzcAmNnXCIUMwAPu3hitWhPJ9vpD3LZ0NUMGDeBnH51HblZ6rEsSkThl7u/o2o9LxcXFXlpaGusyYmr3/hbe91+vcqi1g19+4mImDR8c65JEpJ8zszXuXhxpmfoeEsS+5nY+/OgqGg+18cTfXKhwEJHTplttJIDDbZ3c8ZPVbKs/xMO3FjNrbHasSxKRBKCAiHPtnV188vE1vLGzie/eOJsFk/NjXZKIJAh1McWxri7ni79cx4tb6vjGe2eyeObIWJckIglEexBxyt154JlN/GZtNV+8eio3zR0X65JEJMEoIOLUf/yxnB+/up2PLijiby+dGOtyRCQBKSDi0M9W7OBbL/yF9543mn+4ZpruryQiUaGAiDPPrK/mvqc3cMW04XzzfeeSovsriUiUKCDiyP/9pY7P/mItF4zP5fs3n0daqv73iUj06BsmTry+s4mP/2wNk4YP4Ue3FTMoTTffE5HoUkDEgbd2H+COH69m+NCB/OSOCxiWkRbrkkQkCSgg+rmqpmZufXQVaakp/OyOeQwfMijWJYlIklBA9GMNB1v58KOraG7r4Kd3zGVcXmasSxKRJKIrqfupAy3tfOSx1VTvO8x/f3Qe00YOjXVJIpJkerUHYWYTzWxg8PxSM7s7GBpUoqClvZM7f7qGzTX7+a8PnU9xYW6sSxKRJNTbLqZfAZ1mNgl4FCgCfh61qpJYR2cXn37yDV6raODf3j+Ly84eHuuSRCRJ9TYguty9A/hr4Dvu/llAd4brY+7OP/x6A8s37uYr103nPXNGx7okEUlivQ2IdjO7CbgNeCaYp3Mt+9g3n9/CL0orufvySdw+vyjW5YhIkuttQNwOXAT8k7tvM7Mi4L+jV1byebW8nh/8eSsfmjeOz145JdbliIj07iwmd98E3A1gZjnAEHd/MJqFJZun11YzeOAA/vHa6br5noj0C709i+lFMxtqZrnAOuAxM/tWdEtLHu2dXSzfVMvCacN1Cw0R6Td628U0zN33A+8FHnP384EroldWcllR0cDe5nYWz9BxfxHpP3obEAPMbCTwAd4+SC19pKSslqz0VC6dWhDrUkREjuhtQDwALAe2uvtqM5sAvBW9spJHR2cXyzfWcvm0EepeEpF+pVcB4e6/dPdz3f2TwXSFu7/vROuZ2SIz22Jm5WZ2T4TlHzGzOjNbGzw+FrasM2z+spN5U/Fk5bZGGg+1cc2Ms2JdiojIUXp1FpOZjQH+A5gPOPAy8Gl3rzrOOqnAQ8CVQBWw2syWBWdEhfuFu98VYROH3X12b+qLZyVlNWSkpXLpVF0xLSL9S2+7mB4DlgGjgNHAb4N5xzMXKA/2NtqAJ4HrT7XQRNTZ5aHupbOHk5Gu7iUR6V96GxAF7v6Yu3cEjx8DJzqiOhqoDJuuCub19D4zW29mT5nZ2LD5g8ys1MxWmNl7Ir2Amd0ZtCmtq6vr5VvpP1Zta6T+YBvXzNTZSyLS//Q2IOrN7BYzSw0etwANJ1gn0tVe3mP6t0Chu58L/B74Sdiyce5eDNwMfMfMJr5jY+4Pu3uxuxcXFMTfGUAlZTUMSkvhsrPjr3YRSXy9DYg7CJ3iWgvUADcQuv3G8VQB4XsEY4Dq8Abu3uDurcHkj4Dzw5ZVB/+tAF4E5vSy1rjQ2eU8t6GWy6YOJzNdw3KISP/T27OYdrr7EncvcPfh7v4eQhfNHc9qYLKZFZlZOnAjoeMYRwTXVnRbAmwO5ueEjT+RT+jgeM+D23GtdHsj9Qdb1b0kIv3W6fx0/RzwnWMtdPcOM7uL0PUTqcBSd99oZg8Ape6+DLjbzJYAHUAj8JFg9WnAD82si1CIPRjh7Ke4VlJWw8ABKVyu8R5EpJ86nYA44R3l3L0EKOkx776w5/cC90ZY71Vg5mnU1q91Bd1Ll04tIGugupdEpH/q7TGISHoecJZeWrOziT0H1L0kIv3bcX++mtkBIgeBARlRqSgJPLu+hvQBKSycNiLWpYiIHNNxA8Ldh5ypQpJFV5fz/IZa3jWlgMHqXhKRfux0upjkFLxR2UTt/hbere4lEennFBBnWElZLempKVw+TWcviUj/poA4g7q6nOfKarhkSj5DB6XFuhwRkeNSQJxBa6v2Ur2vRSPHiUhcUECcQc+V1ZCWalwxXWcviUj/p4A4Q9ydkrJa/mpyAcMy1L0kIv2fAuIMWV+1j117D7NYI8eJSJxQQJwhJWU1DEgxrpqugBCR+KCAOAPcnWfLapg/KZ9hmepeEpH4oIA4Azbs2k9V02FdHCcicUUBcQY82929dI7OXhKR+KGAiDJ357kNNVw0MY/szPRYlyMi0msKiCjbWL2fHQ3N6l4SkbijgIiykrIaUlOMq87R2UsiEl8UEFEUujiuhosm5JGbpe4lEYkvCogo2lxzgO0NzRo5TkTikgIiip7bUEOKobOXRCQuKSCipPviuAsn5JE/eGCsyxEROWkKiCjZsvsAFXWH1L0kInFLARElJWW1pBhcrbOXRCRORTUgzGyRmW0xs3IzuyfC8o+YWZ2ZrQ0eHwtbdpuZvRU8botmndFQUlbD3KJcCoaoe0lE4tOAaG3YzFKBh4ArgSpgtZktc/dNPZr+wt3v6rFuLvAVoBhwYE2wblO06u1Lb+0+QPmeg3z4onNiXYqIyCmL5h7EXKDc3SvcvQ14Eri+l+teDbzg7o1BKLwALIpSnX3u2bIazGCRxn4QkTgWzYAYDVSGTVcF83p6n5mtN7OnzGzsyaxrZneaWamZldbV1fVV3aetpKyGCwpzGT5kUKxLERE5ZdEMCIswz3tM/xYodPdzgd8DPzmJdXH3h9292N2LCwoKTqvYvlK+5wB/2X2Qa7T3ICJxLpoBUQWMDZseA1SHN3D3BndvDSZ/BJzf24KLJA0AAAtrSURBVHX7q5KyWgAW6/RWEYlz0QyI1cBkMysys3TgRmBZeAMzC/8WXQJsDp4vB64ysxwzywGuCub1eyVlNRSPz2HEUHUviUh8i9pZTO7eYWZ3EfpiTwWWuvtGM3sAKHX3ZcDdZrYE6AAagY8E6zaa2dcIhQzAA+7eGK1a+0pF3UHerD3AfddOj3UpIiKnLWoBAeDuJUBJj3n3hT2/F7j3GOsuBZZGs76+VlJWA8DimTr+ICLxT1dS96GSslrOG5fNyGEZsS5FROS0KSD6yPb6Q2yq2a97L4lIwlBA9JGSDd3dSwoIEUkMCog+UlJWw+yx2YzOVveSiCQGBUQf2NnQzIZd+3m39h5EJIEoIPpAd/eS7r0kIolEAdEHSspqmDVmGGNzM2NdiohIn1FAnKbKxmbWV+3T2UsiknAUEKfpuaB7SQEhIolGAXGani2rZeZodS+JSOJRQJyGqqZm1lXu1a01RCQhKSBOw/MbQrf21umtIpKIFBCnoaSshnNGDWV8XlasSxER6XMKiFNUvfcwr+/cq4PTIpKwFBCn6Lmge2mxLo4TkQSlgDhFz5XVcPZZQ5hQMDjWpYiIRIUC4hTU7muhdEeTDk6LSEJTQJyC57svjjtXASEiiUsBcQpKymqZOmIIE9W9JCIJTAFxkvbsb2H1jkadvSQiCU8BcZKe31iLO1yjq6dFJMEpIE7Ss+trmDx8MJNHDIl1KSIiUaWAOAl1B1pZtV3dSyKSHKIaEGa2yMy2mFm5md1znHY3mJmbWXEwXWhmh81sbfD4QTTr7K23u5cUECKS+AZEa8Nmlgo8BFwJVAGrzWyZu2/q0W4IcDewsscmtrr77GjVdypK1tcwsSCLKSN09pKIJL5o7kHMBcrdvcLd24AngesjtPsa8C9ASxRrOW31B1tZua2Bd88ciZnFuhwRkaiLZkCMBirDpquCeUeY2RxgrLs/E2H9IjN7w8z+bGZ/FekFzOxOMys1s9K6uro+KzyS5Rtr6XJYrO4lEUkS0QyISD+z/chCsxTg28DnI7SrAca5+xzgc8DPzWzoOzbm/rC7F7t7cUFBQR+VHdlzZbVMyM/i7LN09pKIJIdoBkQVMDZsegxQHTY9BJgBvGhm24ELgWVmVuzure7eAODua4CtwJQo1npcjYfaeK2igcUzz1L3kogkjWgGxGpgspkVmVk6cCOwrHuhu+9z93x3L3T3QmAFsMTdS82sIDjIjZlNACYDFVGs9biWb6yls8t19pKIJJWoncXk7h1mdhewHEgFlrr7RjN7ACh192XHWf0S4AEz6wA6gU+4e2O0aj2RkrIaCvMymT7yHb1cIiIJK2oBAeDuJUBJj3n3HaPtpWHPfwX8Kpq19VbToTZe3drAnZdMUPeSiCQVXUl9Ai9s2k1nl2vsBxFJOgqIE3i2rIaxuRmcM0rdSyKSXBQQx7G3uY1Xyuu5RhfHiUgSUkAcxwubdtOh7iURSVIKiOMoKathTE4GM0cPi3UpIiJnnALiGPYdbudldS+JSBJTQBzD7zftpr3TWTxDI8eJSHJSQBxDSVkNo7MzmD02O9aliIjEhAIigv0t7bz0Vj2LZ+jeSyKSvBQQEfxh827aOrt0a28RSWoKiAhKymoZOWwQc9S9JCJJTAHRw4GWdv78lzoWzxhJSoq6l0QkeSkgevjjm3to6+jimpk6e0lEkpsCooeSshpGDB3IeeNyYl2KiEhMKSDCHGrt4MUt6l4SEQEFxFH++OYeWju6NHKciAgKiKOUlNUwfMhAisere0lERAERaG7r4E9b9rBoxlnqXhIRQQFxxJ/erKOlXd1LIiLdFBCBkrIa8gcP5ILC3FiXIiLSLygggMNtnfzxzT0smjGCVHUviYgACggAXtyyh8PtnepeEhEJo4AAni2rIS8rnbnqXhIROSLpA6KlPdS9dPWMsxiQmvQfh4jIEVH9RjSzRWa2xczKzeye47S7wczczIrD5t0brLfFzK6OVo37D7dzxbQRLJk1KlovISISlwZEa8Nmlgo8BFwJVAGrzWyZu2/q0W4IcDewMmzedOBG4BxgFPB7M5vi7p19XefwoYP43k1z+nqzIiJxL5p7EHOBcnevcPc24Eng+gjtvgb8C9ASNu964El3b3X3bUB5sD0RETlDohkQo4HKsOmqYN4RZjYHGOvuz5zsusH6d5pZqZmV1tXV9U3VIiICRDcgIl1Q4EcWmqUA3wY+f7LrHpnh/rC7F7t7cUFBwSkXKiIi7xS1YxCEfvWPDZseA1SHTQ8BZgAvmhnAWcAyM1vSi3VFRCTKorkHsRqYbGZFZpZO6KDzsu6F7r7P3fPdvdDdC4EVwBJ3Lw3a3WhmA82sCJgMrIpirSIi0kPU9iDcvcPM7gKWA6nAUnffaGYPAKXuvuw46240s/8BNgEdwKeicQaTiIgcm7m/o2s/LhUXF3tpaWmsyxARiStmtsbdiyMt06XDIiISUcLsQZhZHbDjNDaRD9T3UTnxTp/F0fR5HE2fx9sS4bMY7+4RTwNNmIA4XWZWeqzdrGSjz+Jo+jyOps/jbYn+WaiLSUREIlJAiIhIRAqItz0c6wL6EX0WR9PncTR9Hm9L6M9CxyBERCQi7UGIiEhECggREYko6QOit6PeJQMzG2tmfzKzzWa20cw+HeuaYs3MUs3sDTPreUv6pGNm2Wb2lJm9GfyNXBTrmmLJzD4b/DvZYGZPmNmgWNfU15I6IMJGvVsMTAduCkazS1YdwOfdfRpwIfCpJP88AD4NbI51Ef3Ed4Hn3f1sYBZJ/LmY2WhCI2EWu/sMQvebuzG2VfW9pA4Iej/qXVJw9xp3fz14foDQF8A7BmpKFmY2Bng38Eisa4k1MxsKXAI8CuDube6+N7ZVxdwAIMPMBgCZJOCQBMkeEL0auS4ZmVkhMIewscKT0HeAvwO6Yl1IPzABqAMeC7rcHjGzrFgXFSvuvgv4N2AnUAPsc/ffxbaqvpfsAdGrkeuSjZkNBn4FfMbd98e6nlgws2uBPe6+Jta19BMDgPOA/3L3OcAhIGmP2ZlZDqHehiJgFJBlZrfEtqq+l+wBoZHrejCzNELh8Li7/2+s64mh+cASM9tOqOvxcjP779iWFFNVQJW7d+9RPkUoMJLVFcA2d69z93bgf4GLY1xTn0v2gDjuqHfJxkJjvz4KbHb3b8W6nlhy93vdfUww2uGNwB/dPeF+IfaWu9cClWY2NZi1kNCAXslqJ3ChmWUG/24WkoAH7aM5JnW/d6xR72JcVizNB24FysxsbTDv7929JIY1Sf/x/4DHgx9TFcDtMa4nZtx9pZk9BbxO6Oy/N0jA227oVhsiIhJRsncxiYjIMSggREQkIgWEiIhEpIAQEZGIFBAiIhKRAkLkJJhZp5mtDXv02dXEZlZoZhv6ansipyupr4MQOQWH3X12rIsQORO0ByHSB8xsu5l908xWBY9JwfzxZvYHM1sf/HdcMH+Emf3azNYFj+7bNKSa2Y+CcQZ+Z2YZMXtTkvQUECInJ6NHF9MHw5btd/e5wPcJ3QmW4PlP3f1c4HHge8H87wF/dvdZhO5p1H0F/2TgIXc/B9gLvC/K70fkmHQltchJMLOD7j44wvztwOXuXhHc8LDW3fPMrB4Y6e7twfwad883szpgjLu3hm2jEHjB3ScH018C0tz969F/ZyLvpD0Ikb7jx3h+rDaRtIY970THCSWGFBAifeeDYf99LXj+Km8PRfkh4OXg+R+AT8KRca+HnqkiRXpLv05ETk5G2J1uITRGc/eprgPNbCWhH143BfPuBpaa2RcJjcjWfQfUTwMPm9lHCe0pfJLQyGQi/YaOQYj0geAYRLG718e6FpG+oi4mERGJSHsQIiISkfYgREQkIgWEiIhEpIAQEZGIFBAiIhKRAkJERCL6//TFKKQvIT/jAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['kl_loss'])\n",
    "plt.title('Kullback-Liebler Loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc = 'upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3xV5Z3v8c8vOzcSEkJu3AIEAQnICGLEC7Zq6Jm2o8VOpxed4lFHx2nP9LQd2zo650zbcdoZZ9ozY6fTqceq2NbbdNqZju1p67QitkoLhgqoXDRggAASEgiBhJDb7/yxVsIm7kAC2VnJzvf9eu1X9l6XvX/Zgf3dz/OstR5zd0RERPpKi7oAEREZmRQQIiKSkAJCREQSUkCIiEhCCggREUlIASEiIgkpIESGiJm9ZmZXJ+m5/8TM7k/Gc5/mNdeb2QXD+ZoysiggZMDMrNbMjpvZMTN7y8weNbPxUdeViJl90cweS+LzP2pmX4pf5u4XuPuaJLxWJvC/ga+Ej8vNzM3st322KzazdjOrjVt2pZmtNbMjZnbIzF40s0vCdbeYWVf494y/TQ13/ypw72nqusXMXhjiX1dGEAWEDNb73H08sBi4CLgn4nrOigVGy7//64Ft7r63z/JcM1sY9/gPgTd7HphZPvBj4OtAITAN+CvgRNw+v3b38X1u+8J1TwPXmNmUIf59ZJQYLf9BZIRx97eAZwiCAgAzyzKzr5rZbjM7YGYPmNm4uPXXm9lGM2s2sx1m9p5w+VQzezr8hltjZn8ct88Xzex7ZvYdMzsaduNUxq3/czPbG67bbmbLw+f9C+Aj4TfiTeG2a8zsy2b2ItAKnBe2it7V5/Uei3vc8w28ycz2hN+a7wA+CtwVPv+Pwm17nyt8L+43s33h7X4zywrXXW1mdWb2GTOrN7P9Znbrad7u9wLPJ1j+XeDmuMf/HfhO3OPzw7/Vk+7e5e7H3f2/3H3zaV6rl7u3ARuA3x3I9vHO8DddambV4b+DA2b2D+HybDN7zMwaw/f7JTObNNjXlqGjgJCzYmZlBB9cNXGL/47gQ2kxMIfgG+vnw+2XEnx4fQ4oAN4J1Ib7PQnUAVOBDwJ/Y2bL4553BfBUuN/TwD+HzzkP+ARwibvnAe8Gat39Z8DfAP8afiNeFPdcNwF3AHnArjP8jjOAnxJ8Ay8Jf6+N7v4g8Djw9+Hzvy/B7v8LuCzcZxGwlKCbqMdkYEL4Ht0GfMPMJvZTyu8A2xMsfwy4wcxiZjY//J3Wxa1/Hegys2+b2XtP8/ynszWsf7BO9zf9GvA1d88HZgPfC5ffTPCeTAeKgI8Bx8/itWWIKCBksH5oZkeBPUA98AUIumyAPwb+zN0PuftRgg/pG8L9bgMecfefu3u3u+91921mNh24Evhzd29z943AQwQf5D1ecPefuHsXwbfmng+sLiALWGBmGe5e6+47zlD/o+7+mrt3unvHGbb9KPCL8Bt4h7s3hvUNxEeBe9293t0PEnTtxP9OHeH6Dnf/CXAMmNfPcxUARxMsryMIjncRfLjGtx5w92aC99aBbwEHw2/18d/KLwu/rffc+r5/R8PXH7AB/E07gDlmVuzux9z9N3HLi4A5YYtnQ/g7SEQUEDJY7w+/rV8NVADF4fISIAfY0PNhA/wsXA7Bt8JEH95TgZ5A6bGL4Jt1j7fi7rcC2WaW7u41wKeBLwL1ZvZU3ABrf/acYX28/moeiKmc2kLZFS7r0ejunXGPW4H+BvwPE7QOEvkOcAtwI0GL4hTuvtXdb3H3MmBhWEP80VC/cfeCuNvsPk+RBzT189r9OdPf9DaClua2sBvpunD5dwm6LZ8Ku+X+3swyBvnaMoQUEHJW3P154FGCI10AGgi6Ay6I+7CZEA5oQ/DB3PfDB2AfUGhm8R+AM4C+A7L91fGEu18JzCT4pvx3Pav626XP4xaCYOsxOe5+fzWf7vl77Atr6jEjXHY2NhOOJyTwA+BaYKe7n7bLzN23EfzNFp5uuz7mA5sGsT2c4W/q7m+4+41AKcHf6/tmlhu2pv7K3RcAVwDXEYyrSEQUEHIu7gf+m5ktdvdugm6MfzSzUgAzm2Zm7w63fRi4NRxETgvXVbj7HmAt8LfhIOWFBN8wHz/Ti5vZPDOrCgd/2wgCqitcfQAotzMfqbSRoB8/Ixz8/mDcuseBd5nZh80s3cyKzKxnUP4AcN5pnvdJ4H+bWYmZFROMxZztYbc/Aa5KtMLdW4Aq4Pa+68ysIhwILwsfTydoafym77aJhO/rxcDPT7+ZZcffzvQ3NbOVZlYS/pvpaZ10mdk1ZvY7ZhYDmgm6nLoSvKYMEwWEnLWwb/07wF+Gi/6cYND6N2bWDPyCsF/d3dcDtwL/CBwhOCqn5xv2jUA5wTfP/wC+4O6n+1DqkQXcR9B6eYvgG+lfhOv+LfzZaH3OF+jjLwlaCYcJxgmeiPv9dgO/B3wGOEQQJj3jHw8TjH00mdkPEzzvl4Bqgm//rwC/DZedjR8BFf11n7l7dT9jL0eBS4F1ZtZCEAyvhr9Pj8vt7edBXBKuWwGsiTvsNZErCIK592Zm6Zz+b/oe4DUzO0YwYH1DeMTUZOD7BOGwleDfSNLOZZEzM00YJDLyhYfWLnD3Tw/ja64DbnP3V4frNWVkUUCIiEhC6mISEZGEFBAiIpKQAkJERBJKj7qAoVJcXOzl5eVRlyEiMqps2LChwd1LEq1LmYAoLy+nuro66jJEREYVM+v3BEt1MYmISEIKCBERSUgBISIiCaXMGEQiHR0d1NXV0dbWFnUpwyY7O5uysjIyMnQRTBE5NykdEHV1deTl5VFeXk4wXUFqc3caGxupq6tj1qxZUZcjIqNcSncxtbW1UVRUNCbCAcDMKCoqGlMtJhFJnpQOCGDMhEOPsfb7ikjyJD0gwvlyXzazH4ePHzWzNy2YvH5j3PX1++53s5m9Ed5uTrTNUOjs6uZAcxvH2zvPvLGIyBgyHC2ITxFc2z3e59x9cXh72xy/ZlZIMNfxpQSTvX/hLCdcPzOD+uYTNB0/0/TEg9fY2MjixYtZvHgxkydPZtq0ab2P29vbB/Qct956K9u3J5qvXkQkuZI6SB3OZHUt8GXgzkHs+m7g5+5+KHyenxNMMvLkUNeYnpZGblaMo22dTJkwtM9dVFTExo1B/n3xi19k/PjxfPaznz1lG3fH3UlLS5zVq1atGtqiREQGKNktiPuBu4DuPsu/bGabzewfw2kN+5rGqZPL13HqJPZAMImKmVWbWfXBgwfPusi87AzaOrpo7+xbZnLU1NSwcOFCPvaxj7FkyRL279/PHXfcQWVlJRdccAH33ntv77ZXXnklGzdupLOzk4KCAu6++24WLVrE5ZdfTn19/bDUKyJjU9JaEGZ2HVDv7hvM7Oq4VfcQTA+ZCTxIME3lvX13T/CUb5vZyN0fDJ+DysrK08589Fc/eo0t+5oTrut253h7F1npaaTHBp6ZC6bm84X3XTDg7eNt2bKFVatW8cADDwBw3333UVhYSGdnJ9dccw0f/OAHWbBgwSn7HDlyhKuuuor77ruPO++8k0ceeYS77777rF5fRORMktmCWAasMLNa4Cmgyswec/f9HjgBrCIYY+irDpge97iMYG7bpEgzI82Mzu7hm11v9uzZXHLJJb2Pn3zySZYsWcKSJUvYunUrW7Zseds+48aN473vfS8AF198MbW1tcNVroiMQUlrQbj7PQStBcIWxGfdfaWZTXH3/RYcj/l+gknU+3oG+Ju4genf7Xmus3Wmb/r7mo5zqKWdBVPySUtL/qGiubm5vfffeOMNvva1r7F+/XoKCgpYuXJlwnMZMjMze+/HYjE6O3XklYgkTxTnQTxuZq8ArwDFwJcAzKzSzB4CCAen/xp4Kbzd2zNgnSx52el0u3PsxPB/6DY3N5OXl0d+fj779+/nmWeeGfYaRET6GpZLbbj7GmBNeL+qn22qgdvjHj8CPDIM5QGQm5VOmhnNbR3kjxve6xgtWbKEBQsWsHDhQs477zyWLVs2rK8vIpKIuQ9fv3syVVZWet8Jg7Zu3cr8+fMH/By7Gltobe+iYnLeqD4jebC/t4iMXWa2wd0rE61L+UttDEZedgYdXd20dQzP4a4iIiOZAiJOXnbQ43a0bejPqhYRGW1SPiAG04WWEUsjJzNGc9voPTooVboMRSR6KR0Q2dnZNDY2DupDMy87g9b2Tjq7Rl83U898ENnZ2VGXIiIpIKUnDCorK6Ouro7BXIajvbOb+qMn6GjMICdz9L09PTPKiYicq9H3CTgIGRkZg55Zrbvbuexvn+WS8kK+8dHfSVJlIiIjX0p3MZ2NtDSjqqKUX75+kI5R2M0kIjJUFBAJXFNRytETnbxUm9STt0VERjQFRAJXzikmM5bG6q26nLaIjF0KiARys9K5bHYRq7crIERk7FJA9KNqXgk7D7ZQ29ASdSkiIpFQQPSjqmISAKu3qRUhImOTAqIfM4pymFM6XgEhImOWAuI0lleUsu7NxkjmiBARiZoC4jSqKkrp6HJeeGPgZ2KLiKQKBcRpXDxzIvnZ6Tyrw11FZAxSQJxGeiyNq+aV8tz2erq7dZVUERlbFBBnsLyilIZj7byy90jUpYiIDCsFxBlcdX4JaQbP6mgmERljFBBnMDE3kyUzJrJ624GoSxERGVYKiAGoml/Kq3ubOdDcFnUpIiLDRgExAFUVpQA8p24mERlDFBADMG9SHtMKxmkcQkTGFAXEAJgZ11SU8GJNA20dXVGXIyIyLBQQA7S8YhKt7V2se1OTCInI2KCAGKDLZxeRnZHG6q06mklExgYFxABlZ8RYNruY1dvrcddZ1SKS+hQQg1A1v5Q9h45TU38s6lJERJJOATEIPYe76mgmERkLkh4QZhYzs5fN7Md9ln/dzBJ+FTezcjM7bmYbw9sDya5zIKZMGMf8KfmaREhExoT0YXiNTwFbgfyeBWZWCRScYb8d7r44mYWdjeUVpXzz+R0cae1gQk5G1OWIiCRNUlsQZlYGXAs8FLcsBnwFuCuZr50sVfNL6ep2ntckQiKS4pLdxXQ/QRB0xy37BPC0u+8/w76zwq6p583sHYk2MLM7zKzazKoPHhyeD+xFZQUU5mbqcFcRSXlJCwgzuw6od/cNccumAh8Cvn6G3fcDM9z9IuBO4Akzy++7kbs/6O6V7l5ZUlIyhNX3L5ZmXD2vhDWvH6RLkwiJSApLZgtiGbDCzGqBp4Aq4DVgDlATLs8xs5q+O7r7CXdvDO9vAHYA5yex1kFZXjGJptYOXt59OOpSRESSJmkB4e73uHuZu5cDNwCr3X2iu0929/Jweau7z+m7r5mVhGMVmNl5wFxgZ7JqHax3nF9MeprpcFcRSWkj5jwIM1thZveGD98JbDazTcD3gY+5+4i5CFJ+dgaXlBeyeqsCQkRS17AEhLuvcffrEiwfH3f/aXf/fHj/B+5+gbsvcvcl7v6j4ahzMKoqStl+4Ch1h1ujLkVEJClGTAtitKmar0mERCS1KSDO0nnFuZQX5WgcQkRSlgLiLAWTCJWydkcjre2dUZcjIjLkFBDnYHnFJNo7u1lb0xh1KSIiQ04BcQ6WziokNzOmbiYRSUkKiHOQmZ7GO+aW8Nw2TSIkIqlHAXGOquaX8lZzG1v2N0ddiojIkFJAnKNr5gWHu+qkORFJNQqIc1SSl8Wisgms3q6AEJHUooAYAlUVk9i4p4nGYyeiLkVEZMgoIIbA8vmluMOa7ZpESERShwJiCFwwNZ/SvCzNVS0iKUUBMQTMjKqKUn75+kE6urrPvIOIyCiggBgi11SUcvREJy/VjpirkouInBMFxBC5ck4xmbE0He4qIilDATFEcrPSuWx2kcYhRCRlKCCGUNW8EnY2tPBmQ0vUpYiInDMFxBCqqpgEoFaEiKQEBcQQmlGUw9zS8azediDqUkREzpkCYohVVZSy/s1DHG3riLoUEZFzooAYYlUVpXR0OS+80RB1KSIi50QBMcQunjmR/Ox0TSIkIqOeAmKIpcfSuGpeKWu219PdrUmERGT0UkAkwfKKUhqOtbN575GoSxEROWsKiCS46vwS0gxWb9XRTCIyeikgkmBibiZLZkzUJEIiMqopIJKkan4pr+5t5q0jbVGXIiJyVhQQSVJVEcxV/ZxaESIySikgkmTepDymFYzTZTdEZNRSQCRJzyRCL7zRQFtHV9TliIgMWtIDwsxiZvaymf24z/Kvm9mx0+x3j5nVmNl2M3t3sutMhqqKUo53dLHuTU0iJCKjz3C0ID4FbI1fYGaVQEF/O5jZAuAG4ALgPcC/mFksmUUmw+Wzi8jOSNPhriIyKiU1IMysDLgWeChuWQz4CnDXaXa9HnjK3U+4+5tADbA0mbUmQ3ZGjCvnFPPstnrcdVa1iIwuyW5B3E8QBN1xyz4BPO3u+0+z3zRgT9zjunDZKczsDjOrNrPqgwcPDkW9Q+6ailLqDh+npr7f3jQRkREpaQFhZtcB9e6+IW7ZVOBDwNfPtHuCZW/7Cu7uD7p7pbtXlpSUnFO9ydJzuKsu3icio00yWxDLgBVmVgs8BVQBrwFzgJpweY6Z1STYtw6YHve4DNiXxFqTZsqEcSyYks/qrQoIERldkhYQ7n6Pu5e5eznBgPNqd5/o7pPdvTxc3urucxLs/jRwg5llmdksYC6wPlm1JltVRSkbdh+mqbU96lJERAZsxJwHYWYrzOxeAHd/DfgesAX4GfCn7j5qTyaoml9KV7fz/Osjc5xERCSRYQkId1/j7tclWD4+7v7T7v75uMdfdvfZ7j7P3X86HHUmy6KyAopyM3VWtYiMKiOmBZHKYmnGVfNKeP71g3R2dZ95BxGREUABMUyWV0yiqbWDl/c0RV2KiMiAKCCGyTvOLyY9zXhWRzOJyCihgBgm+dkZXFJeyHMahxCRUUIBMYyWzy9l+4Gj7DnUGnUpIiJnNKCAMLPZZpYV3r/azD5pZv1ebE8Su0aTCInIKDLQFsQPgC4zmwM8DMwCnkhaVSnqvOJcyotydLiriIwKAw2IbnfvBH4fuN/d/wyYkryyUlMwidAk1u5opLW9M+pyREROa6AB0WFmNwI3Az0T/2Qkp6TUVlVRSntnN2trGqMuRUTktAYaELcClwNfdvc3w+sjPZa8slLX0lmF5GbGdHVXERnx0geykbtvAT4JYGYTgTx3vy+ZhaWqzPQ03nl+Cc+FkwiZJbqyuYhI9AZ6FNMaM8s3s0JgE7DKzP4huaWlrmsqSnmruY0t+5ujLkVEpF8D7WKa4O7NwAeAVe5+MfCu5JWV2q6ZFxzuqjkiRGQkG2hApJvZFODDnByklrNUkpfFoukFGocQkRFtoAFxL/AMsMPdXzKz84A3kldW6quaV8qmuiYajp2IuhQRkYQGFBDu/m/ufqG7fzx8vNPd/yC5paW25fNLcYc12zWJkIiMTAMdpC4zs/8ws3ozO2BmPzCzsmQXl8oumJpPaV4Wq7cdiLoUEZGEBtrFtIpgnuipwDTgR+EyOUvBWdWl/Or1Bto7NYmQiIw8Aw2IEndf5e6d4e1RoCSJdY0JVRWlHD3RSXXtoahLERF5m4EGRIOZrTSzWHhbCehaEedo2ZxiMmNpOppJREakgQbEHxEc4voWsB/4IMHlN+Qc5Galc9nsIk0iJCIj0kCPYtrt7ivcvcTdS939/QQnzck5Wl5Rys6GFnYePBZ1KSIipziXGeXuHLIqxrCqcBIhzREhIiPNuQSErjI3BKYX5jC3dLxmmROREedcAsKHrIoxrmp+Ket2HuJoW0fUpYiI9DptQJjZUTNrTnA7SnBOhAyBqnmldHY7v3qjIepSRER6nTYg3D3P3fMT3PLcfUBzSciZXTxzIvnZ6RqHEJER5Vy6mGSIpMfSuHpeKWu219PdrZ47ERkZFBAjxPL5pTQca+ffX94bdSkiIoACYsR4z8LJXDqrkLt/sJlnt+oCfiISvaQHRHhpjpfN7Mfh44fNbJOZbTaz75vZ+AT7lJvZcTPbGN4eSHadUctKj/HQzZUsmJrPxx//Lb/eoSuZiEi0hqMF8Slga9zjP3P3Re5+IbAb+EQ/++1w98Xh7WNJr3IEyMvO4NFblzKzMIfbv/0Sm/Y0RV2SiIxhSQ2IcM6Ia4GHepaFc1tjZgaMQ+dTnKIwN5Pv3nYpheMzuXnVel4/cDTqkkRkjEp2C+J+4C7glAkPzGwVwYX/KoCv97PvrLBr6nkze0eiDczsDjOrNrPqgwdTZ2a2yROyefy2y8iMpbHyoXXsbmyNuiQRGYOSFhBmdh1Q7+4b+q5z91sJTrTbCnwkwe77gRnufhHBNZ+eMLP8BM/zoLtXuntlSUlqTU8xoyiH7952Ke1d3Xz04d9woLkt6pJEZIxJZgtiGbDCzGqBp4AqM3usZ6W7dwH/Crxtbmt3P+HujeH9DcAO4Pwk1joizZucx6O3LuXQsXZWPrSOwy3tUZckImNI0gLC3e9x9zJ3LwduAFYDN5nZHOgdg3gfsK3vvmZWYmax8P55wFxgZ7JqHckWTy/gWzdXsutQKzevWq/rNYnIsBnu8yAM+LaZvQK8AkwB7gUwsxVmdm+43TuBzWa2Cfg+8DF3H7Pzcl4xu5h/+cMlvLavmdu/XU1bR1fUJYnIGGDuqXEQUWVlpVdXV0ddRlL958a9fPpfN1I1r5QHbrqYjJjOcxSRc2NmG9y9MtE6fcKMItcvnsZfX7+QZ7fV85nvbaJL120SkSTSFVlHmZWXzeRoWyd/97Nt5GWn86X3LyQYzhERGVoKiFHo41fPprmtg2+u2UFedgZ3v7ci6pJEJAUpIEapu949j+bjHTzw/A7yx6XzP66eE3VJIpJiFBCjlJnx19cv5NiJTv7+Z9vJy87gpstmRl2WiKQQBcQolpZmfPVDizjW1snn//NV8rLSef9F06IuS0RShI5iGuUyYml846NLuHRWIZ/5t038YovmkhCRoaGASAHZGTEeuvkSFk7N53888VvW7miIuiQRSQEKiBQxPiu9dy6JP/52NRs1l4SInCMFRAqZmJvJY7cHc0ncsmo929/SXBIicvYUEClmUv7JuSRuengduxpboi5JREYpBUQKmlGUw2O3X0pHVzcrH17HW0c0l4SIDJ4CIkWdPymPb//RUg63dHDTw+s4pLkkRGSQFBAp7MKyAh66uZLdh1q5RXNJiMggKSBS3GXnFfHNlUvYsq+Z2zSXhIgMggJiDKiqmMT/+fAiXqo9xMcf20B7Z3fUJYnIKKCAGCOuXzyNL71/Ic9tP8id39uouSRE5Ix0LaYx5KOXBnNJ3PfTbeRlZ/A3v6+5JESkfwqIMeZjV82m+XgH/7JmB/nZ6dz93gqFhIgkpIAYgz737nk0t3Xwf3+5k/xxGfzpNZpLQkTeTgExBpkZ965YyNG2Tr7yzHbys9O56fLyqMsSkRFGATFG9cwl0XKik7/8z9cYn53O719UFnVZIjKC6CimMSwjlsY//+ESLj+viM/+22Z+rrkkRCSOAmKMy86I8a2bK1k4NZ8/feK3rK3RXBIiElBASO9cErOKcrn9O9W8vPtw1CWJyAiggBAgmEviu7ctpXh8FisfWse9P9qiS4WLjHEKCOlVmp/Nk3dcRtX8SXzn17Vc/dU13PboS/zqjYO468xrkbHGUuU/fmVlpVdXV0ddRso40NzG47/ZxRPrd9NwrJ3ZJbncckU5H1hSRm6WDn4TSRVmtsHdKxOuU0DI6Zzo7OL/bd7Po2tr2Vx3hLysdD5UOZ2br5jJzKLcqMsTkXOkgJBz5u68vKeJR1+s5Sev7KfLnap5pdyyrJwr5xTrch0io1SkAWFmMaAa2Ovu15nZw0AlYMDrwC3ufizBfvcAtwFdwCfd/ZnTvY4CYvj07X6aUzqemy+fqe4nkVEo6oC4kyAQ8sOAyHf35nDdPwD17n5fn30WAE8CS4GpwC+A892939luFBDDr6f7adWLtbyy9wh52el8uHI6//1ydT+JjBanC4ikHsVkZmXAtcBDPcviwsGAcUCihLoeeMrdT7j7m0ANQVjICJKVHuMDS8p4+hPL+MHHr+CaeaV8e62OfhJJFck+zPV+4C7glCnMzGwV8BZQAXw9wX7TgD1xj+vCZacwszvMrNrMqg8ePDhkRcvgmBkXz5zIP914ES/eXcX/vGYOG/c0cdPD6/lv//hLvvubXbSc6Iy6TBEZpKQFhJldR9B9tKHvOne/laDraCvwkUS7J1j2tq+i7v6gu1e6e2VJScm5lixDYFJ+Nnf+7jzW3lPF//nQIsZlxPjLH77KZX/7LH/9Y518JzKaJHNEcRmwwsx+D8gG8s3sMXdfCeDuXWb2r8DngFV99q0Dpsc9LgP2JbFWGWJZ6TH+4OIyPrBkGr/d3cSja2v59tpaHnnxTZZXlHLzFTr6SWSkG5bDXM3sauCzwPuA2e5eE45BfAXA3T/bZ/sLgCc4OUj9LDBXg9SjW8/RT4+v201jS3j00xXlfOCiaTr6SSQikZ8HERcQK4BfAfkE3UibgI+7e7OZrQAq3f3z4T7/C/gjoBP4tLv/9HSvoYAYPdo6Tp58p6OfRKIVeUAMBwXE6OPuvd1PPw1PvlteUcotV8xi2ZwidT+JDIPTBYTa9RKZnqOfLp45kQPXzu/tfvrF1nXMKR3PDZdM553nlzC3dLzCQiQCakHIiNK3+wmgJC+LK2YXsWx2MVfMKaJsYk7EVYqkDnUxyahUd7iVtTWNvFDTwNodDTQcawegvCiHK+YUs2x2MZfPLqIwNzPiSkVGLwWEjHruzusHjvFiTQMv1jSw7s1DHDvRiRksmJLPsjnFXDG7iKWzCsnJVM+pyEApICTldHR1s7nuCGtrGnihpoGXdzfR3tVNRsy4aMZEls0u5sq5RVxYVkBGTPNiifRHASEp73h7Fy/VHgpaGDsaeG1fM+6Qmxnj0vOKgjGMOcVUTM7TgLdIHB3FJClvXGaMd55fwjvPDy65crilnd/s7Bm/aGT1tnoAisdncvnsYpaFgTG9UAPeIv1RC0LGhH1Nx3vHL17c0cjBoycAmFGYw7I5RVwxOxjDKBqfFXGlIsNLXUwicdydmvpgwPuFmkbW7WzkaHi12Rvyj/cAAAwHSURBVPlT8ntbF0tnFeoSIJLyFBAip9HZ1c0re4+ELYxGNuw6THtXN+lpxkUzClg6q5ALywpYVFbA5AnZUZcrMqQUECKD0NbRRXXt4d7zL17b10xXd/D/ZFJ+VhgWE7iwrIALyyZQkKPzMGT00iC1yCBkZ8S4cm4xV84tBoLAeG1fM5vrmti0p4nNdUf4+ZYDvduXF+X0hsXi6QVcMHUC4zJjUZUvMmQUECJnkJ0R671mVI8jxzt4de8RNu5pYnNdEy/VHuLpTcGUJbE0Y27peBaVFbBoehAc8ybn6XwMGXXUxSQyROqb29hUdyRoaYQ/m1o7AMhKT2PB1PwwNILuqVlFuaSl6ZwMiZbGIEQi4O7sPtQahEXYNfXK3iMc7wjmvcrLTufCcCyjZ0xjyoRsncgnw0pjECIRMDNmFuUysyiXFYumAsERUzUHj7F5zxE21TWxqa6Jb/1yJ53hIHhJXtYpA+CLygqYqIsRSkQUECLDKD2WRsXkfCom5/PhS4Jp19s6uti6v7l3AHxTXRO/2Frfu8+MwhwuLJvA/Cn5zC4Zz5zS8cwsytGYhiSdAkIkYtkZMS6aMZGLZpwcBG9u6+DVuiO9Yxkv727ix5v3965PTzNmFuUwp3T8yVtJHrNLc3U1Wxky+pckMgLlZ2dwxZxirphT3Lvs2IlOdtQfY8fBY9TUB7c36o/xi631vedpAEwrGMd5JblxwRH81GVEZLAUECKjxPisdBZNDw6djdfe2c2uxpZTgqPm4DGeWr+nd0AcYGJORm9ozC4Zz+wwPKYVjNPRVJKQAkJklMtMT2PupDzmTso7ZXl3t7PvyHFq6o+x42BL8LP+GM+8doBDLXt6txuXETvZ4ugJjtLxlBflkpmucY6xTAEhkqLS0oyyiTmUTczh6nmnrjvU0h4Gx8lWR3XtYf5z477ebWJpxszCnN7A6AmPWcW5TBiXMcy/jURBASEyBhXmZrJ0ViFLZxWesry1vZOdB/t0V9UfY832ejq6To5zTMzJoLw4l/KiXGYW5VBelBs+ztG1qVKIAkJEeuVkprNw2gQWTptwyvKOrm52H2plR/0xdjW28mZjC7saW1j/5iF+uHEv8efbThiX0RsWM4tymVUc/CwvymViToZOBBxFFBAickYZsbRgYLtk/NvWtXV0UXe4ldqGVmobW6htbGFXYysbdh3mR5v2EXeAFXnZ6cwqzg0Do6flEQRIUW6mwmOEUUCIyDnJzogxpzSPOaV5b1t3orOLusPHqW1oobaxlV2Nwc9Ne5r4f5v7hEdWOjN7Wxsnu61mFuVQMj5L4REBBYSIJE1Weqzflkd7Zzd7m3rCI2h1vNnQwmt7j/CzV9865dyO3MxYEBxxATKtIIepBdlMLRhHdoYur54MCggRiURmehqzinOZVZz7tnUdXd3sPXz8lODY1djCtv1H+a/XDvReu6pHUW4mUwvG9QbGtIJx4eNgWXFuls71OAsKCBEZcTJiacFAd4Lw6OzqZv+RNvY2HWdfeNvb1Ma+puPsPNjCr95ooLW965R9MtPTmDohOy40xjGtIO7xhHGa5CkBBYSIjCrpsTSmF+YwvTAn4Xp3p/l458kAOXI8vB+EyIs1DRxobqNPI4TC3MygBTJh3CmtkGkTx24rJOkBYWYxoBrY6+7XmdnjQCXQAawH/sTdOxLs1wW8Ej7c7e4rkl2riIx+ZsaEnAwm5GSwYGp+wm06uro50NzG3sNBgOxrOtkiqW1s4cWaBlr6tkJiaUw5JUCymTQhm8KcTCbmZjIxJ5OJuRlMzMlMmSvtDkcL4lPAVqDnL/U4sDK8/wRwO/DNBPsdd/fFyS9PRMaajFha71nmibg7zW2dvV1Y8d1Y+5qO8+sdDbyVoBXSIy8rPQyNjJPhkdPncW4GheH9gpwMstJHXhdXUgPCzMqAa4EvA3cCuPtP4tavB8qSWYOIyGCZGRPGZTBhXAbzpyRuhXR2ddNwrJ3Dre0cbmnncGsHh3rvxy0LL2vS1NrBsROd/b5mbmYsLjwyKczJoCAnMwyRPkETtlSSffRWslsQ9wN3AW87QNrMMoCbCFoYiWSbWTXQCdzn7j9M8Bx3AHcAzJgxY6hqFhE5o/RYGpMnZDN5QvaA9znR2cWR3iDp4HBrO4da2mlqbedQS0fwszUIltqGFg63tHP0NKGSkxljYk4mS2ZO5Os3XjQUv9YpkhYQZnYdUO/uG8zs6gSb/AvwS3f/VT9PMcPd95nZecBqM3vF3XfEb+DuDwIPQjAn9RCWLyIy5LLSY5TmxyjNH3iotHd203S8naawNdLTMulppRxqbWfyIJ5vMJLZglgGrDCz3wOygXwze8zdV5rZF4AS4E/629nd94U/d5rZGuAiYEd/24uIpKLM9DRK87IpzUtOCJxO0oba3f0edy9z93LgBmB1GA63A+8GbnT37kT7mtlEM8sK7xcThM2WZNUqIiJvF8WxWA8Ak4Bfm9lGM/s8gJlVmtlD4TbzgWoz2wQ8RzAGoYAQERlGw3KinLuvAdaE9xO+prtXExzyiruvBX5nOGoTEZHEUuNsDhERGXIKCBERSUgBISIiCSkgREQkIQWEiIgkZO6pcQKymR0Edp3DUxQDDUNUzmin9+JUej9OpffjpFR4L2a6e0miFSkTEOfKzKrdvTLqOkYCvRen0vtxKr0fJ6X6e6EuJhERSUgBISIiCSkgTnow6gJGEL0Xp9L7cSq9Hyel9HuhMQgREUlILQgREUlIASEiIgmN+YAws/eY2XYzqzGzu6OuJ0pmNt3MnjOzrWb2mpn1Nx3smGFmMTN72cx+HHUtUTOzAjP7vpltC/+NXB51TVEysz8L/5+8amZPmtnwz+iTZGM6IMwsBnwDeC+wALjRzBZEW1WkOoHPuPt84DLgT8f4+wHBnOlboy5ihPga8DN3rwAWMYbfFzObBnwSqHT3hUCMYGK0lDKmAwJYCtS4+053bweeAq6PuKbIuPt+d/9teP8owQfAtGirio6ZlQHXAg+dadtUZ2b5wDuBhwHcvd3dm6KtKnLpwDgzSwdygH0R1zPkxnpATAP2xD2uYwx/IMYzs3KCecDXRVtJpO4H7gISTo07xpwHHARWhV1uD5lZbtRFRcXd9wJfBXYD+4Ej7v5f0VY19MZ6QFiCZWP+uF8zGw/8APi0uzdHXU8UzOw6oN7dN0RdywiRDiwBvunuFwEtwJgdszOziQS9DbOAqUCuma2MtqqhN9YDog6YHve4jBRsJg6GmWUQhMPj7v7vUdcToWXACjOrJeh6rDKzx6ItKVJ1QJ2797Qov08QGGPVu4A33f2gu3cA/w5cEXFNQ26sB8RLwFwzm2VmmQSDTE9HXFNkzMwI+pi3uvs/RF1PlNz9Hncvc/dygn8Xq9095b4hDpS7vwXsMbN54aLlwJYIS4rabuAyM8sJ/98sJwUH7dOjLiBK7t5pZp8AniE4CuERd38t4rKitAy4CXjFzDaGy/7C3X8SYU0ycvxP4PHwy9RO4NaI64mMu68zs+8DvyU4+u9lUvCyG7rUhoiIJDTWu5hERKQfCggREUlIASEiIgkpIEREJCEFhIiIJKSAEBkEM+sys41xtyE7m9jMys3s1aF6PpFzNabPgxA5C8fdfXHURYgMB7UgRIaAmdWa2d+Z2frwNidcPtPMnjWzzeHPGeHySWb2H2a2Kbz1XKYhZmbfCucZ+C8zGxfZLyVjngJCZHDG9eli+kjcumZ3Xwr8M8GVYAnvf8fdLwQeB/4pXP5PwPPuvojgmkY9Z/DPBb7h7hcATcAfJPn3EemXzqQWGQQzO+bu4xMsrwWq3H1neMHDt9y9yMwagCnu3hEu3+/uxWZ2EChz9xNxz1EO/Nzd54aP/xzIcPcvJf83E3k7tSBEho73c7+/bRI5EXe/C40TSoQUECJD5yNxP38d3l/LyakoPwq8EN5/Fvg49M57nT9cRYoMlL6diAzOuLgr3UIwR3PPoa5ZZraO4IvXjeGyTwKPmNnnCGZk67kC6qeAB83sNoKWwscJZiYTGTE0BiEyBMIxiEp3b4i6FpGhoi4mERFJSC0IERFJSC0IERFJSAEhIiIJKSBERCQhBYSIiCSkgBARkYT+P2oaAzLtPtPtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['recon_loss'])\n",
    "plt.title('Reconstruction (MSE) Loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc = 'upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anom_df = csv_to_df('bsm_test/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anom_df = anom_df.fillna(0)\n",
    "anom_vars = [entry for entry in df.columns if entry[0] == 'E'] + [entry for entry in df.columns if entry[0:2] == 'pt'] + [entry for entry in df.columns if entry[0:2] == 'et'] + [entry for entry in df.columns if entry[0:2] == 'ph']\n",
    "#Now we one_hot encode the data to make sure we have a class definition\n",
    "anom_df = anom_df[['process_ID']+anom_vars]\n",
    "anom_one_hot = pd.get_dummies(anom_df['process_ID'])\n",
    "anom_processes = anom_one_hot.columns\n",
    "anom_df.drop('process_ID', axis = 'columns', inplace = True)\n",
    "anom_df = pd.concat([anom_df, anom_one_hot], sort = False, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We have created a df of our values and some kind of class label\n",
    "#We should normalize\n",
    "anom_x = anom_df[anom_vars].values\n",
    "anom_scaled = StandardScaler().fit_transform(anom_x)\n",
    "anom_df[anom_vars] = anom_scaled\n",
    "\n",
    "anom_x_train, anom_x_test, anom_y_train, anom_y_test = train_test_split(anom_df[anom_vars].values,\n",
    "\t\t\t\t\t\t    anom_df[anom_processes].values, \n",
    "                                                    shuffle = True,\n",
    "                                                    random_state = 42,\n",
    "                                                    test_size = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reshape the data. This will have to be done explicitly until you think of a clever way of doing it\n",
    "anom_x_train = anom_x_train.reshape((anom_x_train.shape[0], 4, 3, 5))\n",
    "anom_x_test = anom_x_test.reshape((anom_x_test.shape[0], 4, 3, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vae.evaluate(x_train[:2455], anom_x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}