ApplyML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-17 00:16:33.701081: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-03-17 00:16:34.236937: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-03-17 00:16:34.236968: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-03-17 00:16:34.335740: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-03-17 00:16:34.522628: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-03-17 00:16:35.555821: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import BernoulliNB, GaussianNB\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from catboost import CatBoostClassifier\n",
    "import xgboost\n",
    "from xgboost import  XGBClassifier\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns  # for nicer plots\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "apple_data = pd.read_csv(\"apple_processed_data.csv\", index_col=0)\n",
    "tesla_data = pd.read_csv(\"tesla_processed_data.csv\", index_col=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>close</th>\n",
       "      <th>open</th>\n",
       "      <th>lowest</th>\n",
       "      <th>highest</th>\n",
       "      <th>total_vol</th>\n",
       "      <th>mean_vol</th>\n",
       "      <th>std_vol</th>\n",
       "      <th>news</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>is_up</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>14.613</td>\n",
       "      <td>14.865</td>\n",
       "      <td>14.217</td>\n",
       "      <td>14.883</td>\n",
       "      <td>59157390</td>\n",
       "      <td>151685.615385</td>\n",
       "      <td>155760.920466</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-01-05</td>\n",
       "      <td>14.001</td>\n",
       "      <td>14.351</td>\n",
       "      <td>13.811</td>\n",
       "      <td>14.433</td>\n",
       "      <td>68662800</td>\n",
       "      <td>176058.461538</td>\n",
       "      <td>168290.638752</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-01-06</td>\n",
       "      <td>14.079</td>\n",
       "      <td>14.003</td>\n",
       "      <td>13.614</td>\n",
       "      <td>14.280</td>\n",
       "      <td>80752635</td>\n",
       "      <td>207058.038462</td>\n",
       "      <td>152662.243747</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-01-07</td>\n",
       "      <td>14.054</td>\n",
       "      <td>14.219</td>\n",
       "      <td>13.985</td>\n",
       "      <td>14.319</td>\n",
       "      <td>38728110</td>\n",
       "      <td>99558.123393</td>\n",
       "      <td>100907.065350</td>\n",
       "      <td>['BMW revives wireless charging to reduce elec...</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-01-08</td>\n",
       "      <td>14.037</td>\n",
       "      <td>14.216</td>\n",
       "      <td>14.001</td>\n",
       "      <td>14.253</td>\n",
       "      <td>43839960</td>\n",
       "      <td>112699.125964</td>\n",
       "      <td>130930.670726</td>\n",
       "      <td>['How oil at $50 could change almost everythin...</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date   close    open  lowest  highest  total_vol       mean_vol  \\\n",
       "0  2015-01-02  14.613  14.865  14.217   14.883   59157390  151685.615385   \n",
       "1  2015-01-05  14.001  14.351  13.811   14.433   68662800  176058.461538   \n",
       "2  2015-01-06  14.079  14.003  13.614   14.280   80752635  207058.038462   \n",
       "3  2015-01-07  14.054  14.219  13.985   14.319   38728110   99558.123393   \n",
       "4  2015-01-08  14.037  14.216  14.001   14.253   43839960  112699.125964   \n",
       "\n",
       "         std_vol                                               news  month  \\\n",
       "0  155760.920466                                                         1   \n",
       "1  168290.638752                                                         1   \n",
       "2  152662.243747                                                         1   \n",
       "3  100907.065350  ['BMW revives wireless charging to reduce elec...      1   \n",
       "4  130930.670726  ['How oil at $50 could change almost everythin...      1   \n",
       "\n",
       "   day  is_up  \n",
       "0    2      0  \n",
       "1    5      1  \n",
       "2    6      0  \n",
       "3    7      0  \n",
       "4    8      0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tesla_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_valid_test(data, target, split):\n",
    "    train, valid, test = split\n",
    "    n = data.shape[0]\n",
    "    train_end = int(train * n)\n",
    "    valid_end = int(n * (train + valid))\n",
    "    print(valid_end)\n",
    "    x_train, y_train = data[:train_end], target[:train_end]\n",
    "    x_val, y_val = data[train_end:valid_end], target[train_end:valid_end]\n",
    "    x_test, y_test = data[valid_end:], target[valid_end:]\n",
    "\n",
    "    return x_train, y_train, x_val, y_val, x_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(data):\n",
    "    lag = 7\n",
    "    result = data.copy()\n",
    "    result['ema'] = result['close'].ewm(span=lag).mean()\n",
    "    for i in range(1, lag+1):\n",
    "        result['lag_' + str(i)] = result['close'].shift(i)\n",
    "\n",
    "    result = result.iloc[lag:]\n",
    "    # Assuming df is your DataFrame and it has 'day' and 'month' columns\n",
    "    result['month_sin'] = np.sin(2 * np.pi * result['month']/12)\n",
    "    result['month_cos'] = np.cos(2 * np.pi * result['month']/12)\n",
    "    result['day_sin'] = np.sin(2 * np.pi * result['day']/31)\n",
    "    result['day_cos'] = np.cos(2 * np.pi * result['day']/31)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1405/898176073.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['month_sin'] = np.sin(2 * np.pi * data['month']/12)\n",
      "/tmp/ipykernel_1405/898176073.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['month_cos'] = np.cos(2 * np.pi * data['month']/12)\n",
      "/tmp/ipykernel_1405/898176073.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['day_sin'] = np.sin(2 * np.pi * data['day']/31)\n",
      "/tmp/ipykernel_1405/898176073.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['day_cos'] = np.cos(2 * np.pi * data['day']/31)\n"
     ]
    }
   ],
   "source": [
    "data = preprocess_data(apple_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>close</th>\n",
       "      <th>open</th>\n",
       "      <th>lowest</th>\n",
       "      <th>highest</th>\n",
       "      <th>total_vol</th>\n",
       "      <th>mean_vol</th>\n",
       "      <th>std_vol</th>\n",
       "      <th>news</th>\n",
       "      <th>month</th>\n",
       "      <th>...</th>\n",
       "      <th>lag_2</th>\n",
       "      <th>lag_3</th>\n",
       "      <th>lag_4</th>\n",
       "      <th>lag_5</th>\n",
       "      <th>lag_6</th>\n",
       "      <th>lag_7</th>\n",
       "      <th>month_sin</th>\n",
       "      <th>month_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>day_cos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2015-01-13</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.958</td>\n",
       "      <td>24.375</td>\n",
       "      <td>25.297</td>\n",
       "      <td>242520180</td>\n",
       "      <td>621846.615385</td>\n",
       "      <td>402625.780292</td>\n",
       "      <td>Zomato to buy US food portal Urbanspoon for $6...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>23.802</td>\n",
       "      <td>23.807</td>\n",
       "      <td>24.490</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>0.485302</td>\n",
       "      <td>-0.874347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2015-01-14</td>\n",
       "      <td>24.600</td>\n",
       "      <td>24.468</td>\n",
       "      <td>24.283</td>\n",
       "      <td>24.779</td>\n",
       "      <td>166619520</td>\n",
       "      <td>427229.538462</td>\n",
       "      <td>308052.905151</td>\n",
       "      <td>['60,000 mobiles sold in a flash sale but Amaz...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>23.802</td>\n",
       "      <td>23.807</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>0.299363</td>\n",
       "      <td>-0.954139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2015-01-15</td>\n",
       "      <td>23.936</td>\n",
       "      <td>24.521</td>\n",
       "      <td>23.871</td>\n",
       "      <td>24.682</td>\n",
       "      <td>205693220</td>\n",
       "      <td>527418.512821</td>\n",
       "      <td>391931.336059</td>\n",
       "      <td>['60,000 mobiles sold in a flash sale but Amaz...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>23.802</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>0.101168</td>\n",
       "      <td>-0.994869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2015-01-16</td>\n",
       "      <td>23.746</td>\n",
       "      <td>23.990</td>\n",
       "      <td>23.545</td>\n",
       "      <td>24.126</td>\n",
       "      <td>231737276</td>\n",
       "      <td>594198.143590</td>\n",
       "      <td>691319.500876</td>\n",
       "      <td>['60,000 mobiles sold in a flash sale but Amaz...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>24.600</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>-0.101168</td>\n",
       "      <td>-0.994869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2015-01-20</td>\n",
       "      <td>24.358</td>\n",
       "      <td>24.051</td>\n",
       "      <td>23.836</td>\n",
       "      <td>24.395</td>\n",
       "      <td>155383068</td>\n",
       "      <td>398418.123077</td>\n",
       "      <td>419122.611125</td>\n",
       "      <td>['60,000 mobiles sold in a flash sale but Amaz...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>23.936</td>\n",
       "      <td>24.600</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>-0.790776</td>\n",
       "      <td>-0.612106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Date   close    open  lowest  highest  total_vol       mean_vol  \\\n",
       "7   2015-01-13  24.697  24.958  24.375   25.297  242520180  621846.615385   \n",
       "8   2015-01-14  24.600  24.468  24.283   24.779  166619520  427229.538462   \n",
       "9   2015-01-15  23.936  24.521  23.871   24.682  205693220  527418.512821   \n",
       "10  2015-01-16  23.746  23.990  23.545   24.126  231737276  594198.143590   \n",
       "11  2015-01-20  24.358  24.051  23.836   24.395  155383068  398418.123077   \n",
       "\n",
       "          std_vol                                               news  month  \\\n",
       "7   402625.780292  Zomato to buy US food portal Urbanspoon for $6...      1   \n",
       "8   308052.905151  ['60,000 mobiles sold in a flash sale but Amaz...      1   \n",
       "9   391931.336059  ['60,000 mobiles sold in a flash sale but Amaz...      1   \n",
       "10  691319.500876  ['60,000 mobiles sold in a flash sale but Amaz...      1   \n",
       "11  419122.611125  ['60,000 mobiles sold in a flash sale but Amaz...      1   \n",
       "\n",
       "    ...   lag_2   lag_3   lag_4   lag_5   lag_6   lag_7  month_sin  month_cos  \\\n",
       "7   ...  25.095  25.066  24.143  23.802  23.807  24.490        0.5   0.866025   \n",
       "8   ...  24.481  25.095  25.066  24.143  23.802  23.807        0.5   0.866025   \n",
       "9   ...  24.697  24.481  25.095  25.066  24.143  23.802        0.5   0.866025   \n",
       "10  ...  24.600  24.697  24.481  25.095  25.066  24.143        0.5   0.866025   \n",
       "11  ...  23.936  24.600  24.697  24.481  25.095  25.066        0.5   0.866025   \n",
       "\n",
       "     day_sin   day_cos  \n",
       "7   0.485302 -0.874347  \n",
       "8   0.299363 -0.954139  \n",
       "9   0.101168 -0.994869  \n",
       "10 -0.101168 -0.994869  \n",
       "11 -0.790776 -0.612106  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictors = data.drop(['is_up', 'Date', 'news', 'day', 'month', 'total_vol', 'mean_vol', 'std_vol', 'open', 'lowest', 'highest'], axis=1)\n",
    "target = data['is_up']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1772\n"
     ]
    }
   ],
   "source": [
    "x_train, y_train, x_val, y_val, x_test, y_test = train_valid_test(predictors, target, (0.6, 0.2, 0.2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>close</th>\n",
       "      <th>ema</th>\n",
       "      <th>lag_1</th>\n",
       "      <th>lag_2</th>\n",
       "      <th>lag_3</th>\n",
       "      <th>lag_4</th>\n",
       "      <th>lag_5</th>\n",
       "      <th>lag_6</th>\n",
       "      <th>lag_7</th>\n",
       "      <th>month_sin</th>\n",
       "      <th>month_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>day_cos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>24.697</td>\n",
       "      <td>24.598054</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>23.802</td>\n",
       "      <td>23.807</td>\n",
       "      <td>24.490</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>0.485302</td>\n",
       "      <td>-0.874347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>24.600</td>\n",
       "      <td>24.598580</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>23.802</td>\n",
       "      <td>23.807</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>0.299363</td>\n",
       "      <td>-0.954139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>23.936</td>\n",
       "      <td>24.423050</td>\n",
       "      <td>24.600</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>23.802</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>0.101168</td>\n",
       "      <td>-0.994869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>23.746</td>\n",
       "      <td>24.246324</td>\n",
       "      <td>23.936</td>\n",
       "      <td>24.600</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>24.143</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>-0.101168</td>\n",
       "      <td>-0.994869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>24.358</td>\n",
       "      <td>24.275156</td>\n",
       "      <td>23.746</td>\n",
       "      <td>23.936</td>\n",
       "      <td>24.600</td>\n",
       "      <td>24.697</td>\n",
       "      <td>24.481</td>\n",
       "      <td>25.095</td>\n",
       "      <td>25.066</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>-0.790776</td>\n",
       "      <td>-0.612106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     close        ema   lag_1   lag_2   lag_3   lag_4   lag_5   lag_6   lag_7  \\\n",
       "7   24.697  24.598054  24.481  25.095  25.066  24.143  23.802  23.807  24.490   \n",
       "8   24.600  24.598580  24.697  24.481  25.095  25.066  24.143  23.802  23.807   \n",
       "9   23.936  24.423050  24.600  24.697  24.481  25.095  25.066  24.143  23.802   \n",
       "10  23.746  24.246324  23.936  24.600  24.697  24.481  25.095  25.066  24.143   \n",
       "11  24.358  24.275156  23.746  23.936  24.600  24.697  24.481  25.095  25.066   \n",
       "\n",
       "    month_sin  month_cos   day_sin   day_cos  \n",
       "7         0.5   0.866025  0.485302 -0.874347  \n",
       "8         0.5   0.866025  0.299363 -0.954139  \n",
       "9         0.5   0.866025  0.101168 -0.994869  \n",
       "10        0.5   0.866025 -0.101168 -0.994869  \n",
       "11        0.5   0.866025 -0.790776 -0.612106  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictors.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = MinMaxScaler()\n",
    "sc.fit(x_train)\n",
    "x_train_scaled = sc.transform(x_train)\n",
    "x_val_scaled = sc.transform(x_val)\n",
    "x_test_scaled = sc.transform(x_test)\n",
    "\n",
    "\n",
    "X_train_for_compare = np.append(x_train_scaled, x_val_scaled, axis=0)\n",
    "Y_train_for_compare = np.append(y_train, y_val, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline model\n",
    "According to EDA, the market was up ~ 52% of the days, so the simplest model would be always predict up, which could potentially achieve 52% accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.526173285198556"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counting = data['is_up'].value_counts()\n",
    "BASELINE = counting[1] / np.sum(counting)\n",
    "BASELINE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5372460496613995"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_val, [1] * len(y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The baseline model has accuracy of ~ **54%**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Default hyperparameters with some Algo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.5515425131677953\n",
      "Validation accuracy: 0.5146726862302483\n"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression(random_state = 12)\n",
    "lr.fit(x_train_scaled, y_train)\n",
    "\n",
    "print(\"Training accuracy:\", lr.score(x_train_scaled, y_train))\n",
    "print(\"Validation accuracy:\", lr.score(x_val_scaled, y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "K Nearest Neighbors:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.674191121143717\n",
      "Validation accuracy: 0.49435665914221216\n"
     ]
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "knn.fit(x_train_scaled, y_train)\n",
    "\n",
    "print(\"Training accuracy:\", knn.score(x_train_scaled, y_train))\n",
    "print(\"Validation accuracy:\", knn.score(x_val_scaled, y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.580887885628292\n",
      "Validation accuracy: 0.5146726862302483\n"
     ]
    }
   ],
   "source": [
    "svm = SVC()\n",
    "svm.fit(x_train_scaled, y_train)\n",
    "print(\"Training accuracy:\", svm.score(x_train_scaled, y_train))\n",
    "print(\"Validation accuracy:\", svm.score(x_val_scaled, y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.5327313769751693\n",
      "Validation accuracy: 0.5056433408577878\n"
     ]
    }
   ],
   "source": [
    "bnb = BernoulliNB()\n",
    "bnb.fit(x_train_scaled, y_train)\n",
    "print(\"Training accuracy:\", bnb.score(x_train_scaled, y_train))\n",
    "print(\"Validation accuracy:\", bnb.score(x_val_scaled, y_val))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 1.0\n",
      "Validation accuracy: 0.4717832957110609\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(bootstrap=True, random_state = 7)\n",
    "rf.fit(x_train_scaled, y_train)\n",
    "\n",
    "print(\"Training accuracy:\", rf.score(x_train_scaled, y_train))\n",
    "print(\"Validation accuracy:\", rf.score(x_val_scaled, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('close', 0.10043040464661328),\n",
       " ('lag_7', 0.0923772109345861),\n",
       " ('lag_4', 0.09080118022033785),\n",
       " ('lag_1', 0.08971064976791042),\n",
       " ('lag_3', 0.08918385435173042),\n",
       " ('lag_6', 0.08835685626955618),\n",
       " ('lag_5', 0.08765580392616842),\n",
       " ('lag_2', 0.08430168548567064),\n",
       " ('ema', 0.08324976139602493),\n",
       " ('day_sin', 0.0738084676072697),\n",
       " ('day_cos', 0.05805155674978601),\n",
       " ('month_sin', 0.031503648923904774),\n",
       " ('month_cos', 0.030568919720441262)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(list(zip(predictors.columns, rf.feature_importances_)), key= lambda x: -x[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CatBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.8465011286681715\n",
      "Validation accuracy: 0.4785553047404063\n",
      "Test accuracy: 0.46621621621621623\n"
     ]
    }
   ],
   "source": [
    "cat = CatBoostClassifier(silent=True, random_state = 12)\n",
    "cat.fit(x_train_scaled, y_train)\n",
    "\n",
    "print(\"Training accuracy:\", cat.score(x_train_scaled, y_train))\n",
    "print(\"Validation accuracy:\", cat.score(x_val_scaled, y_val))\n",
    "print(\"Test accuracy:\", cat.score(x_test_scaled, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGBoost Classifier "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.9977426636568849\n",
      "Validation accuracy: 0.4582392776523702\n",
      "Test accuracy: 0.4774774774774775\n"
     ]
    }
   ],
   "source": [
    "xg = XGBClassifier()\n",
    "xg.fit(x_train_scaled, y_train)\n",
    "\n",
    "print(\"Training accuracy:\", xg.score(x_train_scaled, y_train))\n",
    "print(\"Validation accuracy:\", xg.score(x_val_scaled, y_val))\n",
    "print(\"Test accuracy:\", xg.score(x_test_scaled, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "1772/1772 [==============================] - 14s 7ms/step - loss: 0.6925 - accuracy: 0.5277 - val_loss: 0.6913 - val_accuracy: 0.5372\n",
      "Epoch 2/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6923 - accuracy: 0.5248 - val_loss: 0.6912 - val_accuracy: 0.5372\n",
      "Epoch 3/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6918 - accuracy: 0.5282 - val_loss: 0.6911 - val_accuracy: 0.5372\n",
      "Epoch 4/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6917 - accuracy: 0.5310 - val_loss: 0.6911 - val_accuracy: 0.5372\n",
      "Epoch 5/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6917 - accuracy: 0.5316 - val_loss: 0.6910 - val_accuracy: 0.5372\n",
      "Epoch 6/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6921 - accuracy: 0.5265 - val_loss: 0.6910 - val_accuracy: 0.5372\n",
      "Epoch 7/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6913 - accuracy: 0.5310 - val_loss: 0.6910 - val_accuracy: 0.5372\n",
      "Epoch 8/15\n",
      "1772/1772 [==============================] - 11s 6ms/step - loss: 0.6918 - accuracy: 0.5288 - val_loss: 0.6909 - val_accuracy: 0.5372\n",
      "Epoch 9/15\n",
      "1772/1772 [==============================] - 11s 6ms/step - loss: 0.6918 - accuracy: 0.5209 - val_loss: 0.6909 - val_accuracy: 0.5372\n",
      "Epoch 10/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6924 - accuracy: 0.5265 - val_loss: 0.6908 - val_accuracy: 0.5372\n",
      "Epoch 11/15\n",
      "1772/1772 [==============================] - 11s 6ms/step - loss: 0.6916 - accuracy: 0.5293 - val_loss: 0.6908 - val_accuracy: 0.5372\n",
      "Epoch 12/15\n",
      "1772/1772 [==============================] - 11s 6ms/step - loss: 0.6915 - accuracy: 0.5305 - val_loss: 0.6908 - val_accuracy: 0.5372\n",
      "Epoch 13/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6917 - accuracy: 0.5271 - val_loss: 0.6908 - val_accuracy: 0.5372\n",
      "Epoch 14/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6916 - accuracy: 0.5293 - val_loss: 0.6907 - val_accuracy: 0.5372\n",
      "Epoch 15/15\n",
      "1772/1772 [==============================] - 12s 7ms/step - loss: 0.6912 - accuracy: 0.5288 - val_loss: 0.6907 - val_accuracy: 0.5372\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "42/42 [==============================] - 0s 3ms/step\n",
      "Training accuracy: 0.5252069224981188\n",
      "Validation accuracy: 0.5372460496613995\n",
      "14/14 [==============================] - 0s 3ms/step - loss: 0.6928 - accuracy: 0.5180\n",
      "Test accuracy: [0.692817211151123, 0.5180180072784424]\n"
     ]
    }
   ],
   "source": [
    "x_train_lstm = np.reshape(X_train_for_compare, (X_train_for_compare.shape[0], X_train_for_compare.shape[1], 1))\n",
    "\n",
    "num_epochs = 15\n",
    "model = tf.keras.Sequential()\n",
    "unit = 50\n",
    "model.add(LSTM(unit, return_sequences=True, input_shape=(x_train_lstm.shape[1], 1)))\n",
    "model.add(LSTM(unit))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(units=25, activation = 'tanh'))\n",
    "model.add(Dense(1, activation = 'sigmoid'))\n",
    "\n",
    "# optimizer = tf.keras.optimizers.Nadam()\n",
    "optimizer = tf.keras.optimizers.Adagrad()\n",
    "\n",
    "model.compile(optimizer=optimizer, loss=\"binary_crossentropy\", metrics=['accuracy'])\n",
    "history = model.fit(x_train_lstm, Y_train_for_compare, validation_data = (x_val_scaled, y_val), batch_size=1, epochs=num_epochs)\n",
    "\n",
    "threshold = 0.5\n",
    "y_val_pred = (model.predict(x_val_scaled) >= threshold).astype(int)\n",
    "print(\"Training accuracy:\", accuracy_score(y_train, (model.predict(x_train_scaled) > threshold).astype(int)))\n",
    "print(\"Validation accuracy:\", accuracy_score(y_val, y_val_pred))\n",
    "print(\"Test accuracy:\", model.evaluate(x_test_scaled, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGwCAYAAACpYG+ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB+3klEQVR4nO3deViUVfsH8O8wMOyLyDYsssiiAoKCKe6mYmiWWrmVaWrlT1uMbDHrzdTETMs2LK183SrrxcxcUlxwSVNScUVERVkEEWTfBmae3x8DoyOIgMwMMN/Pdc2VPPPMc85MMHPPOfe5j0gQBAFEREREesBA1x0gIiIi0hYGPkRERKQ3GPgQERGR3mDgQ0RERHqDgQ8RERHpDQY+REREpDcY+BAREZHeMNR1B1oShUKBGzduwNLSEiKRSNfdISIiogYQBAFFRUVwdnaGgUH9YzoMfO5y48YNuLm56bobRERE1ARpaWlwdXWt9xwGPnextLQEoHzhrKysdNwbIiIiaojCwkK4ubmpPsfrw8DnLjXTW1ZWVgx8iIiIWpmGpKkwuZmIiIj0BgMfIiIi0hsMfIiIiEhvMPAhIiIivcHAh4iIiPRGkwKf6OhoeHp6wsTEBCEhITh06NB9z42Li4NIJKp1u3jxouqczZs3IzQ0FDY2NjA3N0dwcDDWr1+vdh0PD486rzNr1izVOVOmTKl1f69evZryFImIiKgNavRy9k2bNmH27NmIjo5Gnz598N133yEiIgIXLlxAhw4d7vu4pKQktSXi9vb2qn/b2tpi3rx56NSpEyQSCbZt24YXXngBDg4OGDZsGAAgPj4ecrlc9Zhz585h6NCheOaZZ9Taeeyxx7BmzRrVzxKJpLFPkYiIiNookSAIQmMe0LNnT3Tv3h0rV65UHevcuTNGjRqFqKioWufHxcVh0KBByMvLg42NTYPb6d69O0aMGIGFCxfWef/s2bOxbds2JCcnq9btT5kyBfn5+diyZUtjnpJKYWEhrK2tUVBQwDo+RERErURjPr8bNdUlk8lw4sQJhIeHqx0PDw/HkSNH6n1st27dIJVKMXjwYOzfv/++5wmCgL179yIpKQn9+/e/bz82bNiAqVOn1ipWFBcXBwcHB/j6+uLFF19Ednb2fduqqKhAYWGh2o2IiIjarkYFPjk5OZDL5XB0dFQ77ujoiKysrDofI5VKsWrVKsTExGDz5s3w8/PD4MGDcfDgQbXzCgoKYGFhAYlEghEjRuCrr77C0KFD67zmli1bkJ+fjylTpqgdj4iIwMaNG7Fv3z4sX74c8fHxePTRR1FRUVHndaKiomBtba26cZ8uIiKitq1RU103btyAi4sLjhw5grCwMNXxjz/+GOvXr1dLWK7PyJEjIRKJsHXrVtUxhUKBq1evori4GHv37sXChQuxZcsWDBw4sNbjhw0bBolEgj///LPedjIzM+Hu7o5ffvkFY8aMqXV/RUWFWlBUs9cHp7qIiIhaj8ZMdTUqudnOzg5isbjW6E52dnatUaD69OrVCxs2bFA7ZmBgAG9vbwBAcHAwEhMTERUVVSvwuX79Ovbs2YPNmzc/sB2pVAp3d3ckJyfXeb+xsTGMjY0b3G8iIiJq3RoV+EgkEoSEhCA2NhajR49WHY+NjcWTTz7Z4OucOnUKUqm03nMEQahzimrNmjVwcHDAiBEjHthObm4u0tLSHtiWxgkCUFmq2z4QERG1FEZmQAM2FNWERi9nj4yMxKRJkxAaGoqwsDCsWrUKqampmDFjBgBg7ty5yMjIwLp16wAAK1asgIeHB/z9/VVJyTExMYiJiVFdMyoqCqGhoejYsSNkMhl27NiBdevWqa0cA5TTYWvWrMHkyZNhaKje9eLiYsyfPx9PPfUUpFIprl27hvfeew92dnZqQZpOVJYCi5112wciIqKW4r0bgMRcJ003OvAZN24ccnNzsWDBAmRmZiIgIAA7duyAu7s7AGVeTWpqqup8mUyGOXPmICMjA6ampvD398f27dsxfPhw1TklJSWYOXMm0tPTYWpqik6dOmHDhg0YN26cWtt79uxBamoqpk6dWqtfYrEYZ8+exbp165Cfnw+pVIpBgwZh06ZNsLS0bOzTJCIiojao0XV82jKN1fHhVBcREdEdzTzVpbHkZmoikUhnQ3pERER0BzcpJSIiIr3BwIeIiIj0BgMfIiIi0hsMfIiIiEhvMPAhIiIivcHAh4iIiPQGAx8iIiLSGwx8iIiISG8w8CEiIiK9wcCHiIiI9AYDHyIiItIbDHyIiIhIbzDwISIiIr3BwIeIiIj0BgMfIiIi0hsMfIiIiEhvMPAhIiIivcHAh4iIiPQGAx8iIiLSGwx8iIiISG8w8CEiIiK9wcCHiIiI9AYDHyIiItIbDHyIiIhIbzDwISIiIr3BwIeIiIj0BgMfIiIi0hsMfIiIiEhvMPAhIiIivcHAh4iIiPQGAx8iIiLSGwx8iIiISG8w8CEiIiK9wcCHiIiI9AYDHyIiItIbDHyIiIhIbzDwISIiIr3BwIeIiIj0BgMfIiIi0htNCnyio6Ph6ekJExMThISE4NChQ/c9Ny4uDiKRqNbt4sWLqnM2b96M0NBQ2NjYwNzcHMHBwVi/fr3adebPn1/rGk5OTmrnCIKA+fPnw9nZGaamphg4cCDOnz/flKdIREREbVCjA59NmzZh9uzZmDdvHk6dOoV+/fohIiICqamp9T4uKSkJmZmZqpuPj4/qPltbW8ybNw9Hjx7FmTNn8MILL+CFF17Arl271K7h7++vdo2zZ8+q3b906VJ89tln+PrrrxEfHw8nJycMHToURUVFjX2aRERE1AaJBEEQGvOAnj17onv37li5cqXqWOfOnTFq1ChERUXVOj8uLg6DBg1CXl4ebGxsGtxO9+7dMWLECCxcuBCAcsRny5YtSEhIqPN8QRDg7OyM2bNn45133gEAVFRUwNHREZ988glefvnlB7ZZWFgIa2trFBQUwMrKqsF9JSIiIt1pzOd3o0Z8ZDIZTpw4gfDwcLXj4eHhOHLkSL2P7datG6RSKQYPHoz9+/ff9zxBELB3714kJSWhf//+avclJyfD2dkZnp6eGD9+PK5evaq6LyUlBVlZWWp9MzY2xoABA+7bt4qKChQWFqrdiIiIqO1qVOCTk5MDuVwOR0dHteOOjo7Iysqq8zFSqRSrVq1CTEwMNm/eDD8/PwwePBgHDx5UO6+goAAWFhaQSCQYMWIEvvrqKwwdOlR1f8+ePbFu3Trs2rULq1evRlZWFnr37o3c3FwAULXfmL5FRUXB2tpadXNzc2vMy0FEREStjGFTHiQSidR+FgSh1rEafn5+8PPzU/0cFhaGtLQ0LFu2TG1Ex9LSEgkJCSguLsbevXsRGRkJLy8vDBw4EAAQERGhOjcwMBBhYWHo2LEj1q5di8jIyCb1be7cuWqPLSwsZPBDRETUhjUq8LGzs4NYLK41gpKdnV1rpKU+vXr1woYNG9SOGRgYwNvbGwAQHByMxMREREVFqQKfe5mbmyMwMBDJyckAoFrhlZWVBalU2qC+GRsbw9jYuMH9JiIiotatUVNdEokEISEhiI2NVTseGxuL3r17N/g6p06dUgtO6iIIAioqKu57f0VFBRITE1XX8fT0hJOTk1rfZDIZDhw40Ki+ERERUdvV6KmuyMhITJo0CaGhoQgLC8OqVauQmpqKGTNmAFBOH2VkZGDdunUAgBUrVsDDwwP+/v6QyWTYsGEDYmJiEBMTo7pmVFQUQkND0bFjR8hkMuzYsQPr1q1TWzk2Z84cjBw5Eh06dEB2djYWLVqEwsJCTJ48GYByimv27NlYvHgxfHx84OPjg8WLF8PMzAwTJ058qBeJiIiI2oZGBz7jxo1Dbm4uFixYgMzMTAQEBGDHjh1wd3cHAGRmZqrV9JHJZJgzZw4yMjJgamoKf39/bN++HcOHD1edU1JSgpkzZyI9PR2mpqbo1KkTNmzYgHHjxqnOSU9Px4QJE5CTkwN7e3v06tUL//zzj6pdAHj77bdRVlaGmTNnIi8vDz179sTu3bthaWnZpBeHiIiI2pZG1/Fpy1jHh4iIqPXRWB0fIiIiotaMgQ8RERHpDQY+REREpDcY+BAREZHeYOBDREREeoOBDxEREekNBj5ERESkNxj4EBERkd5g4ENERER6g4EPERER6Q0GPkRERKQ3GPgQERGR3mDgQ0RERHqDgQ8RERHpDQY+REREpDcY+BAREZHeYOBDREREeoOBDxEREekNBj5ERESkNxj4EBERkd5g4ENERER6g4EPERER6Q0GPkRERKQ3GPgQERGR3mDgQ0RERHqDgQ8RERHpDQY+REREpDcY+BAREZHeYOBDREREeoOBDxEREekNBj5ERESkNxj4EBERkd5g4ENERER6g4EPERER6Q0GPkRERKQ3GPgQERGR3mDgQ0RERHqDgQ8RERHpDQY+REREpDcY+BAREZHeYOBDREREeqNJgU90dDQ8PT1hYmKCkJAQHDp06L7nxsXFQSQS1bpdvHhRdc7mzZsRGhoKGxsbmJubIzg4GOvXr1e7TlRUFHr06AFLS0s4ODhg1KhRSEpKUjtnypQptdrp1atXU54iERERtUGGjX3Apk2bMHv2bERHR6NPnz747rvvEBERgQsXLqBDhw73fVxSUhKsrKxUP9vb26v+bWtri3nz5qFTp06QSCTYtm0bXnjhBTg4OGDYsGEAgAMHDmDWrFno0aMHqqqqMG/ePISHh+PChQswNzdXXeuxxx7DmjVrVD9LJJLGPkUiIiJqo0SCIAiNeUDPnj3RvXt3rFy5UnWsc+fOGDVqFKKiomqdHxcXh0GDBiEvLw82NjYNbqd79+4YMWIEFi5cWOf9t27dgoODAw4cOID+/fsDUI745OfnY8uWLQ1qo6KiAhUVFaqfCwsL4ebmhoKCArUgjYiIiFquwsJCWFtbN+jzu1FTXTKZDCdOnEB4eLja8fDwcBw5cqTex3br1g1SqRSDBw/G/v3773ueIAjYu3cvkpKSVAFNXQoKCgAoR4vuFhcXBwcHB/j6+uLFF19Ednb2fa8RFRUFa2tr1c3Nza3e50BEREStW6MCn5ycHMjlcjg6Oqodd3R0RFZWVp2PkUqlWLVqFWJiYrB582b4+flh8ODBOHjwoNp5BQUFsLCwgEQiwYgRI/DVV19h6NChdV5TEARERkaib9++CAgIUB2PiIjAxo0bsW/fPixfvhzx8fF49NFH1UZ17jZ37lwUFBSobmlpaY15OYiIiKiVaXSODwCIRCK1nwVBqHWshp+fH/z8/FQ/h4WFIS0tDcuWLVMb0bG0tERCQgKKi4uxd+9eREZGwsvLCwMHDqx1zVdeeQVnzpzB4cOH1Y6PGzdO9e+AgACEhobC3d0d27dvx5gxY2pdx9jYGMbGxg16zkRERNT6NSrwsbOzg1gsrjW6k52dXWsUqD69evXChg0b1I4ZGBjA29sbABAcHIzExERERUXVCnxeffVVbN26FQcPHoSrq2u97UilUri7uyM5ObnBfSMiIqK2q1FTXRKJBCEhIYiNjVU7Hhsbi969ezf4OqdOnYJUKq33HEEQ1KaoBEHAK6+8gs2bN2Pfvn3w9PR8YDu5ublIS0t7YFtERESkHxo91RUZGYlJkyYhNDQUYWFhWLVqFVJTUzFjxgwAyryZjIwMrFu3DgCwYsUKeHh4wN/fHzKZDBs2bEBMTAxiYmJU14yKikJoaCg6duwImUyGHTt2YN26dWorx2bNmoWffvoJf/zxBywtLVWjTtbW1jA1NUVxcTHmz5+Pp556ClKpFNeuXcN7770HOzs7jB49+qFeJCIiImobGh34jBs3Drm5uViwYAEyMzMREBCAHTt2wN3dHQCQmZmJ1NRU1fkymQxz5sxBRkYGTE1N4e/vj+3bt2P48OGqc0pKSjBz5kykp6fD1NQUnTp1woYNG9RydmqCoHunvtasWYMpU6ZALBbj7NmzWLduHfLz8yGVSjFo0CBs2rQJlpaWjX2aRERE1AY1uo5PW9aYOgBERETUMmisjg8RERFRa8bAh4iIiPQGAx8iIiLSGwx8iIiISG8w8CEiIiK9wcCHiIiI9AYDHyIiItIbDHyIiIhIbzDwISIiIr3BwIdICwRBwNVbxSgoq9R1V4iI9Fqj9+oiooZLu12KPxIy8PupDFy5VYIgV2tsmdUHIpFI110jItJLDHyImll+qQzbz2Ziy6kMxF/LU7vvdHoBrtwqhrcDN84lItIFBj5EzaCiSo79F7Px+6kM7L94CzK5AgAgEgG9O7bHqGAX/H4qA0eu5GL3hZsMfIiIdISBD1ETKRQC4q/dxpaEDGw/k4nC8irVfZ2lVhjdzRlPBLnAydoEAFBRpcCRK7mIvXATMwd666rbRER6jYEPUSMl3yzC76cy8EfCDWTkl6mOS61N8GSwC0Z1c0YnJ6tajxvaxRHvbzmHhLR8ZBeVw8HSRJvdJiIiMPAhapDswnJsPX0Dv5/KwPkbharjlsaGiAh0wuhurujpaQsDg/snLTtamSDI1Rqn0wuwNzEbEx7poI2uExHRXRj4EN1HSUUVdp3Pwu+nMvD35RwoBOVxQwMRBvo5YHQ3Fwzu7AATI3GDrzm0iyNOpxcg9sJNBj5ERDrAwIfoLlVyBQ5dzsGWUxnYff4myirlqvtC3NthVDcXjAiUwtZc0qTrD+3ihGW7L+Hw5RyUVFTB3Jh/gkRE2sR3XdJ7giDgTHoBfj+VgW1nbiCnWKa6z9POHKO7ueDJYGe4tzd/6LZ8HS3g3t4M13NLcSj5Fh4LkD70NYmIqOEY+JDeSs0txZaEDGw5lYGrOSWq4+3NJRgZ5IxR3VwQ5GrdrMUGRSIRhnZ2xPeHU7D7wk0GPkREWsbAh/RKXsmd4oL/Xr9TXNDEyADhXZwwupsL+vrYwUisud1chnZRBj77LmajSq6AoQbbIiIidQx8SC/klcgwb8tZxF64iUq5MkvZQAT08bbDqGAXDAtwgoWW8m1C3NuhnZkR8korEX8tD2Ed22ulXV37+3IO4pKy8foQX6291kRE9+K7D+mFRdsTseNsFgCgi9QKo7u54IlgZzhaab+WjqHYAI92ckTMyXTEXripF4GPQiHg7f+dQUZ+GW6XVGL52CBdd4mI9BTH2KnNO5dRgM2n0gEAG6b1xI7X++HF/l46CXpqDO3iCACITcyCIAg664e2nEjNUxV7jDmZjp1nM3XcIyLSVwx8qE0TBAEfb0+EIABPBDmjr4+drrsEAOjvawdjQwOk3S5D0s0iXXdH47Ym3AAAWJooB5nf+/0ssgvLddklItJTDHyoTduflI2jV3MhERvgrWF+uu6OipnEEH29lUFY7PmbOu6NZlXJFdhRPcLz2dhgdJFaIa+0Eu/EnNGL0S4ialkY+FCbVSVXYPGOiwCAF/p4wM3WTMc9UndnuqttBz5/X8lFbokMtuYSDPSzx4rxwZAYGmB/0i38dDxV190jIj3DwIfarF/i03A5uxjtzIwwc1DL2w19cGdHiETAmfQCZBW03Wmfmmmu4YFOMBIbwNfREm9Xj74t2paIlLtqKBERaRoDH2qTisorsWLPJQDA64N9YG1qpOMe1WZvaYxubjYA2u6oT3mlHLvPK1fTPRHkojo+tY8nwrzao6xSjjc2JaBKrtBVF4lIzzDwoTbp2wNXkFMsg6edOSb2dNd1d+5raBcnAEDshbYZ+MQlZaOoogpSaxOEurdTHTcwEGHZ2CBYmhgiIS0f0XFXdNhLItInDHyozbmRX4bvD6UAAN6N6ASJYcv9Na/J8zl6JQdF5ZU67k3z23paOc01MsgZBgbqW3+42Jhi4ZMBAIAv9ibjdFq+trtHRHqo5X4iEDXRst1JqKhS4BEPW4RXBxYtlbeDBbzszFEpF3Dg0i1dd6dZFZVXYm9iNgBlKYG6PBnsjBFdpZArBLzxawLKZHJtdpGI9BADH2pTzmUUYPPJDADAvBGdm3WDUU0Z6l+9uquNTXfFXriJiioFvOzM4e9sVec5IpEIH48KgIOlMa7eKsGSnYla7iXpG7mCJRT0HQMfajMEQcCi7RcAKEcSgqoTh1u6mlGp/RezUdmGknzvnuaqLwC1MZPg02eUW1isPXq9zY18UcuQWVCGST8cQ/CC3TibXqDr7pAOMfChNmPfxWz8c/U2JIYGmBPecooVPkiwWzvYWUhQWF6F4ym3dd2dZnG7RIbDyTkAgCeC657mutsAX3tMDlMmob/122nkl8o02j/SL3+dy0LEF4dwKDkHReVV+Lx6xSfpJwY+1CZUyhVYvEM5TdISixXWR2wgwuBOylGfmqXfrd2Os5moUgjwd7ZCR3uLBj3m3YjO8LI3R3ZRBeZtOceqzvTQymRyvPf7WczYcAL5pZXo5GQJkUj5JSkpq+1vFUN1Y+BDbcIv8Wm4cqsE7cyMMKsFFit8EFUV5ws328QH/p/V01z3S2qui6lEjBXjgmFoIML2M5mqqTKipjh/owCPf3UIPx1TVgd/eYAXtr7SF4/5K0tIfHeQJRT0FQMfavWKyiuxIlY5dD17iC+sTFpescIH6etjB1MjMW4UlOP8jUJdd+ehZBWU4/g15ZTd440IfACgq6sNXhvsAwB4f8s53Kje0Z2ooRQKAT8cTsHob47gyq0SOFgaY8O0npgb0RkSQwPMGNARgLKieAZ/v/QSAx9q9VbGXUFuiQxeduaY2LODrrvTJCZGYvSr3jm+ta/u2nbmBgQB6OHRDi42po1+/MyBHRHsZoOi8irM+e00FFyFQw10q6gCL/w3Hgu3XYBMrsCQzo74a3Z/9K3+2wKAIDcb9PKyRZVCwI+HU3TYW9IVBj7UqmXkl+GHw3eKFRqJW++v9N3TXa3Z1iZMc93NUGyAz8YGwdRIjCNXcrHmyLVm7B21VXFJ2Yj44iAOXLoFY0MDLBwVgNXPh8DWXFLr3JpRn5+PpzKRXg816VMiOjoanp6eMDExQUhICA4dOnTfc+Pi4iASiWrdLl68qDpn8+bNCA0NhY2NDczNzREcHIz169c3ul1BEDB//nw4OzvD1NQUAwcOxPnz55vyFKmVWL6rulihp60qcGitBnd2hIEIuJBZiPS8Ul13p0lSckpwJr0AYgMRhgdKm3wdL3sLzBvRGQDwyV8XcekmE1GpbhVVciz48wKmrIlHTrEMfo6W2PpKX0zq5X7fMgoDfO3RyckSpTI51h+9ruUek641OvDZtGkTZs+ejXnz5uHUqVPo168fIiIikJqaWu/jkpKSkJmZqbr5+Pio7rO1tcW8efNw9OhRnDlzBi+88AJeeOEF7Nq1q1HtLl26FJ999hm+/vprxMfHw8nJCUOHDkVREd8026Kz6QXYfEpZrPD9VlKssD625hKEutsCAPa00lGfmqTmPt52aG9h/FDXerZnBwz0s4esSoHZvyRAVtV2ahxR87icXYRR3xzBj38rR32n9PbAH6/0gZ+TZb2PE4lEqlGf/x65hvJKVgzXJ40OfD777DNMmzYN06dPR+fOnbFixQq4ublh5cqV9T7OwcEBTk5OqptYLFbdN3DgQIwePRqdO3dGx44d8frrr6Nr1644fPhwg9sVBAErVqzAvHnzMGbMGAQEBGDt2rUoLS3FTz/91NinSS2cIAj4eIeyWOGoYGd0dbXRbYeaiWq6qxXu1i4Iwp2ihV2bPtpTQyQSYelTXdHOzAgXMguxgrVXqJogCNh47Doe/+owEjMLYWsuwQ+TQzH/CX+YGIkffAEAI7pK4WJjitwSGX47ka7hHlNL0qjARyaT4cSJEwgPD1c7Hh4ejiNHjtT72G7dukEqlWLw4MHYv3//fc8TBAF79+5FUlIS+vfv3+B2U1JSkJWVpXaOsbExBgwYcN++VVRUoLCwUO1GrcPexLuKFQ5rPcUKH6Qm8Dl29TYKylrXpqWJmUW4nF0MiaEBhgU4Ncs1HaxMEDUmEADw7YEr+Pda2yjwSE2XVyLDjA0nMO/3cyivVKCfjx3+er0fBndu3FS3kdgA0/t5AgBWH7zKrSz0SKMCn5ycHMjlcjg6qv+COTo6Iiur7sJrUqkUq1atQkxMDDZv3gw/Pz8MHjwYBw8eVDuvoKAAFhYWkEgkGDFiBL766isMHTq0we3W/LcxfYuKioK1tbXq5ubm1sBXgnSpUq7A4uo9nab28YRru9ZTrPBBPOzM4eNggSqFgLikbF13p1FqRnsG+dk3a0mBxwKkeKq7KxQCEPnraRRXVDXbtal1OXIlBxFfHMKu8zdhJBZh3vDOWPvCI3CwMmnS9cb1cEM7MyOk3i7FznOZzdxbaqmalNx8by6FIAj3za/w8/PDiy++iO7duyMsLAzR0dEYMWIEli1bpnaepaUlEhISEB8fj48//hiRkZGIi4trdLuN6dvcuXNRUFCguqWlpd33OVPL8cvxVFy9VQJbcwlmDuqo6+40u5pRn92tKM9HEIS7iha6NPv1P3yiC1xsTJF6uxSLtl1o9utTy1YpV2DpXxfx7PfHkFVYDi87c/w+sw9e7O8FA4Om5/aZSQzxfJgHAOWIYlsoHkoP1qjAx87ODmKxuNYISnZ2dq2Rlvr06tULycnJ6h0xMIC3tzeCg4Px5ptv4umnn0ZUVFSD23VyUg6tN6ZvxsbGsLKyUrtRy1ZYXonP9yh/d2YP8WmVxQofJLy6suyBpFuoqGodSZcnU/OQkV8Gc4kYgzs7NPv1rUyMsHxsEEQiZZXu1r7knxruem4Jnv72KKLjrkAQgHGhbvjz1b4IcLFulutP7u0BEyMDnMsoxJEruc1yTWrZGhX4SCQShISEIDY2Vu14bGwsevfu3eDrnDp1ClJp/cmPgiCgoqKiwe16enrCyclJ7RyZTIYDBw40qm/Usn0bdwW3S2TwsjfHhEdaZ7HCB+nqYg0HS2MUV1Thn6utI6dla4JytCfc36nByaWN1curPV7q5wUAeDfmDHKKKzTSDrUMgiBg88l0DP/iEE6n5cPKxBDfTOyOT57uCnNjw2Zrx9ZcgnGhyjSHbw9wGwt90OjfnsjISEyaNAmhoaEICwvDqlWrkJqaihkzZgBQTh9lZGRg3bp1AIAVK1bAw8MD/v7+kMlk2LBhA2JiYhATE6O6ZlRUFEJDQ9GxY0fIZDLs2LED69atU1sp9qB2RSIRZs+ejcWLF8PHxwc+Pj5YvHgxzMzMMHHixId6kahluLtY4dyIzq26WGF9DAxEGNLFET8dS0XshSwM8LXXdZfqVSVXYPtZZX5EU4sWNlRkuC8OXLqFi1lFeDfmLFY/H9LqyxhQbYXllfhgyzn8UR1QP+Jhi8/HBzepEnhDTO/nhQ3HUnEoOQfnMgqabTSJWqZGBz7jxo1Dbm4uFixYgMzMTAQEBGDHjh1wd3cHAGRmZqrV1pHJZJgzZw4yMjJgamoKf39/bN++HcOHD1edU1JSgpkzZyI9PR2mpqbo1KkTNmzYgHHjxjW4XQB4++23UVZWhpkzZyIvLw89e/bE7t27YWlZf00Hah2WVRcr7OlpiyEamE5pSYZWBz57LmRjwRPCQ+UxaNrRq7nIKZahnZmR2tYAmmBsKMbn44Lx5Nd/Y0/iTfz6bxrG9WibI3/66sT1PLz+yymk55VBbCDC64N9MGuQN8Qa/BtwszXDiEAptp6+ge8OXsVXE7pprC3SPZHAbC6VwsJCWFtbo6CggPk+LczZ9AKM/FpZ12nrK33aTN2e+6mokqP7gliUyOT4Y1YfBLnZ6LpL9/XWb6fx24l0TOzZAYtHB2qlze8OXEHUzoswk4ix8/V+cG9vrpV2SXPkCgHf7L+ML/YmQ64Q4NrOFF+MD0ZIdVFPTTt/owAjvjwMAxEQN2cQOrRvO6tF9UFjPr/b5lwBtSmCIGDRduVKntHdXNp80AMoRzYG+CmnuFpyIm9FlRx/nVcuKND0NNfdpvfzwiOetiiVyRH562nWYGnlMvLLMGHVP/gs9hLkCgFPBDljx+v9tBb0AIC/szX6+dhBIQDfH76qtXZJ+xj4UIu3JzEbx1LaXrHCB2kNm5bGJd1CUXkVnKxM8IiH9j6kxAYiLH8mCBbGhjhxPY9Jqa3YjrOZiFhxEMev3Ya5RIzPxgbhi/HBOlmx+X/V21j8+m8acpk832Yx8KEWrVKuQFR1scJpfT01ltzYEg3yc4DYQISkm0VIzW2Zm5bWFC18vKtU63lIbrZmmP+EPwDg89hLOJdRoNX26eGUyqrwzv/OYObGkygsr0KQmw12vN4PY7q76ixhPaxje3R1tUZ5pQJruXlpm8XAh1q0n6uLFbY3l2DmwLZXrLA+NmYS1SjK7gt1Vx/XpZKKKuyt3lPsiWDtTXPd7anuLnjM3wlVCgFvbErgZpOtxLmMAjz+5WFs+jcNIhEwc2BH/G9GmM5ztUQiEV7ur3yfWXf0GkplrBLeFjHwoRarsLwSK+4qVmjZBosVPkhLnu6KvXAT5ZUKeLQ3Q6COlv+KRCIsHhMIOwtjJGcX49NdSTrpBzWMQiFg9cGrGB39N67mlMDJygQbp/fE2491ajHlKR4LcIJHezPkl1bil+Os5t8WtYzfNKI6rLyrWOH4Nlqs8EFqAp/4a7eRVyLTcW/UbVVtUeGs01o6tuYSfPp0VwDAD4dT8PflHJ31he5PEAS8+vMpfLwjEZVyAeFdHLHz9X7o3VGzJRAaS2wgwov9lYUyfzicgkq5Qsc9aj5XbxVjxJeHMO2/8fjz9A29HSFl4EMtUnpeqapY4XttuFjhg7jZmqGTkyUUArDvYsvZtDSvRIaDl24B0N00190GdXLAsz2VwfGc3063up3t9UHshZvYfjYTErEBPh4dgO8mhaCduUTX3arTU91dYWchQUZ+GbaduaHr7jSLSrkCszcl4PyNQuy9mI1Xfz6F0EV7MOe30zhyOUevVkbq56cJtXjLdiVBVqVALy9bjez91JqEt8Dprp3nslClENBZagVvh5ZRIHTeiM7waG+GzIJyfPjHOV13h+5SKVdgyc6LAIBp/TzxbE/3Fl1x28RIjBf6eAIAvjtwtU1sXvrVvss4k14Aa1MjzBjQES42piiuqML/TqRj4vfH0GfJPkTtSERiZqGuu6pxDHyoxTmTno8t1aXq5w3v0qLfILVhaBflpqUHk2+1mKHpraczAGi3ds+DmEkM8dm4YBiIgC0JN1S7xZPu/Xw8FVdzWtcihed6usNcIsbFrCLEVY9utlYnU/Pwzf7LAICPRwfg3YhOOPT2IPw2IwwTe3aAtakRsgrL8d3Bq4j44hAeW3EQ3x64gsyCMh33XDMY+FCLoixWqFy+PqabCwJduWdOgIsVpNYmKJXJceSK7vNXsgrKcSxFuXnqyKD6NxvWtu4d2uGVQd4AgPe3nENWQbmOe0StdZGCtZmRaiPkb+Nab52oUlkVIjclQK4QMCrYGY93VX5ZMTAQoYeHLRaPDsTxeYPx3aQQRAQ4QSI2wMWsIizZeRG9l+zDhFX/4Nf4NBSWt53pYwY+1KLEXriJ4ym3YWxogDf1qFhhfUQiUYta3bX9bCYEAQhxbwfXdi2vrP+rg30Q6GKNgrJKvPW/01DoUe5CSxS9v/UuUpjWzxOGBiIcS7mNU6l5uu5Ok3y8PRHXckshtTbBR08G1HmOsaEYw/ydsPK5EMTPG4KoMYF4xNMWgqDci+/tmDMIXbQHszaexJ4LNyGrat0J3wx8qMVQywPQs2KFD1IT+OxJzNb5B/ndq7laIiOxAT4fFwxjQwMcSs7B+n9YiE5X0vNK8ePfrXeRgtTaFE8GuwBQ5vq0NvsvZmPjMeWm4cufCYK16YNH22pGun59OQyH3xmEt4b5wdvBArIqBbafzcT0df+i5+I9+GDLOZy4ntcq859a128htWk/HbuTB/B/rSQPQFt6eraHpbEhbhVVICE9X2f9uJ5bgtNp+TAQAcMDW9Y01928HSzw3vDOAIConYm4nF2s4x7pp0/bwCKFlwcol7bvupCFq7daz+/R7RIZ3vrfGQDKL5K9vRtfNsC1nRlmDfJG7Bv9se3Vvpje1xP2lsbIK63E+n+u46mVRzDg0zh8FnupVb02DHyoRVDmAVwCAMwe6ttq8gC0RWJogIGdlB8cupzuqkkY7uNtB3tLY531oyEm9XJHPx87lFcqEPlrQpuqx9IanE7Lxx9tYJGCr6MlBndygCAAqw+1jlEfQRAwd/MZ5BRXwMfBAm89ZNqASCRCgIs13n+8C/6ZOxjrpz2CMd1dYCYRI/V2Kb7cm4xHlx/Ak9/8jf/+nYKcFr7PGQMfahGi919BXmklOtqbY0IPN113p0Wqme7afV5321fUTHONbKHTXHczMBDh06eVw/tn0gvw1b7Luu6S3hAEAR/vaDuLFGZUj0DHnMhAdmHLT5iPOZmBXedvwkgswufjgmFiJG62a4sNROjnY4/Pxgbj3/eH4IvxwRjoZw+xgQin0/Ix/88L6Ll4L15Ycxx/JGSgTNYyVqLejYEP6Vza7bvyAIZ3hmErywPQloF+9jASi3DlVolOhpUvZhXi0s1iSMQGGObvpPX2m8LJ2gQfj1YmdH6z/zJOttIE1dZmdxtbpNDDwxYh7u0gkyuw5sg1XXenXmm3SzF/63kAwOwhvgjQ4HYyZhJDPBnsgv++8Aj+mTsYH47sgiBXa8gVAvYn3cLrvyQgdFEsIn9NwOHkllMkkZ8wpHPLdivzAMK82uPRTq0zD0AbrEyM0MurPQDdTHdtrZ62GOhn36AkyZbi8a7OGBXsDLlCQOSmBG48qWFtdZHCy9XbWGz45zqKWujSbrlCwJu/nUZxRRVC3NthxgDt5UraWxrjhT6e+OOVvtj75gC89qg33GxNUSKTY/PJDDz3wzGERe3Fx9sv4PyNAp0mRTPwIZ2qyQMQiZSVd1trHoC26GpZuyAI+PNM65nmutdHTwZAam2Ca7mlWHeUq7w06adjqUhpg4sUhnR2hLeDBYrKq/Dz8VRdd6dOPxy+iuMpt2EuEePzscEQG+jm/bSjvQUiw/1w8K1BiPm/MDzXqwNszIyQXVSB1YdSMOLLw7hZqLs8IAY+pDOCIODj6mKFo7u5aHRItq0Y0lkZ+JxIzdNqAuGptHyk3S6DmUSs6kNrYm1qhGl9lVsQnLjO6S5NKShru4sUDAxEeOmuzUsrqlpW7kpiZiGW7VK+9v8Z2QUd2uu+xpZIJEKIuy0WjQrE8feGYNWkEAwPdEJ/X3s4WZvorF8MfEhndl+4iePXlHkAc8Jbfx6ANjjbmCLAxQqCAOxL1N6mpTXTXEO7OMJU0nyJktrU1dUGAHAuo0C3HWnDouMut+lFCk8GO8PRyhg3Cyvwx6mWsyVKRZUcb2xKgEyuwJDOjhgb2vJee4mhAcL9nRD9bAjWvtBDp31h4EM6cXcewPR+nnBuI3kA2jC0szKxeLeWprvkCgHbz2YCaLlFCxuii7MVRCIgs6C8xS+3bY3Sbpdizd/XALTdRQrGhmLVyOF3B6/ovJhojc92X8LFrCK0N5dgyVOBLT5lQNf9a3u/mdQqbPznOlJySmBnIcH/DfTWdXdalZo8n8OXb2llqeg/V3Nxq6gC1qZG6Odjr/H2NMXC2BCeduYAgLMc9Wl2NcUK2/oihQmPdICliSGu3CrBnkTdbyHzz9VcrKquL7Tkqa6ws2jZ9bVaAgY+pHUFZZX4Ym/NpoW+sDA21HGPWpfOUku42JiivFKBQ8ma3zW6ZppreKATJIat+y0jsDqP7DwDn2aVkJaPraf1Y5GCpYkRnuvlDgD47qBuCxoWllfizV9PQxCA8T3cVF+KqH6t+12MWqWaPABvBwuMb4N5AJomEokQ7q+d1V0VVXLsPKec5mqNq7nuVRP4cMSn+QiCgMV6tkjhhT4ekIgNcOJ6HuKv3dZZPz7aegEZ+WVwszXF+4930Vk/WhsGPqRVabdLsebwNQDAe8M7tck8AG2o+Wa372K2RouCHbyUg8LyKjhYGqOnZ3uNtaMtNR/K5zIKddyTtkMfFyk4WJrgqRDl5qXfxl3RSR92ns1EzMl0GIiAz8cGc+S8EfipQ1r16a4kyOQK9O7YHoP82m4egKY94mELa1Mj5JbINFqNuGaLise7OuusJkhz8ne2AgBk5JfhdolMx71p/WRV+rtI4cV+XhCJgL0Xs3HpZpFW284uLMd7v58FAMwY0BGhHrZabb+1Y+BDWnN3HsB7w9t2HoCmGYoNVAmkmpruKpVVYU/1tZ8Ibv3TXIAyP8OLCc7N5qdj+rtIwcveAsO6KFdYfndAe7k+giDg7ZgzyCutRBepFWYP8dVa220FAx/SCmWxwgsAgDHdXPUiD0DT7q7irIny77EXbqKsUg739mYIauWbTN7NXzXdxcDnYXCRwp3NS/9IyEBmQZlW2tx4LBVxSbcgMTTAivHBrX7BgS7wFSOt2HX+JuKv5SnzAIbxG0pz6O9rD4nYACk5JbiigU1L/6zZib2rc5sanQt0UU53MfB5ONH7uUgh2M0GvbxsUaUQ8MOhFI23l5JToqp2/85jneDraKnxNtsiBj6kcYIg4Mvqb4bT+3lCaq0/eQCaZGFsiN7eyoTjXeebd7orv1SGA5eUS+XbyjRXjQCu7Hpo6sUK9XuRwsvVG4H+fDwVBaWa27y0Sq7AG5sSUFYpR++O7fFCbw+NtdXW6e9vK2nNoeQcXMgshKmRGNP7eum6O22KpjYt/etcFirlAjo5Wba5b5U1gU96XhnymODcJFykcMdAX3t0crJEiUyODcc0twFudNwVJKTlw9LEEMueCYJBG1hsoCsMfEjjvj2gXO45/hE3tDOX6Lg3bUvNhqEJafnILixvtuvWrOZqC7V77mVlYgSP6g0cz93gqE9jcZGCOpFIhBnVoz5r/k5BeWXzV1M/nZavyqdaNCpAr1bPaQIDH9KoM+n5OHIlF2IDEab342hPc3O0MkGQmw0AYE8zbVqaXViOo1dzAbTuvbnqw+mupuEihbqN6CqFi40pcopliDmZ3qzXLpPJ8cavCZArBDzeVdpm/ya1iYEPaVTNMs8ngpzhwm8pGhGumu7KapbrbTuTCUEAunWwgZutWbNcs6UJ4MquJuEihboZiQ0wvZ9y89LVB682a1HRJTsTcfVWCRytjLFoVIDej7A1BwY+pDHXckpU2x28PICjPZpSk+fz95VclFRUPfT1aqa52vI3y0BWcG40ZbFC5YqiF/t5cZHCPcb1cIONmRGu5Zbir3PN8yXkwKVbWHtUmTf06dNBsDFjqkBzYOBDGrP60FUoBGCgnz06OVnpujttlo+DBdzbm0FWpcDBSw+3aWlqbikS0vJhIFIO37dVAc7KwCf1dqlGV+K0JRuPXce13FLYWUhU9WvoDjOJIZ4P8wCgzGt82NpaeSUyvPXbaQDA5DB39Pe1f9guUjUGPqQRt4oq8NsJ5Vx3TeIfaYZIJLpruuvhVnf9eUY52hPWsT0cLE0eum8tlbWZETrYMsG5oe4uVvjGUP0sVtgQk8PcYWJkgLMZBTh6JbfJ1xEEAe9vOYfsogp42Zvj3YjOzdhLYuBDGrH2yDXIqhQIdrNBT0/uI6NpQ6tL5+9LykaVXNHk6/ypB9NcNbhTe8N9s/8y8ksr4eNggXGh+lmssCHaWxhjbPXr8+3Bpm9j8UfCDWw/mwlDAxFWjAuGqUTcXF0kMPAhDSiuqMK6o9cAADMGeDEZTwtC3NvB1lyC/NJKxF9r2qall24W4WJWEYzEIjzm33anuWpwZVfDpN0uxX9VxQo763WxwoZ4sZ8XxAYiHLx0C+ebMJqYkV+GD/44BwB4bbAPurraNHMPib/BWpJbXIEjV3J03Q2t+OV4KgrLq+BlZ64aiSDNEhuIHnrT0q0JytGeAb4OsDYzara+tVQB3LqiQZZWFyvs490eA/2YZ/IgbrZmGBGo/OLQ2M1LFQoBc349jaLyKgS72WAmc6k0okmBT3R0NDw9PWFiYoKQkBAcOnTovufGxcVBJBLVul28eFF1zurVq9GvXz+0a9cO7dq1w5AhQ3D8+HG163h4eNR5nVmzZqnOmTJlSq37e/Xq1ZSn2KwSMwsx8NM4/N+Gk20+kVJWpcAPh5V71rzYX/nNh7RDVcU5MavRiZWCINxZzdXGtqi4n5oE5+u5pSgoa9t/l011KjUPf7JYYaO91F+5inX72Uyk3S5t8OPWHLmGo1dzYWokxufjgjm6piGNflU3bdqE2bNnY968eTh16hT69euHiIgIpKam1vu4pKQkZGZmqm4+Pj6q++Li4jBhwgTs378fR48eRYcOHRAeHo6MjAzVOfHx8WqPj42NBQA888wzau089thjauft2LGjsU+x2fk6WsLZxhQFZZX4al+yrrujUVtP30BmQTnsLY0xupuLrrujV/r52MHY0ABpt8uQdLOoUY89nV6A1NulMDUSY0hn/diCoJ25BK7tlEuymzIl0dYpixUql68/1d0V/s4sVthQAS7W6OdjB7lCwPeHGjbqc+lmET75Szkg8P7jneFpZ67JLuq1Rgc+n332GaZNm4bp06ejc+fOWLFiBdzc3LBy5cp6H+fg4AAnJyfVTSy+k6y1ceNGzJw5E8HBwejUqRNWr14NhUKBvXv3qs6xt7dXe/y2bdvQsWNHDBgwQK0dY2NjtfNsbXWfWCs2EGHu8E4AgLVHryE1t+HfAFoThULAqoPK7Smm9vGEiRET8rTJTGKIfj52AIDYRm5aWjPNNbSLI8wk+rNiJ5CFDO9r1/ks/Hs9DyZGBngznMUKG6tmNeumf9Nw+wF7wsmqFJj9SwJkVQoM8rPHxEc6aKOLeqtRgY9MJsOJEycQHh6udjw8PBxHjhyp97HdunWDVCrF4MGDsX///nrPLS0tRWVl5X2DFplMhg0bNmDq1Km1hl7j4uLg4OAAX19fvPjii8jOvn8Z/4qKChQWFqrdNGWgnwP6+dihUi7gk10XH/yAVmh/UjYu3SyGhbEhJvbkH64u1Ex37W5Eno9cIWDbGf1ZzXW3OwnOLGR4N2WxQuX7FIsVNk3vju0R6GKN8koF1h65Vu+5K/ZcwoXMQrQzM8InT3fllKKGNSrwycnJgVwuh6Ojo9pxR0dHZGXVXalSKpVi1apViImJwebNm+Hn54fBgwfj4MGD923n3XffhYuLC4YMGVLn/Vu2bEF+fj6mTJmidjwiIgIbN27Evn37sHz5csTHx+PRRx9FRUVFndeJioqCtbW16ubmptllmso5cmD7mUycuN60lTctWc1mpM/27ABr07afHNsSPdrJESKRcqVSZkFZgx5zLCUX2UUVsDIxRD9fOw33sGXhiE/dNvxTU6zQGC+zDleT3L156dqj11Aqq7uqevy126r3zqgxgW26flZL0aTMqXujUUEQ7huh+vn54cUXX0T37t0RFhaG6OhojBgxAsuWLavz/KVLl+Lnn3/G5s2bYWJS9y/ADz/8gIiICDg7q387HTduHEaMGIGAgACMHDkSO3fuxKVLl7B9+/Y6rzN37lwUFBSobmlpaQ966g+ls9QKz4S4AgA+3n7hoSt7tiQnrt9G/LU8GIlFeKGPp667o7fsLY3RvUM7AMCeBo761NTuiQiQwthQv6Yna0Z8UnJKUFjOBGcAKCitxJfVuYiRLFb4UB4LcIJ7ezPkl1bi1/jany/FFVWI/DUBCkGZR/VYQNsvI9ESNCrwsbOzg1gsrjW6k52dXWsUqD69evVCcnLtJN9ly5Zh8eLF2L17N7p27VrnY69fv449e/Zg+vTpD2xHKpXC3d29zrYAZT6QlZWV2k3T3gz3g6mRGCdT87GzmfZzaQm+rV62ObqbC5ys+Y1Flxoz3SWrUmDHWeXvob6s5rqbrblEtXnuhRuc7gKAb+LuFCscG+qq6+60amIDEV7sp1zhtfpQCirvKS668M8LSLtdBhcbU3z4RBdddFEvNSrwkUgkCAkJUa2oqhEbG4vevXs3+DqnTp2CVKoe2X766adYuHAh/vrrL4SGht73sWvWrIGDgwNGjBjxwHZyc3ORlpZWqy1dcrQyUS11XLLzIiqq5Dru0cO7nF2sqh3zUn8Oi+taTeDzz9XcB45iHEq+hYKySthbGqOXV3ttdK/FYT2fO9SKFY5gscLm8HSIK+wsJMjIL8OOs5mq47vPZ2HTv2kQiYDlY4NgZcL0AG1p9G91ZGQkvv/+e/z4449ITEzEG2+8gdTUVMyYMQOAcvro+eefV52/YsUKbNmyBcnJyTh//jzmzp2LmJgYvPLKK6pzli5divfffx8//vgjPDw8kJWVhaysLBQXF6u1rVAosGbNGkyePBmGhurDr8XFxZgzZw6OHj2Ka9euIS4uDiNHjoSdnR1Gjx7d2KepUS/194K9pTFSb5diffXOu61ZzUquoV0c4e1goePeUEd7C3jZm6NSLuBAUv2bltbU7hkRKNXbmkvcuuKOT/66CJlcgb7edhjITTGbhYmRGFN6ewBQjowLgoBbRRWYu/ksAOClfl56+6VDVxod+IwbNw4rVqzAggULEBwcjIMHD2LHjh1wd3cHAGRmZqrV9JHJZJgzZw66du2Kfv364fDhw9i+fTvGjBmjOic6OhoymQxPP/00pFKp6nZvHtCePXuQmpqKqVOn1uqXWCzG2bNn8eSTT8LX1xeTJ0+Gr68vjh49CktLy8Y+TY0yNzbEnOrloV/tu4z80vqXOrZkWQXl+P2Ust4SNyNtOYY2YNPSMplcdb8+TnPV4NYVSidT87DtTCaLFWrApF4eMJOIkZhZiAOXbmHu5jPILZGhk5MlIlkqQOtEQlvKsH1IhYWFsLa2RkFBgcbzfeQKAcO/OISkm0WY1tcTHzzeOud3o3Yk4ruDV9HDox1+m9Hw6U7SrBPX8/DUyiOwNDHEyQ+GwqiOKYs/T9/Aqz+fgputKQ6+NUhvP+hyiisQumiPcjXc/GF6mcwrCAKe/vYoTlzPwzMhrvj0mSBdd6nNWbjtAn44nIL25hLklsggERvgj1f6oLNU87ml+qAxn9+cwNURsYEI743oDABYd/QarueW6LhHjVdQVomNx5SjexztaVm6udnAzsIYReVVOHb1dp3n1ExzjezqrLdBDwDYWRhDam0CQQDO6+moz1/nsnBCVazQT9fdaZOm9fWEoYEIudXFDOcM82XQoyMMfHRogK89+vvao1IuYOlfSbruTqP9dCwVxRVV8HW0wCA//djmoLUwMBCptp6IvVB79WBBWaUq/0efp7lq1Ex3ndPDlV2yKgWWVG+V8FI/L67K1BBnG1M8Gazcxqenpy2m9fXScY/0FwMfHXtveCcYiJSb2Z24Xvc385aovFKOH/9Wbkb6Uv+OMNDTxNiW7O48n3tntHedy4JMroCvowU6OfFbpz4XMlz/z3VcZ7FCrfjPyC744PEu+Pa5EL1dTNASMPDRsU5OVngmRFkxetH2xFZT1PD3Uxm4VVQBqbWJ3m1z0Fr08baDqZEYNwrKcf6ekQzVTuz8fwdAf1d2FZRW4su9yjpnb4b7wlwP85u0ydrUCNP6eqKduUTXXdFrDHxagDfDfWFqJMap1HxVMbmWTK4QsPqgsmDhtL6ekBjy16glMjESo3/1FhR3r+7KLirHkSs5AICRDHwA3JnqunKrGCUVdW8t0BZ9vT8ZBWWV8HW0UFWVJ2rr+InVAjhYmeDlAdVFDf9KbPFFDWMvZOFqTgmsTAwxnrsIt2hDuzgBUA98dpzJhEIAgtxs4N7eXFdda1HsLY3hZKVMcL6QqR95Pqm5pVh7RFlH7L3hLFZI+oO/6S3ES/294GBpjLTbZS26qKEgCFhZvT3F82Eeern0tzV5tJMDDETKD/P0vFIAnOa6n5oKzmfT9WO665NdymKF/XzsMIDFCkmPMPBpIcwkhphTvYz0y73JLbao4bGU2zidlg+JoQEmV1cjpZbL1lyCUA9bAMpRn7TbpTiZmg+RCHi8a8vZyqUluLOyq+0HPieu52F7dbHCuREsVkj6hYFPC/JUiCs6OVmisLwKX+27rOvu1OnbA8rtKZ4JcYW9pbGOe0MNEX7X6q4/zyhHe3p5toejFZct301fVnYJgoCPt18AoPw77uLMVX2kXxj4tCBiAxHeG36nqOG1nJZV1DAxsxBxSbdgIIJqx2Fq+WqWtR9LuY3f/k0HwNo9dakJfC5nF6NU1nYTnHeey8LJ1HyYGokROZTFCkn/MPBpYfr72mNATVHDXRd13R01q6pXckUESOFhx6TY1sK9vTl8HS0gVwhIySmBkViEiAAnXXerxXGwMoGDpTEUgjLIb4tkVQos2al8X3mxP4sVkn5i4NMCvTe8MwxEwI6zWS2mqGF6XqkqKbZmBRq1HjWjPgDQ38ceNmasI1IXVT2fNprgvO7oNaTeLoW9pTFe7s+/Y9JPDHxaID8nS4wNbVlFDX84nAK5QkDvju3R1dVG192hRqpZ1g5wmqs+d3Zqb3sjPsUVd3IH3xzKYoWkvxj4tFCRQ31hJlEWNdx+NlOnfckrkeGX42kAuBlpa9XVxRrBbjbwaG+GIZ0dH/wAPVUT+Jxvgyu7DiffQkFZJdxsTfFM9RcrIn3EwKeFcrAywcv9lUHGJ39d1GlRw/X/XEdZpRxdpFbo52Ons35Q0xkYiPD7zN7Y9+ZAftOvR81UV3J2McorW3Yh0cY6cElZrXtwJ0fuE0V6jYFPC/Zif084WimLGq47opuihmUyOf575BoAZW4P6320XiKRiJvJPoCjlTHsLIwhVwhtqoKzIAg4eOkWAKi2MSHSVwx8WjAziSHerC5q+NW+ZOSVaL+o4f9OpOF2iQyu7UwxIpAF76htE4lECKyu4NyW6vlcuVWCjPwySMQG6OXVXtfdIdIpBj4t3FPddVfUsEquwKpDyiXsL/bz4l4+pBfa4squmtGeHp7tYCbhVCfpN36StXBiAxHmjVAWNVz/j3aLGu44l4W022VoZ2akWmVG1NbdWdnVhgKf5OppLh/uyUXEwKcV6Odjj4F+yqKGn/ylnaKGgiDgu+rtKSb39oCpRKyVdol0LaCNJTiXV8rxz9VcAMoCqUT6joFPKzE3QlnUcOe5LPx7TfNFDQ9fzsH5G4UwNRJjcpiHxtsjaimk1iZoby6BXCHgYlaRrrvz0P69lofySgUcLI3RyclS190h0jkGPq2En5MlxvXQXlHDms1Ix/VwQztzVvkl/SESidrUdFfNNFc/H3uuyiQCA59W5Y3qooYJafnYdkZzRQ3Pphfg78u5EBuIMK2vp8baIWqpVDu1t4EE5wNJysBngB+nuYgABj6tioOliapysiaLGn57UDnaM7KrFG62Zhppg6glaysjPlkF5Ui6WQSRCOjnzfo9RAADn1Znej9lUcP0vDKsrS4s2Jyu55ZgZ/UWGS/15/YUpJ8CXZWBz6WbRa06wblmmqurizWnrImqMfBpZcwkhpijKmp4udmLGq4+dBUKARjga48uzlbNem2i1sLZ2gTtzIxQpRBw6WbrTXC+U62Z01xENRj4tEJjuruis9QKReVV+HJfcrNdN6e4Ar/9mw6Am5GSfmsLCc5yhYDDl5X7czHwIbqDgU8rJDYQYd7w6qKGR68jpZmKGq49cg0VVQoEuVqjl5dts1yTqLVSJTi30sDnbEYB8ksrYWlsiGA3G113h6jFYODTSvX1scMgP3tUKQR8svPhixqWVFRh3VHlRqgzBnTkslfSe4GtfMSnZjVXH287GHG7GSIV/jW0YnOHK4sa/nU+C/EPWdTwl/g0FJRVwtPOHOH+Ts3UQ6LWq2aqKymrSGMrKDVJtU0Fp7mI1DDwacV8HS0xrkcHAMqihgpF04oaVsoV+OGuzUjFBhztIXJtZwobMyNUygVcyirWdXcapaCsEglp+QCA/r5cxk50NwY+rdwbQ31gLhHjdFo+tp1tWlHDrQk3cKOgHHYWxhjT3aWZe0jUOolEIgQ4V+f53Ghd011HLudArhDgZW8O13asxUV0NwY+rZxaUcOdFxtdc0QQBHxXXbDwhT4eMDHiZqRENVrryi7uxk50fwx82oDp/bzgZGWCjPwyrDt6rVGP3Z+UjUs3i2EuEeO5Xu6a6SBRK9UaV3YJgoCDl5TL2Acwv4eoFgY+bYCpRIw5w+4UNbzdiKKG3x5Q5vZM7NkB1qZGGukfUWtVE/hczCyCrEqh4940zJVbJcjIL4PE0AA9WZaCqBYGPm3E6G4u6FJT1HBvw4oankzNw/GU2zASizCtr5eGe0jU+rjZmsLa1AgyuaLVVHA+UF2t+REPW5hJDHXcG6KWh4FPGyE2EGHeCGVRww3/NKyo4bdxytyeUcEucLI20Wj/iFojZQVn5dYtrWW66842FVzNRVQXBj5tSB9vOzzayQFVCgFLdibWe+7l7GLEJt4EALw8gKM9RPfTmlZ2lVfKcSwlFwDr9xDdDwOfNmZuRCcYiIBd52/ieMr9ixquPngVggAM6ewIbwdLLfaQqHW5s7KrUMc9ebD4a7dRXqmAo5Ux/Bz5d01UFwY+bYyPoyXGP6Isavjx9gt1FjW8WViO309lAAD+byBHe4jqU5PgnJhZiEp5y05wrpnm6udjz21niO6DgU8b9MYQX2VRw/QC/HnmRq37f/w7BTK5AqHu7RDizlUfRPVxb28GSxNDyKoUSL7Zsis4cxk70YM1KfCJjo6Gp6cnTExMEBISgkOHDt333Li4OIhEolq3ixfvbKy5evVq9OvXD+3atUO7du0wZMgQHD9+XO068+fPr3UNJyf1PaUEQcD8+fPh7OwMU1NTDBw4EOfPn2/KU2zV7C2N8X8DlUUNl/6VpFbUsLC8Ej/9kwoAqsKHRHR/ahWcW3CCc2ZBGZJuFkEkAvp6M7GZ6H4aHfhs2rQJs2fPxrx583Dq1Cn069cPERERSE1NrfdxSUlJyMzMVN18fHxU98XFxWHChAnYv38/jh49ig4dOiA8PBwZGRlq1/D391e7xtmzZ9XuX7p0KT777DN8/fXXiI+Ph5OTE4YOHYqiotaxDLU5Tet7p6jh2iPXVMd/OpaKoooq+DhY4NFODrrrIFErEuja8is4H6oe7enqaoN25hId94ao5Wp04PPZZ59h2rRpmD59Ojp37owVK1bAzc0NK1eurPdxDg4OcHJyUt3E4jtbI2zcuBEzZ85EcHAwOnXqhNWrV0OhUGDv3r1q1zA0NFS7hr39neFcQRCwYsUKzJs3D2PGjEFAQADWrl2L0tJS/PTTT3X2qaKiAoWFhWq3tsJUIsZb1UUNv96vLGpYUSXHj4dTAAAv9feCATcjJWoQf+fqJe0teGXXgeptKgb4cLSHqD6NCnxkMhlOnDiB8PBwtePh4eE4cuRIvY/t1q0bpFIpBg8ejP3799d7bmlpKSorK2Frq55/kpycDGdnZ3h6emL8+PG4evWq6r6UlBRkZWWp9c3Y2BgDBgy4b9+ioqJgbW2turm5udXbr9bm3qKGW05lILuoAk5WJngymJuREjXU3QnOVS0wwVmuEHA4WTniw2XsRPVrVOCTk5MDuVwOR0dHteOOjo7Iysqq8zFSqRSrVq1CTEwMNm/eDD8/PwwePBgHDx68bzvvvvsuXFxcMGTIENWxnj17Yt26ddi1axdWr16NrKws9O7dG7m5ypoVNe03pm9z585FQUGB6paWlvbgF6EVMTAQ4f27ihp+sUdZ0XlaX09IDJnXTtRQHu3NYWFsiPJKBS7fankJzmfS81FQVglLE0MEu9noujtELVqT6pnfu0xSEIT7Lp308/ODn5+f6uewsDCkpaVh2bJl6N+/f63zly5dip9//hlxcXEwMblTTTgiIkL178DAQISFhaFjx45Yu3YtIiMjm9Q3Y2NjGBsb1/NMW7/e3nYY3MkBey9m40ZBOSxNDDGhZwddd4uoVTEwEMHf2QrHUm7jbHoBOjlZ6bpLampWc/X1toOhmF9qiOrTqL8QOzs7iMXiWiMo2dnZtUZa6tOrVy8kJ9feT2rZsmVYvHgxdu/eja5du9Z7DXNzcwQGBqquU7PC62H71hbNHd4J4up8nkm93GFhzP17iBqrJe/UfuBSNgBOcxE1RKMCH4lEgpCQEMTGxqodj42NRe/evRt8nVOnTkEqlaod+/TTT7Fw4UL89ddfCA0NfeA1KioqkJiYqLqOp6cnnJyc1Pomk8lw4MCBRvWtLfJ2sMTciE7o72uPF/uxYCFRU9Ss7Dp3o2UtgigorURCWj4ABj5EDdHor/6RkZGYNGkSQkNDERYWhlWrViE1NRUzZswAoMybycjIwLp16wAAK1asgIeHB/z9/SGTybBhwwbExMQgJiZGdc2lS5figw8+wE8//QQPDw/VqI2FhQUsLCwAAHPmzMHIkSPRoUMHZGdnY9GiRSgsLMTkyZMBKKe4Zs+ejcWLF8PHxwc+Pj5YvHgxzMzMMHHixId7ldqA6f28MJ1BD1GT+VfX8rlwoxByhaAaRdW1v6/kQCEAHe3N4WJjquvuELV4jQ58xo0bh9zcXCxYsACZmZkICAjAjh074O7uDgDIzMxUq+kjk8kwZ84cZGRkwNTUFP7+/ti+fTuGDx+uOic6OhoymQxPP/20Wlsffvgh5s+fDwBIT0/HhAkTkJOTA3t7e/Tq1Qv//POPql0AePvtt1FWVoaZM2ciLy8PPXv2xO7du2FpyT1riOjheNmZw1wiRolMjiu3iuHbQvbCurMbO0d7iBpCJAhC7c2c9FRhYSGsra1RUFAAK6uWlbxIRLo39tujOH7tNpY/E4SnQlx13R0IgoA+S/bhRkE51rzQA4P8WJSU9FNjPr+Z/k9E1EB3dmpvGQnOV24V40ZBOSSGBujl2V7X3SFqFRj4EBE1UKBrdQXnFhL4xCUpp7l6etrCVCJ+wNlEBDDwISJqsJol7RcylQnOunawplqzD/N7iBqKgQ8RUQN52lnATCJGqUyOlBzdVnAur5Tj2FVl5XomNhM1HAMfIqIGEhuI0EWqnO7SdZ7P8ZTbqKhSwMnKBL6OFjrtC1FrwsCHiKgRVAnO6botZFizjL2fj919t+UhotoY+BARNUJL2briYLIy8Bngx2kuosZg4ENE1Ag1W1ecv1EAhY4SnG/kl+HSzWIYiJQbkxJRwzHwISJqhI72FjAxMkCJTI6U3BKd9OFQ9WhPV1cb2JhJdNIHotaKgQ8RUSPcneCsq+mug5eql7FzNRdRozHwISJqpEBVgrP2Ax+5QsDhy8rAZ4Avp7mIGouBDxFRI+ly64rT6fkoKKuEpYkhglxttN4+UWvHwIeIqJHuJDgXaj3B+e5l7IZivoUTNRb/aoiIGsm7OsG5uKIK17Sc4HygOvDhNhVETcPAh4iokQzFBuhck+B8Q3uFDAtKK3E6LR8AE5uJmoqBDxFREwQ4a7+Q4eHLOVAIgLeDBZxtTLXWLlFbwsCHiKgJdLGy6yCnuYgeGgMfIqImqFnZde5GAQRB8wnOgiCotqnoz2XsRE3GwIeIqAl8HC0gMTRAUXkVrueWary9y9nFyCwoh7GhAXp5tdd4e0RtFQMfIqImMLorwVkb9XxqVnM94mkLEyOxxtsjaqsY+BARNVGgS83KLu0FPgO4movooTDwISJqIm2t7CqvlON4ym0AXMZO9LAY+BARNZEqwTmjUKMJzsdSbqOiSgEnKxP4OFhorB0ifcDAh4ioiXwdLSERG6CgrBJpt8s01o5qGbuvHUQikcbaIdIHDHyIiJpIYmiATlJLAJpNcL4T+HCai+hhMfAhInoImt6p/UZ+GZKzi2EgAvp6s34P0cNi4ENE9BBqKjif19DKrprRniA3G9iYSTTSBpE+YeBDRPQQalZ2nc3QTAVnVbVmblNB1CwY+BARPQRfJwsYiUXIL61Eel7zJjhXyRU4nJwDgPk9RM2FgQ8R0UMwNhTDz0mZ4Nzc9XxOpxegsLwKViaGCHK1btZrE+krBj5ERA8pUEMJzjX5PX197GAo5ts1UXPgXxIR0UPS1MoublNB1PwY+BARPaQ7K7uar4JzfqkMZ9LzATC/h6g5MfAhInpIvo6WMDQQ4XaJDDcKypvlmocv50AhAD4OFpBamzbLNYmIgQ8R0UMzMRLD17G6gnN680x3sVozkWYw8CEiagaBLs23U7sgCDh4icvYiTSBgQ8RUTMIcG2+BOfk7GJkFZbD2NAAPT1tH/p6RHQHAx8iomZw94jPwyY410xz9fRqDxMj8UP3jYjuYOBDRNQMOjlZQmwgQm6JDFmFD5fgXLOMvb8PNyUlam4MfIiImoGJkRg+DhYAHi7BuUwmx7GU2wBYv4dIEwyb8qDo6Gh8+umnyMzMhL+/P1asWIF+/frVeW5cXBwGDRpU63hiYiI6deoEAFi9ejXWrVuHc+fOAQBCQkKwePFiPPLII6rzo6KisHnzZly8eBGmpqbo3bs3PvnkE/j5+anOmTJlCtauXavWTs+ePfHPP/805WnWSRAEVFVVQS6XN9s1iVoKIyMjiMWcWmmqQBdrXMwqwrmMAoT7OzXpGsdSciGrUkBqbQLv6kCKiJpPowOfTZs2Yfbs2YiOjkafPn3w3XffISIiAhcuXECHDh3u+7ikpCRYWVmpfra3v/NNJi4uDhMmTEDv3r1hYmKCpUuXIjw8HOfPn4eLiwsA4MCBA5g1axZ69OiBqqoqzJs3D+Hh4bhw4QLMzc1V13rsscewZs0a1c8SiaSxT/G+ZDIZMjMzUVpa2mzXJGpJRCIRXF1dYWHBD9ymCHS1xm8n0h8qwVm1msvHHiKRqLm6RkTVGh34fPbZZ5g2bRqmT58OAFixYgV27dqFlStXIioq6r6Pc3BwgI2NTZ33bdy4Ue3n1atX43//+x/27t2L559/HgDw119/qZ2zZs0aODg44MSJE+jfv7/quLGxMZycmvZNqz4KhQIpKSkQi8VwdnaGRCLhmxK1KYIg4NatW0hPT4ePjw9HfprgztYVygrOTXmPOJjM+j1EmtSowEcmk+HEiRN499131Y6Hh4fjyJEj9T62W7duKC8vR5cuXfD+++/XOf1Vo7S0FJWVlbC1vf8yzoIC5Teqe8+Ji4tTBVkDBgzAxx9/DAcHhzqvUVFRgYqKCtXPhYWF921PJpNBoVDAzc0NZmZm9z2PqDWzt7fHtWvXUFlZycCnCbpIrSA2ECGnuAI3CyvgZG3SqMdn5JfhcnYxDERAX28mNhNpQqOSm3NyciCXy+Ho6Kh23NHREVlZWXU+RiqVYtWqVYiJicHmzZvh5+eHwYMH4+DBg/dt591334WLiwuGDBlS5/2CICAyMhJ9+/ZFQECA6nhERAQ2btyIffv2Yfny5YiPj8ejjz6qFtzcLSoqCtbW1qqbm5vbg14CGBgwH5zaLo5iPpy7E5ybUsiwZhl7sJsNrM2MmrVvRKTUpOTme98c6xvS9fPzU0tADgsLQ1paGpYtW6Y2RVVj6dKl+PnnnxEXFwcTk7q/Lb3yyis4c+YMDh8+rHZ83Lhxqn8HBAQgNDQU7u7u2L59O8aMGVPrOnPnzkVkZKTq58LCwgYFP0RE9+PvrExwPptRgCFdHB/8gLtwmwoizWvU8IWdnR3EYnGt0Z3s7Oxao0D16dWrF5KTk2sdX7ZsGRYvXozdu3eja9eudT721VdfxdatW7F//364urrW245UKoW7u3udbQHKfCArKyu1GxHRwwh0Ub6PNHbEp0quwOHL3KaCSNMaFfhIJBKEhIQgNjZW7XhsbCx69+7d4OucOnUKUqlU7dinn36KhQsX4q+//kJoaGitxwiCgFdeeQWbN2/Gvn374Onp+cB2cnNzkZaWVqstajoPDw+sWLFC190garECm7h1xen0fBSVV8Ha1AhBrjYa6BkRAU2Y6oqMjMSkSZMQGhqKsLAwrFq1CqmpqZgxYwYA5fRRRkYG1q1bB0C56svDwwP+/v6QyWTYsGEDYmJiEBMTo7rm0qVL8cEHH+Cnn36Ch4eHakTJwsJCtax21qxZ+Omnn/DHH3/A0tJSdY61tTVMTU1RXFyM+fPn46mnnoJUKsW1a9fw3nvvwc7ODqNHj364V6mVGzhwIIKDg5slYImPj1crH0BE6rpIrWEgArKLKpBdWA4Hq4YlOB+oXsbe19sOYgPmWhFpSqMDn3HjxiE3NxcLFixAZmYmAgICsGPHDri7uwMAMjMzkZqaqjpfJpNhzpw5yMjIgKmpKfz9/bF9+3YMHz5cdU50dDRkMhmefvpptbY+/PBDzJ8/HwCwcuVKAMoP8butWbMGU6ZMgVgsxtmzZ7Fu3Trk5+dDKpVi0KBB2LRpEywtLRv7NPWKIAiQy+UwNHzwr8Pd9ZfaIplM1qy1n0j/mErE8HawwKWbxTibUYDBDQx8avJ7WK2ZSMMEUikoKBAACAUFBbXuKysrEy5cuCCUlZWpjikUCqGkolInN4VC0aDnNHnyZAGA2m3NmjUCAOGvv/4SQkJCBCMjI2Hfvn3C5cuXhSeeeEJwcHAQzM3NhdDQUCE2Nlbteu7u7sLnn3+u+hmAsHr1amHUqFGCqamp4O3tLfzxxx8N6ltVVZUwdepUwcPDQzAxMRF8fX2FFStW1Drvhx9+ELp06SJIJBLByclJmDVrluq+vLw84cUXXxQcHBwEY2Njwd/fX/jzzz8FQRCEDz/8UAgKClK71ueffy64u7urvT5PPvmksHjxYkEqlaruW79+vRASEiJYWFgIjo6OwoQJE4SbN2+qXevcuXPC8OHDBUtLS8HCwkLo27evcPnyZeHAgQOCoaGhkJmZqXZ+ZGSk0K9fvwa9NrpS1+85Nd4bm04J7u9sE1bEXmrQ+beLKwSPd7cJ7u9sE27kl2q4d0RtT32f3/dq0qouUiqrlKPLf3bppO0LC4bBTPLg/31ffPEFLl26hICAACxYsAAAcP78eQDA22+/jWXLlsHLyws2NjZIT0/H8OHDsWjRIpiYmGDt2rUYOXIkkpKS6q3K/dFHH2Hp0qX49NNP8dVXX+HZZ5/F9evX663DBCiLQrq6uuLXX3+FnZ0djhw5gpdeeglSqRRjx44FoBzpi4yMxJIlSxAREYGCggL8/fffqsdHRESgqKgIGzZsQMeOHXHhwoVG15/Zu3cvrKysEBsbq9pVWyaTYeHChfDz80N2djbeeOMNTJkyBTt27AAAZGRkoH///hg4cCD27dsHKysr/P3336iqqkL//v3h5eWF9evX46233gIAVFVVYcOGDViyZEmj+katU4CzNTafzGhwns/hyzkQBMDX0QJSa1MN945IvzHwaeOsra0hkUhgZmamqmh98eJFAMCCBQswdOhQ1bnt27dHUFCQ6udFixbh999/x9atW/HKK6/ct40pU6ZgwoQJAIDFixfjq6++wvHjx/HYY4/V2zcjIyN89NFHqp89PT1x5MgR/Prrr6rAZ9GiRXjzzTfx+uuvq87r0aMHAGDPnj04fvw4EhMT4evrCwDw8vJ68ItyD3Nzc3z//fdqU1xTp05V/dvLywtffvklHnnkERQXF8PCwgLffPMNrK2t8csvv8DISFlvpaYPADBt2jSsWbNGFfhs374dpaWlqudFbVtNgnNDV3aplrH7cJqLSNMY+DwEUyMxLiwYprO2H9a9q+dKSkrw0UcfYdu2bbhx4waqqqpQVlamlrNVl7tLD5ibm8PS0hLZ2dkN6sO3336L77//HtevX0dZWRlkMhmCg4MBKMsk3LhxA4MHD67zsQkJCXB1dVULOJoiMDCwVl7PqVOnMH/+fCQkJOD27dtQKBQAgNTUVHTp0gUJCQno16+fKui515QpU/D+++/jn3/+Qa9evfDjjz9i7NixTAzXE12kVhCJgKzCctwqqoC9pfF9zxUEgdtUEGkRA5+HIBKJGjTd1FLd+yH81ltvYdeuXVi2bBm8vb1hamqKp59+GjKZrN7r3PvhLxKJVIFCfX799Ve88cYbWL58OcLCwmBpaYlPP/0Ux44dAwCYmtY/5P+g+w0MDFRTVzUqKytrnXfv61BSUoLw8HCEh4djw4YNsLe3R2pqKoYNG6Z6LR7UtoODA0aOHIk1a9bAy8sLO3bsQFxcXL2PobbD3NgQHe0tcDm7GOcyCjCoU93b5gDApZvFuFlYAWNDAzziWf/0MBE9vNb7qU0NJpFIIJfLH3jeoUOHMGXKFNXy/+LiYly7dk1j/Tp06BB69+6NmTNnqo5duXJF9W9LS0t4eHhg7969de7t1rVrV6Snp+PSpUt1jvrY29sjKytLrbJ4QkLCA/t18eJF5OTkYMmSJapK3v/++2+ttteuXYvKysr7jvpMnz4d48ePh6urKzp27Ig+ffo8sG1qOwJdrHE5W7myq77Ap2aaq5dXe5g0w0guEdWPG0/pAQ8PDxw7dgzXrl1DTk7OfUdjvL29sXnzZiQkJOD06dOYOHFig0Zumsrb2xv//vsvdu3ahUuXLuGDDz5AfHy82jnz58/H8uXL8eWXXyI5ORknT57EV199BQAYMGAA+vfvj6eeegqxsbFISUnBzp078ddffwFQlj64desWli5diitXruCbb77Bzp07H9ivDh06QCKR4KuvvsLVq1exdetWLFy4UO2cV155BYWFhRg/fjz+/fdfJCcnY/369UhKSlKdM2zYMFhbW2PRokV44YUXHvblolamZqf2B+X5HOA2FURaxcBHD8yZMwdisRhdunRRTdvU5fPPP0e7du3Qu3dvjBw5EsOGDUP37t011q8ZM2ZgzJgxGDduHHr27Inc3Fy10R8AmDx5MlasWIHo6Gj4+/vj8ccfV9uCJCYmBj169MCECRPQpUsXvP3226rRrc6dOyM6OhrffPMNgoKCcPz4ccyZM+eB/bK3t8d///tf/Pbbb+jSpQuWLFmCZcuWqZ3Tvn177Nu3D8XFxRgwYABCQkKwevVqtdEfAwMDTJkyBXK5HM8///zDvFTUCgU4P3jrijKZHMev3QYADPDlbuxE2iAS7k2C0GOFhYWwtrZGQUFBrX27ysvLkZKSAk9Pz/tunkp0rxdffBE3b97E1q1bdd2VBuHvefMprqhC4PxdEATgxPtD0N6idoLz/qRsvLAmHs7WJvj73Ufvu9kzEdWvvs/ve3HEh0gDCgoKsGfPHmzcuBGvvvqqrrtDOmBhbAhPO2Xi/P3q+dy9GzuDHiLtYOBDGjNjxgzVfmv33mr2dmurnnzySTzxxBN4+eWX1WolkX4JfECez0Hm9xBpHVd1kcYsWLDgvjk1DxqKbO24dJ0AZeDzR8KNOkd80vNKceVWCcQGIvTxZn4PkbYw8CGNcXBwgIPD/ZfxErV1d1Z2Fda672D1buzBbjawNq27JAIRNT9OdRERaUiX6pVdGfllyCtRLwTKbSqIdIOBDxGRhliZGNWZ4FwlV+DvK8oRn/5cxk6kVQx8iIg0qGa66+7AJyEtH0XlVbAxM0JXVxsd9YxIPzHwISLSoECX2oUMa6a5+njbQWzAZexE2sTAh4hIg+oa8TmQrJzmGsBl7ERax8CHGsTDwwMrVqxQ/SwSibBly5b7nn/t2jWIRKIGbQpan+a6DpGu+DsrA5/0vDLkl8pwu0SGM+n5AJjYTKQLXM5OTZKZmYl27do16zWnTJmC/Px8tYDKzc0NmZmZsLNjAii1TtamRnBvb4bruaU4l1GI26UyCALg52gJJ2tuC0KkbQx8qEmcnJy00o5YLNZaWy1NZWWl2qan1HoFuFjjem4pzmYU4MqtYgBczUWkK5zqehiCAMhKdHNr4N6y3333HVxcXKBQKNSOP/HEE5g8eTIA4MqVK3jyySfh6OgICwsL9OjRA3v27Kn3uvdOdR0/fhzdunWDiYkJQkNDcerUKbXz5XI5pk2bBk9PT5iamsLPzw9ffPGF6v758+dj7dq1+OOPPyASiSASiRAXF1fnVNeBAwfwyCOPwNjYGFKpFO+++y6qqqpU9w8cOBCvvfYa3n77bdja2sLJyQnz58+v9/nEx8dj6NChsLOzg7W1NQYMGICTJ0+qnZOfn4+XXnoJjo6OMDExQUBAALZt26a6/++//8aAAQNgZmaGdu3aYdiwYcjLywNQe6oQAIKDg9X6JRKJ8O233+LJJ5+Eubk5Fi1a9MDXrcaPP/4If39/1WvyyiuvAACmTp2Kxx9/XO3cqqoqODk54ccff6z3NaHmE6jK88nHoWRuU0GkSxzxeRiVpcBiZ920/d4NQGL+wNOeeeYZvPbaa9i/fz8GDx4MAMjLy8OuXbvw559/AgCKi4sxfPhwLFq0CCYmJli7di1GjhyJpKQkdOjQ4YFtlJSU4PHHH8ejjz6KDRs2ICUlBa+//rraOQqFAq6urvj1119hZ2eHI0eO4KWXXoJUKsXYsWMxZ84cJCYmorCwEGvWrAEA2Nra4saNG2rXycjIwPDhwzFlyhSsW7cOFy9exIsvvggTExO1IGLt2rWIjIzEsWPHcPToUUyZMgV9+vS5775ZRUVFmDx5Mr788ksAwPLlyzF8+HAkJyfD0tISCoUCERERKCoqwoYNG9CxY0dcuHABYrEYAJCQkIDBgwdj6tSp+PLLL2FoaIj9+/dDLpc/8PW724cffoioqCh8/vnnEIvFD3zdAGDlypWIjIzEkiVLEBERgYKCAvz9998AgOnTp6N///7IzMyEVCoFAOzYsQPFxcWqx5Pm1QQ++y/eQlmlHCZGBujhYavjXhHpJwY+bZytrS0ee+wx/PTTT6rA57fffoOtra3q56CgIAQFBakes2jRIvz+++/YunWrauSgPhs3boRcLsePP/4IMzMz+Pv7Iz09Hf/3f/+nOsfIyAgfffSR6mdPT08cOXIEv/76K8aOHQsLCwuYmpqioqKi3qmt6OhouLm54euvv4ZIJEKnTp1w48YNvPPOO/jPf/4DAwPlIGbXrl3x4YcfAgB8fHzw9ddfY+/evfcNfB599FG1n7/77ju0a9cOBw4cwOOPP449e/bg+PHjSExMhK+vLwDAy8tLdf7SpUsRGhqK6Oho1TF/f/8Hvnb3mjhxIqZOnap2rL7XDVD+/3rzzTfVgs0ePXoAAHr37g0/Pz+sX78eb7/9NgBgzZo1eOaZZ2BhYdHo/lHTBFQnOJdVKgPhXl7tYWIk1mWXiPQWA5+HYWSmHHnRVdsN9Oyzz+Kll15CdHQ0jI2NsXHjRowfP141WlFSUoKPPvoI27Ztw40bN1BVVYWysjKkpqY26PqJiYkICgqCmdmdPoWFhdU679tvv8X333+P69evo6ysDDKZDMHBwQ1+HjVthYWFQSS6U/ukT58+KC4uRnp6umqEqmvXrmqPk0qlyM7Ovu91s7Oz8Z///Af79u3DzZs3IZfLUVpaqnoNEhIS4Orqqgp67pWQkIBnnnmmUc+lLqGhobWO1fe6ZWdn48aNG6ogti7Tp0/HqlWr8PbbbyM7Oxvbt2/H3r17H7qv1HDWZkboYGuG1NulALiai0iXGPg8DJGoQdNNujZy5EgoFAps374dPXr0wKFDh/DZZ5+p7n/rrbewa9cuLFu2DN7e3jA1NcXTTz8NmUxWz1XvEBqQb/Trr7/ijTfewPLlyxEWFgZLS0t8+umnOHbsWKOeiyAIakHP3e3fffzepGCRSFQrz+luU6ZMwa1bt7BixQq4u7vD2NgYYWFhqtfA1NS03n496H4DA4Nar1NlZWWt88zN1X+fHvS6PahdAHj++efx7rvv4ujRozh69Cg8PDzQr1+/Bz6OmleAi9WdwIf5PUQ6w+RmPWBqaooxY8Zg48aN+Pnnn+Hr64uQkBDV/YcOHcKUKVMwevRoBAYGwsnJCdeuXWvw9bt06YLTp0+jrKxMdeyff/5RO+fQoUPo3bs3Zs6ciW7dusHb2xtXrlxRO0cikTwwJ6ZLly44cuSIWhBx5MgRWFpawsXFpcF9vtehQ4fw2muvYfjw4aok4ZycHNX9Xbt2RXp6Oi5dulTn47t27VrvKIq9vT0yMzNVPxcWFiIlJaVB/arvdbO0tISHh0e9bbdv3x6jRo3CmjVrsGbNGrzwwgsPbJeaX00hQxcbU3S0b/lfmIjaKgY+euLZZ5/F9u3b8eOPP+K5555Tu8/b2xubN29GQkICTp8+jYkTJ9Y7OnKviRMnwsDAANOmTcOFCxewY8cOLFu2rFYb//77L3bt2oVLly7hgw8+QHx8vNo5Hh4eOHPmDJKSkpCTk1PniMjMmTORlpaGV199FRcvXsQff/yBDz/8EJGRkar8nqbw9vbG+vXrkZiYiGPHjuHZZ59VG00ZMGAA+vfvj6eeegqxsbFISUnBzp078ddffwEA5s6di/j4eMycORNnzpzBxYsXsXLlSlXw9Oijj2L9+vU4dOgQzp07h8mTJ6umGh/Urwe9bvPnz8fy5cvx5ZdfIjk5GSdPnsRXX32lds706dOxdu1aJCYmqlbzkXaN7OoMHwcLzBjYsdaoJRFpDwMfPfHoo4/C1tYWSUlJmDhxotp9n3/+Odq1a4fevXtj5MiRGDZsGLp3797ga1tYWODPP//EhQsX0K1bN8ybNw+ffPKJ2jkzZszAmDFjMG7cOPTs2RO5ubmYOXOm2jkvvvgi/Pz8EBoaCnt7e9XKpLu5uLhgx44dOH78OIKCgjBjxgxMmzYN77//fiNejdp+/PFH5OXloVu3bpg0aRJee+01ODg4qJ0TExODHj16YMKECejSpQvefvtt1QiVr68vdu/ejdOnT+ORRx5BWFgY/vjjDxgaKmeT586di/79++Pxxx/H8OHDMWrUKHTs2PGB/WrI6zZ58mSsWLEC0dHR8Pf3x+OPP47k5GS1c4YMGQKpVIphw4bB2VlHKxH1nJutGWIjB2BSL3ddd4VIr4mEhiRo6InCwkJYW1ujoKAAVlZWaveVl5cjJSUFnp6eMDFhtVVqXUpLS+Hs7Iwff/wRY8aMue95/D0notaovs/vezG5magNUygUyMrKwvLly2FtbY0nnnhC110iItIpBj5EbVhqaio8PT3h6uqK//73v6qpNyIifcV3QaI2zMPDo0HlBoiI9AWTm4mIiEhvMPBpJH57praMv99E1NYx8GmgmkrApaWlOu4JkebUVKpuSI0hIqLWiDk+DSQWi2FjY6Pa78nMzIxFyKhNUSgUuHXrFszMzJgETURtFt/dGqFm1/D6Nrskas0MDAzQoUMHBvVE1GYx8GkEkUgEqVQKBweHOrdTIGrtJBLJQ239QUTU0jHwaQKxWMwcCCIiolaIX+2IiIhIbzDwISIiIr3BwIeIiIj0BnN87lJTvK2wsFDHPSEiIqKGqvncbkgRVgY+dykqKgIAuLm56bgnRERE1FhFRUWwtrau9xyRwBr1KgqFAjdu3IClpWWz1zEpLCyEm5sb0tLSYGVl1azXZvutow9sX7/bbwl9YPv8HWir7QuCgKKiIjg7Oz+wJAdHfO5iYGAAV1dXjbZhZWWlsz84tt8y+sD29bv9ltAHts/fgbbY/oNGemowuZmIiIj0BgMfIiIi0hsMfLTE2NgYH374IYyNjdm+jui6D2xfv9tvCX1g+/wd0Pf2ASY3ExERkR7hiA8RERHpDQY+REREpDcY+BAREZHeYOBDREREeoOBjxZER0fD09MTJiYmCAkJwaFDh7TW9sGDBzFy5Eg4OztDJBJhy5YtWmsbAKKiotCjRw9YWlrCwcEBo0aNQlJSktbaX7lyJbp27aoqlhUWFoadO3dqrf17RUVFQSQSYfbs2Vprc/78+RCJRGo3JycnrbUPABkZGXjuuefQvn17mJmZITg4GCdOnNBK2x4eHrWev0gkwqxZs7TSflVVFd5//314enrC1NQUXl5eWLBgARQKhVbaB5Rl/GfPng13d3eYmpqid+/eiI+P11h7D3rfEQQB8+fPh7OzM0xNTTFw4ECcP39ea+1v3rwZw4YNg52dHUQiERISEpqt7Qe1X1lZiXfeeQeBgYEwNzeHs7Mznn/+edy4cUMr7QPK94ROnTrB3Nwc7dq1w5AhQ3Ds2LFma78hfbjbyy+/DJFIhBUrVjRrH+6HgY+Gbdq0CbNnz8a8efNw6tQp9OvXDxEREUhNTdVK+yUlJQgKCsLXX3+tlfbudeDAAcyaNQv//PMPYmNjUVVVhfDwcJSUlGilfVdXVyxZsgT//vsv/v33Xzz66KN48sknm/VNtqHi4+OxatUqdO3aVett+/v7IzMzU3U7e/as1trOy8tDnz59YGRkhJ07d+LChQtYvnw5bGxstNJ+fHy82nOPjY0FADzzzDNaaf+TTz7Bt99+i6+//hqJiYlYunQpPv30U3z11VdaaR8Apk+fjtjYWKxfvx5nz55FeHg4hgwZgoyMDI2096D3naVLl+Kzzz7D119/jfj4eDg5OWHo0KGq/RI13X5JSQn69OmDJUuWNEt7jWm/tLQUJ0+exAcffICTJ09i8+bNuHTpEp544gmttA8Avr6++Prrr3H27FkcPnwYHh4eCA8Px61bt7TWhxpbtmzBsWPH4Ozs3GxtP5BAGvXII48IM2bMUDvWqVMn4d1339V6XwAIv//+u9bbvVt2drYAQDhw4IDO+tCuXTvh+++/12qbRUVFgo+PjxAbGysMGDBAeP3117XW9ocffigEBQVprb17vfPOO0Lfvn111v69Xn/9daFjx46CQqHQSnsjRowQpk6dqnZszJgxwnPPPaeV9ktLSwWxWCxs27ZN7XhQUJAwb948jbd/7/uOQqEQnJychCVLlqiOlZeXC9bW1sK3336r8fbvlpKSIgAQTp061eztNqT9GsePHxcACNevX9dJ+wUFBQIAYc+ePc3efn19SE9PF1xcXIRz584J7u7uwueff66R9u/FER8NkslkOHHiBMLDw9WOh4eH48iRIzrqlW4VFBQAAGxtbbXetlwuxy+//IKSkhKEhYVpte1Zs2ZhxIgRGDJkiFbbrZGcnAxnZ2d4enpi/PjxuHr1qtba3rp1K0JDQ/HMM8/AwcEB3bp1w+rVq7XW/t1kMhk2bNiAqVOnNvtGxPfTt29f7N27F5cuXQIAnD59GocPH8bw4cO10n5VVRXkcjlMTEzUjpuamuLw4cNa6cPdUlJSkJWVpfa+aGxsjAEDBuj1+6JIJNLaKOjdZDIZVq1aBWtrawQFBWmtXYVCgUmTJuGtt96Cv7+/1toFuEmpRuXk5EAul8PR0VHtuKOjI7KysnTUK90RBAGRkZHo27cvAgICtNbu2bNnERYWhvLyclhYWOD3339Hly5dtNb+L7/8gpMnT2o0p6I+PXv2xLp16+Dr64ubN29i0aJF6N27N86fP4/27dtrvP2rV69i5cqViIyMxHvvvYfjx4/jtddeg7GxMZ5//nmNt3+3LVu2ID8/H1OmTNFam++88w4KCgrQqVMniMViyOVyfPzxx5gwYYJW2re0tERYWBgWLlyIzp07w9HRET///DOOHTsGHx8frfThbjXvfXW9L16/fl3r/dG18vJyvPvuu5g4caJWNw3dtm0bxo8fj9LSUkilUsTGxsLOzk5r7X/yyScwNDTEa6+9prU2azDw0YJ7v1kKgqC1b5stySuvvIIzZ85o/Vumn58fEhISkJ+fj5iYGEyePBkHDhzQSvCTlpaG119/Hbt37671jVtbIiIiVP8ODAxEWFgYOnbsiLVr1yIyMlLj7SsUCoSGhmLx4sUAgG7duuH8+fNYuXKl1gOfH374AREREVrNJ9i0aRM2bNiAn376Cf7+/khISMDs2bPh7OyMyZMna6UP69evx9SpU+Hi4gKxWIzu3btj4sSJOHnypFbarwvfF5WJzuPHj4dCoUB0dLRW2x40aBASEhKQk5OD1atXY+zYsTh27BgcHBw03vaJEyfwxRdf4OTJkzr5f86pLg2ys7ODWCyuNbqTnZ1d69tOW/fqq69i69at2L9/P1xdXbXatkQigbe3N0JDQxEVFYWgoCB88cUXWmn7xIkTyM7ORkhICAwNDWFoaIgDBw7gyy+/hKGhIeRyuVb6cTdzc3MEBgYiOTlZK+1JpdJaQWbnzp21luBf4/r169izZw+mT5+u1XbfeustvPvuuxg/fjwCAwMxadIkvPHGG4iKitJaHzp27IgDBw6guLgYaWlpOH78OCorK+Hp6am1PtSoWVGo7++LlZWVGDt2LFJSUhAbG6vV0R5A+T7g7e2NXr164YcffoChoSF++OEHrbR96NAhZGdno0OHDqr3xevXr+PNN9+Eh4eHxttn4KNBEokEISEhqlUkNWJjY9G7d28d9Uq7BEHAK6+8gs2bN2Pfvn06eaOtq08VFRVaaWvw4ME4e/YsEhISVLfQ0FA8++yzSEhIgFgs1ko/7lZRUYHExERIpVKttNenT59aJQwuXboEd3d3rbRfY82aNXBwcMCIESO02m5paSkMDNTfasVisVaXs9cwNzeHVCpFXl4edu3ahSeffFLrffD09ISTk5Pa+6JMJsOBAwf05n2xJuhJTk7Gnj17tDLl/CDafF+cNGkSzpw5o/a+6OzsjLfeegu7du3SePuc6tKwyMhITJo0CaGhoQgLC8OqVauQmpqKGTNmaKX94uJiXL58WfVzSkoKEhISYGtriw4dOmi8/VmzZuGnn37CH3/8AUtLS9W3PGtra5iammq8/ffeew8RERFwc3NDUVERfvnlF8TFxeGvv/7SeNuAMr/i3nwmc3NztG/fXmt5TnPmzMHIkSPRoUMHZGdnY9GiRSgsLNTaNMsbb7yB3r17Y/HixRg7diyOHz+OVatWYdWqVVppH1BOt61ZswaTJ0+GoaF23/ZGjhyJjz/+GB06dIC/vz9OnTqFzz77DFOnTtVaH3bt2gVBEODn54fLly/jrbfegp+fH1544QWNtPeg953Zs2dj8eLF8PHxgY+PDxYvXgwzMzNMnDhRK+3fvn0bqampqto5NYG5k5NTs9S4qq99Z2dnPP300zh58iS2bdsGuVyuel+0tbWFRCLRaPvt27fHxx9/jCeeeAJSqRS5ubmIjo5Genp6s5Z4eND/g3uDPSMjIzg5OcHPz6/Z+nBfWlk7pue++eYbwd3dXZBIJEL37t21upR7//79AoBat8mTJ2ul/braBiCsWbNGK+1PnTpV9drb29sLgwcPFnbv3q2Vtu9H28vZx40bJ0ilUsHIyEhwdnYWxowZI5w/f15r7QuCIPz5559CQECAYGxsLHTq1ElYtWqVVtvftWuXAEBISkrSaruCIAiFhYXC66+/LnTo0EEwMTERvLy8hHnz5gkVFRVa68OmTZsELy8vQSKRCE5OTsKsWbOE/Px8jbX3oPcdhUIhfPjhh4KTk5NgbGws9O/fXzh79qzW2l+zZk2d93/44Ycab79mCX1dt/3792u8/bKyMmH06NGCs7OzIJFIBKlUKjzxxBPC8ePHm6XthvShLtpczi4SBEFo/nCKiIiIqOVhjg8RERHpDQY+REREpDcY+BAREZHeYOBDREREeoOBDxEREekNBj5ERESkNxj4EBERkd5g4ENERER6g4EPEbVIAwcOxOzZs3XdjYcWFxcHkUiE/Px8XXeFiMC9uojoIYlEonrvnzx5Mv773/82+rqbN2+GkZFRE3tFRFQ3Bj5E9FAyMzNV/960aRP+85//qO3Gfu9mtJWVlQ0KaGxtbZuvk0RE1TjVRUQPpWZHaycnJ1hbW0MkEql+Li8vh42NDX799VcMHDgQJiYm2LBhA3JzczFhwgS4urrCzMwMgYGB+Pnnn9Wue+9Ul4eHBxYvXoypU6fC0tISHTp0eOAO74IgYOnSpfDy8oKpqSmCgoLwv//9T3V/zTTU9u3bERQUBBMTE/Ts2RNnz55Vu05MTAz8/f1hbGwMDw8PLF++XO3+iooKvP3223Bzc4OxsTF8fHzwww8/qJ1z4sQJhIaGwszMDL1791YLDk+fPo1BgwbB0tISVlZWCAkJwb///tug15+IGoeBDxFp3DvvvIPXXnsNiYmJGDZsGMrLyxESEoJt27bh3LlzeOmllzBp0iQcO3as3ussX74coaGhOHXqFGbOnIn/+7//w8WLF+97/vvvv481a9Zg5cqVOH/+PN544w0899xzOHDggNp5b731FpYtW4b4+Hg4ODjgiSeeQGVlJQBlwDJ27FiMHz8eZ8+exfz58/HBBx+oTd89//zz+OWXX/Dll18iMTER3377LSwsLNTamDdvHpYvX45///0XhoaGmDp1quq+Z599Fq6uroiPj8eJEyfw7rvvcpqPSFO0sgc8EemFNWvWCNbW1qqfU1JSBADCihUrHvjY4cOHC2+++abq5wEDBgivv/666md3d3fhueeeU/2sUCgEBwcHYeXKlXVer7i4WDAxMRGOHDmidnzatGnChAkTBEEQhP379wsAhF9++UV1f25urmBqaips2rRJEARBmDhxojB06FC1a7z11ltCly5dBEEQhKSkJAGAEBsbW2c/atrYs2eP6tj27dsFAEJZWZkgCIJgaWkp/Pe//637hSGiZsURHyLSuNDQULWf5XI5Pv74Y3Tt2hXt27eHhYUFdu/ejdTU1Hqv07VrV9W/a6bUsrOz6zz3woULKC8vx9ChQ2FhYaG6rVu3DleuXFE7NywsTPVvW1tb+Pn5ITExEQCQmJiIPn36qJ3fp08fJCcnQy6XIyEhAWKxGAMGDGhw36VSKQCo+h4ZGYnp06djyJAhWLJkSa3+EVHzYXIzEWmcubm52s/Lly/H559/jhUrViAwMBDm5uaYPXs2ZDJZvde5d/pHJBJBoVDUeW7N8e3bt8PFxUXtPmNj4wf2uWa1miAItVauCYKg+ve9ydv3c3ffa65X08f58+dj4sSJ2L59O3bu3IkPP/wQv/zyC0aPHt2gaxNRw3HEh4i07tChQ3jyySfx3HPPISgoCF5eXkhOTm7WNrp06QJjY2OkpqbC29tb7ebm5qZ27j///KP6d15eHi5duoROnTqprnP48GG1848cOQJfX1+IxWIEBgZCoVDUyhtqLF9fX7zxxhvYvXs3xowZgzVr1jzU9YiobhzxISKt8/b2RkxMDI4cOYJ27drhs88+Q1ZWFjp37txsbVhaWmLOnDl44403oFAo0LdvXxQWFuLIkSOwsLDA5MmTVecuWLAA7du3h6OjI+bNmwc7OzuMGjUKAPDmm2+iR48eWLhwIcaNG4ejR4/i66+/RnR0NADlarPJkydj6tSp+PLLLxEUFITr168jOzsbY8eOfWA/y8rK8NZbb+Hpp5+Gp6cn0tPTER8fj6eeeqrZXgsiuoOBDxFp3QcffICUlBQMGzYMZmZmeOmllzBq1CgUFBQ0azsLFy6Eg4MDoqKicPXqVdjY2KB79+5477331M5bsmQJXn/9dSQnJyMoKAhbt26FRCIBAHTv3h2//vor/vOf/2DhwoWQSqVYsGABpkyZonr8ypUr8d5772HmzJnIzc1Fhw4darVxP2KxGLm5uXj++edx8+ZN2NnZYcyYMfjoo4+a7XUgojtEwt2T1UREeiQuLg6DBg1CXl4ebGxsdN0dItIC5vgQERGR3mDgQ0RERHqDU11ERESkNzjiQ0RERHqDgQ8RERHpDQY+REREpDcY+BAREZHeYOBDREREeoOBDxEREekNBj5ERESkNxj4EBERkd74f+mSBT5Vsl9BAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_accuracy = history.history['accuracy']\n",
    "val_accuracy = history.history['val_accuracy']\n",
    "plt.plot(train_accuracy, label='train_accuracy')\n",
    "plt.plot(val_accuracy, label='validation accuracy')\n",
    "plt.xticks(range(num_epochs))\n",
    "plt.xlabel('Train epochs')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimize hyper params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna import Trial\n",
    "\n",
    "optuna.logging.set_verbosity(optuna.logging.WARNING)\n",
    "\n",
    "n_trials = 100\n",
    "random_state = 7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logis Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best params {'solver': 'lbfgs', 'penalty': 'l2'}\n",
      "Best accuracy 0.5146726862302483\n"
     ]
    }
   ],
   "source": [
    "def lr_optimize(trial: Trial):\n",
    "    solver = trial.suggest_categorical(\"solver\", [\"lbfgs\", 'newton-cholesky'])\n",
    "    penalty = trial.suggest_categorical(\"penalty\", ['l2', None])\n",
    "    lr = LogisticRegression(penalty=penalty, solver=solver, random_state=random_state, max_iter=500, multi_class = 'ovr')\n",
    "    lr.fit(x_train_scaled, y_train)\n",
    "    y_val_pred = lr.predict(x_val_scaled)\n",
    "    return accuracy_score(y_val, y_val_pred)\n",
    "\n",
    "lr_study = optuna.create_study(direction=\"maximize\")\n",
    "lr_study.optimize(lr_optimize, n_trials=n_trials)\n",
    "print(\"Best params\", lr_study.best_params)\n",
    "print(\"Best accuracy\", lr_study.best_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best params {'n_estimators': 39, 'criterion': 'gini', 'bootstrap': False}\n",
      "Best accuracy 0.5146726862302483\n"
     ]
    }
   ],
   "source": [
    "def rf_optimize(trial: Trial):\n",
    "    n_estimators = trial.suggest_int('n_estimators', low=10, high = 100)\n",
    "    criterion = trial.suggest_categorical(\"criterion\", ['gini', 'entropy', 'log_loss'])\n",
    "    bootstrap = trial.suggest_categorical(\"bootstrap\", [True, False])\n",
    "    rf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion, bootstrap=bootstrap)\n",
    "    lr.fit(x_train_scaled, y_train)\n",
    "    y_val_pred = lr.predict(x_val_scaled)\n",
    "    return accuracy_score(y_val, y_val_pred)\n",
    "\n",
    "rf_study = optuna.create_study(direction=\"maximize\")\n",
    "rf_study.optimize(rf_optimize, n_trials=n_trials)\n",
    "print(\"Best params\", rf_study.best_params)\n",
    "print(\"Best accuracy\", rf_study.best_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best params {'iterations': 802, 'depth': 8, 'learning_rate': 0.21097019438094816, 'random_strength': 2, 'bagging_temperature': 0.5197456418335619, 'l2_leaf_reg': 0.6986463160504206, 'border_count': 7}\n",
      "Best accuracy 0.5688487584650113\n"
     ]
    }
   ],
   "source": [
    "def catboost_optimize(trial: Trial):\n",
    "    param = {\n",
    "        'iterations': trial.suggest_int('iterations', 100, 1000),\n",
    "        'depth': trial.suggest_int('depth', 4, 10),\n",
    "        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n",
    "        'random_strength': trial.suggest_int('random_strength', 1, 20),\n",
    "        'bagging_temperature': trial.suggest_float('bagging_temperature', 0.0, 1.0),\n",
    "        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-2, 10.0),\n",
    "        'border_count': trial.suggest_int('border_count', 1, 255),\n",
    "        'loss_function': 'Logloss',\n",
    "        'verbose': False\n",
    "    }\n",
    "    model = CatBoostClassifier(**param)\n",
    "    model.fit(x_train_scaled, y_train, eval_set=[(x_val_scaled, y_val)], early_stopping_rounds=100, verbose=0)\n",
    "    preds = model.predict(x_val_scaled)\n",
    "    accuracy = accuracy_score(y_val, preds)\n",
    "    return accuracy\n",
    "\n",
    "cat_study = optuna.create_study(direction=\"maximize\")\n",
    "cat_study.optimize(catboost_optimize, n_trials=n_trials)\n",
    "print(\"Best params\", cat_study.best_params)\n",
    "print(\"Best accuracy\", cat_study.best_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def create_lstm(units):\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(LSTM(units, return_sequences=True, input_shape=(x_train_lstm.shape[1], 1)))\n",
    "    model.add(LSTM(units))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(units=units//2, activation = 'tanh'))\n",
    "    model.add(Dense(1, activation = 'sigmoid'))\n",
    "    return model\n",
    "\n",
    "\n",
    "\n",
    "def create_lstm_study(trial: Trial):\n",
    "    units = trial.suggest_int(\"units\", low=10, high = 50)\n",
    "    model = create_lstm(units)\n",
    "    optimizer = trial.suggest_categorical('optimizer', \n",
    "                                         [\"Adam\", \"SGD\", \"RMSprop\", \"Adagrad\"])\n",
    "    batch_size = trial.suggest_int('batch_size', low=1, high = 15)\n",
    "    epochs = trial.suggest_int('epochs', low=1, high=3)\n",
    "    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)\n",
    "    model.compile(optimizer=getattr(tf.keras.optimizers, optimizer)(learning_rate=lr), loss=\"binary_crossentropy\", metrics=['accuracy'])\n",
    "    model.fit(x_train_lstm, Y_train_for_compare, validation_data = (x_val_scaled, y_val), batch_size=batch_size, epochs=epochs, verbose=0)\n",
    "    threshold = 0.5\n",
    "    y_val_pred = (model.predict(x_val_scaled) >= threshold).astype(int)\n",
    "    return accuracy_score(y_val, y_val_pred)\n",
    "\n",
    "def run_study(function):\n",
    "    # Create an Optuna study and optimize the objective\n",
    "    study = optuna.create_study(direction='maximize')\n",
    "    study.optimize(function, n_trials=10)\n",
    "    \n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "    \n",
    "    print(f\"Accuracy: {trial.value}\")\n",
    "    print(\"Best hyperparameters:\")\n",
    "    for key, value in trial.params.items():\n",
    "        print(f\"{key}: {value}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 2ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 4ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "Best trial:\n",
      "Accuracy: 0.5598194130925508\n",
      "Best hyperparameters:\n",
      "units: 43\n",
      "optimizer: Adagrad\n",
      "batch_size: 13\n",
      "epochs: 1\n",
      "lr: 0.006957392490504836\n"
     ]
    }
   ],
   "source": [
    "run_study(create_lstm_study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 2ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 4ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 2ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "Best trial:\n",
      "Accuracy: 0.5372460496613995\n",
      "Best hyperparameters:\n",
      "units: 14\n",
      "optimizer: Adam\n",
      "batch_size: 4\n",
      "epochs: 2\n",
      "lr: 3.975159753000951e-05\n"
     ]
    }
   ],
   "source": [
    "def create_lstm_2(units):\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(LSTM(units, return_sequences=True, input_shape=(x_train_lstm.shape[1], 1)))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(LSTM(units))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(units//2, activation = \"tanh\"))\n",
    "    model.add(Dense(1, activation = 'tanh'))\n",
    "    return model\n",
    "\n",
    "def create_lstm_study_2(trial: Trial):\n",
    "    units = trial.suggest_int(\"units\", low=10, high = 50)\n",
    "    model = create_lstm_2(units)\n",
    "    optimizer = trial.suggest_categorical('optimizer', \n",
    "                                         [\"Adam\", \"SGD\", \"RMSprop\", \"Adagrad\"])\n",
    "    batch_size = trial.suggest_int('batch_size', low=1, high = 15)\n",
    "    epochs = trial.suggest_int('epochs', low=1, high=3)\n",
    "    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)\n",
    "    model.compile(optimizer=getattr(tf.keras.optimizers, optimizer)(learning_rate=lr), loss=\"binary_crossentropy\", metrics=['accuracy'])\n",
    "    model.fit(x_train_lstm, Y_train_for_compare, validation_data = (x_val_scaled, y_val), batch_size=batch_size, epochs=epochs, verbose=0)\n",
    "    threshold = 0\n",
    "    y_val_pred = (model.predict(x_val_scaled) >= threshold).astype(int)\n",
    "    return accuracy_score(y_val, y_val_pred)\n",
    "\n",
    "run_study(create_lstm_study_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 2ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 4ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "14/14 [==============================] - 0s 3ms/step\n",
      "Best trial:\n",
      "Accuracy: 0.5372460496613995\n",
      "Best hyperparameters:\n",
      "units: 47\n",
      "optimizer: Adam\n",
      "batch_size: 14\n",
      "epochs: 1\n",
      "lr: 0.002419610503285418\n"
     ]
    }
   ],
   "source": [
    "def create_lstm_gru(units):\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(LSTM(units, return_sequences=True, input_shape=(x_train_lstm.shape[1], 1)))\n",
    "    model.add(LSTM(units))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(GRU(units))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(units//2, activation = \"tanh\"))\n",
    "    model.add(Dense(1, activation = 'sigmoid'))\n",
    "    return model\n",
    "\n",
    "def create_lstm_gru_study(trial: Trial):\n",
    "    units = trial.suggest_int(\"units\", low=10, high = 50)\n",
    "    model = create_lstm_2(units)\n",
    "    optimizer = trial.suggest_categorical('optimizer', \n",
    "                                         [\"Adam\", \"SGD\", \"RMSprop\", \"Adagrad\"])\n",
    "    batch_size = trial.suggest_int('batch_size', low=1, high = 15)\n",
    "    epochs = trial.suggest_int('epochs', low=1, high=3)\n",
    "    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)\n",
    "    model.compile(optimizer=getattr(tf.keras.optimizers, optimizer)(learning_rate=lr), loss=\"binary_crossentropy\", metrics=['accuracy'])\n",
    "    model.fit(x_train_lstm, Y_train_for_compare, validation_data = (x_val_scaled, y_val), batch_size=batch_size, epochs=epochs, verbose=0)\n",
    "    threshold = 0.5\n",
    "    y_val_pred = (model.predict(x_val_scaled) >= threshold).astype(int)\n",
    "    return accuracy_score(y_val, y_val_pred)\n",
    "\n",
    "run_study(create_lstm_gru_study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 2ms/step\n",
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 1ms/step\n",
      "14/14 [==============================] - 0s 984us/step\n",
      "Best trial:\n",
      "Accuracy: 0.5417607223476298\n",
      "Best hyperparameters:\n",
      "optimizer: SGD\n",
      "batch_size: 12\n",
      "epochs: 3\n",
      "lr: 0.010489360088417304\n"
     ]
    }
   ],
   "source": [
    "def create_neural_network_model():\n",
    "    nn = tf.keras.Sequential()\n",
    "    nn.add(Dense(32, activation='relu', input_shape=(len(x_train_scaled[0]),)))\n",
    "    nn.add(Dense(1, activation = \"sigmoid\"))\n",
    "\n",
    "    return nn\n",
    "    \n",
    "def create_neural_network_study(trial: Trial):\n",
    "    model = create_neural_network_model()\n",
    "    optimizer = trial.suggest_categorical('optimizer', \n",
    "                                         [\"Adam\", \"SGD\", \"RMSprop\", \"Adagrad\"])\n",
    "    batch_size = trial.suggest_int('batch_size', low=1, high = 15)\n",
    "    epochs = trial.suggest_int('epochs', low=1, high=5)\n",
    "    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)\n",
    "    model.compile(optimizer=getattr(tf.keras.optimizers, optimizer)(learning_rate=lr), loss=\"binary_crossentropy\", metrics=['accuracy'])\n",
    "    model.fit(x_train_lstm, Y_train_for_compare, validation_data = (x_val_scaled, y_val), batch_size=batch_size, epochs=epochs, verbose=0)\n",
    "    threshold = 0.5\n",
    "    y_val_pred = (model.predict(x_val_scaled) >= threshold).astype(int)\n",
    "    return accuracy_score(y_val, y_val_pred)\n",
    "    \n",
    "run_study(create_neural_network_study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}