6nomads_AutoML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Project 6nomads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment below to install some packages which you're missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install numpy\n",
    "# !pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install scipy \n",
    "# !pip install scikit-learn  \n",
    "# !pip install joblib\n",
    "# !pip install deap update_checker tqdm stopit\n",
    "# !pip install tpot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install xgboost\n",
    "#there are some issues on macos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tpot import TPOTClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import SparsePCA\n",
    "from sklearn.decomposition import FastICA\n",
    "from sklearn.random_projection import GaussianRandomProjection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state=42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pd = pd.read_csv('data/train.csv', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pd = pd.read_csv('data/test.csv', header=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Name the label column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pd.rename(columns={65: 'label'}, inplace=True)\n",
    "test_pd.rename(columns={65: 'label'}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First glance at the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>64</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-47</td>\n",
       "      <td>0</td>\n",
       "      <td>-5</td>\n",
       "      <td>-27</td>\n",
       "      <td>-11</td>\n",
       "      <td>-19</td>\n",
       "      <td>-2</td>\n",
       "      <td>-33</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>-4</td>\n",
       "      <td>-3</td>\n",
       "      <td>4</td>\n",
       "      <td>-23</td>\n",
       "      <td>-68</td>\n",
       "      <td>15</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-3</td>\n",
       "      <td>-1</td>\n",
       "      <td>-3</td>\n",
       "      <td>39</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-7</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>-2</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-4</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>-20</td>\n",
       "      <td>20</td>\n",
       "      <td>-1</td>\n",
       "      <td>-3</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>-2</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "      <td>-3</td>\n",
       "      <td>22</td>\n",
       "      <td>A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-7</td>\n",
       "      <td>-8</td>\n",
       "      <td>12</td>\n",
       "      <td>41</td>\n",
       "      <td>21</td>\n",
       "      <td>-14</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-8</td>\n",
       "      <td>-29</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>-6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>-3</td>\n",
       "      <td>-8</td>\n",
       "      <td>-11</td>\n",
       "      <td>-2</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8673</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>49</td>\n",
       "      <td>123</td>\n",
       "      <td>34</td>\n",
       "      <td>-27</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-4</td>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "      <td>53</td>\n",
       "      <td>15</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8674</td>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>-14</td>\n",
       "      <td>-35</td>\n",
       "      <td>-8</td>\n",
       "      <td>19</td>\n",
       "      <td>-23</td>\n",
       "      <td>-9</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>47</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>-47</td>\n",
       "      <td>-3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8675</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-4</td>\n",
       "      <td>-4</td>\n",
       "      <td>-6</td>\n",
       "      <td>-8</td>\n",
       "      <td>-3</td>\n",
       "      <td>-8</td>\n",
       "      <td>-2</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>-8</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>B</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8676</td>\n",
       "      <td>-29</td>\n",
       "      <td>-24</td>\n",
       "      <td>-4</td>\n",
       "      <td>-10</td>\n",
       "      <td>-9</td>\n",
       "      <td>-6</td>\n",
       "      <td>-4</td>\n",
       "      <td>-11</td>\n",
       "      <td>-10</td>\n",
       "      <td>38</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>-6</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>-10</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8677</td>\n",
       "      <td>-25</td>\n",
       "      <td>-2</td>\n",
       "      <td>-4</td>\n",
       "      <td>-3</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-20</td>\n",
       "      <td>-33</td>\n",
       "      <td>-7</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-37</td>\n",
       "      <td>-2</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>-9</td>\n",
       "      <td>2</td>\n",
       "      <td>-14</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8678 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0   1   2   3   4   5    6   7   8   9  ...  56  57  58  59  60  61  \\\n",
       "0    -47   0  -5 -27 -11 -19   -2 -33  37   0  ...   0   2  -4  -3   4 -23   \n",
       "1      1   1  -2  -3  -1  -3   39  10   0  -1  ...  -7  -2   1   6  -2   4   \n",
       "2      1  -4  -2  -6 -20  20   -1  -3  20   3  ...  14   2  -2   0  23  21   \n",
       "3     10   2   1  -7  -8  12   41  21 -14   3  ...   2   1   2  -2  -8 -29   \n",
       "4      1  -3  -2  -6   1   0   10  -6   4   0  ...   5   0  -3  -8 -11  -2   \n",
       "...   ..  ..  ..  ..  ..  ..  ...  ..  ..  ..  ...  ..  ..  ..  ..  ..  ..   \n",
       "8673   5   4   4   6  20  49  123  34 -27  -1  ...   7   3   0  -4   8  17   \n",
       "8674  16   6  11   2 -14 -35   -8  19 -23  -9  ...  -1  47   9   5 -47  -3   \n",
       "8675  -1  -1  -1  -4  -4  -6   -8  -3  -8  -2  ...   4   0   3   2  -8  22   \n",
       "8676 -29 -24  -4 -10  -9  -6   -4 -11 -10  38  ...   1  -6  -2  -2   6   5   \n",
       "8677 -25  -2  -4  -3   2  -1  -20 -33  -7  -1  ... -37  -2   0   5   6  -9   \n",
       "\n",
       "      62  63  64  label  \n",
       "0    -68  15   C      0  \n",
       "1     22   2   C      0  \n",
       "2     -3  22   A      1  \n",
       "3     29   7   C      0  \n",
       "4     19   4   C      0  \n",
       "...   ..  ..  ..    ...  \n",
       "8673  53  15   C      0  \n",
       "8674   3   0   D      2  \n",
       "8675   4   2   B      3  \n",
       "8676   1 -10   D      2  \n",
       "8677   2 -14   C      0  \n",
       "\n",
       "[8678 rows x 66 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{dtype('int64'): Index([      0,       1,       2,       3,       4,       5,       6,       7,\n",
       "              8,       9,      10,      11,      12,      13,      14,      15,\n",
       "             16,      17,      18,      19,      20,      21,      22,      23,\n",
       "             24,      25,      26,      27,      28,      29,      30,      31,\n",
       "             32,      33,      34,      35,      36,      37,      38,      39,\n",
       "             40,      41,      42,      43,      44,      45,      46,      47,\n",
       "             48,      49,      50,      51,      52,      53,      54,      55,\n",
       "             56,      57,      58,      59,      60,      61,      62,      63,\n",
       "        'label'],\n",
       "       dtype='object'), dtype('O'): Index([64], dtype='object')}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pd.columns.to_series().groupby(train_pd.dtypes).groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>64</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-47</td>\n",
       "      <td>0</td>\n",
       "      <td>-5</td>\n",
       "      <td>-27</td>\n",
       "      <td>-11.0</td>\n",
       "      <td>-19.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-33.0</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>-4</td>\n",
       "      <td>-3</td>\n",
       "      <td>4</td>\n",
       "      <td>-23</td>\n",
       "      <td>-68</td>\n",
       "      <td>15</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-3</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-7</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>-2</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-4</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>-2</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "      <td>-3</td>\n",
       "      <td>22</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-7</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>-14</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-8</td>\n",
       "      <td>-29</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>-3</td>\n",
       "      <td>-8</td>\n",
       "      <td>-11</td>\n",
       "      <td>-2</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2995</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-3</td>\n",
       "      <td>-17.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-7</td>\n",
       "      <td>-3</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>-2</td>\n",
       "      <td>-4</td>\n",
       "      <td>-4</td>\n",
       "      <td>-52</td>\n",
       "      <td>-41</td>\n",
       "      <td>-5</td>\n",
       "      <td>11</td>\n",
       "      <td>D</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2996</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>17.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-21</td>\n",
       "      <td>-12</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>9</td>\n",
       "      <td>-6</td>\n",
       "      <td>-5</td>\n",
       "      <td>0</td>\n",
       "      <td>-17</td>\n",
       "      <td>-9</td>\n",
       "      <td>0</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2997</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>-6</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "      <td>-1</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>-1</td>\n",
       "      <td>-2</td>\n",
       "      <td>C</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2998</td>\n",
       "      <td>18</td>\n",
       "      <td>-3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>15.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-10</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>42</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-2</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2999</td>\n",
       "      <td>-10</td>\n",
       "      <td>17</td>\n",
       "      <td>-4</td>\n",
       "      <td>1</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>-16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4</td>\n",
       "      <td>-9</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>-13</td>\n",
       "      <td>-2</td>\n",
       "      <td>-7</td>\n",
       "      <td>36</td>\n",
       "      <td>-29</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3000 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0   1  2   3     4     5     6     7   8   9  ...  56  57  58  59  60  \\\n",
       "0    -47   0 -5 -27 -11.0 -19.0  -2.0 -33.0  37   0  ...   0   2  -4  -3   4   \n",
       "1      1   1 -2  -3  -1.0  -3.0  39.0  10.0   0  -1  ...  -7  -2   1   6  -2   \n",
       "2      1  -4 -2  -6 -20.0  20.0  -1.0  -3.0  20   3  ...  14   2  -2   0  23   \n",
       "3     10   2  1  -7  -8.0  12.0  41.0  21.0 -14   3  ...   2   1   2  -2  -8   \n",
       "4      1  -3 -2  -6   1.0   0.0  10.0  -6.0   4   0  ...   5   0  -3  -8 -11   \n",
       "...   ..  .. ..  ..   ...   ...   ...   ...  ..  ..  ...  ..  ..  ..  ..  ..   \n",
       "2995   2   1  0  -3 -17.0  -1.0   9.0   3.0  -7  -3  ...   2  -2  -4  -4 -52   \n",
       "2996   2   5  7  13  17.0  62.0   5.0   7.0 -21 -12  ...  16   9  -6  -5   0   \n",
       "2997  -4   0  1  -1   4.0  56.0   6.0   6.0   9   2  ...  -6   0  -2  -1  15   \n",
       "2998  18  -3  1   4  15.0  19.0  -1.0  -1.0 -10   8  ...  42   7   1  -3  14   \n",
       "2999 -10  17 -4   1 -20.0 -16.0   0.0  30.0   4  -9  ...  18 -13  -2  -7  36   \n",
       "\n",
       "      61  62  63  64  label  \n",
       "0    -23 -68  15   A      0  \n",
       "1      4  22   2   C      0  \n",
       "2     21  -3  22   B      1  \n",
       "3    -29  29   7   B      0  \n",
       "4     -2  19   4   C      0  \n",
       "...   ..  ..  ..  ..    ...  \n",
       "2995 -41  -5  11   D      3  \n",
       "2996 -17  -9   0   D      2  \n",
       "2997  15  -1  -2   C      3  \n",
       "2998   1  -3  -2   B      1  \n",
       "2999 -29   4  21   D      2  \n",
       "\n",
       "[3000 rows x 66 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    777\n",
       "2    777\n",
       "3    734\n",
       "0    712\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pd.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    2202\n",
       "0    2177\n",
       "1    2152\n",
       "3    2147\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pd.label.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hot-encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pd = pd.get_dummies(train_pd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>label</th>\n",
       "      <th>64_A</th>\n",
       "      <th>64_B</th>\n",
       "      <th>64_C</th>\n",
       "      <th>64_D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-47</td>\n",
       "      <td>0</td>\n",
       "      <td>-5</td>\n",
       "      <td>-27</td>\n",
       "      <td>-11</td>\n",
       "      <td>-19</td>\n",
       "      <td>-2</td>\n",
       "      <td>-33</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-3</td>\n",
       "      <td>4</td>\n",
       "      <td>-23</td>\n",
       "      <td>-68</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-3</td>\n",
       "      <td>-1</td>\n",
       "      <td>-3</td>\n",
       "      <td>39</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>-2</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-4</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>-20</td>\n",
       "      <td>20</td>\n",
       "      <td>-1</td>\n",
       "      <td>-3</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "      <td>-3</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-7</td>\n",
       "      <td>-8</td>\n",
       "      <td>12</td>\n",
       "      <td>41</td>\n",
       "      <td>21</td>\n",
       "      <td>-14</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>-2</td>\n",
       "      <td>-8</td>\n",
       "      <td>-29</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>-6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-8</td>\n",
       "      <td>-11</td>\n",
       "      <td>-2</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8673</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>49</td>\n",
       "      <td>123</td>\n",
       "      <td>34</td>\n",
       "      <td>-27</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-4</td>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "      <td>53</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8674</td>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>-14</td>\n",
       "      <td>-35</td>\n",
       "      <td>-8</td>\n",
       "      <td>19</td>\n",
       "      <td>-23</td>\n",
       "      <td>-9</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>-47</td>\n",
       "      <td>-3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8675</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-4</td>\n",
       "      <td>-4</td>\n",
       "      <td>-6</td>\n",
       "      <td>-8</td>\n",
       "      <td>-3</td>\n",
       "      <td>-8</td>\n",
       "      <td>-2</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>-8</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8676</td>\n",
       "      <td>-29</td>\n",
       "      <td>-24</td>\n",
       "      <td>-4</td>\n",
       "      <td>-10</td>\n",
       "      <td>-9</td>\n",
       "      <td>-6</td>\n",
       "      <td>-4</td>\n",
       "      <td>-11</td>\n",
       "      <td>-10</td>\n",
       "      <td>38</td>\n",
       "      <td>...</td>\n",
       "      <td>-2</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>-10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8677</td>\n",
       "      <td>-25</td>\n",
       "      <td>-2</td>\n",
       "      <td>-4</td>\n",
       "      <td>-3</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-20</td>\n",
       "      <td>-33</td>\n",
       "      <td>-7</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>-9</td>\n",
       "      <td>2</td>\n",
       "      <td>-14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8678 rows × 69 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0   1   2   3   4   5    6   7   8   9  ...  59  60  61  62  63  label  \\\n",
       "0    -47   0  -5 -27 -11 -19   -2 -33  37   0  ...  -3   4 -23 -68  15      0   \n",
       "1      1   1  -2  -3  -1  -3   39  10   0  -1  ...   6  -2   4  22   2      0   \n",
       "2      1  -4  -2  -6 -20  20   -1  -3  20   3  ...   0  23  21  -3  22      1   \n",
       "3     10   2   1  -7  -8  12   41  21 -14   3  ...  -2  -8 -29  29   7      0   \n",
       "4      1  -3  -2  -6   1   0   10  -6   4   0  ...  -8 -11  -2  19   4      0   \n",
       "...   ..  ..  ..  ..  ..  ..  ...  ..  ..  ..  ...  ..  ..  ..  ..  ..    ...   \n",
       "8673   5   4   4   6  20  49  123  34 -27  -1  ...  -4   8  17  53  15      0   \n",
       "8674  16   6  11   2 -14 -35   -8  19 -23  -9  ...   5 -47  -3   3   0      2   \n",
       "8675  -1  -1  -1  -4  -4  -6   -8  -3  -8  -2  ...   2  -8  22   4   2      3   \n",
       "8676 -29 -24  -4 -10  -9  -6   -4 -11 -10  38  ...  -2   6   5   1 -10      2   \n",
       "8677 -25  -2  -4  -3   2  -1  -20 -33  -7  -1  ...   5   6  -9   2 -14      0   \n",
       "\n",
       "      64_A  64_B  64_C  64_D  \n",
       "0        0     0     1     0  \n",
       "1        0     0     1     0  \n",
       "2        1     0     0     0  \n",
       "3        0     0     1     0  \n",
       "4        0     0     1     0  \n",
       "...    ...   ...   ...   ...  \n",
       "8673     0     0     1     0  \n",
       "8674     0     0     0     1  \n",
       "8675     0     1     0     0  \n",
       "8676     0     0     0     1  \n",
       "8677     0     0     1     0  \n",
       "\n",
       "[8678 rows x 69 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pd = pd.get_dummies(test_pd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>label</th>\n",
       "      <th>64_A</th>\n",
       "      <th>64_B</th>\n",
       "      <th>64_C</th>\n",
       "      <th>64_D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-47</td>\n",
       "      <td>0</td>\n",
       "      <td>-5</td>\n",
       "      <td>-27</td>\n",
       "      <td>-11.0</td>\n",
       "      <td>-19.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-33.0</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-3</td>\n",
       "      <td>4</td>\n",
       "      <td>-23</td>\n",
       "      <td>-68</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-3</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>-2</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-4</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "      <td>-3</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-7</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>-14</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>-2</td>\n",
       "      <td>-8</td>\n",
       "      <td>-29</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-2</td>\n",
       "      <td>-6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-8</td>\n",
       "      <td>-11</td>\n",
       "      <td>-2</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2995</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-3</td>\n",
       "      <td>-17.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-7</td>\n",
       "      <td>-3</td>\n",
       "      <td>...</td>\n",
       "      <td>-4</td>\n",
       "      <td>-52</td>\n",
       "      <td>-41</td>\n",
       "      <td>-5</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2996</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>17.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-21</td>\n",
       "      <td>-12</td>\n",
       "      <td>...</td>\n",
       "      <td>-5</td>\n",
       "      <td>0</td>\n",
       "      <td>-17</td>\n",
       "      <td>-9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2997</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>-1</td>\n",
       "      <td>-2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2998</td>\n",
       "      <td>18</td>\n",
       "      <td>-3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>15.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-10</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>-3</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2999</td>\n",
       "      <td>-10</td>\n",
       "      <td>17</td>\n",
       "      <td>-4</td>\n",
       "      <td>1</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>-16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4</td>\n",
       "      <td>-9</td>\n",
       "      <td>...</td>\n",
       "      <td>-7</td>\n",
       "      <td>36</td>\n",
       "      <td>-29</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3000 rows × 69 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0   1  2   3     4     5     6     7   8   9  ...  59  60  61  62  63  \\\n",
       "0    -47   0 -5 -27 -11.0 -19.0  -2.0 -33.0  37   0  ...  -3   4 -23 -68  15   \n",
       "1      1   1 -2  -3  -1.0  -3.0  39.0  10.0   0  -1  ...   6  -2   4  22   2   \n",
       "2      1  -4 -2  -6 -20.0  20.0  -1.0  -3.0  20   3  ...   0  23  21  -3  22   \n",
       "3     10   2  1  -7  -8.0  12.0  41.0  21.0 -14   3  ...  -2  -8 -29  29   7   \n",
       "4      1  -3 -2  -6   1.0   0.0  10.0  -6.0   4   0  ...  -8 -11  -2  19   4   \n",
       "...   ..  .. ..  ..   ...   ...   ...   ...  ..  ..  ...  ..  ..  ..  ..  ..   \n",
       "2995   2   1  0  -3 -17.0  -1.0   9.0   3.0  -7  -3  ...  -4 -52 -41  -5  11   \n",
       "2996   2   5  7  13  17.0  62.0   5.0   7.0 -21 -12  ...  -5   0 -17  -9   0   \n",
       "2997  -4   0  1  -1   4.0  56.0   6.0   6.0   9   2  ...  -1  15  15  -1  -2   \n",
       "2998  18  -3  1   4  15.0  19.0  -1.0  -1.0 -10   8  ...  -3  14   1  -3  -2   \n",
       "2999 -10  17 -4   1 -20.0 -16.0   0.0  30.0   4  -9  ...  -7  36 -29   4  21   \n",
       "\n",
       "      label  64_A  64_B  64_C  64_D  \n",
       "0         0     1     0     0     0  \n",
       "1         0     0     0     1     0  \n",
       "2         1     0     1     0     0  \n",
       "3         0     0     1     0     0  \n",
       "4         0     0     0     1     0  \n",
       "...     ...   ...   ...   ...   ...  \n",
       "2995      3     0     0     0     1  \n",
       "2996      2     0     0     0     1  \n",
       "2997      3     0     0     1     0  \n",
       "2998      1     0     1     0     0  \n",
       "2999      2     0     0     0     1  \n",
       "\n",
       "[3000 rows x 69 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train_pd.drop('label', axis=1)\n",
    "X_test = test_pd.drop('label', axis=1)\n",
    "y_train = train_pd.label\n",
    "y_test = test_pd.label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dimensionality reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim_reduce = GaussianRandomProjection(n_components=10).fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = dim_reduce.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = dim_reduce.transform(imp_mean.fit_transform(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Acurracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: xgboost.XGBClassifier is not available and will not be used by TPOT.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Optimization Progress', max=30, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: 0.5512773037191591\n",
      "Generation 2 - Current best internal CV score: 0.5542639393806666\n",
      "\n",
      "Best pipeline: ExtraTreesClassifier(input_matrix, bootstrap=False, criterion=gini, max_features=0.3, min_samples_leaf=1, min_samples_split=5, n_estimators=100)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TPOTClassifier(config_dict=None, crossover_rate=0.1, cv=5,\n",
       "               disable_update_check=False, early_stop=None, generations=2,\n",
       "               max_eval_time_mins=5, max_time_mins=None, memory=None,\n",
       "               mutation_rate=0.9, n_jobs=-1, offspring_size=None,\n",
       "               periodic_checkpoint_folder=None, population_size=10,\n",
       "               random_state=None, scoring=None, subsample=1.0, template=None,\n",
       "               use_dask=False, verbosity=2, warm_start=False)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot = TPOTClassifier(generations=2, population_size=10, \n",
    "                      n_jobs=-1, \n",
    "                      verbosity=2)\n",
    "tpot.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9996666666666667"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F1-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: xgboost.XGBClassifier is not available and will not be used by TPOT.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Optimization Progress', max=210, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: 1.0\n",
      "Generation 2 - Current best internal CV score: 1.0\n",
      "Generation 3 - Current best internal CV score: 1.0\n",
      "Generation 4 - Current best internal CV score: 1.0\n",
      "Generation 5 - Current best internal CV score: 1.0\n",
      "Generation 6 - Current best internal CV score: 1.0\n",
      "\n",
      "Best pipeline: ExtraTreesClassifier(input_matrix, bootstrap=False, criterion=entropy, max_features=0.2, min_samples_leaf=8, min_samples_split=6, n_estimators=100)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TPOTClassifier(config_dict=None, crossover_rate=0.1, cv=5,\n",
       "               disable_update_check=False, early_stop=None, generations=6,\n",
       "               max_eval_time_mins=5, max_time_mins=None, memory=None,\n",
       "               mutation_rate=0.9, n_jobs=-1, offspring_size=None,\n",
       "               periodic_checkpoint_folder=None, population_size=30,\n",
       "               random_state=None, scoring='f1_macro', subsample=1.0,\n",
       "               template=None, use_dask=False, verbosity=2, warm_start=False)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot = TPOTClassifier(generations=6, population_size=30, \n",
    "                      n_jobs=-1, \n",
    "                      scoring='f1_macro',\n",
    "                      verbosity=2)\n",
    "tpot.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Imputing missing values in feature set\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.24523022936082453"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}