2021-11-15-dengue-fever-driven-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dengue Fever prediction\n",
    "\n",
    "The competition that this notebook is referring to can be found at this website. \n",
    "\n",
    "https://www.drivendata.org/competitions/44/dengai-predicting-disease-spread/page/80/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from ipywidgets import widgets\n",
    "\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.read_csv('data/dengue_features_train.csv', parse_dates=['week_start_date'])\n",
    "y = pd.read_csv('data/dengue_labels_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>week_start_date</th>\n",
       "      <th>ndvi_ne</th>\n",
       "      <th>ndvi_nw</th>\n",
       "      <th>ndvi_se</th>\n",
       "      <th>ndvi_sw</th>\n",
       "      <th>precipitation_amt_mm</th>\n",
       "      <th>reanalysis_air_temp_k</th>\n",
       "      <th>...</th>\n",
       "      <th>reanalysis_precip_amt_kg_per_m2</th>\n",
       "      <th>reanalysis_relative_humidity_percent</th>\n",
       "      <th>reanalysis_sat_precip_amt_mm</th>\n",
       "      <th>reanalysis_specific_humidity_g_per_kg</th>\n",
       "      <th>reanalysis_tdtr_k</th>\n",
       "      <th>station_avg_temp_c</th>\n",
       "      <th>station_diur_temp_rng_c</th>\n",
       "      <th>station_max_temp_c</th>\n",
       "      <th>station_min_temp_c</th>\n",
       "      <th>station_precip_mm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>18</td>\n",
       "      <td>1990-04-30</td>\n",
       "      <td>0.122600</td>\n",
       "      <td>0.103725</td>\n",
       "      <td>0.198483</td>\n",
       "      <td>0.177617</td>\n",
       "      <td>12.42</td>\n",
       "      <td>297.572857</td>\n",
       "      <td>...</td>\n",
       "      <td>32.00</td>\n",
       "      <td>73.365714</td>\n",
       "      <td>12.42</td>\n",
       "      <td>14.012857</td>\n",
       "      <td>2.628571</td>\n",
       "      <td>25.442857</td>\n",
       "      <td>6.900000</td>\n",
       "      <td>29.4</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>19</td>\n",
       "      <td>1990-05-07</td>\n",
       "      <td>0.169900</td>\n",
       "      <td>0.142175</td>\n",
       "      <td>0.162357</td>\n",
       "      <td>0.155486</td>\n",
       "      <td>22.82</td>\n",
       "      <td>298.211429</td>\n",
       "      <td>...</td>\n",
       "      <td>17.94</td>\n",
       "      <td>77.368571</td>\n",
       "      <td>22.82</td>\n",
       "      <td>15.372857</td>\n",
       "      <td>2.371429</td>\n",
       "      <td>26.714286</td>\n",
       "      <td>6.371429</td>\n",
       "      <td>31.7</td>\n",
       "      <td>22.2</td>\n",
       "      <td>8.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>20</td>\n",
       "      <td>1990-05-14</td>\n",
       "      <td>0.032250</td>\n",
       "      <td>0.172967</td>\n",
       "      <td>0.157200</td>\n",
       "      <td>0.170843</td>\n",
       "      <td>34.54</td>\n",
       "      <td>298.781429</td>\n",
       "      <td>...</td>\n",
       "      <td>26.10</td>\n",
       "      <td>82.052857</td>\n",
       "      <td>34.54</td>\n",
       "      <td>16.848571</td>\n",
       "      <td>2.300000</td>\n",
       "      <td>26.714286</td>\n",
       "      <td>6.485714</td>\n",
       "      <td>32.2</td>\n",
       "      <td>22.8</td>\n",
       "      <td>41.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>21</td>\n",
       "      <td>1990-05-21</td>\n",
       "      <td>0.128633</td>\n",
       "      <td>0.245067</td>\n",
       "      <td>0.227557</td>\n",
       "      <td>0.235886</td>\n",
       "      <td>15.36</td>\n",
       "      <td>298.987143</td>\n",
       "      <td>...</td>\n",
       "      <td>13.90</td>\n",
       "      <td>80.337143</td>\n",
       "      <td>15.36</td>\n",
       "      <td>16.672857</td>\n",
       "      <td>2.428571</td>\n",
       "      <td>27.471429</td>\n",
       "      <td>6.771429</td>\n",
       "      <td>33.3</td>\n",
       "      <td>23.3</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>22</td>\n",
       "      <td>1990-05-28</td>\n",
       "      <td>0.196200</td>\n",
       "      <td>0.262200</td>\n",
       "      <td>0.251200</td>\n",
       "      <td>0.247340</td>\n",
       "      <td>7.52</td>\n",
       "      <td>299.518571</td>\n",
       "      <td>...</td>\n",
       "      <td>12.20</td>\n",
       "      <td>80.460000</td>\n",
       "      <td>7.52</td>\n",
       "      <td>17.210000</td>\n",
       "      <td>3.014286</td>\n",
       "      <td>28.942857</td>\n",
       "      <td>9.371429</td>\n",
       "      <td>35.0</td>\n",
       "      <td>23.9</td>\n",
       "      <td>5.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  city  year  weekofyear week_start_date   ndvi_ne   ndvi_nw   ndvi_se  \\\n",
       "0   sj  1990          18      1990-04-30  0.122600  0.103725  0.198483   \n",
       "1   sj  1990          19      1990-05-07  0.169900  0.142175  0.162357   \n",
       "2   sj  1990          20      1990-05-14  0.032250  0.172967  0.157200   \n",
       "3   sj  1990          21      1990-05-21  0.128633  0.245067  0.227557   \n",
       "4   sj  1990          22      1990-05-28  0.196200  0.262200  0.251200   \n",
       "\n",
       "    ndvi_sw  precipitation_amt_mm  reanalysis_air_temp_k  ...  \\\n",
       "0  0.177617                 12.42             297.572857  ...   \n",
       "1  0.155486                 22.82             298.211429  ...   \n",
       "2  0.170843                 34.54             298.781429  ...   \n",
       "3  0.235886                 15.36             298.987143  ...   \n",
       "4  0.247340                  7.52             299.518571  ...   \n",
       "\n",
       "   reanalysis_precip_amt_kg_per_m2  reanalysis_relative_humidity_percent  \\\n",
       "0                            32.00                             73.365714   \n",
       "1                            17.94                             77.368571   \n",
       "2                            26.10                             82.052857   \n",
       "3                            13.90                             80.337143   \n",
       "4                            12.20                             80.460000   \n",
       "\n",
       "   reanalysis_sat_precip_amt_mm  reanalysis_specific_humidity_g_per_kg  \\\n",
       "0                         12.42                              14.012857   \n",
       "1                         22.82                              15.372857   \n",
       "2                         34.54                              16.848571   \n",
       "3                         15.36                              16.672857   \n",
       "4                          7.52                              17.210000   \n",
       "\n",
       "   reanalysis_tdtr_k  station_avg_temp_c  station_diur_temp_rng_c  \\\n",
       "0           2.628571           25.442857                 6.900000   \n",
       "1           2.371429           26.714286                 6.371429   \n",
       "2           2.300000           26.714286                 6.485714   \n",
       "3           2.428571           27.471429                 6.771429   \n",
       "4           3.014286           28.942857                 9.371429   \n",
       "\n",
       "   station_max_temp_c  station_min_temp_c  station_precip_mm  \n",
       "0                29.4                20.0               16.0  \n",
       "1                31.7                22.2                8.6  \n",
       "2                32.2                22.8               41.4  \n",
       "3                33.3                23.3                4.0  \n",
       "4                35.0                23.9                5.8  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1456 entries, 0 to 1455\n",
      "Data columns (total 24 columns):\n",
      " #   Column                                 Non-Null Count  Dtype         \n",
      "---  ------                                 --------------  -----         \n",
      " 0   city                                   1456 non-null   object        \n",
      " 1   year                                   1456 non-null   int64         \n",
      " 2   weekofyear                             1456 non-null   int64         \n",
      " 3   week_start_date                        1456 non-null   datetime64[ns]\n",
      " 4   ndvi_ne                                1262 non-null   float64       \n",
      " 5   ndvi_nw                                1404 non-null   float64       \n",
      " 6   ndvi_se                                1434 non-null   float64       \n",
      " 7   ndvi_sw                                1434 non-null   float64       \n",
      " 8   precipitation_amt_mm                   1443 non-null   float64       \n",
      " 9   reanalysis_air_temp_k                  1446 non-null   float64       \n",
      " 10  reanalysis_avg_temp_k                  1446 non-null   float64       \n",
      " 11  reanalysis_dew_point_temp_k            1446 non-null   float64       \n",
      " 12  reanalysis_max_air_temp_k              1446 non-null   float64       \n",
      " 13  reanalysis_min_air_temp_k              1446 non-null   float64       \n",
      " 14  reanalysis_precip_amt_kg_per_m2        1446 non-null   float64       \n",
      " 15  reanalysis_relative_humidity_percent   1446 non-null   float64       \n",
      " 16  reanalysis_sat_precip_amt_mm           1443 non-null   float64       \n",
      " 17  reanalysis_specific_humidity_g_per_kg  1446 non-null   float64       \n",
      " 18  reanalysis_tdtr_k                      1446 non-null   float64       \n",
      " 19  station_avg_temp_c                     1413 non-null   float64       \n",
      " 20  station_diur_temp_rng_c                1413 non-null   float64       \n",
      " 21  station_max_temp_c                     1436 non-null   float64       \n",
      " 22  station_min_temp_c                     1442 non-null   float64       \n",
      " 23  station_precip_mm                      1434 non-null   float64       \n",
      "dtypes: datetime64[ns](1), float64(20), int64(2), object(1)\n",
      "memory usage: 273.1+ KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will need to impute missing data for these numeric columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sj    936\n",
       "iq    520\n",
       "Name: city, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X['city'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sj</td>\n",
       "      <td>1990</td>\n",
       "      <td>22</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  city  year  weekofyear  total_cases\n",
       "0   sj  1990          18            4\n",
       "1   sj  1990          19            5\n",
       "2   sj  1990          20            4\n",
       "3   sj  1990          21            3\n",
       "4   sj  1990          22            6"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A plot to show the cases over time.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63e9b12bade94a46b4c197f70f47341e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='city', options={'San Juan': 'sj', 'Iquitos': 'iq'}, value='sj'), O…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def case_plotter(X,y):\n",
    "    def plotter(city):\n",
    "        city_name = {'sj': 'San Juan', 'iq': 'Iquitos'}\n",
    "        plt.plot(pd.to_datetime(X[X['city'] == city]['week_start_date']), y.loc[X[X['city'] == city].index]['total_cases'])\n",
    "        plt.title(f'Cases for {city_name[city]}')\n",
    "        plt.ylabel('Cases')\n",
    "        \n",
    "    return plotter\n",
    "\n",
    "dropdown_values = {'San Juan': 'sj', 'Iquitos': 'iq'}\n",
    "widgets.interact(case_plotter(X,y), city=dropdown_values);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The plan\n",
    "\n",
    "Several things to keep in mind:\n",
    "\n",
    "1.  We have two cities to model.  Do we want one model for both (say, with one-hot encoded values for the city) or different models (or a combined \"meta-model\" that wraps up two models)?\n",
    "1.  There is missing data in almost every column.  Not a lot of missing data, but every numeric column seems to be missing a bit, so this will need to be imputed.  Since we have two cities, we likely want to impute the missing data separately for each city.  \n",
    "1.  There is time inherent in this model, so there is likely some (mild?) seasonal/periodic behavior so likely want to engineer some features based on time.  \n",
    "1.  The data for the two cities covers different time periods.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features = ['city']\n",
    "\n",
    "numeric_features = ['year', 'weekofyear', 'ndvi_ne', 'ndvi_nw',\n",
    "       'ndvi_se', 'ndvi_sw', 'precipitation_amt_mm', 'reanalysis_air_temp_k',\n",
    "       'reanalysis_avg_temp_k', 'reanalysis_dew_point_temp_k',\n",
    "       'reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k',\n",
    "       'reanalysis_precip_amt_kg_per_m2',\n",
    "       'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',\n",
    "       'reanalysis_specific_humidity_g_per_kg', 'reanalysis_tdtr_k',\n",
    "       'station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c',\n",
    "       'station_min_temp_c', 'station_precip_mm']\n",
    "\n",
    "time_features = ['week_start_date']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Imputing missing data\n",
    "\n",
    "Let's just handle imputing the missing data separately.  Not ideal in terms of building a pipeline, but the fact that there are two cities with different means/medians for the numeric data means that either we build our own custom imputer or use two separate imputers.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>ndvi_ne</th>\n",
       "      <th>ndvi_nw</th>\n",
       "      <th>ndvi_se</th>\n",
       "      <th>ndvi_sw</th>\n",
       "      <th>precipitation_amt_mm</th>\n",
       "      <th>reanalysis_air_temp_k</th>\n",
       "      <th>reanalysis_avg_temp_k</th>\n",
       "      <th>reanalysis_dew_point_temp_k</th>\n",
       "      <th>...</th>\n",
       "      <th>reanalysis_precip_amt_kg_per_m2</th>\n",
       "      <th>reanalysis_relative_humidity_percent</th>\n",
       "      <th>reanalysis_sat_precip_amt_mm</th>\n",
       "      <th>reanalysis_specific_humidity_g_per_kg</th>\n",
       "      <th>reanalysis_tdtr_k</th>\n",
       "      <th>station_avg_temp_c</th>\n",
       "      <th>station_diur_temp_rng_c</th>\n",
       "      <th>station_max_temp_c</th>\n",
       "      <th>station_min_temp_c</th>\n",
       "      <th>station_precip_mm</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>iq</th>\n",
       "      <td>2005.000000</td>\n",
       "      <td>26.503846</td>\n",
       "      <td>0.263869</td>\n",
       "      <td>0.238783</td>\n",
       "      <td>0.250126</td>\n",
       "      <td>0.266779</td>\n",
       "      <td>64.245736</td>\n",
       "      <td>297.869538</td>\n",
       "      <td>299.133043</td>\n",
       "      <td>295.492982</td>\n",
       "      <td>...</td>\n",
       "      <td>57.609864</td>\n",
       "      <td>88.639117</td>\n",
       "      <td>64.245736</td>\n",
       "      <td>17.096110</td>\n",
       "      <td>9.206783</td>\n",
       "      <td>27.530933</td>\n",
       "      <td>10.566197</td>\n",
       "      <td>34.004545</td>\n",
       "      <td>21.196680</td>\n",
       "      <td>62.467262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sj</th>\n",
       "      <td>1998.826923</td>\n",
       "      <td>26.503205</td>\n",
       "      <td>0.057925</td>\n",
       "      <td>0.067469</td>\n",
       "      <td>0.177655</td>\n",
       "      <td>0.165956</td>\n",
       "      <td>35.470809</td>\n",
       "      <td>299.163653</td>\n",
       "      <td>299.276920</td>\n",
       "      <td>295.109519</td>\n",
       "      <td>...</td>\n",
       "      <td>30.465419</td>\n",
       "      <td>78.568181</td>\n",
       "      <td>35.470809</td>\n",
       "      <td>16.552409</td>\n",
       "      <td>2.516267</td>\n",
       "      <td>27.006528</td>\n",
       "      <td>6.757373</td>\n",
       "      <td>31.607957</td>\n",
       "      <td>22.600645</td>\n",
       "      <td>26.785484</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             year  weekofyear   ndvi_ne   ndvi_nw   ndvi_se   ndvi_sw  \\\n",
       "city                                                                    \n",
       "iq    2005.000000   26.503846  0.263869  0.238783  0.250126  0.266779   \n",
       "sj    1998.826923   26.503205  0.057925  0.067469  0.177655  0.165956   \n",
       "\n",
       "      precipitation_amt_mm  reanalysis_air_temp_k  reanalysis_avg_temp_k  \\\n",
       "city                                                                       \n",
       "iq               64.245736             297.869538             299.133043   \n",
       "sj               35.470809             299.163653             299.276920   \n",
       "\n",
       "      reanalysis_dew_point_temp_k  ...  reanalysis_precip_amt_kg_per_m2  \\\n",
       "city                               ...                                    \n",
       "iq                     295.492982  ...                        57.609864   \n",
       "sj                     295.109519  ...                        30.465419   \n",
       "\n",
       "      reanalysis_relative_humidity_percent  reanalysis_sat_precip_amt_mm  \\\n",
       "city                                                                       \n",
       "iq                               88.639117                     64.245736   \n",
       "sj                               78.568181                     35.470809   \n",
       "\n",
       "      reanalysis_specific_humidity_g_per_kg  reanalysis_tdtr_k  \\\n",
       "city                                                             \n",
       "iq                                17.096110           9.206783   \n",
       "sj                                16.552409           2.516267   \n",
       "\n",
       "      station_avg_temp_c  station_diur_temp_rng_c  station_max_temp_c  \\\n",
       "city                                                                    \n",
       "iq             27.530933                10.566197           34.004545   \n",
       "sj             27.006528                 6.757373           31.607957   \n",
       "\n",
       "      station_min_temp_c  station_precip_mm  \n",
       "city                                         \n",
       "iq             21.196680          62.467262  \n",
       "sj             22.600645          26.785484  \n",
       "\n",
       "[2 rows x 22 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[categorical_features + numeric_features].groupby('city').mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We build a custom imputer that will impute the missing data for the data based on a column, and returns a new DataFrame.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin\n",
    "\n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GroupByImputer(BaseEstimator, TransformerMixin):\n",
    "    '''Assumes that the feature matrix is a pandas DataFrame.\n",
    "    \n",
    "       column is a column in the input feature matrix \n",
    "       We will group by column and fit a SimpleImputer for\n",
    "       each group in the data, and store in a dictionary\n",
    "       inside the custom transformer.  \n",
    "       \n",
    "       The transform method will return a new pandas DataFrame.  \n",
    "    '''\n",
    "    def __init__(self, column, strategy='mean'):\n",
    "        self.column = column\n",
    "        self.imputers = {}\n",
    "        self.strategy = strategy\n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        grouped = X.groupby(self.column)\n",
    "        cols = [c for c in X.columns if c != self.column]\n",
    "        for k in grouped.groups.keys():\n",
    "            imp = SimpleImputer(strategy=self.strategy)\n",
    "            imp.fit(X.loc[grouped.groups[k]][cols])\n",
    "            self.imputers[k] = imp\n",
    "            \n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        result = []\n",
    "        cols = [c for c in X.columns if c != self.column]\n",
    "        grouped = X.groupby(self.column)\n",
    "        for k in grouped.groups.keys():\n",
    "            df = pd.DataFrame(self.imputers[k].transform(X.loc[grouped.groups[k]][cols]), columns=cols, index=X.loc[grouped.groups[k]].index)\n",
    "            result.append(df)\n",
    "            \n",
    "        return pd.concat(result).sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So now create an imputer and fill in the missing data in the training set.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputer = GroupByImputer('city')\n",
    "imputer.fit(X[categorical_features + numeric_features])\n",
    "\n",
    "X[numeric_features] = imputer.transform(X[categorical_features + numeric_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1456 entries, 0 to 1455\n",
      "Data columns (total 24 columns):\n",
      " #   Column                                 Non-Null Count  Dtype         \n",
      "---  ------                                 --------------  -----         \n",
      " 0   city                                   1456 non-null   object        \n",
      " 1   year                                   1456 non-null   float64       \n",
      " 2   weekofyear                             1456 non-null   float64       \n",
      " 3   week_start_date                        1456 non-null   datetime64[ns]\n",
      " 4   ndvi_ne                                1456 non-null   float64       \n",
      " 5   ndvi_nw                                1456 non-null   float64       \n",
      " 6   ndvi_se                                1456 non-null   float64       \n",
      " 7   ndvi_sw                                1456 non-null   float64       \n",
      " 8   precipitation_amt_mm                   1456 non-null   float64       \n",
      " 9   reanalysis_air_temp_k                  1456 non-null   float64       \n",
      " 10  reanalysis_avg_temp_k                  1456 non-null   float64       \n",
      " 11  reanalysis_dew_point_temp_k            1456 non-null   float64       \n",
      " 12  reanalysis_max_air_temp_k              1456 non-null   float64       \n",
      " 13  reanalysis_min_air_temp_k              1456 non-null   float64       \n",
      " 14  reanalysis_precip_amt_kg_per_m2        1456 non-null   float64       \n",
      " 15  reanalysis_relative_humidity_percent   1456 non-null   float64       \n",
      " 16  reanalysis_sat_precip_amt_mm           1456 non-null   float64       \n",
      " 17  reanalysis_specific_humidity_g_per_kg  1456 non-null   float64       \n",
      " 18  reanalysis_tdtr_k                      1456 non-null   float64       \n",
      " 19  station_avg_temp_c                     1456 non-null   float64       \n",
      " 20  station_diur_temp_rng_c                1456 non-null   float64       \n",
      " 21  station_max_temp_c                     1456 non-null   float64       \n",
      " 22  station_min_temp_c                     1456 non-null   float64       \n",
      " 23  station_precip_mm                      1456 non-null   float64       \n",
      "dtypes: datetime64[ns](1), float64(22), object(1)\n",
      "memory usage: 273.1+ KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Modeling\n",
    "\n",
    "Now let's build some models.  This will involve engineering some features for a time series problem.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler, \\\n",
    "    MaxAbsScaler, PolynomialFeatures, FunctionTransformer\n",
    "\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "from sklearn.linear_model import LinearRegression, Ridge, SGDRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FourierTransformer(BaseEstimator, TransformerMixin):\n",
    "    '''Tranformer that will use the weekofyear as input to compute\n",
    "    sine and cosine terms for a given period.'''\n",
    "    def __init__(self, period):\n",
    "        self.period = period\n",
    "        \n",
    "    def fit(self, X, y = None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return pd.DataFrame({'sin': np.sin(2*np.pi*(X-1)/self.period),\n",
    "                             'cos': np.cos(2*np.pi*(X-1)/self.period)}, index=X.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Custom GroupbyEstimator\n",
    "\n",
    "We want to handle both cities in one estimator, so we build a custom estimator that will do that for us.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GroupbyEstimator(BaseEstimator, RegressorMixin): \n",
    "    def __init__(self, column, estimator_factory):\n",
    "        # column is the value to group by; estimator_factory can be\n",
    "        # called to produce estimators\n",
    "        self.column = column\n",
    "        self.estimator_factory = estimator_factory\n",
    "        self.estimators = dict()\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        # Create an estimator and fit it with the portion in each group\n",
    "        for name, df in X.groupby(self.column):\n",
    "            self.estimators[name] = self.estimator_factory().fit(df, y.loc[df.index])\n",
    "        return self\n",
    "\n",
    "    def _predict(self, df):\n",
    "        name = df[self.column].iloc[0]\n",
    "        return self.estimators[name].predict(df)\n",
    "    \n",
    "    def predict(self, X):\n",
    "        # Call the appropriate predict method for each row of X\n",
    "        retval = pd.Series(dtype=np.float64)\n",
    "        for name, indices in X.groupby(self.column).groups.items():\n",
    "            retval = retval.append(pd.Series(self._predict(X.loc[indices]),\n",
    "                                             indices))\n",
    "        return retval.loc[X.index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  A function to use in FunctionTransformer\n",
    "\n",
    "Let's engineer values from the `week_start_date` to use as a feature for a trend in the model.  We'll use a function we define and the `FunctionTransformer` to turn it into a transformer in the `Pipeline`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_julian(series):\n",
    "    return series.apply(lambda x: pd.Timestamp(x).to_julian_date()).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>week_start_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2448011.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2448018.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2448025.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2448032.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2448039.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1451</th>\n",
       "      <td>2455344.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1452</th>\n",
       "      <td>2455351.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1453</th>\n",
       "      <td>2455358.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454</th>\n",
       "      <td>2455365.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455</th>\n",
       "      <td>2455372.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1456 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      week_start_date\n",
       "0           2448011.5\n",
       "1           2448018.5\n",
       "2           2448025.5\n",
       "3           2448032.5\n",
       "4           2448039.5\n",
       "...               ...\n",
       "1451        2455344.5\n",
       "1452        2455351.5\n",
       "1453        2455358.5\n",
       "1454        2455365.5\n",
       "1455        2455372.5\n",
       "\n",
       "[1456 rows x 1 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_julian(X['week_start_date'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  A factory to spit out pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def factory():\n",
    "    return Pipeline([\n",
    "        ('features', ColumnTransformer([\n",
    "            ('fourier', FeatureUnion([\n",
    "                ('yearly', FourierTransformer(52)),\n",
    "                ('biannual', FourierTransformer(52//2)),\n",
    "                ('semiannual', FourierTransformer(52*2))]), 'weekofyear'),\n",
    "            ('numeric', 'passthrough', \n",
    "             [c for c in numeric_features if c != 'weekofyear']),\n",
    "            ('julian', FunctionTransformer(to_julian), 'week_start_date')\n",
    "        ])),\n",
    "        ('poly', PolynomialFeatures(2, include_bias=False)),\n",
    "        ('scale', StandardScaler()),\n",
    "        ('regressor', \n",
    "             GridSearchCV(Ridge(), \n",
    "                          param_grid={'alpha': np.logspace(-2,2,10)},\n",
    "                          cv=TimeSeriesSplit()))\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fitting the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbe = GroupbyEstimator('city', factory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GroupbyEstimator(column='city',\n",
       "                 estimator_factory=<function factory at 0x7f8c81d55a60>)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbe.fit(X, y['total_cases'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Predictions\n",
    "We can get predictions, but with the additional criteria that case numbers should be non-negative.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = np.maximum(gbe.predict(X), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.33609340647891284"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.r2_score(y['total_cases'], predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16.807059164820828"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.mean_absolute_error(y['total_cases'], predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Plots of predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ab4f8a98e3d4218b93104cf97fa26c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='city', options={'San Juan': 'sj', 'Iquitos': 'iq'}, value='sj'), O…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def case_plotter(X,y,predictions):\n",
    "    def plotter(city):\n",
    "        city_name = {'sj': 'San Juan', 'iq': 'Iquitos'}\n",
    "        plt.plot(pd.to_datetime(X[X['city'] == city]['week_start_date']), y.loc[X[X['city'] == city].index]['total_cases'], label='Real cases', color='r')\n",
    "        plt.plot(pd.to_datetime(X[X['city'] == city]['week_start_date']), predictions.loc[X[X['city'] == city].index], label='Predictions', color='b')\n",
    "        plt.title(f'Cases for {city_name[city]}')\n",
    "        plt.ylabel('Cases')\n",
    "        plt.legend()\n",
    "        \n",
    "    return plotter\n",
    "\n",
    "dropdown_values = {'San Juan': 'sj', 'Iquitos': 'iq'}\n",
    "widgets.interact(case_plotter(X,y,pd.Series(np.maximum(gbe.predict(X),0),index=X.index)), city=dropdown_values);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_format = pd.read_csv('submissions/submission_format.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  city  year  weekofyear  total_cases\n",
       "0   sj  2008          18            0\n",
       "1   sj  2008          19            0\n",
       "2   sj  2008          20            0\n",
       "3   sj  2008          21            0\n",
       "4   sj  2008          22            0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_format.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Read the test data and impute missing values for the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('data/dengue_features_test.csv', parse_dates=['week_start_date'])\n",
    "\n",
    "test[numeric_features] = imputer.transform(test[categorical_features + numeric_features])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Make and save predictions for the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "submissions = sub_format.copy()\n",
    "submissions['total_cases'] = np.round(np.maximum(gbe.predict(test),0)).astype(np.int64)\n",
    "\n",
    "now = datetime.now()\n",
    "filename = 'submissions/submissions-' + now.strftime('%Y-%m-%d-%H%M') + '.csv'\n",
    "submissions.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>21</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>22</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  city  year  weekofyear  total_cases\n",
       "0   sj  2008          18           10\n",
       "1   sj  2008          19            0\n",
       "2   sj  2008          20            0\n",
       "3   sj  2008          21           19\n",
       "4   sj  2008          22            7"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submissions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10,  0, 19,  7, 24,  9, 16,  8,  2, 15, 32, 21, 34, 36, 56, 42, 44,\n",
       "       68, 52, 51, 48, 50, 46, 47, 29, 11, 17,  5, 22, 18,  4,  3, 13, 25,\n",
       "       35, 45, 59, 75, 40, 66, 71, 60, 39, 67, 64, 28, 57, 38, 33, 26, 14,\n",
       "       23, 12, 41, 65,  1,  6, 20, 37, 54, 58, 61])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submissions['total_cases'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      10.345862\n",
       "1       0.000000\n",
       "2       0.000000\n",
       "3      19.160316\n",
       "4       6.601260\n",
       "         ...    \n",
       "411     8.761167\n",
       "412     7.629091\n",
       "413     8.378471\n",
       "414     5.451230\n",
       "415     6.856721\n",
       "Length: 416, dtype: float64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.maximum(gbe.predict(test),0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}