xgb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns; sns.set()\n",
    "import hyperopt\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/train.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "import time\n",
    "\n",
    "hdays = [dt.datetime(2014, 1, day) for day in range(1,9)]\n",
    "hdays = hdays + [dt.datetime(2014, 2, 23), \n",
    "              dt.datetime(2014, 3, 8),\n",
    "              dt.datetime(2014, 3, 9),\n",
    "              dt.datetime(2014, 3, 10),\n",
    "              dt.datetime(2014, 5, 1), \n",
    "              dt.datetime(2014, 5, 2),\n",
    "              dt.datetime(2014, 5, 3),\n",
    "              dt.datetime(2014, 5, 4),\n",
    "              dt.datetime(2014, 5, 9),\n",
    "              dt.datetime(2014, 5, 10),\n",
    "              dt.datetime(2014, 7, 12),\n",
    "              dt.datetime(2014, 7, 13),\n",
    "              dt.datetime(2014, 7, 14),\n",
    "              dt.datetime(2014, 7, 15),\n",
    "              dt.datetime(2014, 11, 3),\n",
    "              dt.datetime(2014, 11, 4)]\n",
    "\n",
    "hdays = [hd.timetuple() for hd in hdays]\n",
    "\n",
    "def check_hday(day, hday):\n",
    "    if day.tm_mon == hday.tm_mon and day.tm_mday == hday.tm_mday:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def map_holidays(date):\n",
    "    tdate = time.strptime(date[:-4], '%Y-%m-%d %H:%M:%S')\n",
    "    hits = [check_hday(tdate, hday) for hday in hdays]\n",
    "    if True in hits:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0 \n",
    "        \n",
    "zz = map(map_holidays, train['due'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((1793300, 2), (1793300,), (1793300, 8))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "\n",
    "vectorizer = DictVectorizer(sparse=False, dtype=np.bool)\n",
    "\n",
    "x_col = [ 'dist','due', 'lat','lon','f_class','s_class','t_class',]\n",
    "X_raw = train[x_col]\n",
    "y = train['burned'].values\n",
    "\n",
    "X_raw.loc[:, 'weekday'] = X_raw['due'].astype('datetime64').map(lambda x: x.weekday())\n",
    "\n",
    "#data_dict = [ {'f_class':f,'s_class':s,'t_class':t}\n",
    "#           for f,s,t in X_raw[['f_class','s_class','t_class']].values ]\n",
    "\n",
    "Xwek = pd.get_dummies(X_raw.weekday).values\n",
    "\n",
    "#Xcat = vectorizer.fit_transform(data_dict)\n",
    "\n",
    "def has_class(train, class_name):\n",
    "    return (train.f_class == class_name) | (train.s_class == class_name) | (train.t_class == class_name)\n",
    "econom = has_class(X_raw, 'econom')\n",
    "business = has_class(X_raw, 'business')\n",
    "vip = has_class(X_raw, 'vip')\n",
    "\n",
    "kmn = KMeans(n_clusters=100)\n",
    "Xkf = kmn.fit_predict(train[['lat', 'lon']].values)\n",
    "\n",
    "conc = econom.map(lambda x: str(x)) + business.map(lambda x: str(x)) + vip.map(lambda x: str(x))\n",
    "\n",
    "Xcon = pd.get_dummies(conc).values\n",
    "\n",
    "Xhol = np.array(zz).reshape(-1, 1)\n",
    "\n",
    "Xhour = pd.get_dummies(X_raw['due'].astype('datetime64').map(lambda x: x.hour)).values\n",
    "\n",
    "real_features = [\"lat\", \"lon\"]\n",
    "Xreal = X_raw[real_features].values\n",
    "Xdist = np.log(X_raw.dist+2).values\n",
    "print(Xreal.shape, Xdist.shape, Xcon.shape)\n",
    "Xfull = np.hstack((Xreal, Xdist.reshape(-1, 1), Xcon, Xwek, Xhol, Xhour, Xkf))\n",
    "\n",
    "wft = ['w'+str(i) for i in range(7)]\n",
    "conc_f = ['c'+str(i) for i in range(Xcon.shape[1])]\n",
    "hor_f = ['h'+str(i) for i in range(Xhour.shape[1])]\n",
    "\n",
    "X_new = pd.DataFrame(Xfull, columns=real_features+['dist']+conc_f+wft+['holiday']+hor_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mask = y.astype('int').astype('str') + X_raw.weekday.astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(mask, n_folds=5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from sklearn.metrics import roc_auc_score,accuracy_score,precision_score,recall_score\n",
    "from sklearn.metrics import roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.585875083734\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-30-b6468ad9f1c3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mRandomForestClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m250\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt2/local/lib/python2.7/site-packages/sklearn/ensemble/forest.pyc\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    288\u001b[0m                     \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrees\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    289\u001b[0m                     verbose=self.verbose, class_weight=self.class_weight)\n\u001b[1;32m--> 290\u001b[1;33m                 for i, t in enumerate(trees))\n\u001b[0m\u001b[0;32m    291\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m             \u001b[1;31m# Collect newly grown trees\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt2/local/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    808\u001b[0m                 \u001b[1;31m# consumption.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    809\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 810\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    811\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    812\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt2/local/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    725\u001b[0m                 \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    726\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 727\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    728\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mexception\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    729\u001b[0m                 \u001b[1;31m# Stop dispatching any new job in the async callback thread\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    550\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    551\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 552\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    553\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    554\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    545\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    546\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 547\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    548\u001b[0m         \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/threading.pyc\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    337\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    338\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 339\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    340\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0m__debug__\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_note\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"%s.wait(): got it\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for train_index, test_index in skf:\n",
    "    X_train, X_test = X_new.iloc[train_index], X_new.iloc[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    \n",
    "    model = RandomForestClassifier(n_estimators=250,n_jobs=-1)\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    preds = model.predict_proba(X_test)[:, 1]\n",
    "    print(roc_auc_score(y_test, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import prettyprint as pp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv('data/test.csv', index_col=0)\n",
    "t_col = [ 'dist','due', 'lat','lon','f_class','s_class','t_class',]\n",
    "t_raw = test[t_col]\n",
    "#y = train['burned'].values\n",
    "\n",
    "t_raw.loc[:, 'weekday'] = t_raw['due'].astype('datetime64').map(lambda x: x.weekday())\n",
    "\n",
    "#data_dict = [ {'f_class':f,'s_class':s,'t_class':t}\n",
    "#           for f,s,t in t_raw[['f_class','s_class','t_class']].values ]\n",
    "\n",
    "twek = pd.get_dummies(t_raw.weekday).values\n",
    "\n",
    "#tcat = vectorizer.transform(data_dict)\n",
    "\n",
    "def has_class(train, class_name):\n",
    "    return (train.f_class == class_name) | (train.s_class == class_name) | (train.t_class == class_name)\n",
    "econom = has_class(t_raw, 'econom')\n",
    "business = has_class(t_raw, 'business')\n",
    "vip = has_class(t_raw, 'vip')\n",
    "\n",
    "conc = econom.map(lambda x: str(x)) + business.map(lambda x: str(x)) + vip.map(lambda x: str(x))\n",
    "\n",
    "tcon = pd.get_dummies(conc).values\n",
    "\n",
    "thol = np.array(map(map_holidays, t_raw['due'])).reshape(-1, 1)\n",
    "\n",
    "thour = pd.get_dummies(t_raw['due'].astype('datetime64').map(lambda x: x.hour)).values\n",
    "\n",
    "real_features = [\"lat\", \"lon\"]\n",
    "treal = t_raw[real_features].values\n",
    "tdist = np.log(t_raw.dist+2).values\n",
    "tfull = np.hstack((treal, tdist.reshape(-1, 1), tcon, twek, thol, thour))\n",
    "\n",
    "wft = ['w'+str(i) for i in range(7)]\n",
    "conc_f = ['c'+str(i) for i in range(tcon.shape[1])]\n",
    "hor_f = ['h'+str(i) for i in range(thour.shape[1])]\n",
    "\n",
    "t_new = pd.DataFrame(tfull, columns=real_features+['dist']+conc_f+wft+['holiday']+hor_f)\n",
    "\n",
    "dtest = xgb.DMatrix(t_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-42-d8abdd2f3c0a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m     gbm = xgb.train(params, dtrain, num_boost_round=5000, maximize=True,\n\u001b[1;32m---> 28\u001b[1;33m           evals=watchlist, early_stopping_rounds=100, verbose_eval=True)\n\u001b[0m\u001b[0;32m     29\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m     \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgbm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mntree_limit\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgbm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_iteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt2/local/lib/python2.7/site-packages/xgboost-0.4-py2.7.egg/xgboost/training.pyc\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, learning_rates, xgb_model)\u001b[0m\n\u001b[0;32m    175\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    176\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m  \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 177\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    178\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    179\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt2/local/lib/python2.7/site-packages/xgboost-0.4-py2.7.egg/xgboost/core.pyc\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m    748\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    749\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 750\u001b[1;33m             \u001b[0m_check_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_LIB\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXGBoosterUpdateOneIter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    751\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    752\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "preds_all = np.zeros((test.shape[0], 5))\n",
    "o = 0\n",
    "for train_index, test_index in skf:\n",
    "    X_train, X_test = X_new.iloc[train_index], X_new.iloc[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    dtrain = xgb.DMatrix(X_train, y_train)\n",
    "    dval = xgb.DMatrix(X_test, y_test)\n",
    "\n",
    "    watchlist = ((dtrain, 'train'), (dval, 'validation'))\n",
    "\n",
    "    params = {\n",
    "        'nthread': 12,\n",
    "        'eval_metric': 'auc',\n",
    "        'eta': .2,\n",
    "        'max_depth': 5,\n",
    "        'min_child_weight': 2,\n",
    "        'subsample': .8,\n",
    "        'colsample_bytree': .8,\n",
    "        'objective': 'binary:logistic',\n",
    "        'seed': 42, \n",
    "        'silent': 1\n",
    "    }\n",
    "\n",
    "    #pp.pp(params)\n",
    "\n",
    "    gbm = xgb.train(params, dtrain, num_boost_round=5000, maximize=True,\n",
    "          evals=watchlist, early_stopping_rounds=100, verbose_eval=True)\n",
    "\n",
    "    preds = gbm.predict(dtest, ntree_limit=gbm.best_iteration)\n",
    "    preds_all[:, o] = preds\n",
    "    o += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pres = preds_all[:, :3].mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(743463,)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pres.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"colsample_bytree\": 0.8, \n",
      "    \"eta\": 0.2, \n",
      "    \"eval_metric\": \"auc\", \n",
      "    \"max_depth\": 5, \n",
      "    \"min_child_weight\": 2, \n",
      "    \"nthread\": 12, \n",
      "    \"objective\": \"binary:logistic\", \n",
      "    \"seed\": 42, \n",
      "    \"silent\": 1, \n",
      "    \"subsample\": 0.8\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_new.iloc[train_index], X_new.iloc[test_index]\n",
    "y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "dtrain = xgb.DMatrix(X_train, y_train)\n",
    "dval = xgb.DMatrix(X_test, y_test)\n",
    "\n",
    "watchlist = ((dtrain, 'train'), (dval, 'validation'))\n",
    "\n",
    "params = {\n",
    "    'nthread': 12,\n",
    "    'eval_metric': 'auc',\n",
    "    'eta': .2,\n",
    "    'max_depth': 5,\n",
    "    'min_child_weight': 2,\n",
    "    'subsample': .8,\n",
    "    'colsample_bytree': .8,\n",
    "    'objective': 'binary:logistic',\n",
    "    'seed': 42, \n",
    "    'silent': 1\n",
    "}\n",
    "\n",
    "pp.pp(params)\n",
    "\n",
    "gbm = xgb.train(params, dtrain, num_boost_round=5000, maximize=True,\n",
    "      evals=watchlist, early_stopping_rounds=300, verbose_eval=True)\n",
    "\n",
    "preds = gbm.predict(dval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = gbm.predict(dval, ntree_limit=gbm.best_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.66759808515858143"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((743463, 2), (743463,), (743463, 12))\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv('data/test.csv', index_col=0)\n",
    "t_col = [ 'dist','due', 'lat','lon','f_class','s_class','t_class',]\n",
    "t_raw = test[t_col]\n",
    "#y = train['burned'].values\n",
    "\n",
    "t_raw.loc[:, 'weekday'] = t_raw['due'].astype('datetime64').map(lambda x: x.weekday())\n",
    "\n",
    "data_dict = [ {'f_class':f,'s_class':s,'t_class':t}\n",
    "           for f,s,t in t_raw[['f_class','s_class','t_class']].values ]\n",
    "\n",
    "twek = pd.get_dummies(t_raw.weekday).values\n",
    "\n",
    "tcat = vectorizer.transform(data_dict)\n",
    "\n",
    "real_features = [\"lat\", \"lon\"]\n",
    "treal = t_raw[real_features].values\n",
    "tdist = np.log(t_raw.dist+2).values\n",
    "print(treal.shape, tdist.shape, tcat.shape)\n",
    "tfull = np.hstack((treal, tdist.reshape(-1, 1), tcat, twek))\n",
    "\n",
    "wft = ['w'+str(i) for i in range(7)]\n",
    "\n",
    "t_new = pd.DataFrame(tfull, columns=real_features+['dist']+vectorizer.feature_names_+wft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dtest = xgb.DMatrix(t_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = gbm.predict(dtest, ntree_limit=gbm.best_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "res = pd.DataFrame(data=preds, index=test.index, columns=['Y_prob'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res.to_csv('res0.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def objective(space):\n",
    "    res = []\n",
    "    \n",
    "    params = {\n",
    "        'nthread': 12,\n",
    "        'eval_metric': 'auc',\n",
    "        'eta': .2,\n",
    "        'max_depth': int(space['max_depth']),\n",
    "        'min_child_weight': space['min_child_weight'],\n",
    "        'subsample': space['subsample'],\n",
    "        'colsample_bytree': space['colsample_bytree'],\n",
    "        'objective': 'binary:logistic',\n",
    "        'seed': 42, \n",
    "        'silent': 1\n",
    "    }\n",
    "\n",
    "    pp.pp(params)\n",
    "        \n",
    "    for train_index, test_index in skf:\n",
    "        X_train, X_test = X_new.iloc[train_index], X_new.iloc[test_index]\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "        dtrain = xgb.DMatrix(X_train, y_train)\n",
    "        dval = xgb.DMatrix(X_test, y_test)\n",
    "\n",
    "        watchlist = ((dtrain, 'train'), (dval, 'validation'))\n",
    "\n",
    "        \n",
    "\n",
    "        gbm = xgb.train(params, dtrain, num_boost_round=5000, maximize=True,\n",
    "              evals=watchlist, early_stopping_rounds=100, verbose_eval=True)\n",
    "\n",
    "        preds = gbm.predict(dval, ntree_limit=gbm.best_iteration)\n",
    "        \n",
    "        res.append(roc_auc_score(y_test, preds))\n",
    "        \n",
    "    print(np.mean(res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from hyperopt import hp, fmin, tpe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "space = {'max_depth': hp.quniform('max_depth', 3, 10, 1),\n",
    "         'min_child_weight': hp.quniform('min_child_weight', 1, 5, 1),\n",
    "         'subsample': hp.uniform('subsample', .5, 1.),\n",
    "         'colsample_bytree': hp.uniform('colsample_bytree', .5, 1.)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"colsample_bytree\": 0.7756573845414456, \n",
      "    \"eta\": 0.2, \n",
      "    \"eval_metric\": \"auc\", \n",
      "    \"max_depth\": 5, \n",
      "    \"min_child_weight\": 2.0, \n",
      "    \"nthread\": 12, \n",
      "    \"objective\": \"binary:logistic\", \n",
      "    \"seed\": 42, \n",
      "    \"silent\": 1, \n",
      "    \"subsample\": 0.8482345927989308\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "best = fmin(objective, space, algo=tpe.suggest, max_evals=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res = pd.DataFrame(data=pres, index=test.index, columns=['Y_prob'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "res.to_csv('res1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}