train_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T05:39:53.348806Z",
     "start_time": "2018-05-15T05:39:53.345777Z"
    }
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:40.937012Z",
     "start_time": "2018-05-15T07:25:29.984509Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "# from helpers import *\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn import metrics\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "sys.path.append(\".\")\n",
    "sys.path.append(\"..\")\n",
    "# Use the Azure Machine Learning data preparation package\n",
    "# from azureml.dataprep import package"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:40.945018Z",
     "start_time": "2018-05-15T07:25:40.939016Z"
    }
   },
   "outputs": [],
   "source": [
    "column_to_predict = \"ticket_type\"\n",
    "# Supported datasets:\n",
    "# ticket_type\n",
    "# business_service\n",
    "# category\n",
    "# impact\n",
    "# urgency\n",
    "# sub_category1\n",
    "# sub_category2\n",
    "\n",
    "classifier = \"NB\"  # Supported algorithms # \"SVM\" # \"NB\"\n",
    "use_grid_search = True  # grid search is used to find hyperparameters. Searching for hyperparameters is time consuming\n",
    "remove_stop_words = True  # removes stop words from processed text\n",
    "stop_words_lang = 'english'  # used with 'remove_stop_words' and defines language of stop words collection\n",
    "use_stemming = False  # word stemming using nltk\n",
    "fit_prior = True  # if use_stemming == True then it should be set to False ?? double check\n",
    "min_data_per_class = 1  # used to determine number of samples required for each class.Classes with less than that will be excluded from the dataset. default value is 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:41.218329Z",
     "start_time": "2018-05-15T07:25:40.947014Z"
    }
   },
   "outputs": [],
   "source": [
    "# loading dataset from dprep in Workbench    \n",
    "# dfTickets = package.run('AllTickets.dprep', dataflow_idx=0) \n",
    "\n",
    "# loading dataset from csv\n",
    "dfTickets = pd.read_csv(\n",
    "    './datasets/all_tickets.csv',\n",
    "    dtype=str\n",
    ")  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select 'TEXT' column and remove poorly represented classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:41.321758Z",
     "start_time": "2018-05-15T07:25:41.220101Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of dataset before removing classes with less then 1 rows: (48549, 9)\n",
      "Number of classes before removing classes with less then 1 rows: 2\n",
      "Shape of dataset after removing classes with less then 1 rows: (48549, 9)\n",
      "Number of classes after removing classes with less then 1 rows: 2\n"
     ]
    }
   ],
   "source": [
    "text_columns = \"body\"  # \"title\" - text columns used for TF-IDF\n",
    "\n",
    "# Removing rows related to classes represented by low amount of data\n",
    "print(\"Shape of dataset before removing classes with less then \" + str(min_data_per_class) + \" rows: \"+str(dfTickets.shape))\n",
    "print(\"Number of classes before removing classes with less then \" + str(min_data_per_class) + \" rows: \"+str(len(np.unique(dfTickets[column_to_predict]))))\n",
    "bytag = dfTickets.groupby(column_to_predict).aggregate(np.count_nonzero)\n",
    "tags = bytag[bytag.body > min_data_per_class].index\n",
    "dfTickets = dfTickets[dfTickets[column_to_predict].isin(tags)]\n",
    "print(\n",
    "    \"Shape of dataset after removing classes with less then \"\n",
    "    + str(min_data_per_class) + \" rows: \"\n",
    "    + str(dfTickets.shape)\n",
    ")\n",
    "print(\n",
    "    \"Number of classes after removing classes with less then \"\n",
    "    + str(min_data_per_class) + \" rows: \"\n",
    "    + str(len(np.unique(dfTickets[column_to_predict])))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data and labels and split them to train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:41.365785Z",
     "start_time": "2018-05-15T07:25:41.324755Z"
    }
   },
   "outputs": [],
   "source": [
    "labelData = dfTickets[column_to_predict]\n",
    "data = dfTickets[text_columns]\n",
    "\n",
    "# Split dataset into training and testing data\n",
    "train_data, test_data, train_labels, test_labels = train_test_split(\n",
    "    data, labelData, test_size=0.2\n",
    ")  # split data to train/test sets with 80:20 ratio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting features from text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:41.374133Z",
     "start_time": "2018-05-15T07:25:41.368126Z"
    }
   },
   "outputs": [],
   "source": [
    "class StemmedCountVectorizer(CountVectorizer):\n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(StemmedCountVectorizer, self).build_analyzer()\n",
    "        stemmer = SnowballStemmer(\"english\", ignore_stopwords=True)\n",
    "        return lambda doc: ([stemmer.stem(w) for w in analyzer(doc)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:42.972298Z",
     "start_time": "2018-05-15T07:25:41.377130Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38839, 11253)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count vectorizer\n",
    "if remove_stop_words:\n",
    "    count_vect = CountVectorizer(stop_words=stop_words_lang)\n",
    "elif use_stemming:\n",
    "    count_vect = StemmedCountVectorizer(stop_words=stop_words_lang)\n",
    "else:\n",
    "    count_vect = CountVectorizer()\n",
    "\n",
    "vectorized_data = count_vect.fit_transform(train_data)\n",
    "vectorized_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:43.026267Z",
     "start_time": "2018-05-15T07:25:42.975265Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38839, 11253)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf = TfidfTransformer()\n",
    "features = tfidf.fit_transform(vectorized_data)\n",
    "features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using pipeline to preprocess data and train classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:25:44.786016Z",
     "start_time": "2018-05-15T07:25:43.028264Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training NB classifier\n"
     ]
    }
   ],
   "source": [
    "# Fitting the training data into a data processing pipeline and eventually into the model itself\n",
    "if classifier == \"NB\":\n",
    "    print(\"Training NB classifier\")\n",
    "    # Building a pipeline: We can write less code and do all of the above, by building a pipeline as follows:\n",
    "    # The names ‘vect’ , ‘tfidf’ and ‘clf’ are arbitrary but will be used later.\n",
    "    # We will be using the 'text_clf' going forward.\n",
    "\n",
    "    text_clf = Pipeline([\n",
    "        ('vect', count_vect),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        ('clf', MultinomialNB(fit_prior=fit_prior))\n",
    "    ])\n",
    "    text_clf = text_clf.fit(train_data, train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use GridSearchCV to search for best set of parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:26:20.271405Z",
     "start_time": "2018-05-15T07:25:44.789019Z"
    }
   },
   "outputs": [],
   "source": [
    "if use_grid_search:\n",
    "    # Grid Search\n",
    "    # Here, we are creating a list of parameters for which we would like to do performance tuning.\n",
    "    # All the parameters name start with the classifier name (remember the arbitrary name we gave).\n",
    "    # E.g. vect__ngram_range; here we are telling to use unigram and bigrams and choose the one which is optimal.\n",
    "\n",
    "    # NB parameters\n",
    "    parameters = {\n",
    "        'vect__ngram_range': [(1, 1), (1, 2)],\n",
    "        'tfidf__use_idf': (True, False),\n",
    "        'clf__alpha': (1e-2, 1e-3)\n",
    "    }\n",
    "\n",
    "    # Next, we create an instance of the grid search by passing the classifier, parameters\n",
    "    # and n_jobs=-1 which tells to use multiple cores from user machine.\n",
    "    gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)\n",
    "    gs_clf = gs_clf.fit(train_data, train_labels)\n",
    "\n",
    "    # To see the best mean score and the params, run the following code\n",
    "    gs_clf.best_score_\n",
    "    gs_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:26:20.280680Z",
     "start_time": "2018-05-15T07:26:20.274677Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__alpha': 0.001, 'tfidf__use_idf': False, 'vect__ngram_range': (1, 1)}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:26:21.189731Z",
     "start_time": "2018-05-15T07:26:20.282676Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model\n",
      "Confusion matrix without GridSearch:\n",
      "[[2655  150]\n",
      " [  85 6820]]\n",
      "Mean without GridSearch: 0.975798146241\n",
      "Confusion matrix with GridSearch:\n",
      "[[2747   58]\n",
      " [  99 6806]]\n",
      "Mean with GridSearch: 0.983831101957\n"
     ]
    }
   ],
   "source": [
    "print(\"Evaluating model\")\n",
    "# Score and evaluate model on test data using model without hyperparameter tuning\n",
    "predicted = text_clf.predict(test_data)\n",
    "prediction_acc = np.mean(predicted == test_labels)\n",
    "print(\"Confusion matrix without GridSearch:\")\n",
    "print(metrics.confusion_matrix(test_labels, predicted))\n",
    "print(\"Mean without GridSearch: \" + str(prediction_acc))\n",
    "\n",
    "# Score and evaluate model on test data using model WITH hyperparameter tuning\n",
    "if use_grid_search:\n",
    "    predicted = gs_clf.predict(test_data)\n",
    "    prediction_acc = np.mean(predicted == test_labels)\n",
    "    print(\"Confusion matrix with GridSearch:\")\n",
    "    print(metrics.confusion_matrix(test_labels, predicted))\n",
    "    print(\"Mean with GridSearch: \" + str(prediction_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ploting confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:26:21.627104Z",
     "start_time": "2018-05-15T07:26:21.192694Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAEFCAYAAADqlvKRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEvRJREFUeJzt3Wl4VdXZxvH/SUICBFA0RSaRwfKgCAgyKijiUAZrUdAKaCsOr4IVraLFgqCt1lbU2qq1IrRqpVaFYkXBAURAlEnKIMoCEQQBQQRUICQhOf1wQszqm4QtZWeHk/t3XbnC3vucdZ7zgTtrD2utWDweR0TkgJSoCxCRikWhICIehYKIeBQKIuJRKIiIJy3qAkryTIPLdUvkCHL1F7OiLkEOQV7uplhJ+9VTEBGPQkFEPAoFEfEoFETEo1AQEY9CQUQ8CgUR8SgURMSjUBARj0JBRDwKBRHxKBRExKNQEBGPQkFEPAoFEfEoFETEo1AQEY9CQUQ8CgUR8SgURMSjUBARj0JBRDwKBRHxKBRExKNQEBGPQkFEPAoFEfEoFETEo1AQEY9CQUQ8CgUR8SgURMSjUBARj0JBRDwKBRHxKBRExKNQEBGPQkFEPAoFEfEoFETEo1AQEY9CQUQ8CgUR8SgURMSjUBARj0JBRDwKBRHxKBRExKNQEBGPQkFEPGlRF3Cka3LxGbQc0hvisD87h0V3/o2m/btyXOcWRa+pXrc22Vt3MfW8XxbtSz+qOn2m38P79z7HhlcXUa9bS067c2DR8dSq6RzVrB6v9BzFjhXry/MrVVo3DB3MkKGD2Ze9j49WrWHYsJEAPProfbRp05K9e/by9NPP89if/hpxpeFSKPwPajWrx2mjBvBqz1Fkb9tFgx5t6D7+JiZ3vLnoNZkNs+g55U7eufnP3nvPePh60mtVK9reMnclr5w/smj7rHHD2DBtkQKhnJx11ukMH34DXbv9kE2btjBoUD8ef/x+9u7Zy57de2jdujupqalMnjSBdes3Mm3ajKhLDk3opw9mlrSnKPk5ebx323iyt+0C4Mtl66j6vaNJqZJa9JouY6/mw3HT2blyQ9G+Vjf3Zeeqjez86LMS221y8RlkNsxi6dhJ4X4BKdKuXSveemsumzZtAWDKlGlc0OdcTmvfhmcnTqagoIC8vDymT59Jv4v7RFxtuEL5D2tmTc3sJTP7DPjEzDaY2atm1jyMz4vKns+2s2nm0qLt9mMG8dmbSyjIyweg/tmtyWyQxaoJrxe9pl63U6jbuQXLSvkPn1IllXYjLmHxmGeJ5xeE+wWkyMKF/6Z79zNo1KgBAFf+9MdkZGSwYP77XD6oH2lpaWRmVueii/pQt26diKsNV1h/xccD9znnGjrnGjvnGgG/BpLyZCytWgZnPnEjNZscx7vDxxftP/naXnzw6MvEC+IAZNY/lvZjBjH3xseL9v23E/p05JtPv2DbotXlUrskzJu3kHvueYhJL05g/nvTKCiI8+WXOxlxx73E43EWLXqdyZP+woyZc8jNy4u63FCFFQpVnXMLiu9wzs0P6bMilVn/WHq+PJp4fgFvXHIveV/vBSDjmJpktW3Gp1MXFr32hB92JK1aOudOvJ0L3riXY9s04bRRA2h+RY+i1zS+sDMfvzCn3L9HZVejRiZz5s6nY6eedO7Sm5envl64vzoj7riXtm3PoWevy4jFYqz9eH20xYYsrAuNy8zsL8BrwFdATaA3sDykz4tEWmZVzp80krUvzmX576d4x+p0aM6Xyz5hf3ZO0b4Pn5jOh09ML9o+/8WRrHrqDTa8uujb93VqwYKRT4dfvHjq16/L6689T+s23fnmm93cMWIYzz//EtdeewW1atbgpptHUadOFlcNHsCgQUOiLjdUYYXCUKAv0BWoBXwNvAJMKetNR5oWg88js2EWjXq1p1Gv9kX73/zxfdRqUpfdG7d/p/YyjqlJlcwM9m7ZcbhLlYNYvXotY8c+yrx5r5CSksK78xYy7KZRpKWl8tRTf+Tf/55JLBbj7l89wOL3l0Vdbqhi8XjJ57ZReqbB5RWvKCnV1V/MiroEOQR5uZtiJe1P2tuFInJoFAoi4lEoiIhHoSAiHoWCiHgUCiLiUSiIiEehICIehYKIeBQKIuJRKIiIR6EgIh6Fgoh4FAoi4lEoiIin1ElWzOzMst7onNOcYSJJqKyZl+4u41gc6FHGcRE5QpUaCs65s8uzEBGpGA46R6OZnUBiyvbGQDfg78BVzrn1oVYmIpEIcqHxCWAssBvYCjwHPBNmUSISnSChkOWcewPAORd3zj1JYoZmEUlCQUIh28wakri4iJl1BXLKfouIHKmCrPvwcxJrNjQzs6XAMcAloVYlIpE5aCg45xabWQegOYmehXPO5YZemYhE4qCnD2Z2FPBbEhcXnwTuNLPqYRcmItEIck1hApAPXAlcR2JdyHEh1iQiEQpyTeFE51z/Yts3m1lSLRQrIt8K0lNwZnb6gQ0zawOsCa8kEYlSWQOi1pG4DVkN6G9mq0icRpyEQkEkaZV1+tC9vIoQkYqjrAFRnwKYWQbQG6gBxIBUoAkwujwKFJHyFeRC43NAbeBEYC5wNvBOmEWJSHSCXGhsTWLuhCnA/cAZJEZMikgSChIK25xzcWAV0No59wmQHm5ZIhKVIKcPH5jZI8DjwEQzq0/i2oKIJKEgPYUhwAvOuQ9JXFysBwwMtSoRiUzgiVsLt78CJpMYKSkiSUgTt4qIRxO3iohHi8GIiEehICIehYKIeMq6+zCLwslaS+Kc04VGkSRU1t2Huwp/XwtkA08D+4EBJIZTi0gSKuvuw2wAM3vAOdeh2KH5ZrY49MpEJBJBrilUM7PmBzbMrBVQJbySRCRKQcY+3AK8bWabSIRIHRKnECKShIKs+/CGmTUGWpG48LjcObc/7MJEJBpB1n2oDTxGYpHZ9cC4wn0ikoSCnD48CbwBdCSx8vQW4FmgT1hFXfXFrLCalhBkb54bdQlyGAW50NjEOTcOKHDO5TrnRgINQ65LRCISJBT2Fy4dd2DV6e8DBaFWJSKRCXL6MAZ4G2hkZi8BXYCrwixKRKIT5O7Da4UPK3UiMb37dcDOsAsTkWgcNBTM7D3nXBfg1cLtFGAZiVuUIpJkyhoQ9RaFq0SZWQHfDo7KB14OvTIRiURZYx96AJjZH5xzN5VfSSISpSB3H8ab2T8AzOwkM5tjZhZyXSISkSCh8CSJYdM45z4Cfg1MCLMoEYlOkFDIdM5NP7DhnHsTyAyvJBGJUpDnFLaZ2fUkHm0GuAzYGl5JIhKlID2FwcAFJMY8bCAx5uGaMIsSkejE4vFSp2GMTFp6g4pXlJRKA6KOTFWympa4JmxZzym84py7wMzWUcIErs65poexPhGpIMq6pnBt4e/u5VCHiFQQZYXCeQd5HOGZw1yLiFQAZYXCgbUkmwEnAtNITPHeE1iJQkEkKZX1mPNgKFoUprVzbnvhdm3gpfIpT0TKW5BbkvWBHcW29wD1wilHRKIW5OGlV4E3zeyfQAy4FHg+1KpEJDKBnlMws34k7kLEgRnOuVCHTus5hSOLnlM4MpX2nELQVac/J3Fx8TZg1+EqSkQqniDrPtwE3ENipahM4AkzGx52YSISjSA9hSuBHwB7nHM7gA5o4laRpBUkFPKdc7nFtveRmJJNRJJQkFCYbWYPAJlm1pfE/Iwzwy1LRKISJBRuA9aQmMH5JySebNQ1BZEkFeQ5henOuR8AT4RdjIhEL0hPobqZHR96JSJSIQTpKWQB681sG5BN4qnGuOZTEElOQUKhZ+hViEiFESQUNgM3AD1IDJ2ehqZ4F0laQUJhPFCNxPoPKSTuQJwC3BxiXSISkSCh0Mk51+LAhplNBT4IryQRiVKQuw/rzOzEYtvHAZtCqkdEIhakp1AFWGZmc0hcU+gKbClclbpoIVoRSQ5BQuHX/7X9QBiFiEjFcNBQcM7NLo9CRKRiCDrJiohUEgoFEfEoFETEo1AQEY9CQUQ8CgUR8SgURMSjUBARj0JBRDwKBRHxKBRExBNkQJQcgrG/G02/fhewY2di6c3Vq9cycNAQRt95C5dcciH5+fksWbKCIUN/QU5OTsTVVh6r167jN79/nN2795CSksqY22+kZYvv89iEZ3lt5hxSU1I42U5kzO3DyMhIZ8fOXfzyngfZ/PlWUmIpjPnFMNq2OrnMto506imEpEuX9gy6fCjtO5xP+w7nM3DQEM46swuXXvojOnTsyaltz6FWrRr87IbBUZdaaWTv28f//XwkVw3sz6SnHuP6wQMYcff9LFyynNdmzObFvz7ClL89zu69e5k4KbGw+r0P/Yl2rVvy8sRx/Hb0bdw66jdk79tXalvJQD2FEKSnp3PqqS0ZPnwITZuewJo1n3Dr8LtITU2lakYG1apVJT8/n6oZGezbp15CeXl34RKOb1CPM0/vCMDZXTvToF5ddu7aRU5uLjk5uaSmpJCbm0dGehX2789n9ryFjLxlKAAtmjej0fH1eWf++6SkxEpsKxkoFEJQv/5xzJo1j9Fj7mflSsett1zPPyf/lQ4df8CMmXNYt3Yhubm5rF79CeOefDbqciuNTzduIuuY2tx53+9xa9ZRq2Ymtwy9ms7t29KlQzvOvfgnVElLo0mjhlzatze7vvqKgngBx9Q+uqiN476XxdYvtpObm1tiW8lApw8hWL9+Iz/80U9YudIB8OBDf6ZZ0xMYfOVlNGnciIaN2tKwUTvWrd/A2PtHR1xt5ZG3fz9z31vMJRf24oW//JGB/S9kyPDRvPDSNDZt+Zy3/zWRt1+eSIP6dRn7yJMUFMSJEfMbicdJTUkpta3c3NySP/wIEkoomNksM3v3v37eM7N3w/i8iqZVq5MYNKifty8Wi3HZj/vy9+f+ye7de8jNzWX8+Il07356RFVWPnWyjqVp4+Np3TIxD3GPbl0oKMjn9bfm0Of8s8nMrE56ejr9L+zFwiXLOab20cSJ89XX3xS1sW37Do6rk1VqWxs3fx7JdzucwuopjABqAFcAAwp/Liv8nfQKCgp4+KFf0bhxYrW966/7KStWfMR78xdz0UW9SU1NBaBv314sWLAkylIrlW6d2/PZ5s9ZuWoNAIuXriBGjJPt+8yYPY/9+/OJx+PMmD2P1i1bkJaWypldOvLiv6YD4D5ex9r1G+jQtnWpbTVMgusKsXg8HkrDZnYb8LFzbsp3fW9aeoNwiipHAwdezO233UBqaiqbPtvCtdfdyrZtX/LA2DGcc043cnJyWL78Q24cNpKvi/0lOhJlb54bdQmBLV66ggcfm0B29j7S06sw4qbraNmiOfc/Mo75i5eSXqUKzU9swqhbb6BmjUy279jJmN8+zKbNW4nFYgz/2TWc0em0Uttq1+aUiL9hcFWymsZK2h9aKPwvkiEUKpMjKRTkW6WFgi40iohHoSAiHoWCiHgUCiLiUSiIiEehICIehYKIeBQKIuJRKIiIR6EgIh6Fgoh4FAoi4lEoiIhHoSAiHoWCiHgUCiLiUSiIiEehICIehYKIeBQKIuJRKIiIR6EgIh6Fgoh4FAoi4lEoiIhHoSAiHoWCiHgUCiLiUSiIiEehICIehYKIeBQKIuJRKIiIR6EgIh6Fgoh4FAoi4lEoiIhHoSAiHoWCiHgUCiLiUSiIiEehICIehYKIeBQKIuJRKIiIR6EgIh6Fgoh4FAoi4lEoiIhHoSAinlg8Ho+6BhGpQNRTEBGPQkFEPAoFEfEoFETEo1AQEY9CQUQ8CgUR8aRFXUBlYGYpwJ+ANkAOcI1z7uNoq5IgzKwT8DvnXPeoaykv6imUj75AVedcF2AE8GDE9UgAZnY7MB6oGnUt5UmhUD66Aq8BOOfmA+2jLUcCWgtcHHUR5U2hUD5qAV8V2843M526VXDOuclAXtR1lDeFQvn4GqhZbDvFObc/qmJEyqJQKB/zgN4AZtYZWBFtOSKlUxe2fEwBzjOzd4EYMDjiekRKpaHTIuLR6YOIeBQKIuJRKIiIR6EgIh6Fgoh4FAqViJkdZWZTQmi3sZmtP8hr7jKzuw5nmxIOhULlUhtoG3URUrHp4aXK5Y9A/cLews9JDNLaDmQDE4HuzrkrAczsbeAu59zbZjYCuBRIBV4HfuGcK/EBFzM7BXgEqAHUAe5zzv258HBHM1tQeGycc+4Phe/5f+0f5u8t34F6CpXLMGCzc+6iwm0DLnfOnVfaG8ysJ3Aa0IFEL6MBMKiMz7gGuMc51wE4Gxhb7Fg9oAfQBfiZmZ16CO1LyNRTqNy2OefWH+Q15wKdgPcLt6sBG8p4/a1ATzO7A2hFoldwwD+cc3sAzGwqcBZwfCntvxP8a8jhpFCo3LKL/TtOYlzGAVUKf6cCDzvnHgIws6OBskZ4vgDsBKYC/wAGFDtW/H0pJIYll9Z+1nf9MnJ46PShctlP6X8ItgMnmVnMzJoArQv3vwVcYWY1CueAeAnoX8ZnnAeMds79C+gFYGaphcf6m1mGmdUGLgBmHUL7EjKFQuWyFdhgZrNKODYD2Ag44A8Udt+dc1OBycAC4ANgKfB0GZ9xF/COmX0IdAPWA00Kj31KYhj5O8BvnHMfHUL7EjKNkhQRj3oKIuJRKIiIR6EgIh6Fgoh4FAoi4lEoiIhHoSAinv8A/WEdUvxKzwwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b1eb529ef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Ploting confusion matrix with 'seaborn' module\n",
    "# Use below line only with Jupyter Notebook\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "mat = confusion_matrix(test_labels, predicted)\n",
    "plt.figure(figsize=(4, 4))\n",
    "sns.set()\n",
    "sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False,\n",
    "            xticklabels=np.unique(test_labels),\n",
    "            yticklabels=np.unique(test_labels))\n",
    "plt.xlabel('true label')\n",
    "plt.ylabel('predicted label')\n",
    "# Save confusion matrix to outputs in Workbench\n",
    "# plt.savefig(os.path.join('.', 'outputs', 'confusion_matrix.png'))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Printing classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-15T07:26:21.684170Z",
     "start_time": "2018-05-15T07:26:21.631088Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      0.98      0.97      2805\n",
      "          1       0.99      0.99      0.99      6905\n",
      "\n",
      "avg / total       0.98      0.98      0.98      9710\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(test_labels, predicted,\n",
    "                            target_names=np.unique(test_labels)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}