Performance metric.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute performance metrics for the given Y and Y_score without sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "# other than these two you should not import any other packages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "<font color='red'><b>A.</b></font> Compute performance metrics for the given data <strong>5_a.csv</strong>\n",
    "   <b>Note 1:</b> in this data you can see number of positive points >> number of negatives points\n",
    "   <b>Note 2:</b> use pandas or numpy to read the data from <b>5_a.csv</b>\n",
    "   <b>Note 3:</b> you need to derive the class labels from given score</pre> $y^{pred}= \\text{[0 if y_score < 0.5 else 1]}$\n",
    "\n",
    "<pre>\n",
    "<ol>\n",
    "<li> Compute Confusion Matrix </li>\n",
    "<li> Compute F1 Score </li>\n",
    "<li> Compute AUC Score</li>\n",
    "<li> Compute Accuracy Score </li>\n",
    "</ol>\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10100, 2)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv(\"5_a.csv\")\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(df,y,thresh_hold):\n",
    "    y_pred=[]\n",
    "    for label in df[y]:\n",
    "        if label<thresh_hold:\n",
    "            y_pred.append(0)\n",
    "        else:\n",
    "            y_pred.append(1)\n",
    "    return y_pred\n",
    "    \n",
    "    \n",
    "# confusion matrix\n",
    "def cal_vals(df,y,y_pred):\n",
    "    tp=0\n",
    "    tn=0\n",
    "    fn=0\n",
    "    fp=0\n",
    "    for val1,val2 in enumerate(df['y']):\n",
    "        if(df.y_pred[val1]==1) and df.y[val1]==1:\n",
    "            tp=tp+1\n",
    "        if(df.y_pred[val1]==0) and df.y[val1]==0:\n",
    "            tn=tn+1\n",
    "        if(df.y_pred[val1]==0) and df.y[val1]==1:\n",
    "            fn=fn+1\n",
    "        if(df.y_pred[val1]==1) and df.y[val1]==0:\n",
    "            fp=fp+1\n",
    "    return {'tn':tn,'tp':tp,'fn':fn,'fp':fp}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh_hold=0.5\n",
    "data['y_pred']=predict(data,'proba',thresh_hold)\n",
    "confusion_matrix=cal_vals(data,'y','y_pred')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the confusion matrix is:  {'tn': 0, 'tp': 10000, 'fn': 0, 'fp': 100}\n"
     ]
    }
   ],
   "source": [
    "# confusion matrix values \n",
    "print(\"the confusion matrix is: \",confusion_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the F1 score is:  0.9950248756218906\n"
     ]
    }
   ],
   "source": [
    "# F1 score\n",
    "x=data.y.value_counts()\n",
    "P=x[1]\n",
    "\n",
    "precision=confusion_matrix['tp']/(confusion_matrix['tp']+confusion_matrix['fp'])\n",
    "recall=confusion_matrix['tp']/P\n",
    "\n",
    "F1=2*precision*recall/(precision+recall)\n",
    "print('the F1 score is: ',F1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the accuracy is:  0.9900990099009901\n"
     ]
    }
   ],
   "source": [
    "# Accuracy\n",
    "Acc=(confusion_matrix['tp']+confusion_matrix['tn'])/data.shape[0]\n",
    "print('the accuracy is: ',Acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AUC score funtion\n",
    "from tqdm import tqdm_notebook      # purpose of import is to just see progress\n",
    "def auc(df):\n",
    "    s = df['y'].value_counts()\n",
    "    P = s[1]\n",
    "    N = s[0]\n",
    "    tpr = []\n",
    "    fpr = []\n",
    "    for elem in tqdm_notebook(df['proba']):\n",
    "        df['y_pred']=predict(df,'proba',elem)\n",
    "        confusion_matrix=cal_vals(df,'y','y_pred')\n",
    "        tpr.append(confusion_matrix['tp']/P)\n",
    "        fpr.append(confusion_matrix['fp']/N)\n",
    "        df.drop(columns=['y_pred'])\n",
    "    return np.trapz(tpr,fpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y</th>\n",
       "      <th>proba</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1664</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.899965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2099</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.899828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1028</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.899825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9592</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.899812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8324</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.899768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8294</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1630</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7421</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>805</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5012</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        y     proba\n",
       "1664  1.0  0.899965\n",
       "2099  1.0  0.899828\n",
       "1028  1.0  0.899825\n",
       "9592  1.0  0.899812\n",
       "8324  1.0  0.899768\n",
       "...   ...       ...\n",
       "8294  1.0  0.500081\n",
       "1630  1.0  0.500058\n",
       "7421  1.0  0.500058\n",
       "805   1.0  0.500047\n",
       "5012  1.0  0.500019\n",
       "\n",
       "[10100 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=data.sort_values(by='proba',ascending=False)\n",
    "data.drop(columns=['y_pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a822bfb4f9fd4bde883090ba30c24492",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10100), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "the AUC Score is : 0.48829900000000004\n"
     ]
    }
   ],
   "source": [
    "AUC_score=auc(data)\n",
    "print ('the AUC Score is :',AUC_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "<font color='red'><b>B.</b></font> Compute performance metrics for the given data <strong>5_b.csv</strong>\n",
    "   <b>Note 1:</b> in this data you can see number of positive points << number of negatives points\n",
    "   <b>Note 2:</b> use pandas or numpy to read the data from <b>5_b.csv</b>\n",
    "   <b>Note 3:</b> you need to derive the class labels from given score</pre> $y^{pred}= \\text{[0 if y_score < 0.5 else 1]}$\n",
    "\n",
    "<pre>\n",
    "<ol>\n",
    "<li> Compute Confusion Matrix </li>\n",
    "<li> Compute F1 Score </li>\n",
    "<li> Compute AUC Score </li>\n",
    "</ol>\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10100, 2)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_B=pd.read_csv('5_b.csv')\n",
    "data_B.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh_hold=0.5\n",
    "data_B['y_pred']=predict(data_B,'proba',thresh_hold)\n",
    "confusion_matrix_B=cal_vals(data_B,'y','y_pred')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the confusion matrix is : {'tn': 9761, 'tp': 55, 'fn': 45, 'fp': 239}\n"
     ]
    }
   ],
   "source": [
    "# confusion matrix values \n",
    "print('the confusion matrix is :', confusion_matrix_B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the F1 Score is :  0.2791878172588833\n"
     ]
    }
   ],
   "source": [
    "# F1 score\n",
    "x=data_B.y.value_counts()\n",
    "P=x[1]\n",
    "\n",
    "precision_B=confusion_matrix_B['tp']/(confusion_matrix_B['tp']+confusion_matrix_B['fp'])\n",
    "recall_B=confusion_matrix_B['tp']/P\n",
    "\n",
    "F1_B=2*precision_B*recall_B/(precision_B+recall_B)\n",
    "print('the F1 Score is : ',F1_B)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the Accuracy is : 0.9718811881188119\n"
     ]
    }
   ],
   "source": [
    "# Accuracy\n",
    "Acc_B=(confusion_matrix_B['tp']+confusion_matrix_B['tn'])/data_B.shape[0]\n",
    "print('the Accuracy is :',Acc_B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aca8c027884f4023bfc0b5ae75d820ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10100), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "the AUC Score is:  0.9377570000000001\n"
     ]
    }
   ],
   "source": [
    "#AUC score\n",
    "data_B=data_B.sort_values(by='proba',ascending=False)\n",
    "data_B.drop(columns=['y_pred'])\n",
    "AUC_score_B=auc(data_B)\n",
    "print('the AUC Score is: ',AUC_score_B)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color='red'><b>C.</b></font> Compute the best threshold (similarly to ROC curve computation) of probability which gives lowest values of metric <b>A</b> for the given data <strong>5_c.csv</strong>\n",
    "<br>\n",
    "\n",
    "you will be predicting label of a data points like this: $y^{pred}= \\text{[0 if y_score < threshold  else 1]}$\n",
    "\n",
    "$ A = 500 \\times \\text{number of false negative} + 100 \\times \\text{numebr of false positive}$\n",
    "\n",
    "<pre>\n",
    "   <b>Note 1:</b> in this data you can see number of negative points > number of positive points\n",
    "   <b>Note 2:</b> use pandas or numpy to read the data from <b>5_c.csv</b>\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min metric function\n",
    "\n",
    "def min_metric(data):\n",
    "    s = data['y'].value_counts()\n",
    "    P = s[1]\n",
    "    N = s[0]\n",
    "    tpr = []\n",
    "    fpr = []\n",
    "    metric={}\n",
    "    for elem in tqdm_notebook(data['prob']):\n",
    "        data['y_pred']=predict(data,'prob',elem)\n",
    "        confusion_matrix=cal_vals(data,'y','y_pred')\n",
    "#         tpr.append(confusion_matrix['tp']/P)\n",
    "#         fpr.append(confusion_matrix['fp']/N)\n",
    "        metric_val=(500*confusion_matrix['fn'])+(100*confusion_matrix['fp'])\n",
    "        metric[elem]=metric_val\n",
    "        data.drop(columns=['y_pred'])\n",
    "    return(metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   y      prob\n",
      "0  0  0.458521\n",
      "1  0  0.505037\n",
      "2  0  0.418652\n",
      "3  0  0.412057\n",
      "4  0  0.375579\n",
      "(2852, 2)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90d392427f6a4e1bbef068a2f5b4e93a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2852), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data=pd.read_csv('5_c.csv')\n",
    "print(data.head())\n",
    "print(data.shape)\n",
    "data=data.sort_values(by='prob',ascending=False)\n",
    "result=min_metric(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the key:value pair for min value of the specified metric is- [0.2300390278970873] 141000\n"
     ]
    }
   ],
   "source": [
    "temp = min(result.values()) \n",
    "res = [key for key in result if result[key] == temp]\n",
    "print('the key:value pair for min value of the specified metric is-',res,temp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "<font color='red'><b>D.</b></font> Compute performance metrics(for regression) for the given data <strong>5_d.csv</strong>\n",
    "    <b>Note 2:</b> use pandas or numpy to read the data from <b>5_d.csv</b>\n",
    "    <b>Note 1:</b> <b>5_d.csv</b> will having two columns Y and predicted_Y both are real valued features\n",
    "<ol>\n",
    "<li> Compute Mean Square Error </li>\n",
    "<li> Compute MAPE: https://www.youtube.com/watch?v=ly6ztgIkUxk</li>\n",
    "<li> Compute R^2 error: https://en.wikipedia.org/wiki/Coefficient_of_determination#Definitions </li>\n",
    "</ol>\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>120.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>131.0</td>\n",
       "      <td>113.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>164.0</td>\n",
       "      <td>125.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>154.0</td>\n",
       "      <td>152.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       y   pred\n",
       "0  101.0  100.0\n",
       "1  120.0  100.0\n",
       "2  131.0  113.0\n",
       "3  164.0  125.0\n",
       "4  154.0  152.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_d=pd.read_csv(\"5_d.csv\")\n",
    "data_d.shape\n",
    "data_d.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    " \n",
    "def error(df,col1,col2):\n",
    "    val=[]\n",
    "    for index, (value1, value2) in enumerate(zip(df[col1], df[col2])):\n",
    "        val.append(value1-value2)\n",
    "    return val\n",
    "    \n",
    "def absolute_error(df,col):\n",
    "    val=[]\n",
    "    for index,value in enumerate(df[col]):\n",
    "        val.append(abs(value))\n",
    "    return val\n",
    "\n",
    "def mean_sq_error(df,col):\n",
    "    return ss_res(df,col)/len(df[col])\n",
    "\n",
    "def mape(df,col1,col2):\n",
    "    val=sum(df[col1])/sum(df[col2])\n",
    "    return val\n",
    "\n",
    "def ss_res(df,col):\n",
    "    val=0\n",
    "    for index,value in enumerate(df[col]):\n",
    "        val=val+(value*value)\n",
    "    return val\n",
    "\n",
    "def ss_tot(df,col):\n",
    "    val=0\n",
    "    mean_val=data_d['y'].mean()\n",
    "    for index,value in enumerate(df[col]):\n",
    "        val=val+ (value-mean_val)*(value-mean_val)\n",
    "    return val\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_d['error']=error(data_d,'y','pred')\n",
    "data_d['abs_error']=absolute_error(data_d,'error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the Mean squared error is :  177.16569974554707\n"
     ]
    }
   ],
   "source": [
    "MSE=mean_sq_error(data_d,'error')\n",
    "print(\"the Mean squared error is : \", MSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the MAPE value is : 0.1291202994009687\n"
     ]
    }
   ],
   "source": [
    "MAPE=mape(data_d,'abs_error','y')\n",
    "print('the MAPE value is :', MAPE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the Co-efficient of determination value is:  0.9563582786990964\n"
     ]
    }
   ],
   "source": [
    "SS_RES=ss_res(data_d,'error')\n",
    "SS_TOT=ss_tot(data_d,'y')\n",
    "R_square= 1- (SS_RES/SS_TOT)\n",
    "print('the Co-efficient of determination value is: ',R_square)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}