NonLinearSVM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bec18a4d",
   "metadata": {},
   "source": [
    "# Non linear SVM\n",
    "\n",
    "Prima di provare tutte le combinazioni di attributi e parametri, controlliamo come i parametri influenzano il modello"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bba5455c",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "#from sklearn.model_selection import train_test_split, cross_val_score \n",
    "\n",
    "from sklearn.metrics import accuracy_score, f1_score, classification_report\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "156c3776",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Temperature</th>\n",
       "      <th>Humidity</th>\n",
       "      <th>Light</th>\n",
       "      <th>CO2</th>\n",
       "      <th>HumidityRatio</th>\n",
       "      <th>Occupancy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23.7000</td>\n",
       "      <td>26.272</td>\n",
       "      <td>585.200000</td>\n",
       "      <td>749.200000</td>\n",
       "      <td>0.004764</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23.7180</td>\n",
       "      <td>26.290</td>\n",
       "      <td>578.400000</td>\n",
       "      <td>760.400000</td>\n",
       "      <td>0.004773</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23.7300</td>\n",
       "      <td>26.230</td>\n",
       "      <td>572.666667</td>\n",
       "      <td>769.666667</td>\n",
       "      <td>0.004765</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23.7225</td>\n",
       "      <td>26.125</td>\n",
       "      <td>493.750000</td>\n",
       "      <td>774.750000</td>\n",
       "      <td>0.004744</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23.7540</td>\n",
       "      <td>26.200</td>\n",
       "      <td>488.600000</td>\n",
       "      <td>779.000000</td>\n",
       "      <td>0.004767</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Temperature  Humidity       Light         CO2  HumidityRatio  Occupancy\n",
       "0      23.7000    26.272  585.200000  749.200000       0.004764          1\n",
       "1      23.7180    26.290  578.400000  760.400000       0.004773          1\n",
       "2      23.7300    26.230  572.666667  769.666667       0.004765          1\n",
       "3      23.7225    26.125  493.750000  774.750000       0.004744          1\n",
       "4      23.7540    26.200  488.600000  779.000000       0.004767          1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_name = 'Occupancy'\n",
    "df_training = pd.read_csv('training.csv', skipinitialspace=True, na_values='?', keep_default_na=True)\n",
    "df_test = pd.read_csv(\"test.csv\", skipinitialspace=True, na_values='?', keep_default_na=True)\n",
    "\n",
    "\n",
    "columns2remove = ['date', 'Unnamed: 0', 'cumulative_hour', 'cumulative_minute', 'day', 'hour', 'minute', 'day_minute', 'weekend']\n",
    "df_training.drop(columns2remove, inplace=True, axis=1)\n",
    "df_training.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c577f97a",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "attributes = [col for col in df_training.columns if col != class_name]\n",
    "\n",
    "X_train = scaler.fit_transform(df_training[attributes].values)\n",
    "y_train = df_training[class_name]\n",
    "\n",
    "X_test = scaler.fit_transform(df_test[attributes].values)\n",
    "y_test = df_test[class_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a7b958bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=2)\n",
    "pca.fit(X_train)\n",
    "X_pca = pca.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "61530093",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.9868677042801557\n",
      "F1-score [0.99195071 0.96436428]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99      5071\n",
      "           1       0.93      1.00      0.96      1097\n",
      "\n",
      "    accuracy                           0.99      6168\n",
      "   macro avg       0.97      0.99      0.98      6168\n",
      "weighted avg       0.99      0.99      0.99      6168\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf = SVC(gamma='auto')\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred = clf.predict(X_test)\n",
    "\n",
    "print('Accuracy %s' % accuracy_score(y_test, y_pred))\n",
    "print('F1-score %s' % f1_score(y_test, y_pred, average=None))\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "365c7041",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.9917315175097277\n",
      "F1-score [0.99494699 0.97726259]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      0.99      5071\n",
      "           1       0.96      1.00      0.98      1097\n",
      "\n",
      "    accuracy                           0.99      6168\n",
      "   macro avg       0.98      0.99      0.99      6168\n",
      "weighted avg       0.99      0.99      0.99      6168\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf = SVC(gamma='auto', C=0.1, kernel='rbf', random_state=42)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred = clf.predict(X_test)\n",
    "\n",
    "print('Accuracy %s' % accuracy_score(y_test, y_pred))\n",
    "print('F1-score %s' % f1_score(y_test, y_pred, average=None))\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "600bd2b6",
   "metadata": {},
   "source": [
    "## Effect of different kernels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "124edb1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kernel: poly\n",
      "\n",
      "Accuracy 0.9925421530479897\n",
      "F1-score [0.99544464 0.97944593]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      1.00      5071\n",
      "           1       0.96      1.00      0.98      1097\n",
      "\n",
      "    accuracy                           0.99      6168\n",
      "   macro avg       0.98      1.00      0.99      6168\n",
      "weighted avg       0.99      0.99      0.99      6168\n",
      "\n",
      "---------------------------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "Kernel: rbf\n",
      "\n",
      "Accuracy 0.9917315175097277\n",
      "F1-score [0.99494699 0.97726259]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      0.99      5071\n",
      "           1       0.96      1.00      0.98      1097\n",
      "\n",
      "    accuracy                           0.99      6168\n",
      "   macro avg       0.98      0.99      0.99      6168\n",
      "weighted avg       0.99      0.99      0.99      6168\n",
      "\n",
      "---------------------------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "Kernel: sigmoid\n",
      "\n",
      "Accuracy 0.8425745784695201\n",
      "F1-score [0.89506106 0.68504703]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.82      0.90      5071\n",
      "           1       0.53      0.96      0.69      1097\n",
      "\n",
      "    accuracy                           0.84      6168\n",
      "   macro avg       0.76      0.89      0.79      6168\n",
      "weighted avg       0.91      0.84      0.86      6168\n",
      "\n",
      "---------------------------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "CPU times: total: 4.61 s\n",
      "Wall time: 4.61 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scaler = StandardScaler()\n",
    "y_train = df_training[class_name]\n",
    "y_test = df_test[class_name]\n",
    "\n",
    "for kernel in ['poly', 'rbf', 'sigmoid']:\n",
    "        print(\"Kernel: {}\".format(kernel))\n",
    "        X_train = scaler.fit_transform(df_training[['Temperature', 'Light', 'CO2']].values)\n",
    "        X_test = scaler.fit_transform(df_test[['Temperature', 'Light', 'CO2']].values)\n",
    "        \n",
    "        clf = SVC(gamma='auto', C=0.06, kernel=kernel, random_state=42)\n",
    "        clf.fit(X_train, y_train)\n",
    "\n",
    "        y_pred = clf.predict(X_test)\n",
    "\n",
    "        print()\n",
    "        print('Accuracy %s' % accuracy_score(y_test, y_pred))\n",
    "        print('F1-score %s' % f1_score(y_test, y_pred, average=None))\n",
    "        print(classification_report(y_test, y_pred))\n",
    "        report = classification_report(y_test, y_pred)\n",
    "        \n",
    "        print(\"---------------------------------------------------------\\n\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "527862d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit the model with linear kernel\n",
      "Fit the model with rbf kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\andre\\AppData\\Local\\Temp\\ipykernel_7520\\3009108544.py:39: UserWarning: No contour levels were found within the data range.\n",
      "  plt.contour(XX, YY, Z, colors=['k', 'k', 'k'],\n"
     ]
    }
   ],
   "source": [
    "X = pd.concat([df_training[attributes], df_test[attributes]], axis = 0)\n",
    "y = pd.concat([df_training[class_name], df_test[class_name]], axis = 0)\n",
    "\n",
    "pca.fit(X)\n",
    "X_pca = pca.transform(X)\n",
    "pca.fit(X_train)\n",
    "# per il modello uso X_train e X_test come al solito\n",
    "X_train_pca = pca.transform(X_train)\n",
    "pca.fit(X_test)\n",
    "X_test_pca = pca.transform(X_test)\n",
    "\n",
    "# fit the model\n",
    "for kernel in ('linear', 'rbf', 'poly'):\n",
    "    clf = SVC(kernel=kernel, gamma=10)\n",
    "    clf.fit(X_train_pca, y_train)\n",
    "    print(\"Fit the model with {} kernel\".format(kernel))\n",
    "\n",
    "    plt.figure()\n",
    "    plt.clf()\n",
    "    plt.scatter(X_pca[:, 0], X_pca[:, 1], c=y, zorder=10, cmap=plt.cm.Paired,\n",
    "                edgecolor='k', s=20)\n",
    "\n",
    "    # Circle out the test data\n",
    "    plt.scatter(X_test_pca[:, 0], X_test_pca[:, 1], s=80, facecolors='none',\n",
    "                zorder=10, edgecolor='k')\n",
    "\n",
    "    plt.axis('tight')\n",
    "    x_min = X_pca[:, 0].min()\n",
    "    x_max = X_pca[:, 0].max()\n",
    "    y_min = X_pca[:, 1].min()\n",
    "    y_max = X_pca[:, 1].max()\n",
    "\n",
    "    XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]\n",
    "    Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])\n",
    "\n",
    "    # Put the result into a color plot\n",
    "    Z = Z.reshape(XX.shape)\n",
    "    plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.Paired)\n",
    "    plt.contour(XX, YY, Z, colors=['k', 'k', 'k'],\n",
    "                linestyles=['--', '-', '--'], levels=[-.5, 0, .5])\n",
    "    plt.title(kernel)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e681357f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(summary2)):\n",
    "    print(i, summary2[i]['f1-score [1]'])\n",
    "\n",
    "print()\n",
    "for i in range(len(summary2)):\n",
    "    print(i, summary2[i]['precision [1]'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fc0ea92",
   "metadata": {},
   "outputs": [],
   "source": [
    "for c in range(1, 11):\n",
    "        # 'attr' è la combinazione di attributi da testare per ogni iterazione\n",
    "        print(\"C =\", c/100)\n",
    "        X_train = scaler.fit_transform(df_training[['Temperature', 'Light', 'CO2']].values)\n",
    "        X_test = scaler.fit_transform(df_test[['Temperature', 'Light', 'CO2']].values)\n",
    "        \n",
    "        #clf = SVC(gamma='auto', C=0.1, kernel='rbf', random_state=42)\n",
    "        #clf.fit(X_train, y_train)\n",
    "\n",
    "        #y_pred = clf.predict(X_test)\n",
    "        \n",
    "        clf = SVC(C=c/100, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
    "                     decision_function_shape='ovr', degree=3, gamma='auto', kernel='poly',\n",
    "                     max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
    "                     tol=0.001, verbose=False)\n",
    "\n",
    "        clf.fit(X_train, y_train)\n",
    "        y_pred = clf.predict(X_test)\n",
    "\n",
    "        print()\n",
    "        print('Accuracy %s' % accuracy_score(y_test, y_pred))\n",
    "        print('F1-score %s' % f1_score(y_test, y_pred, average=None))\n",
    "        print(classification_report(y_test, y_pred))\n",
    "        report = classification_report(y_test, y_pred, output_dict=True)\n",
    "        print('precision', report['1']['precision'])\n",
    "        print('precision', report['1']['recall'])\n",
    "        print(\"---------------------------------------------------------\\n\\n\\n\")\n",
    "        \n",
    "        i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c393697c",
   "metadata": {},
   "outputs": [],
   "source": [
    "attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89182a64",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "attributes = [col for col in df_training.columns if col != class_name]\n",
    "\n",
    "df = pd.concat([df_training, df_test], axis = 0)\n",
    "y = df['Occupancy'].values\n",
    "\n",
    "X = scaler.fit_transform(df[['Temperature', 'Light', 'CO2']].values)\n",
    "X_train = scaler.fit_transform(df_training[['Temperature', 'Light', 'CO2']].values)\n",
    "X_test = scaler.fit_transform(df_test[['Temperature', 'Light', 'CO2']].values)\n",
    "\n",
    "pca = PCA(n_components=2) \n",
    "pca.fit(X)\n",
    "\n",
    "X_pca = pca.transform(X)\n",
    "X_train_pca = pca.transform(X_train)\n",
    "X_test_pca = pca.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a645604c",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = SVC(C=0.06, break_ties=False, cache_size=2048, class_weight=None, coef0=0.0,\n",
    "             decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
    "             max_iter=10000, probability=False, random_state=None, shrinking=True,\n",
    "             tol=0.001, verbose=True).fit(X_train_pca, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e42b5af8",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.axis('tight')\n",
    "x_min = X_pca[:, 0].min()\n",
    "x_max = X_pca[:, 0].max()\n",
    "y_min = X_pca[:, 1].min()\n",
    "y_max = X_pca[:, 1].max()\n",
    "\n",
    "XX, YY = np.meshgrid(np.linspace(x_min, x_max, 100), np.linspace(y_min, y_max, 100))\n",
    "Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])\n",
    "Z = Z.reshape(XX.shape)\n",
    "#contours = plt.contour(XX, YY, Z, cmap=plt.cm.coolwarm, alpha=0.8)\n",
    "\n",
    "plt.contourf(XX, YY, Z, cmap=plt.cm.coolwarm, alpha=0.8)\n",
    "plt.scatter(X_pca[:, 0], X_pca[:, 1], s=30, c=y, cmap=plt.cm.Paired,\n",
    "            edgecolors='k')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4955ade0",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (8, 6))\n",
    "Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])\n",
    "Z = Z.reshape(XX.shape)\n",
    "plt.contourf(XX, YY, Z, cmap=plt.cm.coolwarm, alpha=1)\n",
    "\n",
    "sns.scatterplot(x=X_test_pca[:, 0], y=X_test_pca[:, 1], hue = y_test, palette=['blue', 'red'], linewidth=0)\n",
    "plt.title(\"Nonlinear SVM\", fontsize = 15)\n",
    "plt.tick_params(labelsize = 15)\n",
    "plt.legend(fontsize = 15)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}