FOOD Recommender System!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init('/home/arul/spark-2.4.3-bin-hadoop2.7/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark= SparkSession.builder.appName('rec').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.recommendation import ALS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=spark.read.csv('Food Survey.csv',inferSchema=True,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Timestamp='2020/01/15 9:55:13 pm GMT+5:30', What's your name?='Sushant khatri', Year?='I', Select one?='Hosteller', Prefer, North Indian or South Indian?='North Indian', Vegetarian?='No', What would you rather prefer?='Take-away', Preferred place for Breakfast? (except mess :p)='Butty', Preferred place for Lunch ?(except mess :P)='Chickwalk', Preferred place for Dinner ?(except mess :P)='Being Punjabi', Select One:='Street Indian', Preferred Chinese Place='Blanc Cafe', Preferred Italian Place='Cheesy Juicy Burgers', Preferred North Indian Place='Baski Chef', Preferred South Indian Place='Amana cafe', Preferred Continental Place='Golocandas Restaurant', Tea/Coffee='Chai kings', Maggie='Chai kings', Juices/Milkshakes/Lime='Hot Chips', Rolls='Nawab choice restraunt', Samosa/Puff='Evergreen', Pani Puri/Chat='Delhi 6', Shawarma='Classic Chettinadu restraunt', Sugar Rush='Bliss cafe', Preferred choice='Spicy', Spicy 🌶️='Classic Chettinadu restraunt', Sweet 🍬='Baski Chef', Name the food Sanctum='Google', Food sanctum that doesn't vibes with you=\"I dont know the name of that shop which is in potheri where the merchant selling beef over there,so as im a hindu boy and i worship cow so how can i eat it thats why my vibes doesn't even want to visit that place\", _c29='Classic biryani')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Timestamp',\n",
       " \"What's your name?\",\n",
       " 'Year?',\n",
       " 'Select one?',\n",
       " 'Prefer, North Indian or South Indian?',\n",
       " 'Vegetarian?',\n",
       " 'What would you rather prefer?',\n",
       " 'Preferred place for Breakfast? (except mess :p)',\n",
       " 'Preferred place for Lunch ?(except mess :P)',\n",
       " 'Preferred place for Dinner ?(except mess :P)',\n",
       " 'Select One:',\n",
       " 'Preferred Chinese Place',\n",
       " 'Preferred Italian Place',\n",
       " 'Preferred North Indian Place',\n",
       " 'Preferred South Indian Place',\n",
       " 'Preferred Continental Place',\n",
       " 'Tea/Coffee',\n",
       " 'Maggie',\n",
       " 'Juices/Milkshakes/Lime',\n",
       " 'Rolls',\n",
       " 'Samosa/Puff',\n",
       " 'Pani Puri/Chat',\n",
       " 'Shawarma',\n",
       " 'Sugar Rush',\n",
       " 'Preferred choice',\n",
       " 'Spicy 🌶️',\n",
       " 'Sweet 🍬',\n",
       " 'Name the food Sanctum',\n",
       " \"Food sanctum that doesn't vibes with you\",\n",
       " '_c29']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1=data.select('Year?',\n",
    " 'Select one?',\n",
    " 'Prefer, North Indian or South Indian?',\n",
    " 'Vegetarian?',\n",
    " 'What would you rather prefer?',\n",
    " 'Preferred place for Breakfast? (except mess :p)',\n",
    " 'Preferred place for Lunch ?(except mess :P)',\n",
    " 'Preferred place for Dinner ?(except mess :P)',\n",
    " 'Select One:',\n",
    " 'Preferred Chinese Place',\n",
    " 'Preferred Italian Place',\n",
    " 'Preferred North Indian Place',\n",
    " 'Preferred South Indian Place',\n",
    " 'Preferred Continental Place',\n",
    " 'Tea/Coffee',\n",
    " 'Maggie',\n",
    " 'Juices/Milkshakes/Lime',\n",
    " 'Rolls',\n",
    " 'Samosa/Puff',\n",
    " 'Sugar Rush',\n",
    " 'Preferred choice',\n",
    " 'Spicy 🌶️',\n",
    " 'Sweet 🍬',\n",
    " 'Name the food Sanctum',\n",
    " )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2=data1.na.drop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import (VectorAssembler,VectorIndexer,\n",
    "                                OneHotEncoder,StringIndexer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_indexer = StringIndexer(inputCol='Year?',outputCol='y_index')\n",
    "y_encoder = OneHotEncoder(inputCol='y_index',outputCol='y_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "live_indexer = StringIndexer(inputCol='Select one?',outputCol='live_index')\n",
    "live_encoder = OneHotEncoder(inputCol='live_index',outputCol='live_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pref_indexer = StringIndexer(inputCol='Prefer, North Indian or South Indian?',outputCol='pref_index')\n",
    "pref_encoder = OneHotEncoder(inputCol='pref_index',outputCol='pref_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "veg_indexer = StringIndexer(inputCol='Vegetarian?',outputCol='veg_index')\n",
    "veg_encoder = OneHotEncoder(inputCol='veg_index',outputCol='veg_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "or_indexer = StringIndexer(inputCol='What would you rather prefer?',outputCol='or_index')\n",
    "or_encoder = OneHotEncoder(inputCol='or_index',outputCol='or_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "break_indexer = StringIndexer(inputCol='Preferred place for Breakfast? (except mess :p)',outputCol='break_index')\n",
    "break_encoder = OneHotEncoder(inputCol='break_index',outputCol='break_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lunch_indexer = StringIndexer(inputCol='Preferred place for Lunch ?(except mess :P)',outputCol='lunch_index')\n",
    "lunch_encoder = OneHotEncoder(inputCol='lunch_index',outputCol='lunch_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "din_indexer = StringIndexer(inputCol='Preferred place for Dinner ?(except mess :P)',outputCol='din_index')\n",
    "din_encoder = OneHotEncoder(inputCol='din_index',outputCol='din_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cus_indexer = StringIndexer(inputCol='Select One:',outputCol='cus_index')\n",
    "cus_encoder = OneHotEncoder(inputCol='cus_index',outputCol='cus_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "chi_indexer = StringIndexer(inputCol='Preferred Chinese Place',outputCol='chi_index')\n",
    "chi_encoder = OneHotEncoder(inputCol='chi_index',outputCol='chi_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "it_indexer = StringIndexer(inputCol='Preferred Italian Place',outputCol='it_index')\n",
    "it_encoder = OneHotEncoder(inputCol='it_index',outputCol='it_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "north_indexer = StringIndexer(inputCol='Preferred North Indian Place',outputCol='north_index')\n",
    "north_encoder = OneHotEncoder(inputCol='north_index',outputCol='north_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "south_indexer = StringIndexer(inputCol='Preferred South Indian Place',outputCol='south_index')\n",
    "south_encoder = OneHotEncoder(inputCol='south_index',outputCol='south_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "conti_indexer = StringIndexer(inputCol='Preferred Continental Place',outputCol='conti_index')\n",
    "conti_encoder = OneHotEncoder(inputCol='conti_index',outputCol='conti_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "tea_indexer = StringIndexer(inputCol='Tea/Coffee',outputCol='tea_index')\n",
    "tea_encoder = OneHotEncoder(inputCol='tea_index',outputCol='tea_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "mag_indexer = StringIndexer(inputCol='Maggie',outputCol='mag_index')\n",
    "mag_encoder = OneHotEncoder(inputCol='mag_index',outputCol='mag_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "ju_indexer = StringIndexer(inputCol='Juices/Milkshakes/Lime',outputCol='ju_index')\n",
    "ju_encoder = OneHotEncoder(inputCol='ju_index',outputCol='ju_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "roll_indexer = StringIndexer(inputCol='Rolls',outputCol='roll_index')\n",
    "roll_encoder = OneHotEncoder(inputCol='roll_index',outputCol='roll_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "sam_indexer = StringIndexer(inputCol='Samosa/Puff',outputCol='sam_index')\n",
    "sam_encoder = OneHotEncoder(inputCol='sam_index',outputCol='sam_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "sug_indexer = StringIndexer(inputCol='Sugar Rush',outputCol='sug_index')\n",
    "sug_encoder = OneHotEncoder(inputCol='sug_index',outputCol='sug_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "flav_indexer = StringIndexer(inputCol='Preferred choice',outputCol='flav_index')\n",
    "flav_encoder = OneHotEncoder(inputCol='flav_index',outputCol='flav_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "spice_indexer = StringIndexer(inputCol='Spicy 🌶️',outputCol='spice_index')\n",
    "spice_encoder = OneHotEncoder(inputCol='spice_index',outputCol='spice_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "sweet_indexer = StringIndexer(inputCol='Sweet 🍬',outputCol='sweet_index')\n",
    "sweet_encoder = OneHotEncoder(inputCol='sweet_index',outputCol='sweet_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "sanc_indexer = StringIndexer(inputCol='Name the food Sanctum',outputCol='sanc_index')\n",
    "sanc_encoder = OneHotEncoder(inputCol='sanc_index',outputCol='sanc_Vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "assembler = VectorAssembler(inputCols=['live_Vec','pref_Vec','veg_Vec','or_Vec','break_Vec','lunch_Vec','din_Vec','cus_Vec','chi_Vec','it_Vec','north_Vec',\n",
    "  'south_Vec','conti_Vec','tea_Vec','mag_Vec','ju_Vec','roll_Vec','sam_Vec','sug_Vec','flav_Vec','spice_Vec','sweet_Vec'                                    \n",
    "],outputCol='features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline=Pipeline(stages=[live_indexer,live_encoder,y_indexer,pref_indexer,pref_encoder,veg_indexer,veg_encoder,\n",
    "                            or_indexer,or_encoder,break_indexer,break_encoder,lunch_indexer,lunch_encoder,din_indexer,din_encoder,cus_indexer,cus_encoder,\n",
    "                            it_indexer,it_encoder,chi_indexer,chi_encoder,north_indexer,north_encoder,south_indexer,south_encoder,conti_indexer,conti_encoder,\n",
    "                            tea_indexer,tea_encoder,mag_indexer,mag_encoder,ju_indexer,ju_encoder,roll_indexer,roll_encoder,sam_indexer,sam_encoder,sug_indexer,sug_encoder,\n",
    "                            flav_indexer,flav_encoder,spice_indexer,spice_encoder,sweet_indexer,sweet_encoder ,\n",
    "                           sanc_indexer,sanc_encoder])\n",
    "                            \n",
    "                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "output=pipeline.fit(data2).transform(data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Year?',\n",
       " 'Select one?',\n",
       " 'Prefer, North Indian or South Indian?',\n",
       " 'Vegetarian?',\n",
       " 'What would you rather prefer?',\n",
       " 'Preferred place for Breakfast? (except mess :p)',\n",
       " 'Preferred place for Lunch ?(except mess :P)',\n",
       " 'Preferred place for Dinner ?(except mess :P)',\n",
       " 'Select One:',\n",
       " 'Preferred Chinese Place',\n",
       " 'Preferred Italian Place',\n",
       " 'Preferred North Indian Place',\n",
       " 'Preferred South Indian Place',\n",
       " 'Preferred Continental Place',\n",
       " 'Tea/Coffee',\n",
       " 'Maggie',\n",
       " 'Juices/Milkshakes/Lime',\n",
       " 'Rolls',\n",
       " 'Samosa/Puff',\n",
       " 'Sugar Rush',\n",
       " 'Preferred choice',\n",
       " 'Spicy 🌶️',\n",
       " 'Sweet 🍬',\n",
       " 'Name the food Sanctum',\n",
       " 'live_index',\n",
       " 'live_Vec',\n",
       " 'y_index',\n",
       " 'pref_index',\n",
       " 'pref_Vec',\n",
       " 'veg_index',\n",
       " 'veg_Vec',\n",
       " 'or_index',\n",
       " 'or_Vec',\n",
       " 'break_index',\n",
       " 'break_Vec',\n",
       " 'lunch_index',\n",
       " 'lunch_Vec',\n",
       " 'din_index',\n",
       " 'din_Vec',\n",
       " 'cus_index',\n",
       " 'cus_Vec',\n",
       " 'it_index',\n",
       " 'it_Vec',\n",
       " 'chi_index',\n",
       " 'chi_Vec',\n",
       " 'north_index',\n",
       " 'north_Vec',\n",
       " 'south_index',\n",
       " 'south_Vec',\n",
       " 'conti_index',\n",
       " 'conti_Vec',\n",
       " 'tea_index',\n",
       " 'tea_Vec',\n",
       " 'mag_index',\n",
       " 'mag_Vec',\n",
       " 'ju_index',\n",
       " 'ju_Vec',\n",
       " 'roll_index',\n",
       " 'roll_Vec',\n",
       " 'sam_index',\n",
       " 'sam_Vec',\n",
       " 'sug_index',\n",
       " 'sug_Vec',\n",
       " 'flav_index',\n",
       " 'flav_Vec',\n",
       " 'spice_index',\n",
       " 'spice_Vec',\n",
       " 'sweet_index',\n",
       " 'sweet_Vec',\n",
       " 'sanc_index',\n",
       " 'sanc_Vec']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+---------+--------+-----------+-----------+---------+---------+--------+---------+-----------+\n",
      "|pref_index|veg_index|or_index|break_index|lunch_index|din_index|cus_index|it_index|chi_index|north_index|\n",
      "+----------+---------+--------+-----------+-----------+---------+---------+--------+---------+-----------+\n",
      "|       0.0|      0.0|     0.0|        1.0|       14.0|      0.0|      6.0|     4.0|     22.0|       14.0|\n",
      "|       1.0|      0.0|     0.0|        1.0|       19.0|      1.0|      3.0|     3.0|      1.0|        1.0|\n",
      "|       0.0|      0.0|     0.0|       13.0|        9.0|      4.0|      4.0|     0.0|     13.0|       20.0|\n",
      "|       1.0|      1.0|     1.0|       29.0|        9.0|      3.0|      0.0|     0.0|      9.0|       16.0|\n",
      "|       1.0|      0.0|     1.0|        1.0|       10.0|     23.0|      0.0|     3.0|      8.0|       10.0|\n",
      "|       0.0|      0.0|     1.0|       28.0|        2.0|      6.0|      2.0|     6.0|      0.0|        0.0|\n",
      "|       0.0|      0.0|     1.0|       22.0|        0.0|     13.0|      6.0|     1.0|      0.0|       11.0|\n",
      "|       0.0|      1.0|     1.0|       31.0|       11.0|      6.0|      4.0|    16.0|      7.0|        5.0|\n",
      "|       0.0|      0.0|     0.0|        4.0|        0.0|      2.0|      3.0|    21.0|      0.0|        2.0|\n",
      "|       1.0|      1.0|     1.0|        8.0|        9.0|      9.0|      0.0|     4.0|      5.0|       15.0|\n",
      "|       0.0|      0.0|     0.0|       15.0|        2.0|     16.0|      0.0|     3.0|      4.0|        0.0|\n",
      "|       0.0|      0.0|     0.0|        5.0|       18.0|      4.0|      0.0|     1.0|     16.0|        9.0|\n",
      "|       0.0|      0.0|     0.0|        4.0|       17.0|      2.0|      5.0|     5.0|      2.0|        2.0|\n",
      "|       1.0|      1.0|     0.0|        7.0|        0.0|     20.0|      2.0|     0.0|     19.0|        2.0|\n",
      "|       1.0|      1.0|     0.0|        0.0|       31.0|      7.0|      0.0|    10.0|     20.0|        5.0|\n",
      "|       0.0|      0.0|     0.0|       27.0|        4.0|      8.0|      2.0|     2.0|      6.0|        2.0|\n",
      "|       0.0|      0.0|     0.0|       26.0|       24.0|     22.0|      0.0|     2.0|     23.0|        7.0|\n",
      "|       0.0|      0.0|     0.0|       11.0|       23.0|      2.0|      6.0|     9.0|     14.0|        0.0|\n",
      "|       0.0|      0.0|     1.0|       11.0|        4.0|      6.0|      4.0|    12.0|      2.0|       11.0|\n",
      "|       0.0|      1.0|     1.0|       10.0|       12.0|      7.0|      0.0|     0.0|      3.0|        0.0|\n",
      "+----------+---------+--------+-----------+-----------+---------+---------+--------+---------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "output.select(['pref_index',\n",
    " 'veg_index',\n",
    " 'or_index',\n",
    " 'break_index','lunch_index','din_index','cus_index','it_index','chi_index','north_index']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "final=assembler.transform(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|            features|\n",
      "+--------------------+\n",
      "|(442,[0,1,3,4,6,5...|\n",
      "|(442,[2,3,4,6,57,...|\n",
      "|(442,[0,1,3,4,18,...|\n",
      "|(442,[2,34,47,73,...|\n",
      "|(442,[0,2,3,6,48,...|\n",
      "|(442,[0,1,3,33,40...|\n",
      "|(442,[0,1,3,27,38...|\n",
      "|(442,[0,1,36,49,7...|\n",
      "|(442,[1,3,4,9,38,...|\n",
      "|(442,[0,2,13,47,7...|\n",
      "|(442,[0,1,3,4,20,...|\n",
      "|(442,[1,3,4,10,56...|\n",
      "|(442,[1,3,4,9,55,...|\n",
      "|(442,[0,2,4,12,38...|\n",
      "|(442,[0,2,4,5,69,...|\n",
      "|(442,[0,1,3,4,32,...|\n",
      "|(442,[0,1,3,4,31,...|\n",
      "|(442,[0,1,3,4,16,...|\n",
      "|(442,[0,1,3,16,42...|\n",
      "|(442,[0,1,15,50,7...|\n",
      "+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "final.select([\"features\"]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "als=ALS(maxIter=5,regParam=0.01,userCol=\"y_index\",itemCol=\"flav_index\",ratingCol=\"spice_index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data,test_data=final.randomSplit([9.0,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fit_model=als.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions=fit_model.transform(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------+-----------+----------+\n",
      "|y_index|flav_index|spice_index|prediction|\n",
      "+-------+----------+-----------+----------+\n",
      "|    1.0|       1.0|       13.0|  5.606282|\n",
      "|    0.0|       1.0|       24.0| 4.9960294|\n",
      "|    0.0|       1.0|        4.0| 4.9960294|\n",
      "|    1.0|       0.0|       28.0| 11.094643|\n",
      "|    1.0|       0.0|       12.0| 11.094643|\n",
      "|    1.0|       0.0|       32.0| 11.094643|\n",
      "|    2.0|       0.0|       21.0| 12.692442|\n",
      "|    2.0|       0.0|       11.0| 12.692442|\n",
      "|    0.0|       0.0|        3.0|  8.333258|\n",
      "|    0.0|       0.0|        0.0|  8.333258|\n",
      "+-------+----------+-----------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select([\"y_index\",\"flav_index\",\"spice_index\",\"prediction\"]).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval=RegressionEvaluator(metricName='rmse',labelCol='spice_index',predictionCol='prediction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse=eval.evaluate(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.446490657580144"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import RandomForestClassifier,DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc = DecisionTreeClassifier(labelCol='sanc_index',featuresCol='features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_model=dtc.fit(final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SparseVector(442, {0: 0.0769, 3: 0.0427, 4: 0.1197, 9: 0.0864, 18: 0.0656, 22: 0.0733, 39: 0.0941, 72: 0.0908, 137: 0.0748, 382: 0.1483, 391: 0.1274})"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtc_model.featureImportances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc= RandomForestClassifier(featuresCol='features',labelCol='sanc_index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_model=rfc.fit(final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SparseVector(442, {0: 0.0098, 1: 0.0057, 3: 0.0037, 5: 0.0204, 6: 0.0074, 14: 0.0103, 16: 0.0061, 17: 0.0028, 25: 0.0122, 33: 0.0123, 35: 0.0047, 39: 0.0193, 40: 0.0096, 45: 0.0046, 47: 0.0059, 51: 0.0057, 59: 0.0099, 62: 0.0044, 64: 0.0064, 70: 0.0113, 71: 0.0106, 72: 0.018, 75: 0.0024, 76: 0.0058, 88: 0.0054, 99: 0.0111, 101: 0.0065, 102: 0.0051, 104: 0.0151, 105: 0.004, 106: 0.0113, 107: 0.0211, 108: 0.0036, 112: 0.0144, 118: 0.0026, 120: 0.01, 126: 0.0095, 128: 0.0095, 129: 0.0087, 134: 0.0067, 137: 0.0109, 138: 0.0049, 140: 0.0048, 143: 0.006, 144: 0.0054, 146: 0.0204, 150: 0.0023, 160: 0.0048, 164: 0.0083, 165: 0.006, 166: 0.0151, 169: 0.0084, 170: 0.0023, 171: 0.002, 178: 0.0064, 181: 0.0076, 183: 0.0074, 184: 0.0047, 187: 0.0167, 188: 0.0035, 194: 0.0023, 195: 0.0154, 196: 0.0094, 201: 0.0077, 208: 0.0038, 211: 0.0106, 223: 0.0056, 232: 0.0101, 236: 0.0059, 237: 0.0048, 238: 0.0031, 242: 0.0087, 244: 0.0097, 253: 0.0171, 255: 0.0061, 256: 0.0025, 263: 0.0071, 264: 0.008, 283: 0.0028, 285: 0.0107, 286: 0.0071, 288: 0.0076, 290: 0.0057, 294: 0.0074, 298: 0.0098, 301: 0.0074, 310: 0.0078, 311: 0.0067, 313: 0.0182, 320: 0.003, 322: 0.0027, 330: 0.0036, 333: 0.0043, 334: 0.0023, 335: 0.0125, 337: 0.0044, 338: 0.006, 339: 0.0062, 342: 0.0054, 345: 0.0065, 356: 0.0053, 357: 0.0184, 360: 0.0057, 362: 0.0026, 364: 0.0067, 367: 0.0069, 372: 0.0061, 382: 0.0317, 383: 0.0037, 386: 0.0104, 387: 0.0174, 391: 0.0127, 393: 0.0095, 396: 0.0044, 402: 0.0044, 403: 0.0041, 404: 0.0047, 408: 0.008, 409: 0.0103, 412: 0.0082, 422: 0.0074, 424: 0.0059, 425: 0.0071, 428: 0.0058, 433: 0.0004, 437: 0.0041})"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc_model.featureImportances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}