Content_based_filtering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df=pd.read_csv(\"ml-100k/item_cc_genre.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['movie_title']=df['movie_title'].apply(lambda x:x.lower().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['director']=df[\"director\"].astype(str)\n",
    "df['director']=df['director'].apply(lambda x: x.lower().replace(\" \",\"\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['actor1']=df[\"actor1\"].astype(str)\n",
    "df['actor1']=df['actor1'].apply(lambda x: x.lower().replace(\" \",\"\"))\n",
    "#ensuring Case variasion in the actor's name is not identified as a different actor "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['actor2']=df[\"actor2\"].astype(str)\n",
    "df['actor2']=df['actor2'].apply(lambda x: x.lower().replace(\" \",\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['director'] = df['director'].apply(lambda x: x+\" \"+x+\" \"+x )\n",
    "#adding weight to director by replicating the direcor name thrice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['year']=df['year'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['hybrid'] = df['director'] +\" \"+ df['actor1'] +\" \"+ df['actor2'] +\" \"+ df['genre']+\" \"+df['year']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['hybrid']=df['hybrid'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"johnlasseter johnlasseter johnlasseter tomhanks timallen Animation Children's Comedy 1995\""
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['hybrid'][0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Paramters considered for content based filtering\n",
    "The name of the director is used 3 times to add more weight to director feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "from ast import literal_eval\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.metrics.pairwise import linear_kernel, cosine_similarity\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=0,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words='english',\n",
       "        strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = CountVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0, stop_words='english')\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1681x9105 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 19634 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_matrix = count.fit_transform(df['hybrid'])\n",
    "count_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>9095</th>\n",
       "      <th>9096</th>\n",
       "      <th>9097</th>\n",
       "      <th>9098</th>\n",
       "      <th>9099</th>\n",
       "      <th>9100</th>\n",
       "      <th>9101</th>\n",
       "      <th>9102</th>\n",
       "      <th>9103</th>\n",
       "      <th>9104</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 9105 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0     1     2     3     4     5     6     7     8     9     ...   9095  \\\n",
       "0     0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "1     0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "2     0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "3     0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "4     0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "\n",
       "   9096  9097  9098  9099  9100  9101  9102  9103  9104  \n",
       "0     0     0     0     0     0     0     0     0     0  \n",
       "1     0     0     0     0     0     0     0     0     0  \n",
       "2     0     0     0     0     0     0     0     0     0  \n",
       "3     0     0     0     0     0     0     0     0     0  \n",
       "4     0     0     0     0     0     0     0     0     0  \n",
       "\n",
       "[5 rows x 9105 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.sparse import csc_matrix\n",
    "csc = csc_matrix(count_matrix)\n",
    "pd.DataFrame(csc.todense()).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1681, 9)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The count_matrix is a class numpy matrix with 1681 rows which is the number of unique movies and 9105 columns where \n",
    "each column is a distinct feature as identified by Count Vectorizer.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1681 1681\n"
     ]
    }
   ],
   "source": [
    "cosine_sim = cosine_similarity(count_matrix, count_matrix)\n",
    "print(len(cosine_sim),len(cosine_sim[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each cell in the cosine_sim gives the extent of similarity between the two movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "titles = df['movie_title']\n",
    "indices = pd.Series(df.index, index=df['movie_title'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_recommendations(title):\n",
    "    try:\n",
    "        title=title.lower().strip()\n",
    "        idx = indices[title]\n",
    "        sim_scores = list(enumerate(cosine_sim[idx])) \n",
    "        #cosine_sim[idx] is the row of cosine_sim matrix of that particular movie\n",
    "        sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)\n",
    "        sim_scores = sim_scores[1:31]\n",
    "        movie_indices = [i[0] for i in sim_scores]\n",
    "        d=df[df[\"movie_title\"]==title][\"director\"].to_string().split()[1]\n",
    "        a1=df[df[\"movie_title\"]==title][\"actor1\"].to_string().split()[1]\n",
    "        a2=df[df[\"movie_title\"]==title][\"actor2\"].to_string().split()[1]\n",
    "       \n",
    "        return titles.iloc[movie_indices].head(15),d,a1,a2\n",
    "    \n",
    "    except:\n",
    "        return \"movie not in the database\"\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1217                                       goofy movie, a\n",
       " 420                       aladdin and the king of thieves\n",
       " 994                                        big green, the\n",
       " 94                                                aladdin\n",
       " 818                                             space jam\n",
       " 991                                              hercules\n",
       " 843                                    that thing you do!\n",
       " 62                                      santa clause, the\n",
       " 1064                                                balto\n",
       " 1410    land before time iii: the time of the great gi...\n",
       " 7                                                    babe\n",
       " 540                                            pocahontas\n",
       " 1468                                     gumby: the movie\n",
       " 558                         kid in king arthur's court, a\n",
       " 25                                 brothers mcmullen, the\n",
       " Name: movie_title, dtype: object, 'johnlasseter', 'tomhanks', 'timallen')"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_recommendations(\"toy story\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}