diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..5023175 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,35 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: markdouthwaite + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**System:** + - OS: [e.g. iOS] + - Version [e.g. 22] + +**Xanthus Version:** + - 0.1.0a1 + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 4e1ef42..cf4caf3 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -22,6 +22,7 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine + pip install -r requirements.txt - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} diff --git a/README.md b/README.md index e56eb00..fe162c2 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,10 @@ [**Quickstart**](#quickstart) | [**Install guide**](#installation) -| [**Contributing**](docs/contributing.md) -| [**Known issues**](#known-performance-issues) ## What is Xanthus? -Xanthus is a package that was started as personal project to translate an academic ML paper into a 'production ready' software package. Itprovides the tools and model architectures necessary to utilise the techniques outlined in [He et al's work](https://dl.acm.org/doi/10.1145/3038912.3052569) on Neural Collaborative Filtering in your own projects. Over time, you'll find work from other research finding it's way in here. The aim of this package is to make state-of-the-art research into neural recommenders accessible and ultimately ready for deployment. You might find that it's not quite there yet on that latter point, but sit tight, it will be. +Xanthus is a package that was started as personal project to translate an academic ML paper into a 'production ready' software package. It provides the tools and model architectures necessary to utilise the techniques outlined in [He et al's work](https://dl.acm.org/doi/10.1145/3038912.3052569) on Neural Collaborative Filtering in your own projects. Over time, you'll find work from other research finding it's way in here. The aim of this package is to make state-of-the-art research into neural recommenders accessible and ultimately ready for deployment. You might find that it's not quite there yet on that latter point, but sit tight, it will be. Sound good? Great, here goes. @@ -23,7 +21,7 @@ Want to get straight into the code? Here's an [introductory notebook](docs/getti You can also find some examples of how to use Xanthus with sample datasets in this repo's docs. These include: -* [A minimal example using the Movielens (100k) dataset.](examples/basics.py) +* [A minimal example using the Movielens (100k) dataset.](docs/getting-started.ipynb) * [An example using the meta-data features of Xanthus on the Movielens (100k) dataset.](examples/metadata.py) If you're interested in seeing the results of benchmarking of Xanthus' models against 'classic' @@ -41,17 +39,7 @@ pip install xanthus That's it, you're good to go. Well, except for one thing... -The package makes extensive use of [Tensorflow 2.0]() and the [Keras]() API. If +The package makes extensive use of [Tensorflow 2.0](https://www.tensorflow.org/tutorials/quickstart/beginner) and the [Keras](https://keras.io/) API. If you want to make use of the GPU acceleration provided by Tensorflow, you'll need to -follow the [Tensorflow team's guide]() for setting that up. If you don't need GPUs +follow the [Tensorflow team's guide](https://www.tensorflow.org/install/gpu) for setting that up. If you don't need GPUs right now, then great, you _really_ are all set. - -## Known performance issues - -This is a pre-release. There's likely to be a few performance issues. Here's a couple that are being worked on: - -* The negative sampling `xanthus.datasets.utils.negative_sampling` approach is expensive and can therefore be slow for large datasets. -* The inference method (`NeuralRecommenderModel.predict`) is currently pretty slow. An alternative is on its way. -* The training method (`NeuralRecommenderModel.fit`) is a little crude, again, improvements are inbound. - -You will likely find other performance issues on very large datasets. Hold tight for some optimizations, it'll get there. diff --git a/docs/getting-started.ipynb b/docs/getting-started.ipynb index 3399f8d..5049010 100644 --- a/docs/getting-started.ipynb +++ b/docs/getting-started.ipynb @@ -8,11 +8,11 @@ "\n", "## What is Xanthus?\n", "\n", - "Xanthus is a Neural Recommender package written in Python. It started life as a personal project to take an academic ML paper and translate it into a 'production-ready' software package and to replicate the results of the paper along the way. It uses Tensorflow 2.0 under the hood, and makes extensive use of the Keras API. If you're interested, the original authors of [the paper that inspired this project]() provided code for their experiments, and this proved valuable when starting this project. \n", + "Xanthus is a Neural Recommender package written in Python. It started life as a personal project to take an academic ML paper and translate it into a 'production-ready' software package and to replicate the results of the paper along the way. It uses Tensorflow 2.0 under the hood, and makes extensive use of the Keras API. If you're interested, the original authors of [the paper that inspired this project](https://dl.acm.org/doi/10.1145/3038912.3052569) provided code for their experiments, and this proved valuable when starting this project. \n", "\n", "However, while it is great that they provided their code, the repository isn't maintained, the code uses an old versions of Keras (and Theano!), it can be a little hard for beginners to get to grips with, and it's very much tailored to produce the results in their paper. All fair enough, they wrote a great paper and published their workings. Admirable stuff. Xanthus aims to make it super easy to get started with the work of building a neural recommendation system, and to scale the techniques in the original paper (hopefully) gracefully with you as the complexity of your applications increase.\n", "\n", - "This notebook will walk you through a basic example of using Xanthus to predict previously unseen movies to a set of users using the classic 'Movielens' recommender dataset. The [original paper]() tests the architectures in this paper as part of an _implicit_ recommendation problem. You'll find out more about what this means later in the notebook. In the meantime, it is worth remembering that the examples in this notebook make the same assumption.\n", + "This notebook will walk you through a basic example of using Xanthus to predict previously unseen movies to a set of users using the classic 'Movielens' recommender dataset. The [original paper](https://dl.acm.org/doi/10.1145/3038912.3052569) tests the architectures in this paper as part of an _implicit_ recommendation problem. You'll find out more about what this means later in the notebook. In the meantime, it is worth remembering that the examples in this notebook make the same assumption.\n", "\n", "Ready for some code?\n", "\n", @@ -23,13 +23,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "from xanthus.datasets import download\n", + "from xanthus import datasets\n", "\n", - "download.movielens(version=\"latest-small\", output_dir=\"data\")" + "datasets.movielens.download(version=\"ml-latest-small\", output_dir=\"data\")" ] }, { @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -60,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -140,7 +140,7 @@ "4 Comedy " ] }, - "execution_count": 6, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -160,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -239,7 +239,7 @@ "4 1 50 5.0 964982931" ] }, - "execution_count": 7, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -257,7 +257,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -266,7 +266,7 @@ "'2000-07-30 19:45:03'" ] }, - "execution_count": 8, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -295,7 +295,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -311,7 +311,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -366,7 +366,7 @@ "1 1 Grumpier Old Men (1995) 4.0 964981247" ] }, - "execution_count": 10, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -386,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -408,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -436,16 +436,16 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 13, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -466,7 +466,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -475,7 +475,7 @@ "{'items': array([1694], dtype=int32)}" ] }, - "execution_count": 14, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -493,7 +493,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -513,85 +513,75 @@ "\n", "## Getting neural\n", "\n", - "With your datasets ready, you can build and fit your model. In the example, the `GeneralizedMatrixFactorizationModel` (or `GMFModel`) is used. If you're not sure what a GMF model is, be sure to check out the original paper, and the GMF class itself in the Xanthus docs. Anyway, here's how you set it up: " + "With your datasets ready, you can build and fit your model. In the example, the `GeneralizedMatrixFactorization` (or `GMFModel`) is used. If you're not sure what a GMF model is, be sure to check out the original paper, and the GMF class itself in the Xanthus docs. Anyway, here's how you set it up: " ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ - "from xanthus.models import GeneralizedMatrixFactorizationModel as GMFModel\n", + "from xanthus.models import GeneralizedMatrixFactorization as GMFModel\n", "\n", - "fit_params = dict(epochs=10, batch_size=256)\n", - "\n", - "model = GMFModel(\n", - " fit_params=fit_params, n_factors=32, negative_samples=4\n", - ")" + "model = GMFModel(train_ds.user_dim, train_ds.item_dim, factors=64)\n", + "model.compile(optimizer=\"adam\", loss=\"binary_crossentropy\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "What is going on here, you ask? Good question. First, you import the `GeneralizedMatrixFactorizationModel` as any other object. You then define `fit_params` -- fit parameters -- to define the training loop used by the Keras optimizer. All Xanthus neural recommender models inherit from the base `NeuralRecommenderModel` class. By default, this class (and therefore all child classes) utilize the `Adam` optimizer. You can configure this to use any optimizer you wish though!\n", - "\n", - "After the `fit_param`, the `GeneralizedMatrixFactorizationModel` is initialized. There are two further keyword arguments here, `n_factors` and `negative_samples`. In the former case, `n_factors` refers to the size of the latent factor space encoded by the model. The larger the number, the more expressive the model -- to a point. In the latter case, `negative_samples` configures the sampling pointwise sampling policy outlined by [He et al](). In practice, the model will be trained by sampling 'negative' instances for each positive instance in the set. In other words: for each user-item pair with a positive rating (in this case one -- remember `utils.as_implicit`?), a given number of `negative_samples` will be drawn that the user _did not_ interact with. This is resampled in each epoch. This helps the model learn more general patterns, and to avoid overfitting. Empirically, it makes quite a difference over other sampling approaches. If you're interested, you should look at the [pairwise loss used in Bayesian Personalized Ranking (BPR)]().\n", + "So what's going on here? Well, `GMFModel` is a _subclass_ of the Keras `Model` class. Consequently, is shares the same interface. You will initialize your model with specific information (in this case information related to the size of the user and item input vectors and the size of the latent factors you're looking to compute), compile the model with a given loss and optimizer, and then train it. Straightforward enough, eh? In principle, you can use `GMFModel` however you'd use a 'normal' Keras model.\n", "\n", "You're now ready to fit your model. You can do this with:" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.4895 - val_loss: 0.3513\n", - "Epoch 2/2\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.3343 - val_loss: 0.3284\n", - "Epoch 3/3\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.3087 - val_loss: 0.3057\n", - "Epoch 4/4\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.2892 - val_loss: 0.2919\n", + "Epoch 1/5\n", + "5729/5729 [==============================] - 7s 1ms/step - loss: 0.5001\n", + "Epoch 2/5\n", + "5729/5729 [==============================] - 7s 1ms/step - loss: 0.3685\n", + "Epoch 3/5\n", + "5729/5729 [==============================] - 7s 1ms/step - loss: 0.2969\n", + "Epoch 4/5\n", + "5729/5729 [==============================] - 7s 1ms/step - loss: 0.2246\n", "Epoch 5/5\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.2753 - val_loss: 0.2737\n", - "Epoch 6/6\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.2568 - val_loss: 0.2583\n", - "Epoch 7/7\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.2385 - val_loss: 0.2387\n", - "Epoch 8/8\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.2179 - val_loss: 0.2195\n", - "Epoch 9/9\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.2000 - val_loss: 0.1994\n", - "Epoch 10/10\n", - "1075/1075 [==============================] - 2s 2ms/step - loss: 0.1824 - val_loss: 0.1822\n" + "5729/5729 [==============================] - 7s 1ms/step - loss: 0.1581\n" ] }, { "data": { "text/plain": [ - "GeneralizedMatrixFactorizationModel()" + "" ] }, - "execution_count": 17, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "model.fit(train_ds)" + "# prepare training data\n", + "users_x, items_x, y = train_ds.to_components(\n", + " negative_samples=4\n", + ")\n", + "model.fit([users_x, items_x], y, epochs=5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Remember that (as with any ML model) you'll want to tweak your hyperparameters (e.g. `n_factor`, regularization, etc.) to optimize your model's performance on your given dataset. The example model here is just a quick un-tuned model to show you the ropes." + "Remember that (as with any ML model) you'll want to tweak your hyperparameters (e.g. `factors`, regularization, etc.) to optimize your model's performance on your given dataset. The example model here is just a quick un-tuned model to show you the ropes." ] }, { @@ -605,32 +595,45 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "from xanthus.evaluate import he_sampling\n", + "from xanthus.evaluate import create_rankings\n", "\n", - "_, test_items, _ = test_ds.to_components(shuffle=False)\n", - "users, items = he_sampling(test_ds, train_ds, n_samples=200)" + "users, items = create_rankings(\n", + " test_ds, train_ds, output_dim=1, n_samples=100, unravel=True\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "So, what's going on here? First, you're importing the `he_sampling` function. This implements a sampling approach used be [He et al.]() in their work. The idea is that you evaluate your model on the user-item pairs in your test set, and for each 'true' user-item pair, you sample `n_samples` negative instances for that user (i.e. items they haven't interacted with). In the case of the `he_sampling` function, this produces and array of shape `n_users, n_samples + 1`. Concretely, for each user, you'll get an array where the first element is a positive sample (something they _did_ interact with) and `n_samples` negative samples (things they _did not_ interact with). \n", + "So, what's going on here? First, you're importing the `create_rankings` function. This implements a sampling approach used be _He et al_ in their work. The idea is that you evaluate your model on the user-item pairs in your test set, and for each 'true' user-item pair, you sample `n_samples` negative instances for that user (i.e. items they haven't interacted with). In the case of the `create_rankings` function, this produces and array of shape `n_users, n_samples + 1`. Concretely, for each user, you'll get an array where the first element is a positive sample (something they _did_ interact with) and `n_samples` negative samples (things they _did not_ interact with). \n", "\n", "The rationale here is that by having the model rank these `n_samples + 1` items for each user, you'll be able to determine whether your model is learning an effective ranking function -- the positive sample _should_ appear higher in the recommendations than the negative results if the model is doing it's job. Here's how you can rank these sampled items:" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "240/240 [==============================] - 0s 540us/step\n" + ] + } + ], "source": [ - "recommended = model.predict(test_ds, users=users, items=items, n=10)" + "from xanthus.models import utils\n", + "test_users, test_items, _ = test_ds.to_components(shuffle=False)\n", + "\n", + "scores = model.predict([users, items], verbose=1, batch_size=256)\n", + "recommended = utils.reshape_recommended(users.reshape(-1, 1), items.reshape(-1, 1), scores, 10, mode=\"array\")" ] }, { @@ -642,15 +645,15 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "t-nDCG 0.4727691131482932\n", - "HR@k 0.6973684210526315\n" + "t-nDCG 0.4719391834962755\n", + "HR@k 0.7351973684210527\n" ] } ], @@ -679,11 +682,20 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "240/240 [==============================] - 0s 578us/step\n" + ] + } + ], "source": [ - "recommended = model.predict(users=users, items=items[:, 1:], n=5)" + "scores = model.predict([users, items], verbose=1, batch_size=256)\n", + "recommended = utils.reshape_recommended(users.reshape(-1, 1), items.reshape(-1, 1), scores, 10, mode=\"array\")" ] }, { @@ -695,7 +707,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -725,233 +737,363 @@ " item_2\n", " item_3\n", " item_4\n", + " item_5\n", + " item_6\n", + " item_7\n", + " item_8\n", + " item_9\n", " \n", " \n", " \n", " \n", " 0\n", " 1\n", - " There's Something About Mary (1998)\n", - " Crow, The (1994)\n", - " Star Trek II: The Wrath of Khan (1982)\n", - " Casablanca (1942)\n", - " Heathers (1989)\n", + " Saint, The (1997)\n", + " Fisher King, The (1991)\n", + " Lost Boys, The (1987)\n", + " West Side Story (1961)\n", + " Harry Potter and the Sorcerer's Stone (a.k.a. ...\n", + " Courage Under Fire (1996)\n", + " Thing, The (1982)\n", + " Tin Cup (1996)\n", + " Mask of Zorro, The (1998)\n", + " On Her Majesty's Secret Service (1969)\n", " \n", " \n", " 1\n", " 2\n", - " Lord of the Rings: The Return of the King, The...\n", - " Logan (2017)\n", - " Avatar (2009)\n", - " Sherlock Holmes (2009)\n", - " Truman Show, The (1998)\n", + " Seven (a.k.a. Se7en) (1995)\n", + " Django Unchained (2012)\n", + " Kung Fury (2015)\n", + " There's Something About Mary (1998)\n", + " Hanna (2011)\n", + " Crash (2004)\n", + " The Boss Baby (2017)\n", + " Unbreakable (2000)\n", + " Finding Dory (2016)\n", + " Dr. Horrible's Sing-Along Blog (2008)\n", " \n", " \n", " 2\n", " 3\n", - " Blade Runner (1982)\n", - " River Wild, The (1994)\n", - " Quiz Show (1994)\n", - " First Knight (1995)\n", - " Ref, The (1994)\n", + " Four Weddings and a Funeral (1994)\n", + " African Queen, The (1951)\n", + " Jeremiah Johnson (1972)\n", + " Fantastic Voyage (1966)\n", + " Cobra (1986)\n", + " Notorious (1946)\n", + " Monsoon Wedding (2001)\n", + " Heartbreak Ridge (1986)\n", + " Miracle on 34th Street (1947)\n", + " Raise the Titanic (1980)\n", " \n", " \n", " 3\n", " 4\n", - " Boot, Das (Boat, The) (1981)\n", - " Sling Blade (1996)\n", - " Terminator, The (1984)\n", - " Rain Man (1988)\n", - " Little Big Man (1970)\n", + " Killing Fields, The (1984)\n", + " Dracula (Bram Stoker's Dracula) (1992)\n", + " Midnight Cowboy (1969)\n", + " Pi (1998)\n", + " Truman Show, The (1998)\n", + " Out of Sight (1998)\n", + " Last Emperor, The (1987)\n", + " Once Were Warriors (1994)\n", + " Everyone Says I Love You (1996)\n", + " Nightmare Before Christmas, The (1993)\n", " \n", " \n", " 4\n", " 5\n", - " Much Ado About Nothing (1993)\n", - " Star Trek: First Contact (1996)\n", - " Batman (1989)\n", - " One Flew Over the Cuckoo's Nest (1975)\n", - " Sling Blade (1996)\n", + " Harry Potter and the Sorcerer's Stone (a.k.a. ...\n", + " This Is Spinal Tap (1984)\n", + " Mask, The (1994)\n", + " Airplane! (1980)\n", + " Friday (1995)\n", + " Star Trek IV: The Voyage Home (1986)\n", + " Kelly's Heroes (1970)\n", + " Inception (2010)\n", + " What Dreams May Come (1998)\n", + " For Love of the Game (1999)\n", " \n", " \n", " 5\n", " 6\n", - " Crimson Tide (1995)\n", - " Searching for Bobby Fischer (1993)\n", - " James and the Giant Peach (1996)\n", - " Ronin (1998)\n", - " G.I. Jane (1997)\n", + " Home Alone (1990)\n", + " Supercop (Police Story 3: Supercop) (Jing cha ...\n", + " Dragonheart (1996)\n", + " Funny People (2009)\n", + " Kazaam (1996)\n", + " Red Dawn (1984)\n", + " Patch Adams (1998)\n", + " Ruthless People (1986)\n", + " Footloose (1984)\n", + " Sleepers (1996)\n", " \n", " \n", " 6\n", " 7\n", - " Unbreakable (2000)\n", - " Love Actually (2003)\n", - " Illusionist, The (2006)\n", - " Training Day (2001)\n", - " Hero (Ying xiong) (2002)\n", + " There's Something About Mary (1998)\n", + " Gladiator (2000)\n", + " Ferris Bueller's Day Off (1986)\n", + " Crimson Tide (1995)\n", + " Die Hard: With a Vengeance (1995)\n", + " Shakespeare in Love (1998)\n", + " Young Frankenstein (1974)\n", + " Batman (1989)\n", + " Game, The (1997)\n", + " Christmas Story, A (1983)\n", " \n", " \n", " 7\n", " 8\n", - " Sixth Sense, The (1999)\n", - " Ghost (1990)\n", - " Pocahontas (1995)\n", - " Thomas Crown Affair, The (1999)\n", - " Air Force One (1997)\n", + " Firm, The (1993)\n", + " Dangerous Minds (1995)\n", + " Few Good Men, A (1992)\n", + " Who Framed Roger Rabbit? (1988)\n", + " Bonnie and Clyde (1967)\n", + " Superman (1978)\n", + " Carlito's Way (1993)\n", + " Rocky III (1982)\n", + " Trainspotting (1996)\n", + " What's Love Got to Do with It? (1993)\n", " \n", " \n", " 8\n", " 9\n", - " Shrek (2001)\n", - " Indiana Jones and the Last Crusade (1989)\n", - " Ocean's Eleven (2001)\n", - " South Park: Bigger, Longer and Uncut (1999)\n", - " Pirates of the Caribbean: The Curse of the Bla...\n", + " Cinema Paradiso (Nuovo cinema Paradiso) (1989)\n", + " Lord of the Rings: The Fellowship of the Ring,...\n", + " Finding Nemo (2003)\n", + " Hangover, The (2009)\n", + " Running Man, The (1987)\n", + " Ben-Hur (1959)\n", + " Talented Mr. Ripley, The (1999)\n", + " Sliding Doors (1998)\n", + " Kagemusha (1980)\n", + " Some Like It Hot (1959)\n", " \n", " \n", " 9\n", " 10\n", - " Lion King, The (1994)\n", - " Zombieland (2009)\n", - " Rush Hour 2 (2001)\n", - " 300 (2007)\n", - " Horrible Bosses 2 (2014)\n", + " Young Frankenstein (1974)\n", + " Batman & Robin (1997)\n", + " Tangled (2010)\n", + " Louis C.K.: Hilarious (2010)\n", + " Pacific Rim (2013)\n", + " Planet of the Apes (2001)\n", + " American Pie (1999)\n", + " Guardians of the Galaxy (2014)\n", + " X-Men (2000)\n", + " 28 Days Later (2002)\n", " \n", " \n", " 10\n", " 11\n", - " X-Men (2000)\n", - " Piano, The (1993)\n", - " American History X (1998)\n", - " Waterworld (1995)\n", - " Snatch (2000)\n", + " Heat (1995)\n", + " Analyze This (1999)\n", + " To Wong Foo, Thanks for Everything! Julie Newm...\n", + " Mystery, Alaska (1999)\n", + " I, Robot (2004)\n", + " Invincible (2006)\n", + " Gandhi (1982)\n", + " Galaxy Quest (1999)\n", + " Training Day (2001)\n", + " Romy and Michele's High School Reunion (1997)\n", " \n", " \n", " 11\n", " 12\n", - " Beauty and the Beast (1991)\n", - " Birdcage, The (1996)\n", - " Perfect Storm, The (2000)\n", - " Braveheart (1995)\n", - " Courage Under Fire (1996)\n", + " 'burbs, The (1989)\n", + " Hercules (1997)\n", + " Payback (1999)\n", + " Three Men and a Baby (1987)\n", + " Enemy of the State (1998)\n", + " Top Gun (1986)\n", + " White Squall (1996)\n", + " Dumbo (1941)\n", + " Amistad (1997)\n", + " Quiz Show (1994)\n", " \n", " \n", " 12\n", " 13\n", " Die Hard (1988)\n", - " Lion King, The (1994)\n", - " Pirates of the Caribbean: The Curse of the Bla...\n", - " Jumanji (1995)\n", - " Casino (1995)\n", + " Outbreak (1995)\n", + " Zootopia (2016)\n", + " I Am Legend (2007)\n", + " Kate & Leopold (2001)\n", + " Lost in Translation (2003)\n", + " Battle Royale (Batoru rowaiaru) (2000)\n", + " Mallrats (1995)\n", + " Gangs of New York (2002)\n", + " Lethal Weapon (1987)\n", " \n", " \n", " 13\n", " 14\n", - " Terminator 2: Judgment Day (1991)\n", - " Godfather, The (1972)\n", - " Babe (1995)\n", - " First Knight (1995)\n", - " Die Hard (1988)\n", + " Don Juan DeMarco (1995)\n", + " Jungle Book, The (1994)\n", + " River Wild, The (1994)\n", + " National Treasure (2004)\n", + " Super Troopers (2001)\n", + " Amistad (1997)\n", + " Django Unchained (2012)\n", + " Son in Law (1993)\n", + " Chocolat (1988)\n", + " Doctor Zhivago (1965)\n", " \n", " \n", " 14\n", " 15\n", - " Good Will Hunting (1997)\n", - " Truman Show, The (1998)\n", - " Kiss Kiss Bang Bang (2005)\n", - " Zodiac (2007)\n", - " Heat (1995)\n", + " Raiders of the Lost Ark (Indiana Jones and the...\n", + " Silence of the Lambs, The (1991)\n", + " Hobbit: The Desolation of Smaug, The (2013)\n", + " 50 First Dates (2004)\n", + " Prometheus (2012)\n", + " Vertigo (1958)\n", + " Fear and Loathing in Las Vegas (1998)\n", + " Billy Madison (1995)\n", + " Knocked Up (2007)\n", + " Eraser (1996)\n", " \n", " \n", " 15\n", " 16\n", - " Indiana Jones and the Last Crusade (1989)\n", - " Saving Private Ryan (1998)\n", - " Gladiator (2000)\n", - " Birds, The (1963)\n", - " Pianist, The (2002)\n", + " Life Is Beautiful (La Vita è bella) (1997)\n", + " Ed Wood (1994)\n", + " Wallace & Gromit: A Close Shave (1995)\n", + " Pinocchio (1940)\n", + " Fast Times at Ridgemont High (1982)\n", + " Corpse Bride (2005)\n", + " Basic Instinct (1992)\n", + " Billy Elliot (2000)\n", + " Before Sunrise (1995)\n", + " Gia (1998)\n", " \n", " \n", " 16\n", " 17\n", - " Spider-Man (2002)\n", - " Fifth Element, The (1997)\n", - " Master and Commander: The Far Side of the Worl...\n", - " Field of Dreams (1989)\n", - " Team America: World Police (2004)\n", + " Dr. Strangelove or: How I Learned to Stop Worr...\n", + " RoboCop (1987)\n", + " The Devil's Advocate (1997)\n", + " eXistenZ (1999)\n", + " Robin Hood: Men in Tights (1993)\n", + " Bourne Supremacy, The (2004)\n", + " Blair Witch Project, The (1999)\n", + " Sicario (2015)\n", + " The Count of Monte Cristo (2002)\n", + " Indiana Jones and the Kingdom of the Crystal S...\n", " \n", " \n", " 17\n", " 18\n", - " Princess Mononoke (Mononoke-hime) (1997)\n", - " 28 Days Later (2002)\n", - " Adjustment Bureau, The (2011)\n", - " Stardust (2007)\n", - " Ghostbusters (a.k.a. Ghost Busters) (1984)\n", + " Run Lola Run (Lola rennt) (1998)\n", + " WALL·E (2008)\n", + " Fury (2014)\n", + " Fish Called Wanda, A (1988)\n", + " Peter Pan (1953)\n", + " Braveheart (1995)\n", + " Scanner Darkly, A (2006)\n", + " The Hobbit: The Battle of the Five Armies (2014)\n", + " Day After Tomorrow, The (2004)\n", + " Cowboy Bebop: The Movie (Cowboy Bebop: Tengoku...\n", " \n", " \n", " 18\n", " 19\n", - " GoldenEye (1995)\n", - " Lethal Weapon (1987)\n", - " Cool Hand Luke (1967)\n", - " Crimson Tide (1995)\n", - " Analyze This (1999)\n", + " Like Water for Chocolate (Como agua para choco...\n", + " Crocodile Dundee (1986)\n", + " Speed (1994)\n", + " Congo (1995)\n", + " Ruthless People (1986)\n", + " 101 Dalmatians (One Hundred and One Dalmatians...\n", + " Pleasantville (1998)\n", + " Killing Fields, The (1984)\n", + " Shanghai Noon (2000)\n", + " *batteries not included (1987)\n", " \n", " \n", " 19\n", " 20\n", - " Shakespeare in Love (1998)\n", - " Crash (2004)\n", - " Walk the Line (2005)\n", - " March of the Penguins (Marche de l'empereur, L...\n", - " Cars (2006)\n", + " Dodgeball: A True Underdog Story (2004)\n", + " Harry Potter and the Goblet of Fire (2005)\n", + " Citizen Kane (1941)\n", + " 13 Going on 30 (2004)\n", + " Star Wars: Episode III - Revenge of the Sith (...\n", + " Alien: Resurrection (1997)\n", + " Lord of the Rings, The (1978)\n", + " Mask, The (1994)\n", + " 20,000 Leagues Under the Sea (1954)\n", + " Over the Hedge (2006)\n", " \n", " \n", " 20\n", " 21\n", - " Wreck-It Ralph (2012)\n", - " Atlantis: The Lost Empire (2001)\n", - " Harry Potter and the Order of the Phoenix (2007)\n", - " True Lies (1994)\n", - " Happy Gilmore (1996)\n", + " Gone Girl (2014)\n", + " Whiplash (2014)\n", + " Grown Ups 2 (2013)\n", + " Mercury Rising (1998)\n", + " (500) Days of Summer (2009)\n", + " Apocalypto (2006)\n", + " Red Riding Hood (2011)\n", + " Creepshow (1982)\n", + " World Is Not Enough, The (1999)\n", + " Dear Zachary: A Letter to a Son About His Fath...\n", " \n", " \n", " 21\n", " 22\n", - " Pirates of the Caribbean: The Curse of the Bla...\n", - " E.T. the Extra-Terrestrial (1982)\n", - " WALL·E (2008)\n", - " Harry Potter and the Sorcerer's Stone (a.k.a. ...\n", + " Bowling for Columbine (2002)\n", + " Memento (2000)\n", " Blow (2001)\n", + " Pianist, The (2002)\n", + " Star Wars: Episode II - Attack of the Clones (...\n", + " Apollo 13 (1995)\n", + " Rudy (1993)\n", + " Romeo and Juliet (1968)\n", + " Beetlejuice (1988)\n", + " X-Men: Days of Future Past (2014)\n", " \n", " \n", " 22\n", " 23\n", - " Insider, The (1999)\n", - " Streetcar Named Desire, A (1951)\n", - " Wizard of Oz, The (1939)\n", - " Midnight Cowboy (1969)\n", - " Jurassic Park (1993)\n", + " Spirited Away (Sen to Chihiro no kamikakushi) ...\n", + " Excalibur (1981)\n", + " Once Upon a Time in America (1984)\n", + " Austin Powers: The Spy Who Shagged Me (1999)\n", + " Jerk, The (1979)\n", + " Cat on a Hot Tin Roof (1958)\n", + " Wallace & Gromit: The Wrong Trousers (1993)\n", + " You Can Count on Me (2000)\n", + " Out of Sight (1998)\n", + " Harry Potter and the Deathly Hallows: Part 1 (...\n", " \n", " \n", " 23\n", " 24\n", - " Requiem for a Dream (2000)\n", - " Star Wars: Episode III - Revenge of the Sith (...\n", - " Blood Diamond (2006)\n", - " Howl's Moving Castle (Hauru no ugoku shiro) (2...\n", - " Corpse Bride (2005)\n", + " Léon: The Professional (a.k.a. The Professiona...\n", + " King Kong (2005)\n", + " Lethal Weapon 3 (1992)\n", + " Star Trek Beyond (2016)\n", + " Star Trek: Nemesis (2002)\n", + " Road to Perdition (2002)\n", + " To Kill a Mockingbird (1962)\n", + " A-Team, The (2010)\n", + " Home (2015)\n", + " Stripes (1981)\n", " \n", " \n", " 24\n", " 25\n", - " Ex Machina (2015)\n", - " Usual Suspects, The (1995)\n", - " The Lego Movie (2014)\n", - " Clear and Present Danger (1994)\n", - " Pirates of the Caribbean: The Curse of the Bla...\n", + " Shawshank Redemption, The (1994)\n", + " WALL·E (2008)\n", + " Walk the Line (2005)\n", + " Town, The (2010)\n", + " Cars (2006)\n", + " Elite Squad: The Enemy Within (Tropa de Elite ...\n", + " Mulholland Falls (1996)\n", + " Motorcycle Diaries, The (Diarios de motociclet...\n", + " Rainmaker, The (1997)\n", + " Dallas Buyers Club (2013)\n", " \n", " \n", "\n", @@ -959,148 +1101,283 @@ ], "text/plain": [ " id item_0 \\\n", - "0 1 There's Something About Mary (1998) \n", - "1 2 Lord of the Rings: The Return of the King, The... \n", - "2 3 Blade Runner (1982) \n", - "3 4 Boot, Das (Boat, The) (1981) \n", - "4 5 Much Ado About Nothing (1993) \n", - "5 6 Crimson Tide (1995) \n", - "6 7 Unbreakable (2000) \n", - "7 8 Sixth Sense, The (1999) \n", - "8 9 Shrek (2001) \n", - "9 10 Lion King, The (1994) \n", - "10 11 X-Men (2000) \n", - "11 12 Beauty and the Beast (1991) \n", + "0 1 Saint, The (1997) \n", + "1 2 Seven (a.k.a. Se7en) (1995) \n", + "2 3 Four Weddings and a Funeral (1994) \n", + "3 4 Killing Fields, The (1984) \n", + "4 5 Harry Potter and the Sorcerer's Stone (a.k.a. ... \n", + "5 6 Home Alone (1990) \n", + "6 7 There's Something About Mary (1998) \n", + "7 8 Firm, The (1993) \n", + "8 9 Cinema Paradiso (Nuovo cinema Paradiso) (1989) \n", + "9 10 Young Frankenstein (1974) \n", + "10 11 Heat (1995) \n", + "11 12 'burbs, The (1989) \n", "12 13 Die Hard (1988) \n", - "13 14 Terminator 2: Judgment Day (1991) \n", - "14 15 Good Will Hunting (1997) \n", - "15 16 Indiana Jones and the Last Crusade (1989) \n", - "16 17 Spider-Man (2002) \n", - "17 18 Princess Mononoke (Mononoke-hime) (1997) \n", - "18 19 GoldenEye (1995) \n", - "19 20 Shakespeare in Love (1998) \n", - "20 21 Wreck-It Ralph (2012) \n", - "21 22 Pirates of the Caribbean: The Curse of the Bla... \n", - "22 23 Insider, The (1999) \n", - "23 24 Requiem for a Dream (2000) \n", - "24 25 Ex Machina (2015) \n", + "13 14 Don Juan DeMarco (1995) \n", + "14 15 Raiders of the Lost Ark (Indiana Jones and the... \n", + "15 16 Life Is Beautiful (La Vita è bella) (1997) \n", + "16 17 Dr. Strangelove or: How I Learned to Stop Worr... \n", + "17 18 Run Lola Run (Lola rennt) (1998) \n", + "18 19 Like Water for Chocolate (Como agua para choco... \n", + "19 20 Dodgeball: A True Underdog Story (2004) \n", + "20 21 Gone Girl (2014) \n", + "21 22 Bowling for Columbine (2002) \n", + "22 23 Spirited Away (Sen to Chihiro no kamikakushi) ... \n", + "23 24 Léon: The Professional (a.k.a. The Professiona... \n", + "24 25 Shawshank Redemption, The (1994) \n", "\n", " item_1 \\\n", - "0 Crow, The (1994) \n", - "1 Logan (2017) \n", - "2 River Wild, The (1994) \n", - "3 Sling Blade (1996) \n", - "4 Star Trek: First Contact (1996) \n", - "5 Searching for Bobby Fischer (1993) \n", - "6 Love Actually (2003) \n", - "7 Ghost (1990) \n", - "8 Indiana Jones and the Last Crusade (1989) \n", - "9 Zombieland (2009) \n", - "10 Piano, The (1993) \n", - "11 Birdcage, The (1996) \n", - "12 Lion King, The (1994) \n", - "13 Godfather, The (1972) \n", - "14 Truman Show, The (1998) \n", - "15 Saving Private Ryan (1998) \n", - "16 Fifth Element, The (1997) \n", - "17 28 Days Later (2002) \n", - "18 Lethal Weapon (1987) \n", - "19 Crash (2004) \n", - "20 Atlantis: The Lost Empire (2001) \n", - "21 E.T. the Extra-Terrestrial (1982) \n", - "22 Streetcar Named Desire, A (1951) \n", - "23 Star Wars: Episode III - Revenge of the Sith (... \n", - "24 Usual Suspects, The (1995) \n", + "0 Fisher King, The (1991) \n", + "1 Django Unchained (2012) \n", + "2 African Queen, The (1951) \n", + "3 Dracula (Bram Stoker's Dracula) (1992) \n", + "4 This Is Spinal Tap (1984) \n", + "5 Supercop (Police Story 3: Supercop) (Jing cha ... \n", + "6 Gladiator (2000) \n", + "7 Dangerous Minds (1995) \n", + "8 Lord of the Rings: The Fellowship of the Ring,... \n", + "9 Batman & Robin (1997) \n", + "10 Analyze This (1999) \n", + "11 Hercules (1997) \n", + "12 Outbreak (1995) \n", + "13 Jungle Book, The (1994) \n", + "14 Silence of the Lambs, The (1991) \n", + "15 Ed Wood (1994) \n", + "16 RoboCop (1987) \n", + "17 WALL·E (2008) \n", + "18 Crocodile Dundee (1986) \n", + "19 Harry Potter and the Goblet of Fire (2005) \n", + "20 Whiplash (2014) \n", + "21 Memento (2000) \n", + "22 Excalibur (1981) \n", + "23 King Kong (2005) \n", + "24 WALL·E (2008) \n", "\n", " item_2 \\\n", - "0 Star Trek II: The Wrath of Khan (1982) \n", - "1 Avatar (2009) \n", - "2 Quiz Show (1994) \n", - "3 Terminator, The (1984) \n", - "4 Batman (1989) \n", - "5 James and the Giant Peach (1996) \n", - "6 Illusionist, The (2006) \n", - "7 Pocahontas (1995) \n", - "8 Ocean's Eleven (2001) \n", - "9 Rush Hour 2 (2001) \n", - "10 American History X (1998) \n", - "11 Perfect Storm, The (2000) \n", - "12 Pirates of the Caribbean: The Curse of the Bla... \n", - "13 Babe (1995) \n", - "14 Kiss Kiss Bang Bang (2005) \n", - "15 Gladiator (2000) \n", - "16 Master and Commander: The Far Side of the Worl... \n", - "17 Adjustment Bureau, The (2011) \n", - "18 Cool Hand Luke (1967) \n", - "19 Walk the Line (2005) \n", - "20 Harry Potter and the Order of the Phoenix (2007) \n", - "21 WALL·E (2008) \n", - "22 Wizard of Oz, The (1939) \n", - "23 Blood Diamond (2006) \n", - "24 The Lego Movie (2014) \n", + "0 Lost Boys, The (1987) \n", + "1 Kung Fury (2015) \n", + "2 Jeremiah Johnson (1972) \n", + "3 Midnight Cowboy (1969) \n", + "4 Mask, The (1994) \n", + "5 Dragonheart (1996) \n", + "6 Ferris Bueller's Day Off (1986) \n", + "7 Few Good Men, A (1992) \n", + "8 Finding Nemo (2003) \n", + "9 Tangled (2010) \n", + "10 To Wong Foo, Thanks for Everything! Julie Newm... \n", + "11 Payback (1999) \n", + "12 Zootopia (2016) \n", + "13 River Wild, The (1994) \n", + "14 Hobbit: The Desolation of Smaug, The (2013) \n", + "15 Wallace & Gromit: A Close Shave (1995) \n", + "16 The Devil's Advocate (1997) \n", + "17 Fury (2014) \n", + "18 Speed (1994) \n", + "19 Citizen Kane (1941) \n", + "20 Grown Ups 2 (2013) \n", + "21 Blow (2001) \n", + "22 Once Upon a Time in America (1984) \n", + "23 Lethal Weapon 3 (1992) \n", + "24 Walk the Line (2005) \n", "\n", - " item_3 \\\n", - "0 Casablanca (1942) \n", - "1 Sherlock Holmes (2009) \n", - "2 First Knight (1995) \n", - "3 Rain Man (1988) \n", - "4 One Flew Over the Cuckoo's Nest (1975) \n", - "5 Ronin (1998) \n", - "6 Training Day (2001) \n", - "7 Thomas Crown Affair, The (1999) \n", - "8 South Park: Bigger, Longer and Uncut (1999) \n", - "9 300 (2007) \n", - "10 Waterworld (1995) \n", - "11 Braveheart (1995) \n", - "12 Jumanji (1995) \n", - "13 First Knight (1995) \n", - "14 Zodiac (2007) \n", - "15 Birds, The (1963) \n", - "16 Field of Dreams (1989) \n", - "17 Stardust (2007) \n", - "18 Crimson Tide (1995) \n", - "19 March of the Penguins (Marche de l'empereur, L... \n", - "20 True Lies (1994) \n", - "21 Harry Potter and the Sorcerer's Stone (a.k.a. ... \n", - "22 Midnight Cowboy (1969) \n", - "23 Howl's Moving Castle (Hauru no ugoku shiro) (2... \n", - "24 Clear and Present Danger (1994) \n", + " item_3 \\\n", + "0 West Side Story (1961) \n", + "1 There's Something About Mary (1998) \n", + "2 Fantastic Voyage (1966) \n", + "3 Pi (1998) \n", + "4 Airplane! (1980) \n", + "5 Funny People (2009) \n", + "6 Crimson Tide (1995) \n", + "7 Who Framed Roger Rabbit? (1988) \n", + "8 Hangover, The (2009) \n", + "9 Louis C.K.: Hilarious (2010) \n", + "10 Mystery, Alaska (1999) \n", + "11 Three Men and a Baby (1987) \n", + "12 I Am Legend (2007) \n", + "13 National Treasure (2004) \n", + "14 50 First Dates (2004) \n", + "15 Pinocchio (1940) \n", + "16 eXistenZ (1999) \n", + "17 Fish Called Wanda, A (1988) \n", + "18 Congo (1995) \n", + "19 13 Going on 30 (2004) \n", + "20 Mercury Rising (1998) \n", + "21 Pianist, The (2002) \n", + "22 Austin Powers: The Spy Who Shagged Me (1999) \n", + "23 Star Trek Beyond (2016) \n", + "24 Town, The (2010) \n", "\n", - " item_4 \n", - "0 Heathers (1989) \n", - "1 Truman Show, The (1998) \n", - "2 Ref, The (1994) \n", - "3 Little Big Man (1970) \n", - "4 Sling Blade (1996) \n", - "5 G.I. Jane (1997) \n", - "6 Hero (Ying xiong) (2002) \n", - "7 Air Force One (1997) \n", - "8 Pirates of the Caribbean: The Curse of the Bla... \n", - "9 Horrible Bosses 2 (2014) \n", - "10 Snatch (2000) \n", - "11 Courage Under Fire (1996) \n", - "12 Casino (1995) \n", - "13 Die Hard (1988) \n", - "14 Heat (1995) \n", - "15 Pianist, The (2002) \n", - "16 Team America: World Police (2004) \n", - "17 Ghostbusters (a.k.a. Ghost Busters) (1984) \n", - "18 Analyze This (1999) \n", - "19 Cars (2006) \n", - "20 Happy Gilmore (1996) \n", - "21 Blow (2001) \n", - "22 Jurassic Park (1993) \n", - "23 Corpse Bride (2005) \n", - "24 Pirates of the Caribbean: The Curse of the Bla... " + " item_4 \\\n", + "0 Harry Potter and the Sorcerer's Stone (a.k.a. ... \n", + "1 Hanna (2011) \n", + "2 Cobra (1986) \n", + "3 Truman Show, The (1998) \n", + "4 Friday (1995) \n", + "5 Kazaam (1996) \n", + "6 Die Hard: With a Vengeance (1995) \n", + "7 Bonnie and Clyde (1967) \n", + "8 Running Man, The (1987) \n", + "9 Pacific Rim (2013) \n", + "10 I, Robot (2004) \n", + "11 Enemy of the State (1998) \n", + "12 Kate & Leopold (2001) \n", + "13 Super Troopers (2001) \n", + "14 Prometheus (2012) \n", + "15 Fast Times at Ridgemont High (1982) \n", + "16 Robin Hood: Men in Tights (1993) \n", + "17 Peter Pan (1953) \n", + "18 Ruthless People (1986) \n", + "19 Star Wars: Episode III - Revenge of the Sith (... \n", + "20 (500) Days of Summer (2009) \n", + "21 Star Wars: Episode II - Attack of the Clones (... \n", + "22 Jerk, The (1979) \n", + "23 Star Trek: Nemesis (2002) \n", + "24 Cars (2006) \n", + "\n", + " item_5 \\\n", + "0 Courage Under Fire (1996) \n", + "1 Crash (2004) \n", + "2 Notorious (1946) \n", + "3 Out of Sight (1998) \n", + "4 Star Trek IV: The Voyage Home (1986) \n", + "5 Red Dawn (1984) \n", + "6 Shakespeare in Love (1998) \n", + "7 Superman (1978) \n", + "8 Ben-Hur (1959) \n", + "9 Planet of the Apes (2001) \n", + "10 Invincible (2006) \n", + "11 Top Gun (1986) \n", + "12 Lost in Translation (2003) \n", + "13 Amistad (1997) \n", + "14 Vertigo (1958) \n", + "15 Corpse Bride (2005) \n", + "16 Bourne Supremacy, The (2004) \n", + "17 Braveheart (1995) \n", + "18 101 Dalmatians (One Hundred and One Dalmatians... \n", + "19 Alien: Resurrection (1997) \n", + "20 Apocalypto (2006) \n", + "21 Apollo 13 (1995) \n", + "22 Cat on a Hot Tin Roof (1958) \n", + "23 Road to Perdition (2002) \n", + "24 Elite Squad: The Enemy Within (Tropa de Elite ... \n", + "\n", + " item_6 \\\n", + "0 Thing, The (1982) \n", + "1 The Boss Baby (2017) \n", + "2 Monsoon Wedding (2001) \n", + "3 Last Emperor, The (1987) \n", + "4 Kelly's Heroes (1970) \n", + "5 Patch Adams (1998) \n", + "6 Young Frankenstein (1974) \n", + "7 Carlito's Way (1993) \n", + "8 Talented Mr. Ripley, The (1999) \n", + "9 American Pie (1999) \n", + "10 Gandhi (1982) \n", + "11 White Squall (1996) \n", + "12 Battle Royale (Batoru rowaiaru) (2000) \n", + "13 Django Unchained (2012) \n", + "14 Fear and Loathing in Las Vegas (1998) \n", + "15 Basic Instinct (1992) \n", + "16 Blair Witch Project, The (1999) \n", + "17 Scanner Darkly, A (2006) \n", + "18 Pleasantville (1998) \n", + "19 Lord of the Rings, The (1978) \n", + "20 Red Riding Hood (2011) \n", + "21 Rudy (1993) \n", + "22 Wallace & Gromit: The Wrong Trousers (1993) \n", + "23 To Kill a Mockingbird (1962) \n", + "24 Mulholland Falls (1996) \n", + "\n", + " item_7 \\\n", + "0 Tin Cup (1996) \n", + "1 Unbreakable (2000) \n", + "2 Heartbreak Ridge (1986) \n", + "3 Once Were Warriors (1994) \n", + "4 Inception (2010) \n", + "5 Ruthless People (1986) \n", + "6 Batman (1989) \n", + "7 Rocky III (1982) \n", + "8 Sliding Doors (1998) \n", + "9 Guardians of the Galaxy (2014) \n", + "10 Galaxy Quest (1999) \n", + "11 Dumbo (1941) \n", + "12 Mallrats (1995) \n", + "13 Son in Law (1993) \n", + "14 Billy Madison (1995) \n", + "15 Billy Elliot (2000) \n", + "16 Sicario (2015) \n", + "17 The Hobbit: The Battle of the Five Armies (2014) \n", + "18 Killing Fields, The (1984) \n", + "19 Mask, The (1994) \n", + "20 Creepshow (1982) \n", + "21 Romeo and Juliet (1968) \n", + "22 You Can Count on Me (2000) \n", + "23 A-Team, The (2010) \n", + "24 Motorcycle Diaries, The (Diarios de motociclet... \n", + "\n", + " item_8 \\\n", + "0 Mask of Zorro, The (1998) \n", + "1 Finding Dory (2016) \n", + "2 Miracle on 34th Street (1947) \n", + "3 Everyone Says I Love You (1996) \n", + "4 What Dreams May Come (1998) \n", + "5 Footloose (1984) \n", + "6 Game, The (1997) \n", + "7 Trainspotting (1996) \n", + "8 Kagemusha (1980) \n", + "9 X-Men (2000) \n", + "10 Training Day (2001) \n", + "11 Amistad (1997) \n", + "12 Gangs of New York (2002) \n", + "13 Chocolat (1988) \n", + "14 Knocked Up (2007) \n", + "15 Before Sunrise (1995) \n", + "16 The Count of Monte Cristo (2002) \n", + "17 Day After Tomorrow, The (2004) \n", + "18 Shanghai Noon (2000) \n", + "19 20,000 Leagues Under the Sea (1954) \n", + "20 World Is Not Enough, The (1999) \n", + "21 Beetlejuice (1988) \n", + "22 Out of Sight (1998) \n", + "23 Home (2015) \n", + "24 Rainmaker, The (1997) \n", + "\n", + " item_9 \n", + "0 On Her Majesty's Secret Service (1969) \n", + "1 Dr. Horrible's Sing-Along Blog (2008) \n", + "2 Raise the Titanic (1980) \n", + "3 Nightmare Before Christmas, The (1993) \n", + "4 For Love of the Game (1999) \n", + "5 Sleepers (1996) \n", + "6 Christmas Story, A (1983) \n", + "7 What's Love Got to Do with It? (1993) \n", + "8 Some Like It Hot (1959) \n", + "9 28 Days Later (2002) \n", + "10 Romy and Michele's High School Reunion (1997) \n", + "11 Quiz Show (1994) \n", + "12 Lethal Weapon (1987) \n", + "13 Doctor Zhivago (1965) \n", + "14 Eraser (1996) \n", + "15 Gia (1998) \n", + "16 Indiana Jones and the Kingdom of the Crystal S... \n", + "17 Cowboy Bebop: The Movie (Cowboy Bebop: Tengoku... \n", + "18 *batteries not included (1987) \n", + "19 Over the Hedge (2006) \n", + "20 Dear Zachary: A Letter to a Son About His Fath... \n", + "21 X-Men: Days of Future Past (2014) \n", + "22 Harry Potter and the Deathly Hallows: Part 1 (... \n", + "23 Stripes (1981) \n", + "24 Dallas Buyers Club (2013) " ] }, - "execution_count": 22, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "recommended_df = encoder.to_df(users, recommended)\n", + "recommended_df = encoder.to_df(test_users.flatten(), recommended)\n", "recommended_df.head(25)" ] }, @@ -1130,18 +1407,18 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.4" + "version": "3.7.8" }, "pycharm": { "stem_cell": { "cell_type": "raw", - "source": [], "metadata": { "collapsed": false - } + }, + "source": [] } } }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/docs/getting-started.md b/docs/getting-started.md index 9f32a1b..e149f04 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,11 +2,11 @@ ## What is Xanthus? -Xanthus is a Neural Recommender package written in Python. It started life as a personal project to take an academic ML paper and translate it into a 'production-ready' software package and to replicate the results of the paper along the way. It uses Tensorflow 2.0 under the hood, and makes extensive use of the Keras API. If you're interested, the original authors of [the paper that inspired this project]() provided code for their experiments, and this proved valuable when starting this project. +Xanthus is a Neural Recommender package written in Python. It started life as a personal project to take an academic ML paper and translate it into a 'production-ready' software package and to replicate the results of the paper along the way. It uses Tensorflow 2.0 under the hood, and makes extensive use of the Keras API. If you're interested, the original authors of [the paper that inspired this project](https://dl.acm.org/doi/10.1145/3038912.3052569) provided code for their experiments, and this proved valuable when starting this project. -However, while it is great that they provided their code, the repository isn't maintained, the code uses an old versions of Keras (and Theano!), it can be a little hard for beginners to get to grips with, and it's very much tailored to produce the results in their paper. All fair enough, they wrote a great paper and published their workings. Admirable stuff. Xanthus aims to make it super easy to get started with the work of building a neural recommenation system, and to scale the techniques in the original paper (hopefully) gracefully with you as the complexity of your applications increase. +However, while it is great that they provided their code, the repository isn't maintained, the code uses an old versions of Keras (and Theano!), it can be a little hard for beginners to get to grips with, and it's very much tailored to produce the results in their paper. All fair enough, they wrote a great paper and published their workings. Admirable stuff. Xanthus aims to make it super easy to get started with the work of building a neural recommendation system, and to scale the techniques in the original paper (hopefully) gracefully with you as the complexity of your applications increase. -This notebook will walk you through a basic example of using Xanthus to predict previously unseen movies to a set of users using the classic 'Movielens' recommender dataset. The [original paper]() tests the architectures in this paper as part of an _implicit_ recommendation problem. You'll find out more about what this means later in the notebook. In the meantime, it is worth remembering that the examples in this notebook make the same assumption. +This notebook will walk you through a basic example of using Xanthus to predict previously unseen movies to a set of users using the classic 'Movielens' recommender dataset. The [original paper](https://dl.acm.org/doi/10.1145/3038912.3052569) tests the architectures in this paper as part of an _implicit_ recommendation problem. You'll find out more about what this means later in the notebook. In the meantime, it is worth remembering that the examples in this notebook make the same assumption. Ready for some code? @@ -16,9 +16,9 @@ Ah, the beginning of a brand new ML problem. You'll need to download the dataset ```python -from xanthus.datasets import download +from xanthus import datasets -download.movielens(version="latest-small", output_dir="data") +datasets.movielens.download(version="ml-latest-small", output_dir="data") ``` Time to crack out Pandas and load some CSVs. You know the drill. @@ -308,7 +308,7 @@ encoder.fit(ratings["user"], ratings["item"]) - + @@ -342,59 +342,49 @@ And that is it for preparing your datasets for modelling, at least for now. Time ## Getting neural -With your datasets ready, you can build and fit your model. In the example, the `GeneralizedMatrixFactorizationModel` (or `GMFModel`) is used. If you're not sure what a GMF model is, be sure to check out the original paper, and the GMF class itself in the Xanthus docs. Anyway, here's how you set it up: +With your datasets ready, you can build and fit your model. In the example, the `GeneralizedMatrixFactorization` (or `GMFModel`) is used. If you're not sure what a GMF model is, be sure to check out the original paper, and the GMF class itself in the Xanthus docs. Anyway, here's how you set it up: ```python -from xanthus.models import GeneralizedMatrixFactorizationModel as GMFModel +from xanthus.models import GeneralizedMatrixFactorization as GMFModel -fit_params = dict(epochs=10, batch_size=256) - -model = GMFModel( - fit_params=fit_params, n_factors=32, negative_samples=4 -) +model = GMFModel(train_ds.user_dim, train_ds.item_dim, factors=64) +model.compile(optimizer="adam", loss="binary_crossentropy") ``` -What is going on here, you ask? Good question. First, you import the `GeneralizedMatrixFactorizationModel` as any other object. You then define `fit_params` -- fit parameters -- to define the training loop used by the Keras optimizer. All Xanthus neural recommender models inherit from the base `NeuralRecommenderModel` class. By default, this class (and therefore all child classes) utilize the `Adam` optimizer. You can configure this to use any optimizer you wish though! - -After the `fit_param`, the `GeneralizedMatrixFactorizationModel` is initialized. There are two further keyword arguments here, `n_factors` and `negative_samples`. In the former case, `n_factors` refers to the size of the latent factor space encoded by the model. The larger the number, the more expressive the model -- to a point. In the latter case, `negative_samples` configures the sampling pointwise sampling policy outlined by [He et al](). In practice, the model will be trained by sampling 'negative' instances for each positive instance in the set. In other words: for each user-item pair with a positive rating (in this case one -- remember `utils.as_implicit`?), a given number of `negative_samples` will be drawn that the user _did not_ interact with. This is resampled in each epoch. This helps the model learn more general patterns, and to avoid overfitting. Empirically, it makes quite a difference over other sampling approaches. If you're interested, you should look at the [pairwise loss used in Bayesian Personalized Ranking (BPR)](). +So what's going on here? Well, `GMFModel` is a _subclass_ of the Keras `Model` class. Consequently, is shares the same interface. You will initialize your model with specific information (in this case information related to the size of the user and item input vectors and the size of the latent factors you're looking to compute), compile the model with a given loss and optimizer, and then train it. Straightforward enough, eh? In principle, you can use `GMFModel` however you'd use a 'normal' Keras model. You're now ready to fit your model. You can do this with: ```python -model.fit(train_ds) +# prepare training data +users_x, items_x, y = train_ds.to_components( + negative_samples=4 +) +model.fit([users_x, items_x], y, epochs=5) ``` - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.4895 - val_loss: 0.3513 - Epoch 2/2 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.3343 - val_loss: 0.3284 - Epoch 3/3 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.3087 - val_loss: 0.3057 - Epoch 4/4 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.2892 - val_loss: 0.2919 + Epoch 1/5 + 5729/5729 [==============================] - 7s 1ms/step - loss: 0.5001 + Epoch 2/5 + 5729/5729 [==============================] - 7s 1ms/step - loss: 0.3685 + Epoch 3/5 + 5729/5729 [==============================] - 7s 1ms/step - loss: 0.2969 + Epoch 4/5 + 5729/5729 [==============================] - 7s 1ms/step - loss: 0.2246 Epoch 5/5 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.2753 - val_loss: 0.2737 - Epoch 6/6 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.2568 - val_loss: 0.2583 - Epoch 7/7 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.2385 - val_loss: 0.2387 - Epoch 8/8 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.2179 - val_loss: 0.2195 - Epoch 9/9 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.2000 - val_loss: 0.1994 - Epoch 10/10 - 1075/1075 [==============================] - 2s 2ms/step - loss: 0.1824 - val_loss: 0.1822 + 5729/5729 [==============================] - 7s 1ms/step - loss: 0.1581 - GeneralizedMatrixFactorizationModel() + -Remember that (as with any ML model) you'll want to tweak your hyperparameters (e.g. `n_factor`, regularization, etc.) to optimize your model's performance on your given dataset. The example model here is just a quick un-tuned model to show you the ropes. +Remember that (as with any ML model) you'll want to tweak your hyperparameters (e.g. `factors`, regularization, etc.) to optimize your model's performance on your given dataset. The example model here is just a quick un-tuned model to show you the ropes. ## Evaluating the model @@ -402,21 +392,29 @@ Now to diagnose how well your model has done. The evaluation protocol here is se ```python -from xanthus.evaluate import he_sampling +from xanthus.evaluate import create_rankings -_, test_items, _ = test_ds.to_components(shuffle=False) -users, items = he_sampling(test_ds, train_ds, n_samples=200) +users, items = create_rankings( + test_ds, train_ds, output_dim=1, n_samples=100, unravel=True +) ``` -So, what's going on here? First, you're importing the `he_sampling` function. This implements a sampling approach used be [He et al.]() in their work. The idea is that you evaluate your model on the user-item pairs in your test set, and for each 'true' user-item pair, you sample `n_samples` negative instances for that user (i.e. items they haven't interacted with). In the case of the `he_sampling` function, this produces and array of shape `n_users, n_samples + 1`. Concretely, for each user, you'll get an array where the first element is a positive sample (something they _did_ interact with) and `n_samples` negative samples (things they _did not_ interact with). +So, what's going on here? First, you're importing the `create_rankings` function. This implements a sampling approach used be _He et al_ in their work. The idea is that you evaluate your model on the user-item pairs in your test set, and for each 'true' user-item pair, you sample `n_samples` negative instances for that user (i.e. items they haven't interacted with). In the case of the `create_rankings` function, this produces and array of shape `n_users, n_samples + 1`. Concretely, for each user, you'll get an array where the first element is a positive sample (something they _did_ interact with) and `n_samples` negative samples (things they _did not_ interact with). The rationale here is that by having the model rank these `n_samples + 1` items for each user, you'll be able to determine whether your model is learning an effective ranking function -- the positive sample _should_ appear higher in the recommendations than the negative results if the model is doing it's job. Here's how you can rank these sampled items: ```python -recommended = model.predict(test_ds, users=users, items=items, n=10) +from xanthus.models import utils +test_users, test_items, _ = test_ds.to_components(shuffle=False) + +scores = model.predict([users, items], verbose=1, batch_size=256) +recommended = utils.reshape_recommended(users.reshape(-1, 1), items.reshape(-1, 1), scores, 10, mode="array") ``` + 240/240 [==============================] - 0s 540us/step + + And finally for the evaluation, you can use the `score` function and the provided `metrics` in the Xanthus `evaluate` subpackage. Here's how you can use them: @@ -427,8 +425,8 @@ print("t-nDCG", score(metrics.truncated_ndcg, test_items, recommended).mean()) print("HR@k", score(metrics.precision_at_k, test_items, recommended).mean()) ``` - t-nDCG 0.4727691131482932 - HR@k 0.6973684210526315 + t-nDCG 0.4719391834962755 + HR@k 0.7351973684210527 Looking okay. Good work. Going into detail on how the metrics presented here work is beyond the scope of this notebook. If you're interested in what is going on here, make sure to check out the docs (docstrings) in the Xanthus package itself. @@ -439,14 +437,18 @@ After all of that, it is time to see what you've won. Exciting times. You can ge ```python -recommended = model.predict(users=users, items=items[:, 1:], n=5) +scores = model.predict([users, items], verbose=1, batch_size=256) +recommended = utils.reshape_recommended(users.reshape(-1, 1), items.reshape(-1, 1), scores, 10, mode="array") ``` + 240/240 [==============================] - 0s 578us/step + + Recall that the first 'column' in the `items` array corresponds to positive the positive sample for a user. You can skip that here. So now you have a great big array of integers. Not as exciting as you'd hoped? Fair enough. Xanthus provides a utility to convert the outputs of your model predictions into a more readable Pandas `DataFrame`. Specifically, your `DatasetEncoder` has the handy `to_df` method for just this job. Give it a set of _encoded_ users and a list of _encoded_ items for each user, and it'll build you a nice `DataFrame`. Here's how: ```python -recommended_df = encoder.to_df(users, recommended) +recommended_df = encoder.to_df(test_users.flatten(), recommended) recommended_df.head(25) ``` @@ -477,233 +479,363 @@ recommended_df.head(25) item_2 item_3 item_4 + item_5 + item_6 + item_7 + item_8 + item_9 0 1 - There's Something About Mary (1998) - Crow, The (1994) - Star Trek II: The Wrath of Khan (1982) - Casablanca (1942) - Heathers (1989) + Saint, The (1997) + Fisher King, The (1991) + Lost Boys, The (1987) + West Side Story (1961) + Harry Potter and the Sorcerer's Stone (a.k.a. ... + Courage Under Fire (1996) + Thing, The (1982) + Tin Cup (1996) + Mask of Zorro, The (1998) + On Her Majesty's Secret Service (1969) 1 2 - Lord of the Rings: The Return of the King, The... - Logan (2017) - Avatar (2009) - Sherlock Holmes (2009) - Truman Show, The (1998) + Seven (a.k.a. Se7en) (1995) + Django Unchained (2012) + Kung Fury (2015) + There's Something About Mary (1998) + Hanna (2011) + Crash (2004) + The Boss Baby (2017) + Unbreakable (2000) + Finding Dory (2016) + Dr. Horrible's Sing-Along Blog (2008) 2 3 - Blade Runner (1982) - River Wild, The (1994) - Quiz Show (1994) - First Knight (1995) - Ref, The (1994) + Four Weddings and a Funeral (1994) + African Queen, The (1951) + Jeremiah Johnson (1972) + Fantastic Voyage (1966) + Cobra (1986) + Notorious (1946) + Monsoon Wedding (2001) + Heartbreak Ridge (1986) + Miracle on 34th Street (1947) + Raise the Titanic (1980) 3 4 - Boot, Das (Boat, The) (1981) - Sling Blade (1996) - Terminator, The (1984) - Rain Man (1988) - Little Big Man (1970) + Killing Fields, The (1984) + Dracula (Bram Stoker's Dracula) (1992) + Midnight Cowboy (1969) + Pi (1998) + Truman Show, The (1998) + Out of Sight (1998) + Last Emperor, The (1987) + Once Were Warriors (1994) + Everyone Says I Love You (1996) + Nightmare Before Christmas, The (1993) 4 5 - Much Ado About Nothing (1993) - Star Trek: First Contact (1996) - Batman (1989) - One Flew Over the Cuckoo's Nest (1975) - Sling Blade (1996) + Harry Potter and the Sorcerer's Stone (a.k.a. ... + This Is Spinal Tap (1984) + Mask, The (1994) + Airplane! (1980) + Friday (1995) + Star Trek IV: The Voyage Home (1986) + Kelly's Heroes (1970) + Inception (2010) + What Dreams May Come (1998) + For Love of the Game (1999) 5 6 - Crimson Tide (1995) - Searching for Bobby Fischer (1993) - James and the Giant Peach (1996) - Ronin (1998) - G.I. Jane (1997) + Home Alone (1990) + Supercop (Police Story 3: Supercop) (Jing cha ... + Dragonheart (1996) + Funny People (2009) + Kazaam (1996) + Red Dawn (1984) + Patch Adams (1998) + Ruthless People (1986) + Footloose (1984) + Sleepers (1996) 6 7 - Unbreakable (2000) - Love Actually (2003) - Illusionist, The (2006) - Training Day (2001) - Hero (Ying xiong) (2002) + There's Something About Mary (1998) + Gladiator (2000) + Ferris Bueller's Day Off (1986) + Crimson Tide (1995) + Die Hard: With a Vengeance (1995) + Shakespeare in Love (1998) + Young Frankenstein (1974) + Batman (1989) + Game, The (1997) + Christmas Story, A (1983) 7 8 - Sixth Sense, The (1999) - Ghost (1990) - Pocahontas (1995) - Thomas Crown Affair, The (1999) - Air Force One (1997) + Firm, The (1993) + Dangerous Minds (1995) + Few Good Men, A (1992) + Who Framed Roger Rabbit? (1988) + Bonnie and Clyde (1967) + Superman (1978) + Carlito's Way (1993) + Rocky III (1982) + Trainspotting (1996) + What's Love Got to Do with It? (1993) 8 9 - Shrek (2001) - Indiana Jones and the Last Crusade (1989) - Ocean's Eleven (2001) - South Park: Bigger, Longer and Uncut (1999) - Pirates of the Caribbean: The Curse of the Bla... + Cinema Paradiso (Nuovo cinema Paradiso) (1989) + Lord of the Rings: The Fellowship of the Ring,... + Finding Nemo (2003) + Hangover, The (2009) + Running Man, The (1987) + Ben-Hur (1959) + Talented Mr. Ripley, The (1999) + Sliding Doors (1998) + Kagemusha (1980) + Some Like It Hot (1959) 9 10 - Lion King, The (1994) - Zombieland (2009) - Rush Hour 2 (2001) - 300 (2007) - Horrible Bosses 2 (2014) + Young Frankenstein (1974) + Batman & Robin (1997) + Tangled (2010) + Louis C.K.: Hilarious (2010) + Pacific Rim (2013) + Planet of the Apes (2001) + American Pie (1999) + Guardians of the Galaxy (2014) + X-Men (2000) + 28 Days Later (2002) 10 11 - X-Men (2000) - Piano, The (1993) - American History X (1998) - Waterworld (1995) - Snatch (2000) + Heat (1995) + Analyze This (1999) + To Wong Foo, Thanks for Everything! Julie Newm... + Mystery, Alaska (1999) + I, Robot (2004) + Invincible (2006) + Gandhi (1982) + Galaxy Quest (1999) + Training Day (2001) + Romy and Michele's High School Reunion (1997) 11 12 - Beauty and the Beast (1991) - Birdcage, The (1996) - Perfect Storm, The (2000) - Braveheart (1995) - Courage Under Fire (1996) + 'burbs, The (1989) + Hercules (1997) + Payback (1999) + Three Men and a Baby (1987) + Enemy of the State (1998) + Top Gun (1986) + White Squall (1996) + Dumbo (1941) + Amistad (1997) + Quiz Show (1994) 12 13 Die Hard (1988) - Lion King, The (1994) - Pirates of the Caribbean: The Curse of the Bla... - Jumanji (1995) - Casino (1995) + Outbreak (1995) + Zootopia (2016) + I Am Legend (2007) + Kate & Leopold (2001) + Lost in Translation (2003) + Battle Royale (Batoru rowaiaru) (2000) + Mallrats (1995) + Gangs of New York (2002) + Lethal Weapon (1987) 13 14 - Terminator 2: Judgment Day (1991) - Godfather, The (1972) - Babe (1995) - First Knight (1995) - Die Hard (1988) + Don Juan DeMarco (1995) + Jungle Book, The (1994) + River Wild, The (1994) + National Treasure (2004) + Super Troopers (2001) + Amistad (1997) + Django Unchained (2012) + Son in Law (1993) + Chocolat (1988) + Doctor Zhivago (1965) 14 15 - Good Will Hunting (1997) - Truman Show, The (1998) - Kiss Kiss Bang Bang (2005) - Zodiac (2007) - Heat (1995) + Raiders of the Lost Ark (Indiana Jones and the... + Silence of the Lambs, The (1991) + Hobbit: The Desolation of Smaug, The (2013) + 50 First Dates (2004) + Prometheus (2012) + Vertigo (1958) + Fear and Loathing in Las Vegas (1998) + Billy Madison (1995) + Knocked Up (2007) + Eraser (1996) 15 16 - Indiana Jones and the Last Crusade (1989) - Saving Private Ryan (1998) - Gladiator (2000) - Birds, The (1963) - Pianist, The (2002) + Life Is Beautiful (La Vita è bella) (1997) + Ed Wood (1994) + Wallace & Gromit: A Close Shave (1995) + Pinocchio (1940) + Fast Times at Ridgemont High (1982) + Corpse Bride (2005) + Basic Instinct (1992) + Billy Elliot (2000) + Before Sunrise (1995) + Gia (1998) 16 17 - Spider-Man (2002) - Fifth Element, The (1997) - Master and Commander: The Far Side of the Worl... - Field of Dreams (1989) - Team America: World Police (2004) + Dr. Strangelove or: How I Learned to Stop Worr... + RoboCop (1987) + The Devil's Advocate (1997) + eXistenZ (1999) + Robin Hood: Men in Tights (1993) + Bourne Supremacy, The (2004) + Blair Witch Project, The (1999) + Sicario (2015) + The Count of Monte Cristo (2002) + Indiana Jones and the Kingdom of the Crystal S... 17 18 - Princess Mononoke (Mononoke-hime) (1997) - 28 Days Later (2002) - Adjustment Bureau, The (2011) - Stardust (2007) - Ghostbusters (a.k.a. Ghost Busters) (1984) + Run Lola Run (Lola rennt) (1998) + WALL·E (2008) + Fury (2014) + Fish Called Wanda, A (1988) + Peter Pan (1953) + Braveheart (1995) + Scanner Darkly, A (2006) + The Hobbit: The Battle of the Five Armies (2014) + Day After Tomorrow, The (2004) + Cowboy Bebop: The Movie (Cowboy Bebop: Tengoku... 18 19 - GoldenEye (1995) - Lethal Weapon (1987) - Cool Hand Luke (1967) - Crimson Tide (1995) - Analyze This (1999) + Like Water for Chocolate (Como agua para choco... + Crocodile Dundee (1986) + Speed (1994) + Congo (1995) + Ruthless People (1986) + 101 Dalmatians (One Hundred and One Dalmatians... + Pleasantville (1998) + Killing Fields, The (1984) + Shanghai Noon (2000) + *batteries not included (1987) 19 20 - Shakespeare in Love (1998) - Crash (2004) - Walk the Line (2005) - March of the Penguins (Marche de l'empereur, L... - Cars (2006) + Dodgeball: A True Underdog Story (2004) + Harry Potter and the Goblet of Fire (2005) + Citizen Kane (1941) + 13 Going on 30 (2004) + Star Wars: Episode III - Revenge of the Sith (... + Alien: Resurrection (1997) + Lord of the Rings, The (1978) + Mask, The (1994) + 20,000 Leagues Under the Sea (1954) + Over the Hedge (2006) 20 21 - Wreck-It Ralph (2012) - Atlantis: The Lost Empire (2001) - Harry Potter and the Order of the Phoenix (2007) - True Lies (1994) - Happy Gilmore (1996) + Gone Girl (2014) + Whiplash (2014) + Grown Ups 2 (2013) + Mercury Rising (1998) + (500) Days of Summer (2009) + Apocalypto (2006) + Red Riding Hood (2011) + Creepshow (1982) + World Is Not Enough, The (1999) + Dear Zachary: A Letter to a Son About His Fath... 21 22 - Pirates of the Caribbean: The Curse of the Bla... - E.T. the Extra-Terrestrial (1982) - WALL·E (2008) - Harry Potter and the Sorcerer's Stone (a.k.a. ... + Bowling for Columbine (2002) + Memento (2000) Blow (2001) + Pianist, The (2002) + Star Wars: Episode II - Attack of the Clones (... + Apollo 13 (1995) + Rudy (1993) + Romeo and Juliet (1968) + Beetlejuice (1988) + X-Men: Days of Future Past (2014) 22 23 - Insider, The (1999) - Streetcar Named Desire, A (1951) - Wizard of Oz, The (1939) - Midnight Cowboy (1969) - Jurassic Park (1993) + Spirited Away (Sen to Chihiro no kamikakushi) ... + Excalibur (1981) + Once Upon a Time in America (1984) + Austin Powers: The Spy Who Shagged Me (1999) + Jerk, The (1979) + Cat on a Hot Tin Roof (1958) + Wallace & Gromit: The Wrong Trousers (1993) + You Can Count on Me (2000) + Out of Sight (1998) + Harry Potter and the Deathly Hallows: Part 1 (... 23 24 - Requiem for a Dream (2000) - Star Wars: Episode III - Revenge of the Sith (... - Blood Diamond (2006) - Howl's Moving Castle (Hauru no ugoku shiro) (2... - Corpse Bride (2005) + Léon: The Professional (a.k.a. The Professiona... + King Kong (2005) + Lethal Weapon 3 (1992) + Star Trek Beyond (2016) + Star Trek: Nemesis (2002) + Road to Perdition (2002) + To Kill a Mockingbird (1962) + A-Team, The (2010) + Home (2015) + Stripes (1981) 24 25 - Ex Machina (2015) - Usual Suspects, The (1995) - The Lego Movie (2014) - Clear and Present Danger (1994) - Pirates of the Caribbean: The Curse of the Bla... + Shawshank Redemption, The (1994) + WALL·E (2008) + Walk the Line (2005) + Town, The (2010) + Cars (2006) + Elite Squad: The Enemy Within (Tropa de Elite ... + Mulholland Falls (1996) + Motorcycle Diaries, The (Diarios de motociclet... + Rainmaker, The (1997) + Dallas Buyers Club (2013) diff --git a/examples/advanced_training.py b/examples/advanced_training.py index a0be3d7..3be88ab 100644 --- a/examples/advanced_training.py +++ b/examples/advanced_training.py @@ -4,57 +4,95 @@ Copyright (c) 2018-2020 Mark Douthwaite """ +import fire + import numpy as np import pandas as pd from tensorflow.keras import callbacks -from xanthus.models import neural -from xanthus.evaluate import he_sampling, score, metrics -from xanthus.utils import create_datasets -from xanthus.models import GeneralizedMatrixFactorizationModel +from sklearn.model_selection import train_test_split + +from xanthus import datasets, models +from xanthus.evaluate import create_rankings, score, metrics + np.random.seed(42) -# setup your Tensorboard callback. This will write logs to the `/logs` directory. -# you can start Tensorboard while your model is training by running: -# tensorboard --logdir=examples/logs -# Note - you may need to install Tensorboard first! -tensorboard = callbacks.TensorBoard(log_dir="./logs", profile_batch=5) - -# setup your Early Stopping protocol. This will terminate your training early if the -# validation loss for your model does not improve after a given period of time. -early_stop = callbacks.EarlyStopping( - monitor="val_loss", - min_delta=1e-4, - patience=5, - verbose=0, - mode="auto", - baseline=None, - restore_best_weights=True, -) -# you can add custom callbacks too! - -# and now to the old-school model training bit. -df = pd.read_csv("../data/movielens-100k/ratings.csv") -df = df.rename(columns={"userId": "user", "movieId": "item"}) - -# for expedience, you can use `create_datasets` to do a lot of the setup for you -# - at the loss of flexibility and transparency, of course. -train_dataset, test_dataset = create_datasets(df, policy="leave_one_out") - -users, items = he_sampling(test_dataset, train_dataset) -_, test_items, _ = test_dataset.to_components(shuffle=False) - -model = neural.GeneralizedMatrixFactorizationModel( - fit_params=dict(epochs=10, batch_size=256), - n_factors=8, - negative_samples=4, -) - -# make sure to pass your 'callbacks' arguments in here as a list! -model.fit(train_dataset, callbacks=[tensorboard, early_stop]) - -recommended = model.predict(test_dataset, users=users, items=items, n=10) - -print("t-nDCG", score(metrics.truncated_ndcg, test_items, recommended).mean()) -print("HR@k", score(metrics.hit_ratio, test_items, recommended).mean()) + +def run(version="ml-latest-small", samples=4, batch_size=256, epochs=1): + + # setup your Tensorboard callback. This will write logs to the `/logs` directory. + # you can start Tensorboard while your model is training by running: + # tensorboard --logdir=examples/logs + # Note - you may need to install Tensorboard first! + tensorboard = callbacks.TensorBoard(log_dir="./logs", profile_batch=5) + + # setup your Early Stopping protocol. This will terminate your training early if the + # validation loss for your model does not improve after a given period of time. + early_stop = callbacks.EarlyStopping( + monitor="val_loss", + min_delta=1e-4, + patience=5, + verbose=0, + mode="auto", + baseline=None, + restore_best_weights=True, + ) + # remember: you can add custom callbacks too! + + # and now to the old-school model training bit. + datasets.movielens.download(version) + + df = pd.read_csv(f"data/{version}/ratings.csv") + df = df.rename(columns={"userId": "user", "movieId": "item"}) + + # for expedience, you can use `datasets.build` to do a lot of the setup for you + # - at the loss of flexibility and transparency, of course. + train_dataset, test_dataset = datasets.build(df, policy="leave_one_out") + + # prepare model + model = models.GeneralizedMatrixFactorization( + n=train_dataset.user_dim, m=train_dataset.item_dim + ) + model.compile(optimizer="adam", loss="binary_crossentropy") + + # get training data + user_x, item_x, y = train_dataset.to_components( + negative_samples=samples, aux_matrix=test_dataset.interactions + ) + ( + train_user_x, + val_user_x, + train_item_x, + val_item_x, + train_y, + val_y, + ) = train_test_split(user_x, item_x, y, test_size=0.2) + + model.fit( + [train_user_x, train_item_x], + train_y, + epochs=epochs, + batch_size=batch_size, + validation_data=([val_user_x, val_item_x], val_y), + callbacks=[tensorboard, early_stop], + ) + + # get evaluation data + users, items = create_rankings( + test_dataset, train_dataset, n_samples=100, unravel=True + ) + _, test_items, _ = test_dataset.to_components(shuffle=False) + + # generate scores and evaluate + scores = model.predict([users, items], verbose=1) + recommended = models.utils.reshape_recommended( + users.reshape(-1, 1), items.reshape(-1, 1), scores, 10, mode="array" + ) + + print("nDCG", score(metrics.truncated_ndcg, test_items, recommended).mean()) + print("HR@k", score(metrics.hit_ratio, test_items, recommended).mean()) + + +if __name__ == "__main__": + fire.Fire(run) diff --git a/examples/basic.py b/examples/basic.py new file mode 100644 index 0000000..fb5435c --- /dev/null +++ b/examples/basic.py @@ -0,0 +1,19 @@ +import numpy as np +import pandas as pd +from xanthus.datasets.build import build +from xanthus.models.baseline import MatrixFactorization as MFModel +from xanthus.evaluate import create_rankings, score, metrics + +df = pd.read_csv("../data/ml-latest-small/ratings.csv") +df = df.rename(columns={"movieId": "item", "userId": "user"}) +train, val = build(df) + +users, items, _ = val.to_components(shuffle=False) +test_users, test_items = create_rankings(val, train) + +model = MFModel(factors=32, iterations=15) +model.fit(train) + +recommended = model.predict(val, users=users, items=test_items, n=10) + +print(score(metrics.pak, items, recommended).mean()) diff --git a/examples/metadata.py b/examples/metadata.py index 159a221..d1c5a27 100644 --- a/examples/metadata.py +++ b/examples/metadata.py @@ -4,42 +4,66 @@ Copyright (c) 2018-2020 Mark Douthwaite """ +import fire + import numpy as np import pandas as pd -from xanthus.datasets import utils -from xanthus.models import neural -from xanthus.evaluate import he_sampling, score, metrics -from xanthus.utils import create_datasets +from xanthus import datasets +from xanthus.datasets.build import build +from xanthus.models import GeneralizedMatrixFactorization, utils +from xanthus.evaluate import create_rankings, score, metrics np.random.seed(42) -df = pd.read_csv("../data/movielens-100k/ratings.csv") -item_df = pd.read_csv("../data/movielens-100k/movies.csv") -item_df = item_df.rename(columns={"movieId": "item"}) +def run(version="ml-latest-small", samples=4, input_dim=3, batch_size=256, epochs=1): + + # download the dataset + datasets.movielens.download(version=version) + + # load and prepare datasets + df = pd.read_csv(f"data/{version}/ratings.csv") + + item_df = pd.read_csv(f"data/{version}/movies.csv") + item_df = item_df.rename(columns={"movieId": "item"}) + + item_df = datasets.utils.fold( + item_df, "item", ["genres"], fn=lambda s: (t.lower() for t in s.split("|")) + ) + + df = df.rename(columns={"userId": "user", "movieId": "item"}) + + train_dataset, test_dataset = build( + df, item_df=item_df, policy="leave_one_out" + ) -item_df = utils.fold( - item_df, "item", ["genres"], fn=lambda s: (t.lower() for t in s.split("|")) -) + n, m = train_dataset.user_dim, train_dataset.item_dim -df = df.rename(columns={"userId": "user", "movieId": "item"}) + # build training arrays (you can also use 'batched' models too. + users_x, items_x, y = train_dataset.to_components( + negative_samples=samples, output_dim=input_dim + ) -train_dataset, test_dataset = create_datasets( - df, item_df=item_df, policy="leave_one_out" -) + # initialize, compile and train the model + model = GeneralizedMatrixFactorization(n, m) + model.compile(optimizer="adam", loss="binary_crossentropy") + model.fit([users_x, items_x], y, epochs=epochs) -_, test_items, _ = test_dataset.to_components(shuffle=False) + # evaluate the model as described in He et al. + users, items = create_rankings( + test_dataset, train_dataset, output_dim=input_dim, n_samples=100, unravel=True + ) + _, test_items, _ = test_dataset.to_components(shuffle=False) -model = neural.GeneralizedMatrixFactorizationModel( - fit_params=dict(epochs=1, batch_size=256), n_factors=8, negative_samples=4, n_meta=2 -) + scores = model.predict([users, items], verbose=1, batch_size=batch_size) + recommended = utils.reshape_recommended(users, items, scores, 10, mode="array") -model.fit(train_dataset) + ndcg = score(metrics.truncated_ndcg, test_items, recommended).mean() + hr = score(metrics.hit_ratio, test_items, recommended).mean() -users, items = he_sampling(test_dataset, train_dataset) + print(f"NDCG={ndcg}, HitRatio={hr}") -recommended = model.predict(test_dataset, users=users, items=items, n=10) -print("t-nDCG", score(metrics.truncated_ndcg, test_items, recommended).mean()) -print("HR@k", score(metrics.hit_ratio, test_items, recommended).mean()) +if __name__ == "__main__": + fire.Fire(run) diff --git a/notebooks/benchmarks.ipynb b/notebooks/benchmarks.ipynb new file mode 100644 index 0000000..f28ece3 --- /dev/null +++ b/notebooks/benchmarks.ipynb @@ -0,0 +1,443 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "\n", + "\n", + "def load_results(path=\"../scripts/data/benchmarks-2\"):\n", + "\n", + " _results = {}\n", + "\n", + " for root, dirs, files in os.walk(path):\n", + " for file in files:\n", + " path, factors = os.path.split(root)\n", + " _, model = os.path.split(path)\n", + " key=f\"{model}-{factors}\"\n", + " _results[key] = pd.read_csv(os.path.join(root, file), index_col=0)\n", + "\n", + " return _results\n", + "\n", + "\n", + "def get_best_stats(results, key, filtered=lambda _: False):\n", + " stats = []\n", + " for k, df in results.items():\n", + " print(k, filtered(k))\n", + " if not filtered(k):\n", + " best_vals = [df[_].max() for _ in df.columns if _.startswith(key)]\n", + " stats.append(pd.DataFrame(data=zip(list(range(1, len(best_vals))), best_vals, [k for _ in range(len(best_vals))]), columns=[\"k\", key, \"key\"]))\n", + "\n", + " return pd.concat(stats)\n", + "\n", + "\n", + "def get_best_stats_by_factor(results, k, filtered=lambda _: False):\n", + " stats = []\n", + " for key, df in results.items():\n", + " if not filtered(key):\n", + " label, factors = key.split(\"-\")\n", + " stats.append([label, int(factors), df[f\"ndcg{k}\"].max(), df[f\"hr{k}\"].max()])\n", + "\n", + " return pd.DataFrame(data=stats, columns=[\"key\", \"factors\", \"ndcg\", \"hr\"])\n", + "\n", + "\n", + "def plot_best_stats(results, key, **kwargs):\n", + " stats = get_best_stats(results, key, **kwargs)\n", + " return alt.Chart(stats).mark_line().encode(\n", + " x=\"k:O\",\n", + " y=alt.Y(f'{key}:Q',\n", + " scale=alt.Scale(zero=False)\n", + " ),\n", + " color=\"key\"\n", + " )\n", + "\n", + "\n", + "def plot_best_stats_by_factor(results, key, k):\n", + " stats = get_best_stats_by_factor(results, k)\n", + "\n", + " return alt.Chart(stats.sort_values(by=\"factors\")).mark_line().encode(\n", + " x=alt.X('factors:O',\n", + " scale=alt.Scale(zero=False)\n", + " ),\n", + " y=alt.Y(f'{key}:Q',\n", + " scale=alt.Scale(zero=False)\n", + " ),\n", + " color=\"key\"\n", + " )\n", + "\n", + "\n", + "def filter_on(factors):\n", + " return lambda _: False if int(_.split(\"-\")[1]) == factors else True\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gmf-8 False\n", + "nmf-8 False\n", + "als-8 False\n", + "mlp-8 False\n", + "bpr-8 False\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + "" + ], + "text/plain": [ + "alt.Chart(...)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import altair as alt\n", + "\n", + "results = load_results()\n", + "\n", + "plot_best_stats(results, \"ndcg\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gmf-8 False\n", + "nmf-8 False\n", + "als-8 False\n", + "mlp-8 False\n", + "bpr-8 False\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + "" + ], + "text/plain": [ + "alt.Chart(...)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "plot_best_stats(results, \"hr\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + "" + ], + "text/plain": [ + "alt.Chart(...)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "plot_best_stats_by_factor(results, \"ndcg\", 10)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + "" + ], + "text/plain": [ + "alt.Chart(...)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "plot_best_stats_by_factor(results, \"hr\", 10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.8" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "metadata": { + "collapsed": false + }, + "source": [] + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/requirements.txt b/requirements.txt index 8260502..ae854cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ fire==0.3.1 black==19.10b0 -tensorflow==2.2.0 +tensorflow==2.5.0 implicit==0.4.0 scikit-learn==0.23.1 jupyter==1.0.0 diff --git a/sandbox.py b/sandbox.py deleted file mode 100644 index 777e381..0000000 --- a/sandbox.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import time -import numpy as np -import pandas as pd - -from xanthus.models import GeneralizedMatrixFactorizationModel as GMFModel -from xanthus.datasets import Dataset, DatasetEncoder, utils -from xanthus.evaluate import leave_one_out, score, metrics, he_sampling - - -ratings = pd.read_csv("data/movielens-100k/ratings.csv") -movies = pd.read_csv("data/movielens-100k/movies.csv") -title_mapping = dict(zip(movies["movieId"], movies["title"])) - -ratings = ratings.rename(columns={"userId": "user", "movieId": "item"}) -ratings.loc[:, "item"] = ratings["item"].apply(lambda _: title_mapping[_]) - -ratings = ratings[ratings["rating"] > 3.0] - -train_df, test_df = leave_one_out(ratings) - -encoder = DatasetEncoder() -encoder.fit(ratings["user"], ratings["item"]) - -train_ds = Dataset.from_df(train_df, normalize=utils.as_implicit, encoder=encoder) -test_ds = Dataset.from_df(test_df, normalize=utils.as_implicit, encoder=encoder) - -model = GMFModel( - fit_params=dict(epochs=10, batch_size=256), n_factors=32, negative_samples=4 -) - -model.fit(train_ds) - -_, test_items, _ = test_ds.to_components(shuffle=False) - -# evaluate -users, items = he_sampling(test_ds, train_ds, n_samples=200) - -recommended = model.predict(test_ds, users=users, items=items, n=10) - -print("t-nDCG", score(metrics.truncated_ndcg, test_items, recommended).mean()) -print("HR@k", score(metrics.precision_at_k, test_items, recommended).mean()) - -# results -recommended = model.predict(users=users, items=items[:, 1:], n=3) -recommended_df = encoder.to_df(users, recommended) - -recommended_df.to_csv("recs.csv", index=False) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index d009e9c..bb8b1fd 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -1,150 +1,44 @@ -""" -The MIT License - -Copyright (c) 2018-2020 Mark Douthwaite -""" - -import os -from typing import Optional, Any - import fire -import numpy as np -import pandas as pd -from tensorflow.keras import callbacks -from xanthus.models import baseline, neural -from xanthus.evaluate import he_sampling, score, metrics -from xanthus.utils import create_datasets - - -np.random.seed(42) - - -def _run_trials( - models, - configs, - train, - test, - sampled_users, - sampled_items, - held_out_items, - n_trials=3, -): - results = [] - - for i in range(n_trials): - for config in configs: - for model in models: - m = model(**config) - m.fit(train) - recommended = m.predict(test, users=sampled_users, items=sampled_items) - ndcg = score(metrics.truncated_ndcg, held_out_items, recommended).mean() - hit_ratio = score(metrics.hit_ratio, held_out_items, recommended).mean() - - result = dict(name=m, trial=i, ndcg=ndcg, hit_ratio=hit_ratio,) - for key, value in config.items(): - if isinstance(value, dict): - result.update(value) - else: - result[key] = value - results.append(result) - - return results - +import logging -def _dump_results(results, path): - df = pd.DataFrame.from_records(results) - - if not os.path.exists(path): - os.makedirs(os.path.split(path)[0], exist_ok=True) - - df.to_csv(path, index=False) - - -def ncf( - input_path: str = "data/movielens-100k/ratings.csv", - output_path: str = "data/benchmarking/ncf2.csv", - n_trials: int = 1, - policy: str = "leave_one_out", - **kwargs: Optional[Any] -): - df = pd.read_csv(input_path) - df = df.rename(columns={"userId": "user", "movieId": "item"}) - - train_dataset, test_dataset = create_datasets(df, policy=policy, **kwargs) - - _, test_items, _ = test_dataset.to_components(shuffle=False) - - neural_models = [ - neural.GeneralizedMatrixFactorizationModel, - neural.MultiLayerPerceptronModel, - ] - - neural_configs = [ - { - "n_factors": 8, - "negative_samples": 1, - "fit_params": {"epochs": 25, "batch_size": 256}, - }, - ] - - users, items = he_sampling(test_dataset, train_dataset) - - results = _run_trials( - neural_models, - neural_configs, - train_dataset, - test_dataset, - users, - items, - test_items, - n_trials=n_trials, - ) - - _dump_results(results, output_path) +import pandas as pd +from xanthus.datasets.build import build +from xanthus.datasets.movielens import download +from xanthus.utils.benchmarking import ( + benchmark, + save, +) -def baselines( - input_path: str = "data/movielens-100k/ratings.csv", - output_path: str = "data/benchmarking/baselines.csv", - n_trials: int = 1, - policy: str = "leave_one_out", - **kwargs: Optional[Any] -): +from managers import NeuralModelManager, BaselineModelManager - df = pd.read_csv(input_path) - df = df.rename(columns={"userId": "user", "movieId": "item"}) - train_dataset, test_dataset = create_datasets(df, policy=policy, **kwargs) +logging.basicConfig(level=logging.INFO) - _, test_items, _ = test_dataset.to_components(shuffle=False) - baseline_models = [ - baseline.AlternatingLeastSquaresModel, - baseline.BayesianPersonalizedRankingModel, - ] +def run(experiment="benchmarks-2", factors=(8,), epochs=15, root="data"): + if isinstance(factors, (int, str)): + factors = (int(factors),) - users, items = he_sampling(test_dataset, train_dataset) + download() - baseline_configs = [ - {"factors": 8, "iterations": 15}, - {"factors": 16, "iterations": 15}, - {"factors": 32, "iterations": 15}, - {"factors": 64, "iterations": 15}, - ] + df = pd.read_csv("data/ml-latest-small/ratings.csv") + df = df.rename(columns={"movieId": "item", "userId": "user"}) + train, val = build(df) - results = _run_trials( - baseline_models, - baseline_configs, - train_dataset, - test_dataset, - users, - items, - test_items, - n_trials=n_trials, - ) + for factor in factors: + managers = [ + BaselineModelManager("als", factors=factor, datasets=(train, val)), + BaselineModelManager("bpr", factors=factor, datasets=(train, val)), + NeuralModelManager("nmf", factors=factor, datasets=(train, val)), + NeuralModelManager("gmf", factors=factor, datasets=(train, val)), + NeuralModelManager("mlp", factors=factor, datasets=(train, val)), + ] - _dump_results(results, output_path) + for manager in managers: + results, info = benchmark(manager, epochs) + save(experiment, manager, results, info, root=root, identifier=str(factor)) if __name__ == "__main__": - fire.Fire() + fire.Fire(run) diff --git a/scripts/managers.py b/scripts/managers.py new file mode 100644 index 0000000..465c6fb --- /dev/null +++ b/scripts/managers.py @@ -0,0 +1,158 @@ +from typing import Dict + +import numpy as np + +from sklearn.model_selection import train_test_split + +from xanthus.evaluate import metrics, create_rankings, score +from xanthus.models.baseline import MatrixFactorization as MFModel +from xanthus.models import ( + MultiLayerPerceptron, + GeneralizedMatrixFactorization, + NeuralMatrixFactorization, + utils, +) + +from xanthus.utils.benchmarking import ModelManager + + +class BaselineModelManager(ModelManager): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._model = MFModel(self.name, **self._params, iterations=1) + + def update(self, epochs: int) -> None: + self._model.fit(self.train) + + def metrics(self, min_k: int = 1, max_k: int = 10) -> Dict[str, float]: + + users, items, _ = self.val.to_components(shuffle=False) + + _, ranked_items = create_rankings(self.val, self.train) + + recommended = self._model.predict( + self.val, users=users, items=ranked_items, n=10 + ) + + recommended = np.asarray(recommended) + + pak = {} + ndcg = {} + + for k in range(min_k, max_k + 1): + pak[f"hr{k}"] = score(metrics.pak, items, recommended[:, :k]).mean() + ndcg[f"ndcg{k}"] = score( + metrics.truncated_ndcg, items, recommended[:, :k] + ).mean() + + return dict(**pak, **ndcg) + + +class NeuralModelManager(ModelManager): + + _models = { + "gmf": GeneralizedMatrixFactorization, + "nmf": NeuralMatrixFactorization, + "mlp": MultiLayerPerceptron, + } + + def __init__( + self, + *args, + optimizer="adam", + loss="binary_crossentropy", + samples=3, + batch_size=256, + factors=8, + **kwargs, + ) -> None: + super().__init__(*args, **kwargs) + self.samples = samples + self.batch_size = batch_size + self._model = self.get_model(factors, optimizer, loss) + + self.users, self.items = create_rankings( + self.val, self.train, n_samples=100, unravel=True + ) + + _, test_items, _ = self.val.to_components(shuffle=False) + self.test_items = test_items + + def get_model(self, factors, optimizer, loss): + + model_class = self._models[self.name] + + if self.name == "mlp": + model = model_class( + n=self.train.user_dim, + m=self.train.item_dim, + layers=self.get_layers(factors), + ) + elif self.name == "nmf": + model = model_class( + self.train.user_dim, + m=self.train.item_dim, + factors=factors, + layers=self.get_layers(factors), + ) + else: + model = model_class( + n=self.train.user_dim, m=self.train.item_dim, factors=factors + ) + + model.compile(optimizer=optimizer, loss=loss) + + return model + + def get_layers(self, factors, n=3): + layers = [factors] + + for i in range(2, n + 1): + layers.append(layers[-1] * 2) + + return tuple(layers[::-1]) + + def update(self, epochs: int) -> None: + user_x, item_x, y = self.train.to_components( + negative_samples=self.samples, aux_matrix=self.val.interactions + ) + ( + train_user_x, + val_user_x, + train_item_x, + val_item_x, + train_y, + val_y, + ) = train_test_split(user_x, item_x, y, test_size=0.2) + + self._model.fit( + [train_user_x, train_item_x], + train_y, + epochs=1, + batch_size=self.batch_size, + validation_data=([val_user_x, val_item_x], val_y), + ) + + def metrics(self, min_k: int = 1, max_k: int = 10) -> Dict[str, float]: + scores = self._model.predict([self.users, self.items]) + + recommended = utils.reshape_recommended( + self.users.reshape(-1, 1), + self.items.reshape(-1, 1), + scores, + max_k, + mode="array", + ) + + pak = {} + ndcg = {} + + for k in range(min_k, max_k + 1): + pak[f"hr{k}"] = score( + metrics.pak, self.test_items, recommended[:, :k] + ).mean() + ndcg[f"ndcg{k}"] = score( + metrics.truncated_ndcg, self.test_items, recommended[:, :k] + ).mean() + + return dict(**pak, **ndcg) diff --git a/setup.py b/setup.py index 539c32d..9618b3e 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ author_email="mark@douthwaite.io", url="https://github.com/markdouthwaite/xanthus", license="MIT", - install_requires=["tensorflow==2.2.0", "jupyter==1.0.0", "pandas==1.0.4", "numpy==1.18.5", "h5py==2.10.0", "scipy==1.4.1", "scikit-learn==0.23.1", "requests==2.23.0", "implicit==0.4.0"], + install_requires=["tensorflow==2.5.0", "jupyter==1.0.0", "pandas==1.0.4", "numpy==1.18.5", "h5py==2.10.0", "scipy==1.4.1", "scikit-learn==0.23.1", "requests==2.23.0", "implicit==0.4.0"], extras_require={"tests": ["pytest", "pandas", "requests", "markdown", "black", "tensorflow"]}, classifiers=[ # 'Development Status :: 5 - Production/Stable', diff --git a/tests/test_dataset.py b/tests/test_dataset.py index e0c0f98..a1b576c 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -4,8 +4,6 @@ Copyright (c) 2018-2020 Mark Douthwaite """ -import pytest - import numpy as np import pandas as pd @@ -134,4 +132,4 @@ def test_to_arrays_relative_negative_sample_shapes(sample_dataframes): # sampled_users, sampled_items, _ = dataset.to_arrays(negative_samples=i, # sampling_mode="absolute") # assert sampled_users.shape[0] == (i * n_users) + users.shape[0] - # assert sampled_items.shape[0] == (i * n_users) + users.shape[0] \ No newline at end of file + # assert sampled_items.shape[0] == (i * n_users) + users.shape[0] diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py index 29216b1..c3d5f98 100644 --- a/tests/test_evaluate.py +++ b/tests/test_evaluate.py @@ -28,7 +28,7 @@ score, ndcg, utils, - he_sampling, + create_rankings, split, coverage_at_k, precision_at_k, @@ -121,7 +121,7 @@ def test_he_sampling_correctness(sample_dataframes): test, encoder=encoder, normalize=lambda _: ones_like(_) ) - users, items = he_sampling(test_dataset, train_dataset) + users, items = create_rankings(test_dataset, train_dataset) a, b, _ = dataset.to_components() all_users, all_items = groupby(a, b) diff --git a/xanthus/__init__.py b/xanthus/__init__.py index 67a8325..20cfed5 100644 --- a/xanthus/__init__.py +++ b/xanthus/__init__.py @@ -4,7 +4,5 @@ Copyright (c) 2018-2020 Mark Douthwaite """ -from .utils import create_datasets - -__version__ = "0.1.0a6" +__version__ = "0.1.0rc2" __author__ = "Mark Douthwaite" diff --git a/xanthus/datasets/__init__.py b/xanthus/datasets/__init__.py index 3bee4f1..840cc88 100644 --- a/xanthus/datasets/__init__.py +++ b/xanthus/datasets/__init__.py @@ -7,3 +7,14 @@ from .core import Dataset from .encoder import DatasetEncoder from .utils import groupby, fold, sample_negatives +from . import movielens + + +__all__ = [ + "groupby", + "fold", + "sample_negatives", + "Dataset", + "DatasetEncoder", + "movielens", +] diff --git a/xanthus/utils.py b/xanthus/datasets/build.py similarity index 93% rename from xanthus/utils.py rename to xanthus/datasets/build.py index 25e5d2b..360fc28 100644 --- a/xanthus/utils.py +++ b/xanthus/datasets/build.py @@ -8,11 +8,12 @@ from pandas import DataFrame -from .datasets import Dataset, DatasetEncoder, utils -from .evaluate import leave_one_out, split +from .core import Dataset, DatasetEncoder +from . import utils +from ..evaluate import leave_one_out, split -def create_datasets( +def build( df: DataFrame, user_df: Optional[DataFrame] = None, item_df: Optional[DataFrame] = None, @@ -20,7 +21,7 @@ def create_datasets( **kwargs: Optional[Any] ) -> Tuple[Dataset, Dataset]: """ - Utility function for creating train and test datasets. + Utility function for building train and test datasets for recommender models. Parameters ---------- diff --git a/xanthus/datasets/core.py b/xanthus/datasets/core.py index b29581b..2d84241 100644 --- a/xanthus/datasets/core.py +++ b/xanthus/datasets/core.py @@ -6,6 +6,7 @@ from typing import Optional, Any, List, Tuple, Callable, Iterator from collections import defaultdict +from functools import lru_cache from numpy import ndarray from pandas import DataFrame @@ -16,7 +17,7 @@ from .encoder import DatasetEncoder -from .utils import construct_coo_matrix, sample_negatives, SamplerCallable +from .utils import construct_coo_matrix, sample_negatives, SamplerCallable, batched class Dataset: @@ -137,6 +138,20 @@ def all_items(self) -> ndarray: return np.arange(self.interactions.shape[1]) + @property + def user_dim(self): + if self.user_meta is None: + return self.all_users.shape[0] + else: + return self.all_users.shape[0] + self.user_meta.shape[1] + + @property + def item_dim(self): + if self.item_meta is None: + return self.all_items.shape[0] + else: + return self.all_items.shape[0] + self.item_meta.shape[1] + @property def history(self) -> ndarray: """Get the history (items a user has interacted with) of each user.""" @@ -332,11 +347,14 @@ def _iter_meta(ids: ndarray, meta: csr_matrix, n_dim: int) -> Iterator[List[int] for _id, _tag in zip(_ids, tags): groups[_id].append(_tag) - for _id in ids: - group = groups[_id] + @lru_cache(maxsize=None) # dangerous for large datasets? + def _fetch(_): + group = groups[_] padding = [0] * max(0, n_dim - len(group)) - features = [_id, *group, *padding][:n_dim] - yield features + return np.asarray([_, *group, *padding][:n_dim]) + + for _id in ids: + yield _fetch(_id) @classmethod def from_df( @@ -443,6 +461,9 @@ def from_df( interactions, user_meta=user_meta, item_meta=item_meta, encoder=encoder ) + def batched(self, *args, **kwargs): + return BatchedDataset(self, *args, **kwargs) + def to_components( self, *args: Optional[Any], **kwargs: Optional[Any] ) -> Tuple[ndarray, ...]: @@ -454,3 +475,85 @@ def __iter__(self) -> Iterator[Tuple[ndarray, ndarray, float]]: """Iterate over the dataset.""" yield from self.iter() + + +class BatchedDataset: + """ + Iterate batches of a dataset. + + This class exposes the method `flow` which provides a generator that yields batches + from the underlying dataset. It can reduce memory load when using large datasets + with user and item metadata. + + """ + + def __init__( + self, + dataset: Dataset, + batch_size: int, + negative_samples: int = 0, + **kwargs: Any + ): + """ + Initialise a batched dataset. + + Parameters + ---------- + dataset: Dataset + The underlying dataset. + batch_size: int + The size of the batches you'd like to use. + negative_samples: int + The total number of negative samples (for each positive sample) you wish + to take from the provided interactions set (and auxiliary matrix, if + provided). + kwargs: Any + Arguments to be passed to the `Dataset.iter` method when called. + + """ + + self.dataset = dataset + self.batch_size = batch_size + self.negative_samples = negative_samples + self.kwargs = kwargs + + @property + def n(self) -> int: + """Get the total number of records in the dataset.""" + return self.dataset.interactions.data.shape[0] + + @property + def steps(self) -> int: + """Get the total number of steps needed to process all batches in the set.""" + return self.n * (1 + self.negative_samples) // self.batch_size + + def flow(self) -> Iterator[Tuple[List[ndarray], ndarray]]: + """ + Iterate over batches. + + Note that this will loop infinitely. It is designed to allow you (and Keras) + to extract the batches you need for each training loop. + """ + yield from self + + def _iter_dataset(self) -> Iterator[Tuple[ndarray, ndarray, ndarray]]: + """Iterate through batches from the dataset.""" + + yield from ( + tuple(map(np.asarray, zip(*_))) + for _ in batched( + self.dataset.iter( + negative_samples=self.negative_samples, **self.kwargs + ), + self.batch_size, + ) + ) + + def __iter__(self) -> Iterator[Tuple[List[ndarray], ndarray]]: + """ + Iterate through batches from the dataset, packaging in a format ready for Keras. + """ + + while True: + yield from (([a, b], c) for (a, b, c) in (self._iter_dataset())) + continue diff --git a/xanthus/datasets/download.py b/xanthus/datasets/download.py deleted file mode 100644 index b3e3ea8..0000000 --- a/xanthus/datasets/download.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import io -import requests -import zipfile - - -def movielens( - version: str = "latest-small", - base_url: str = "http://files.grouplens.org/datasets/movielens/ml-{version}.zip", - output_dir: str = "data", - unzip: bool = True, -) -> None: - """Download a given movielens dataset.""" - - response = requests.get(base_url.format(version=version)) - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - if unzip: - zipfile.ZipFile(io.BytesIO(response.content)).extractall(output_dir) - else: - with open(os.path.join(output_dir, f"ml-{version}", "wb")) as file: - file.write(response.content) diff --git a/xanthus/datasets/encoder.py b/xanthus/datasets/encoder.py index e8cf34b..1223bad 100644 --- a/xanthus/datasets/encoder.py +++ b/xanthus/datasets/encoder.py @@ -298,8 +298,8 @@ def to_df( if len(targets) != len(items): raise ValueError( - f"The total number of users ('{len(targets)}') does not match the total " - f"number of item recommendation rows ('{len(items)}')." + f"The total number of users ('{len(targets)}') does not match the " + f"total number of item recommendation rows ('{len(items)}')." ) # create column headers. diff --git a/xanthus/datasets/movielens.py b/xanthus/datasets/movielens.py new file mode 100644 index 0000000..8094fb1 --- /dev/null +++ b/xanthus/datasets/movielens.py @@ -0,0 +1,122 @@ +""" +The MIT License + +Copyright (c) 2018-2020 Mark Douthwaite +""" + +import os +import io +import zipfile +import warnings +from typing import Tuple, Any, NoReturn, TypeVar + +import requests +import pandas as pd + +from .utils import fold +from .build import build + + +Dataset = TypeVar("Dataset") + + +def download( + version: str = "ml-latest-small", + base_url: str = "http://files.grouplens.org/datasets/movielens/{version}.zip", + output_dir: str = "data", + unzip: bool = True, +) -> NoReturn: + """Download a given movielens dataset.""" + + path = os.path.join(output_dir, f"ml-{version}") + response = requests.get(base_url.format(version=version)) + + if os.path.exists(path): + warnings.warn(f"Dataset already exists on path '{path}'. Aborting download.") + else: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + if unzip: + zipfile.ZipFile(io.BytesIO(response.content)).extractall(output_dir) + else: + with open(os.path.join(output_dir, f"ml-{version}"), "wb") as file: + file.write(response.content) + + +def _load_100k(path: str, policy: str, **kwargs: Any) -> Tuple[Dataset, Dataset]: + """Load the ml-latest-small dataset.""" + + df = pd.read_csv(os.path.join(path, "ratings.csv")) + + item_df = pd.read_csv(os.path.join(path, "movies.csv")) + item_df = item_df.rename(columns={"movieId": "item"}) + item_df = fold( + item_df, "item", ["genres"], fn=lambda s: (t.lower() for t in s.split("|")) + ) + + df = df.rename(columns={"userId": "user", "movieId": "item"}) + + train_ds, test_ds = build(df, item_df=item_df, policy=policy, **kwargs) + return train_ds, test_ds + + +def _load_1m(path: str, policy: str, **kwargs: Any) -> Tuple[Dataset, Dataset]: + """Load the ml-1m dataset.""" + + df = pd.read_csv( + os.path.join(path, "ratings.dat"), + names=["userId", "movieId", "rating", "timestamp"], + delimiter="::", + engine="python", + ) + + item_df = pd.read_csv( + os.path.join(path, "movies.dat"), + names=["movieId", "title", "genres"], + delimiter="::", + engine="python", + ) + item_df = item_df.rename(columns={"movieId": "item"}) + item_df = fold( + item_df, "item", ["genres"], fn=lambda s: (t.lower() for t in s.split("|")) + ) + + user_df = pd.read_csv( + os.path.join(path, "users.dat"), + names=["userId", "gender", "age", "job", "zip"], + delimiter="::", + engine="python", + ) + user_df = user_df.rename(columns={"userId": "user"}) + user_df = user_df[["user", "gender", "age"]] + user_df = fold(user_df, "user", ["gender", "age"]) + + df = df.rename(columns={"userId": "user", "movieId": "item"}) + + train_ds, test_ds = build( + df, user_df=user_df, item_df=item_df, policy=policy, **kwargs + ) + return train_ds, test_ds + + +def load( + dirname: str = "data", + version: str = "ml-latest-small", + policy: str = "leave_one_out", + **kwargs: Any, +) -> Tuple[Dataset, Dataset]: + """Load a chosen Movielens dataset as train and test datasets.""" + + path = os.path.join(dirname, version) + + if not os.path.exists(path): + raise FileNotFoundError( + f"Could not find '{path}'. Try downloading the data first with the " + f"`xanthus.datasets.movielens.download` function." + ) + + if version == "ml-latest-small": + return _load_100k(path, policy, **kwargs) + elif version == "ml-1m": + return _load_1m(path, policy, **kwargs) diff --git a/xanthus/datasets/utils.py b/xanthus/datasets/utils.py index 8c2736d..8ff095f 100644 --- a/xanthus/datasets/utils.py +++ b/xanthus/datasets/utils.py @@ -4,7 +4,7 @@ Copyright (c) 2018-2020 Mark Douthwaite """ -from typing import Optional, Union, List, Set, Iterator, Tuple, Any, Callable +from typing import Optional, Union, List, Set, Iterator, Tuple, Any, Callable, Iterable from itertools import islice from pandas import DataFrame @@ -293,6 +293,7 @@ def sample_negatives( if aux_matrix is not None: interactions += aux_matrix + # Todo: this could be a generator too -- no need to pause... neg_users, neg_items, neg_ratings = unpack_negative_samples( users, interactions, negative_samples, mode=sampling_mode ) @@ -438,3 +439,13 @@ def fold( output = DataFrame(data=pairs, columns=[key, "tag"]) return output.drop_duplicates([key, "tag"]) + + +def batched(i: Iterable[Any], n: int) -> Iterable[Any]: + """Batch an iterable 'i' into batches of 'n'.""" + + g = (_ for _ in i) + c = list(islice(g, n)) + while c: + yield c + c = list(islice(g, n)) diff --git a/xanthus/evaluate/__init__.py b/xanthus/evaluate/__init__.py index b63937b..ec0fcf7 100644 --- a/xanthus/evaluate/__init__.py +++ b/xanthus/evaluate/__init__.py @@ -14,4 +14,18 @@ pak, ) -from .utils import split, he_sampling, leave_one_out +from .utils import split, leave_one_out, create_rankings + + +__all__ = [ + "score", + "ndcg", + "normalized_discounted_cumulative_gain", + "precision_at_k", + "coverage_at_k", + "cak", + "pak", + "leave_one_out", + "split", + "create_rankings", +] diff --git a/xanthus/evaluate/utils.py b/xanthus/evaluate/utils.py index 31d5aec..1724af9 100644 --- a/xanthus/evaluate/utils.py +++ b/xanthus/evaluate/utils.py @@ -5,20 +5,14 @@ """ import warnings -from typing import Tuple, List +from typing import Tuple from pandas import DataFrame, concat -from numpy import ( - in1d, - concatenate, - ndarray, - unique, - c_, -) +from numpy import in1d, concatenate, ndarray, asarray, c_, unique from numpy.random import choice -from ..datasets import Dataset, groupby +from xanthus.datasets import Dataset, groupby def _ignore(df: DataFrame, elements: ndarray, key: str, frac: float) -> DataFrame: @@ -114,7 +108,7 @@ def split( References ---------- - [1] Inspired by https://docs.microsoft.com/en-us/azure/machine-learning/studio-module-reference/split-data-using-recommender-split#:~:text=The%20Recommender%20Split%20option%20is,user%2Ditem%2Drating%20triples. + [1] Inspired by https://bit.ly/3fCL9Xc (Azure Recommender Split) Notes ----- @@ -197,8 +191,8 @@ def split( return train_df[columns], test_df[columns] -def he_sampling( - a: Dataset, b: Dataset, n_samples: int = 100 +def create_rankings( + a: Dataset, b: Dataset, n_samples: int = 100, unravel: bool = False, **kwargs: int ) -> Tuple[ndarray, ndarray]: """ Sample a dataset 'a' with 'n' negative samples given interactions in dataset 'a' @@ -224,12 +218,21 @@ def he_sampling( The total number of negative samples per user to generate. For example, if the dataset 'a' was generated from a leave-one-out split, and n_samples=100, that user would receive 101 samples. + unravel: bool + If 'True', the function will return two arrays, where the first element of the + first array corresponds to the user _vector_ (i.e. user ID + optional metadata), + the first element of the first array corresponds to an associated sampled item + vector(i.e. item ID + optional metadata). Returns ------- output: (ndarray, List[ndarray]) - The first element corresponds to an array of _ordered_ user ids, the second - the per-user samples. + If 'unravel=False', the first element corresponds to an array of _ordered_ user + ids, the second the `n_samples+1`per-user samples. + If `unravel=True`, the first element corresponds to an array of _ordered_ user + vectors, the second to each individual item vector. See `unravel` argument and + `_unravel_ranked`, below. This function is provided for use when evaluating + Keras Models with the `predict` method. References ---------- @@ -251,8 +254,36 @@ def he_sampling( items[len(unique_users) :], ) - groups, grouped = groupby(sampled_users, sampled_items) + _, grouped = groupby(sampled_users, sampled_items) - grouped = c_[items[: len(unique_users)], grouped] + grouped = c_[grouped, items[: len(unique_users)]] - return unique_users, grouped + if unravel: + return _unravel_sampled(unique_users, grouped, a, **kwargs) + else: + return unique_users, grouped + + +def _unravel_sampled( + users: ndarray, ranked: ndarray, a: Dataset, output_dim: int = 1 +) -> Tuple[ndarray, ndarray]: + """ + Unravel two arrays of the form: + user_{i}, [item_{i,0}, ..., item_{i, j} + Into the form: + user_{i}, item_{i, 0} + ... + user_{i}, item_{i, j} + """ + + z = list([user, item] for i, user in enumerate(users) for item in ranked[i]) + z = asarray(z) + + if output_dim > 1: + users = asarray(list(a.iter_user(z[:, 0], n_dim=output_dim))) + items = asarray(list(a.iter_item(z[:, 1], n_dim=output_dim))) + else: + users = z[:, 0] + items = z[:, 1] + + return users, items diff --git a/xanthus/models/__init__.py b/xanthus/models/__init__.py index 39a97d0..55215fa 100644 --- a/xanthus/models/__init__.py +++ b/xanthus/models/__init__.py @@ -5,8 +5,18 @@ """ from xanthus.models.neural import ( - GeneralizedMatrixFactorizationModel, - NeuralMatrixFactorizationModel, - MultiLayerPerceptronModel, + MultiLayerPerceptron, + GeneralizedMatrixFactorization, + NeuralMatrixFactorization, ) -from xanthus.models.baseline import MatrixFactorizationModel, PopRankModel +from xanthus.models.baseline import MatrixFactorization, PopRank +from . import utils + +__all__ = [ + "MatrixFactorization", + "PopRank", + "MultiLayerPerceptron", + "GeneralizedMatrixFactorization", + "NeuralMatrixFactorization", + "utils", +] diff --git a/xanthus/models/baseline/__init__.py b/xanthus/models/baseline/__init__.py index 8e443ef..e6bc299 100644 --- a/xanthus/models/baseline/__init__.py +++ b/xanthus/models/baseline/__init__.py @@ -6,8 +6,15 @@ from functools import partial -from .pop_rank import PopRankModel -from .mf import MatrixFactorizationModel +from .pop_rank import PopRank +from .mf import MatrixFactorization -AlternatingLeastSquaresModel = partial(MatrixFactorizationModel, method="als") -BayesianPersonalizedRankingModel = partial(MatrixFactorizationModel, method="bpr") +AlternatingLeastSquares = partial(MatrixFactorization, method="als") +BayesianPersonalizedRanking = partial(MatrixFactorization, method="bpr") + +__all__ = [ + "MatrixFactorization", + "AlternatingLeastSquares", + "BayesianPersonalizedRanking", + "PopRank", +] diff --git a/xanthus/models/baseline/mf.py b/xanthus/models/baseline/mf.py index 7008d66..1a252d9 100644 --- a/xanthus/models/baseline/mf.py +++ b/xanthus/models/baseline/mf.py @@ -15,7 +15,7 @@ from xanthus.datasets import Dataset -class MatrixFactorizationModel: +class MatrixFactorization: """ A simple adapter for 'non-neural' matrix factorization algorithms. @@ -46,7 +46,7 @@ def __init__(self, method: str = "als", **kwargs: Optional[Any]) -> None: self._mat = None self._method = method - def fit(self, dataset: Dataset) -> "MatrixFactorizationModel": + def fit(self, dataset: Dataset) -> "MatrixFactorization": """ Fit the model to a provided Dataset. @@ -57,7 +57,7 @@ def fit(self, dataset: Dataset) -> "MatrixFactorizationModel": Returns ------- - output: MatrixFactorizationModel + output: MatrixFactorization Returns itself. How fun. See Also diff --git a/xanthus/models/baseline/pop_rank.py b/xanthus/models/baseline/pop_rank.py index 00920ed..7adc318 100644 --- a/xanthus/models/baseline/pop_rank.py +++ b/xanthus/models/baseline/pop_rank.py @@ -9,10 +9,10 @@ from numpy import ndarray from xanthus.datasets import Dataset -from xanthus.models import base +from xanthus.models.legacy import base -class PopRankModel(base.RecommenderModel): +class PopRank(base.RecommenderModel): """ A Popularity Ranking recommender model. This class implements some simple popularity-based recommendation approaches. By default, it will simply return @@ -55,7 +55,7 @@ def __init__( self._alpha = alpha self._limit = limit - def fit(self, dataset: Dataset) -> "PopRankModel": + def fit(self, dataset: Dataset) -> "PopRank": """ Fit the model to a provided Dataset. @@ -66,7 +66,7 @@ def fit(self, dataset: Dataset) -> "PopRankModel": Returns ------- - output: PopRankModel + output: PopRank Returns itself. How fun. See Also diff --git a/xanthus/models/legacy/__init__.py b/xanthus/models/legacy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xanthus/models/base.py b/xanthus/models/legacy/base.py similarity index 98% rename from xanthus/models/base.py rename to xanthus/models/legacy/base.py index 5003656..c9ed7d7 100644 --- a/xanthus/models/base.py +++ b/xanthus/models/legacy/base.py @@ -213,10 +213,6 @@ def predict( list of 'users' was provided, this will be ordered by this list. If 'users' are not provided, it will be ordered by 'dataset.all_users'. - # Todo: - # * This needs an overhaul. Some of that will involve re-working the - # dataset generators, but this feels v. clunky & extraordinarily slow. - """ recommended = [] diff --git a/xanthus/models/legacy/neural.py b/xanthus/models/legacy/neural.py new file mode 100644 index 0000000..8d2877a --- /dev/null +++ b/xanthus/models/legacy/neural.py @@ -0,0 +1,321 @@ +""" +The MIT License + +Copyright (c) 2018-2020 Mark Douthwaite +""" + +from typing import Optional, Any, Tuple +from tensorflow.keras import Model +from tensorflow.keras.layers import Multiply, Dense, Concatenate +from tensorflow.keras.initializers import lecun_uniform +from tensorflow.keras.regularizers import l2 + +from xanthus.datasets import Dataset +from xanthus.models import utils +from xanthus.models.legacy import base + + +class MultiLayerPerceptronModel(base.NeuralRecommenderModel): + """ + An implementation of a Multilayer Perceptron (MLP) model in Keras. + + Parameters + ---------- + layers: tuple + A tuple, where each element corresponds to the number of units in each of the + layers of the MLP. + activations: str + The activation function to use for each of the layers in the MLP. + l2_reg: float + The L2 regularization to be applied to each of the layers in the MLP. + + References + ---------- + [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 + + See Also + -------- + xanthus.models.base.NeuralRecommenderModel + + """ + + def __init__( + self, + *args: Optional[Any], + layers: Tuple[int, ...] = (64, 32, 16, 8), + activations: str = "relu", + l2_reg: float = 1e-3, + **kwargs: Optional[Any] + ): + """Initialize a MultiLayerPerceptronModel.""" + + super().__init__(*args, **kwargs) + self._activations = activations + self._layers = layers + self._l2_reg = l2_reg + + def _build_model( + self, + dataset: Dataset, + n_user_dim: int = 1, + n_item_dim: int = 1, + n_factors: int = 50, + **kwargs: Optional[Any] + ) -> Model: + """ + Build a Keras model, in this case a MultiLayerPerceptronModel (MLP) + model. See [1] for more info. The original code released with [1] can be + found at [2]. + + Parameters + ---------- + dataset: Dataset + The input dataset. This is used to specify the 'vocab' size of each of the + 'embedding blocks' (of which there are two in this architecture). + n_user_dim: int + The dimensionality of the user input vector. When using metadata, you should + make sure to set this to the size of each of these vectors. + n_item_dim: int + The dimensionality of the item input vector. When using metadata, you should + make sure to set this to the size of each of these vectors. + n_factors: int + The dimensionality of the latent feature space _for both users and items_ + for the GMF component of the architecture. + + Returns + ------- + output: Model + The 'complete' Keras Model object. + + References + ---------- + [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 + [2] https://github.com/hexiangnan/neural_collaborative_filtering + """ + + n_user_vocab = dataset.all_users.shape[0] + n_item_vocab = dataset.all_items.shape[0] + + if dataset.user_meta is not None: + n_user_vocab += dataset.user_meta.shape[1] + if dataset.item_meta is not None: + n_item_vocab += dataset.item_meta.shape[1] + + # mlp block + user_input, user_bias, user_factors = utils.get_embedding_block( + n_user_vocab, n_user_dim, int(self._layers[0] / 2) + ) + item_input, item_bias, item_factors = utils.get_embedding_block( + n_item_vocab, n_item_dim, int(self._layers[0] / 2) + ) + + body = Concatenate()([user_factors, item_factors]) + + for layer in self._layers: + body = Dense( + layer, + activity_regularizer=l2(self._l2_reg), + activation=self._activations, + )(body) + + output = Dense(1, activation="sigmoid", kernel_initializer=lecun_uniform())( + body + ) + + return Model(inputs=[user_input, item_input], outputs=output) + + +class NeuralMatrixFactorizationModel(base.NeuralRecommenderModel): + """ + An implementation of a Neural Matrix Factorization (NeuMF) model in Keras. + + Parameters + ---------- + layers: tuple + A tuple, where each element corresponds to the number of units in each of the + layers of the MLP. + activations: str + The activation function to use for each of the layers in the MLP. + l2_reg: float + The L2 regularization to be applied to each of the layers in the MLP. + + References + ---------- + [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 + + See Also + -------- + xanthus.models.base.NeuralRecommenderModel + + """ + + def __init__( + self, + *args: Optional[Any], + layers: Tuple[int, ...] = (64, 32, 16, 8), + activations: str = "relu", + l2_reg: float = 1e-3, + **kwargs: Optional[Any] + ): + """Initialize a MultiLayerPerceptronModel.""" + + super().__init__(*args, **kwargs) + self._activations = activations + self._layers = layers + self._l2_reg = l2_reg + + def _build_model( + self, + dataset: Dataset, + n_user_dim: int = 1, + n_item_dim: int = 1, + n_factors: int = 50, + **kwargs: Optional[Any] + ) -> Model: + """ + Build a Keras model, in this case a NeuralMatrixFactorizationModel (NeuMF) + model. This is a recommender model with two input branches (one half the same + architecture as in GeneralizedMatrixFactorizationModel, the other the same + architecture as in MultiLayerPerceptronModel. See [1] for more info. The + original code released with [1] can be found at [2]. + + Parameters + ---------- + dataset: Dataset + The input dataset. This is used to specify the 'vocab' size of each of the + 'embedding blocks' (of which there are four in this architecture). + n_user_dim: int + The dimensionality of the user input vector. When using metadata, you should + make sure to set this to the size of each of these vectors. + n_item_dim: int + The dimensionality of the item input vector. When using metadata, you should + make sure to set this to the size of each of these vectors. + n_factors: int + The dimensionality of the latent feature space _for both users and items_ + for the GMF component of the architecture. + + Returns + ------- + output: Model + The 'complete' Keras Model object. + + References + ---------- + [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 + [2] https://github.com/hexiangnan/neural_collaborative_filtering + + """ + + n_user_vocab = dataset.all_users.shape[0] + n_item_vocab = dataset.all_items.shape[0] + + if dataset.user_meta is not None: + n_user_vocab += dataset.user_meta.shape[1] + if dataset.item_meta is not None: + n_item_vocab += dataset.item_meta.shape[1] + + # mlp block + user_input, mlp_user_bias, mlp_user_factors = utils.get_embedding_block( + n_user_vocab, n_user_dim, int(self._layers[0] / 2) + ) + item_input, mlp_item_bias, mlp_item_factors = utils.get_embedding_block( + n_item_vocab, n_item_dim, int(self._layers[0] / 2) + ) + + mlp_body = Concatenate()([mlp_user_factors, mlp_item_factors]) + + for layer in self._layers: + mlp_body = Dense( + layer, + activity_regularizer=l2(self._l2_reg), + activation=self._activations, + )(mlp_body) + + # mf block + user_input, mf_user_bias, mf_user_factors = utils.get_embedding_block( + n_user_vocab, n_user_dim, n_factors, inputs=user_input, + ) + item_input, mf_item_bias, mf_item_factors = utils.get_embedding_block( + n_item_vocab, n_item_dim, n_factors, inputs=item_input, + ) + mf_body = Multiply()([mf_user_factors, mf_item_factors]) + + body = Concatenate()([mf_body, mlp_body]) + + output = Dense(1, activation="sigmoid", kernel_initializer=lecun_uniform())( + body + ) + + return Model(inputs=[user_input, item_input], outputs=output) + + +class GeneralizedMatrixFactorizationModel(base.NeuralRecommenderModel): + """ + An implementation of a Generalized Matrix Factorization (GMF) model in Keras. + + References + ---------- + [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 + + """ + + def _build_model( + self, + dataset: Dataset, + n_user_dim: int = 1, + n_item_dim: int = 1, + n_factors: int = 50, + **kwargs: Optional[Any] + ) -> Model: + """ + Build a Keras model, in this case a GeneralizedMatrixFactorizationModel (GMF) + model. See [1] for more info. The original code released with [1] can be + found at [2]. + + Parameters + ---------- + dataset: Dataset + The input dataset. This is used to specify the 'vocab' size of each of the + 'embedding blocks' (of which there are two in this architecture). + n_user_dim: int + The dimensionality of the user input vector. When using metadata, you should + make sure to set this to the size of each of these vectors. + n_item_dim: int + The dimensionality of the item input vector. When using metadata, you should + make sure to set this to the size of each of these vectors. + n_factors: int + The dimensionality of the latent feature space _for both users and items_ + for the GMF component of the architecture. + + Returns + ------- + output: Model + The 'complete' Keras Model object. + + References + ---------- + [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 + [2] https://github.com/hexiangnan/neural_collaborative_filtering + """ + + n_user_vocab = dataset.all_users.shape[0] + n_item_vocab = dataset.all_items.shape[0] + + if dataset.user_meta is not None: + n_user_vocab += dataset.user_meta.shape[1] + if dataset.item_meta is not None: + n_item_vocab += dataset.item_meta.shape[1] + + user_input, user_bias, user_factors = utils.get_embedding_block( + n_user_vocab, n_user_dim, n_factors, **kwargs + ) + item_input, item_bias, item_factors = utils.get_embedding_block( + n_item_vocab, n_item_dim, n_factors, **kwargs + ) + + body = Multiply()([user_factors, item_factors]) + output = Dense(1, activation="sigmoid", kernel_initializer=lecun_uniform())( + body + ) + + return Model(inputs=[user_input, item_input], outputs=output) diff --git a/xanthus/models/neural.py b/xanthus/models/neural.py index d194ce5..5f7aab9 100644 --- a/xanthus/models/neural.py +++ b/xanthus/models/neural.py @@ -4,317 +4,294 @@ Copyright (c) 2018-2020 Mark Douthwaite """ -from typing import Optional, Any, Tuple + +from typing import Optional, Any, NoReturn, Iterable, List, Union, Callable, Tuple + +from tensorflow import Tensor from tensorflow.keras import Model -from tensorflow.keras.layers import Multiply, Dense, Concatenate +from tensorflow.keras.layers import Multiply, Dense, Concatenate, Layer from tensorflow.keras.initializers import lecun_uniform -from tensorflow.keras.regularizers import l2 +from tensorflow.keras.regularizers import Regularizer +from .utils import InputEmbeddingBlock -from xanthus.datasets import Dataset -from xanthus.models import utils, base +Activation = Callable[[Tensor], Tensor] -class MultiLayerPerceptronModel(base.NeuralRecommenderModel): +class GeneralizedMatrixFactorization(Model): """ - An implementation of a Multilayer Perceptron (MLP) model in Keras. - - Parameters - ---------- - layers: tuple - A tuple, where each element corresponds to the number of units in each of the - layers of the MLP. - activations: str - The activation function to use for each of the layers in the MLP. - l2_reg: float - The L2 regularization to be applied to each of the layers in the MLP. + A Keras model implementing Generalized Matrix Factorization (GMF) architecture + from [1]. References ---------- [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 - - See Also - -------- - xanthus.models.base.NeuralRecommenderModel - """ def __init__( self, - *args: Optional[Any], - layers: Tuple[int, ...] = (64, 32, 16, 8), - activations: str = "relu", - l2_reg: float = 1e-3, - **kwargs: Optional[Any] - ): - """Initialize a MultiLayerPerceptronModel.""" - - super().__init__(*args, **kwargs) - self._activations = activations - self._layers = layers - self._l2_reg = l2_reg - - def _build_model( - self, - dataset: Dataset, - n_user_dim: int = 1, - n_item_dim: int = 1, - n_factors: int = 50, - **kwargs: Optional[Any] - ) -> Model: + n: int, + m: int, + factors: int = 32, + *args: Any, + embedding_regularizer: Optional[Regularizer] = None, + **kwargs: Any, + ) -> None: """ - Build a Keras model, in this case a MultiLayerPerceptronModel (MLP) - model. See [1] for more info. The original code released with [1] can be - found at [2]. + Initialize a GMF model. Parameters ---------- - dataset: Dataset - The input dataset. This is used to specify the 'vocab' size of each of the - 'embedding blocks' (of which there are two in this architecture). - n_user_dim: int - The dimensionality of the user input vector. When using metadata, you should - make sure to set this to the size of each of these vectors. - n_item_dim: int - The dimensionality of the item input vector. When using metadata, you should - make sure to set this to the size of each of these vectors. - n_factors: int - The dimensionality of the latent feature space _for both users and items_ - for the GMF component of the architecture. - - Returns - ------- - output: Model - The 'complete' Keras Model object. - - References - ---------- - [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 - [2] https://github.com/hexiangnan/neural_collaborative_filtering - """ - - n_user_vocab = dataset.all_users.shape[0] - n_item_vocab = dataset.all_items.shape[0] + n: int + The size of the 'vocabulary' of users (i.e. unique user IDs + metadata tags) + m: int + The size of the 'vocabulary' of items (i.e. unique item IDs + metadata tags) + factors: int + The size of 'predictive factors' (analogous to latent features) of the MF + model. + embedding_regularizer: Regularizer + A regularizer to be applied to Embdeddings. See keras.layers.Embedding + args: Any, optional + Optional args to be passed to base keras.Model. + kwargs: Any + Optional kwargs to be passed to base keras.Model - if dataset.user_meta is not None: - n_user_vocab += dataset.user_meta.shape[1] - if dataset.item_meta is not None: - n_item_vocab += dataset.item_meta.shape[1] + """ - # mlp block - user_input, user_bias, user_factors = utils.get_embedding_block( - n_user_vocab, n_user_dim, int(self._layers[0] / 2) + super().__init__(*args, **kwargs) + self.user_embedding = InputEmbeddingBlock( + n, factors, name="user_embeddings", regularizer=embedding_regularizer ) - item_input, item_bias, item_factors = utils.get_embedding_block( - n_item_vocab, n_item_dim, int(self._layers[0] / 2) + self.item_embedding = InputEmbeddingBlock( + m, factors, name="item_embeddings", regularizer=embedding_regularizer ) - - body = Concatenate()([user_factors, item_factors]) - - for layer in self._layers: - body = Dense( - layer, - activity_regularizer=l2(self._l2_reg), - activation=self._activations, - )(body) - - output = Dense(1, activation="sigmoid", kernel_initializer=lecun_uniform())( - body + self.multiply = Multiply(name="multiply") + self.prediction = Dense( + 1, + activation="sigmoid", + kernel_initializer=lecun_uniform(), + name="prediction", ) - return Model(inputs=[user_input, item_input], outputs=output) + def call( + self, + inputs: Union[List[Tensor], Tensor], + training: Optional[bool] = None, + mask: Optional[Union[List[Tensor], Tensor]] = None, + ) -> Union[List[Tensor], Tensor]: + """Call the model.""" + user_z = self.user_embedding(inputs[0]) + item_z = self.item_embedding(inputs[1]) + z = self.multiply([user_z, item_z]) + return self.prediction(z) -class NeuralMatrixFactorizationModel(base.NeuralRecommenderModel): - """ - An implementation of a Neural Matrix Factorization (NeuMF) model in Keras. - Parameters - ---------- - layers: tuple - A tuple, where each element corresponds to the number of units in each of the - layers of the MLP. - activations: str - The activation function to use for each of the layers in the MLP. - l2_reg: float - The L2 regularization to be applied to each of the layers in the MLP. +class MultiLayerPerceptron(Model): + """ + A Keras model implementing Multilayer Perceptron Model (MLP) recommendation model + architecture described in [1]. References ---------- [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 - - See Also - -------- - xanthus.models.base.NeuralRecommenderModel - """ def __init__( self, - *args: Optional[Any], - layers: Tuple[int, ...] = (64, 32, 16, 8), - activations: str = "relu", - l2_reg: float = 1e-3, - **kwargs: Optional[Any] - ): - """Initialize a MultiLayerPerceptronModel.""" - - super().__init__(*args, **kwargs) - self._activations = activations - self._layers = layers - self._l2_reg = l2_reg - - def _build_model( - self, - dataset: Dataset, - n_user_dim: int = 1, - n_item_dim: int = 1, - n_factors: int = 50, - **kwargs: Optional[Any] - ) -> Model: + n: int, + m: int, + layers: Tuple[int, ...] = (32, 16, 8), + regularizer: Optional[Regularizer] = None, + embedding_regularizer: Optional[Regularizer] = None, + activation: Union[str, Activation] = "relu", + *args: Any, + **kwargs: Any, + ) -> None: """ - Build a Keras model, in this case a NeuralMatrixFactorizationModel (NeuMF) - model. This is a recommender model with two input branches (one half the same - architecture as in GeneralizedMatrixFactorizationModel, the other the same - architecture as in MultiLayerPerceptronModel. See [1] for more info. The - original code released with [1] can be found at [2]. + Initialize a GMF model. Parameters ---------- - dataset: Dataset - The input dataset. This is used to specify the 'vocab' size of each of the - 'embedding blocks' (of which there are four in this architecture). - n_user_dim: int - The dimensionality of the user input vector. When using metadata, you should - make sure to set this to the size of each of these vectors. - n_item_dim: int - The dimensionality of the item input vector. When using metadata, you should - make sure to set this to the size of each of these vectors. - n_factors: int - The dimensionality of the latent feature space _for both users and items_ - for the GMF component of the architecture. - - Returns - ------- - output: Model - The 'complete' Keras Model object. - - References - ---------- - [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 - [2] https://github.com/hexiangnan/neural_collaborative_filtering + n: int + The size of the 'vocabulary' of users (i.e. unique user IDs + metadata tags) + m: int + The size of the 'vocabulary' of items (i.e. unique item IDs + metadata tags) + layers: tuple + A tuple, where each element corresponds to the number of units in each of + the layers of the MLP. + regularizer: Regularizer + A regularizer to be applied to hidden layers. See keras.layers.Dense + embedding_regularizer: Regularizer + A regularizer to be applied to Embdeddings. See keras.layers.Embedding + activation: str, Regularizer + The activation function to use for hidden layers. + args: Any, optional + Optional args to be passed to base keras.Model. + kwargs: Any + Optional kwargs to be passed to base keras.Model """ - n_user_vocab = dataset.all_users.shape[0] - n_item_vocab = dataset.all_items.shape[0] - - if dataset.user_meta is not None: - n_user_vocab += dataset.user_meta.shape[1] - if dataset.item_meta is not None: - n_item_vocab += dataset.item_meta.shape[1] + super().__init__(*args, **kwargs) - # mlp block - user_input, mlp_user_bias, mlp_user_factors = utils.get_embedding_block( - n_user_vocab, n_user_dim, int(self._layers[0] / 2) + self.user_embedding = InputEmbeddingBlock( + n, layers[0] // 2, name="user_embeddings", regularizer=embedding_regularizer ) - item_input, mlp_item_bias, mlp_item_factors = utils.get_embedding_block( - n_item_vocab, n_item_dim, int(self._layers[0] / 2) + self.item_embedding = InputEmbeddingBlock( + m, layers[0] // 2, name="item_embeddings", regularizer=embedding_regularizer + ) + self.concat = Concatenate(name="concat") + self.hidden = list(self._build_layers(layers, activation, regularizer)) + self.prediction = Dense( + 1, + activation="sigmoid", + kernel_initializer=lecun_uniform(), + name="prediction", ) - mlp_body = Concatenate()([mlp_user_factors, mlp_item_factors]) + @staticmethod + def _build_layers( + layers: Iterable[int], + activation: Union[str, Activation], + regularizer: Regularizer, + ) -> Iterable[Layer]: + """Build the model's hidden layers.""" - for layer in self._layers: - mlp_body = Dense( + for i, layer in enumerate(layers): + yield Dense( layer, - activity_regularizer=l2(self._l2_reg), - activation=self._activations, - )(mlp_body) + activity_regularizer=regularizer, + activation=activation, + name=f"layer{i+1}", + ) - # mf block - user_input, mf_user_bias, mf_user_factors = utils.get_embedding_block( - n_user_vocab, n_user_dim, n_factors, inputs=user_input, - ) - item_input, mf_item_bias, mf_item_factors = utils.get_embedding_block( - n_item_vocab, n_item_dim, n_factors, inputs=item_input, - ) - mf_body = Multiply()([mf_user_factors, mf_item_factors]) + def call( + self, + inputs: Union[List[Tensor], Tensor], + training: Optional[bool] = None, + mask: Optional[Union[List[Tensor], Tensor]] = None, + ) -> Union[List[Tensor], Tensor]: + """Invoke the model. A single 'forward pass'.""" - body = Concatenate()([mf_body, mlp_body]) + user_z = self.user_embedding(inputs[0]) + item_z = self.item_embedding(inputs[1]) + z = self.concat([user_z, item_z]) - output = Dense(1, activation="sigmoid", kernel_initializer=lecun_uniform())( - body - ) + for layer in self.hidden: + z = layer(z) - return Model(inputs=[user_input, item_input], outputs=output) + return self.prediction(z) -class GeneralizedMatrixFactorizationModel(base.NeuralRecommenderModel): +class NeuralMatrixFactorization(Model): """ - An implementation of a Generalized Matrix Factorization (GMF) model in Keras. + A Keras model implementing Neural Matrix Factorization (GMF) architecture + from [1]. References ---------- [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 - """ - def _build_model( + def __init__( self, - dataset: Dataset, - n_user_dim: int = 1, - n_item_dim: int = 1, - n_factors: int = 50, - **kwargs: Optional[Any] - ) -> Model: + n: int, + m: int, + factors: int = 32, + layers: Tuple[int, ...] = (32, 16, 8), + activation: Union[str, Activation] = "relu", + regularizer: Optional[Regularizer] = None, + embedding_regularizer: Optional[Regularizer] = None, + *args: Any, + **kwargs: Any, + ) -> None: """ - Build a Keras model, in this case a GeneralizedMatrixFactorizationModel (GMF) - model. See [1] for more info. The original code released with [1] can be - found at [2]. + Initialize a NMF model. Parameters ---------- - dataset: Dataset - The input dataset. This is used to specify the 'vocab' size of each of the - 'embedding blocks' (of which there are two in this architecture). - n_user_dim: int - The dimensionality of the user input vector. When using metadata, you should - make sure to set this to the size of each of these vectors. - n_item_dim: int - The dimensionality of the item input vector. When using metadata, you should - make sure to set this to the size of each of these vectors. - n_factors: int - The dimensionality of the latent feature space _for both users and items_ - for the GMF component of the architecture. - - Returns - ------- - output: Model - The 'complete' Keras Model object. - - References - ---------- - [1] He et al. https://dl.acm.org/doi/10.1145/3038912.3052569 - [2] https://github.com/hexiangnan/neural_collaborative_filtering - """ - - n_user_vocab = dataset.all_users.shape[0] - n_item_vocab = dataset.all_items.shape[0] + n: int + The size of the 'vocabulary' of users (i.e. unique user IDs + metadata tags) + m: int + The size of the 'vocabulary' of items (i.e. unique item IDs + metadata tags) + layers: tuple + A tuple, where each element corresponds to the number of units in each of + the layers of the MLP. + regularizer: Regularizer + A regularizer to be applied to hidden layers. See keras.layers.Dense + embedding_regularizer: Regularizer + A regularizer to be applied to Embdeddings. See keras.layers.Embedding + activation: str, Regularizer + The activation function to use for hidden layers. + args: Any, optional + Optional args to be passed to base keras.Model. + kwargs: Any + Optional kwargs to be passed to base keras.Model - if dataset.user_meta is not None: - n_user_vocab += dataset.user_meta.shape[1] - if dataset.item_meta is not None: - n_item_vocab += dataset.item_meta.shape[1] + """ - user_input, user_bias, user_factors = utils.get_embedding_block( - n_user_vocab, n_user_dim, n_factors, **kwargs + super().__init__(*args, **kwargs) + self.gmf_user_embedding = InputEmbeddingBlock( + n, factors, regularizer=embedding_regularizer ) - item_input, item_bias, item_factors = utils.get_embedding_block( - n_item_vocab, n_item_dim, n_factors, **kwargs + self.gmf_item_embedding = InputEmbeddingBlock( + m, factors, regularizer=embedding_regularizer ) + self.gmf_multiply = Multiply() - body = Multiply()([user_factors, item_factors]) - output = Dense(1, activation="sigmoid", kernel_initializer=lecun_uniform())( - body + # check units -- this could cause issues. + self.mlp_user_embedding = InputEmbeddingBlock( + n, layers[0] // 2, regularizer=embedding_regularizer + ) + self.mlp_item_embedding = InputEmbeddingBlock( + m, layers[0] // 2, regularizer=embedding_regularizer + ) + self.mlp_concat = Concatenate() + self.mlp_hidden = list(self._build_layers(layers, activation, regularizer)) + + self.concat = Concatenate() + self.prediction = Dense( + 1, + activation="sigmoid", + kernel_initializer=lecun_uniform(), + name="prediction", ) - return Model(inputs=[user_input, item_input], outputs=output) + @staticmethod + def _build_layers( + layers: Iterable[int], + activation: Union[str, Activation], + regularizer: Regularizer, + ) -> Iterable[Dense]: + """Build the model's hidden layers.""" + + for i, layer in enumerate(layers): + yield Dense( + layer, + activity_regularizer=regularizer, + activation=activation, + name=f"layer{i+1}", + ) + + def call( + self, + inputs: Union[List[Tensor], Tensor], + training: Optional[bool] = None, + mask: Optional[Union[List[Tensor], Tensor]] = None, + ) -> Union[List[Tensor], Tensor]: + """Invoke the model. A single 'forward pass'.""" + + mlp_user_z = self.mlp_user_embedding(inputs[0]) + mlp_item_z = self.mlp_item_embedding(inputs[1]) + mlp_z = self.mlp_concat([mlp_user_z, mlp_item_z]) + + for layer in self.mlp_hidden: + mlp_z = layer(mlp_z) + + gmf_user_z = self.gmf_user_embedding(inputs[0]) + gmf_item_z = self.gmf_item_embedding(inputs[1]) + gmf_z = self.gmf_multiply([gmf_user_z, gmf_item_z]) + + z = self.concat([gmf_z, mlp_z]) + + return self.prediction(z) diff --git a/xanthus/models/utils.py b/xanthus/models/utils.py index dad968a..1f39a3e 100644 --- a/xanthus/models/utils.py +++ b/xanthus/models/utils.py @@ -4,8 +4,10 @@ Copyright (c) 2018-2020 Mark Douthwaite """ -from itertools import islice -from typing import Optional, Tuple, Iterable, Any +from typing import Optional, Tuple, Any, Union, Dict, List + +import numpy as np +from numpy import ndarray from tensorflow.keras.layers import ( Layer, @@ -13,10 +15,54 @@ Flatten, Embedding, ) -from tensorflow.keras.regularizers import l2 +from tensorflow.keras.regularizers import l2, Regularizer from tensorflow.keras.initializers import RandomNormal, Initializer +class InputEmbeddingBlock(Layer): + """An input embedding block that flattens the output.""" + + def __init__( + self, + n_vocab: int, + n_factors: int, + *args: Any, + regularizer: Optional[Regularizer] = None, + **kwargs: Any, + ) -> None: + """Initialize the block!""" + + super().__init__(*args, **kwargs) + self._n_factors = n_factors + self._n_vocab = n_vocab + self._embedding: Optional[Embedding] = None + self._output: Optional[Flatten] = None + self._regularizer: Optional[Union[Regularizer, str]] = regularizer + + def build(self, input_shape: Tuple[int, ...]) -> None: + """Build the block!""" + + self._embedding = Embedding( + input_dim=self._n_vocab, + output_dim=self._n_factors, + input_length=input_shape, + embeddings_initializer=RandomNormal(stddev=0.01), + embeddings_regularizer=self._regularizer, + ) + self._output = Flatten() + + def call(self, inputs: ndarray, **kwargs: Any) -> ndarray: + """Call the block.""" + + if self._embedding is None or self._output is None: + raise ValueError( + "You must call 'build' on an InputEmbeddingBlock before 'call'." + ) + else: + x = self._embedding(inputs) + return self._output(x) + + def get_embedding_block( n_vocab: int, n_dim: int, @@ -66,11 +112,27 @@ def get_embedding_block( return inputs, Flatten()(bias_embedding), factors -def batched(i: Iterable[Any], n: int) -> Iterable[Any]: - """Batch an iterable 'i' into batches of 'n'.""" +def reshape_recommended( + users: ndarray, items: ndarray, scores: ndarray, n: int, mode: str = "array" +) -> Union[ndarray, Dict[int, List[Tuple[int, float]]]]: + """ + Reshape recommendations from 'long' format into 'wide' format. + """ + + recommended: Dict[int, List[Tuple[int, float]]] = {k: [] for k in users[:, 0]} + + for user, item, rating in zip(users[:, 0], items[:, 0], scores.flatten()): + recommended[user].append((item, rating)) + + if mode == "dict": + return recommended - g = (_ for _ in i) - c = list(islice(g, n)) - while c: - yield c - c = list(islice(g, n)) + elif mode == "array": + return np.asarray( + [ + [e for e, _ in sorted(recommended[_], key=lambda x: -x[1])][:n] + for _ in recommended.keys() + ] + ) + else: + raise ValueError(f"Unknown create recommended mode '{mode}'.") diff --git a/xanthus/utils/__init__.py b/xanthus/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xanthus/utils/benchmarking/__init__.py b/xanthus/utils/benchmarking/__init__.py new file mode 100644 index 0000000..0d0fefb --- /dev/null +++ b/xanthus/utils/benchmarking/__init__.py @@ -0,0 +1,2 @@ +from .core import benchmark, save +from .managers import ModelManager diff --git a/xanthus/utils/benchmarking/core.py b/xanthus/utils/benchmarking/core.py new file mode 100644 index 0000000..755ed87 --- /dev/null +++ b/xanthus/utils/benchmarking/core.py @@ -0,0 +1,50 @@ +import uuid +import json +import logging +from pathlib import Path +from datetime import datetime + +import pandas as pd + + +TIMESTAMP_FORMAT = "%H:%M:%S.%f %y-%m-%d" + + +def benchmark(manager, epochs, **kwargs): + logger = logging.getLogger(f"Benchmark ({manager.name}|{kwargs})") + start = datetime.now() + records = [] + for epoch in range(epochs): + logger.info(f"Running epoch {epoch + 1} of {epochs}...") + manager.update(1) + metrics = manager.metrics(**kwargs) + metrics["epoch"] = epoch + 1 + records.append(metrics) + + end = datetime.now() + info = dict( + start=start.strftime(TIMESTAMP_FORMAT), + end=end.strftime(TIMESTAMP_FORMAT), + elapsed=(end - start).seconds, + params=manager.params(), + ) + + return records, info + + +def save(experiment, manager, records, info=None, root=None, identifier=None): + + identifier = identifier or uuid.uuid4().hex[:6] + + if root is not None: + path = Path(root) / experiment / manager.name / identifier + else: + path = Path(experiment) / manager.name / identifier + + path.mkdir(parents=True, exist_ok=True) + + df = pd.DataFrame.from_records(records) + df.to_csv(path / "results.csv") + + if info is not None: + json.dump(info, (path / "info.json").open("w")) diff --git a/xanthus/utils/benchmarking/managers.py b/xanthus/utils/benchmarking/managers.py new file mode 100644 index 0000000..67c0ff0 --- /dev/null +++ b/xanthus/utils/benchmarking/managers.py @@ -0,0 +1,21 @@ +from typing import Any, Dict, Tuple +from abc import ABC, abstractmethod + + +class ModelManager(ABC): + def __init__(self, name: str, datasets: Tuple, **params: Any) -> None: + self.name = name + self._params = params + self.train = datasets[0] + self.val = datasets[1] + + @abstractmethod + def update(self, epochs: int) -> None: + pass + + @abstractmethod + def metrics(self, **kwargs) -> Dict[str, float]: + pass + + def params(self): + return self._params