pandas_delete_duplicates.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:82b187c9f2fb38540a4104a291b895983e05c0fa68aa4bf4cd8f3891fe173b4d"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Delete Duplicates In Pandas\n",
      "\n",
      "This snippit was written by [Chris R. Albon](http://www.chrisralbon.com/) and is part of his collection of [well-documented Python snippits](https://github.com/chrisalbon/code_py). All code is written in Python 3 in iPython notebook and offered under the [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### import modules"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Create dataframe with duplicates"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "raw_data = {'first_name': ['Jason', 'Jason', 'Tina', 'Jake', 'Amy'], \n",
      "        'last_name': ['Miller', 'Miller', 'Ali', 'Milner', 'Cooze'], \n",
      "        'age': [42, 42, 36, 24, 73], \n",
      "        'preTestScore': [4, 4, 31, 2, 3],\n",
      "        'postTestScore': [25, 25, 57, 62, 70]}\n",
      "df = pd.DataFrame(raw_data, columns = ['first_name', 'last_name', 'age', 'preTestScore', 'postTestScore'])\n",
      "df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>first_name</th>\n",
        "      <th>last_name</th>\n",
        "      <th>age</th>\n",
        "      <th>preTestScore</th>\n",
        "      <th>postTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> Jason</td>\n",
        "      <td> Miller</td>\n",
        "      <td> 42</td>\n",
        "      <td>  4</td>\n",
        "      <td> 25</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> Jason</td>\n",
        "      <td> Miller</td>\n",
        "      <td> 42</td>\n",
        "      <td>  4</td>\n",
        "      <td> 25</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  Tina</td>\n",
        "      <td>    Ali</td>\n",
        "      <td> 36</td>\n",
        "      <td> 31</td>\n",
        "      <td> 57</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  Jake</td>\n",
        "      <td> Milner</td>\n",
        "      <td> 24</td>\n",
        "      <td>  2</td>\n",
        "      <td> 62</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>   Amy</td>\n",
        "      <td>  Cooze</td>\n",
        "      <td> 73</td>\n",
        "      <td>  3</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>5 rows \u00d7 5 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "  first_name last_name  age  preTestScore  postTestScore\n",
        "0      Jason    Miller   42             4             25\n",
        "1      Jason    Miller   42             4             25\n",
        "2       Tina       Ali   36            31             57\n",
        "3       Jake    Milner   24             2             62\n",
        "4        Amy     Cooze   73             3             70\n",
        "\n",
        "[5 rows x 5 columns]"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Identify which observations are duplicates"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.duplicated()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "0    False\n",
        "1     True\n",
        "2    False\n",
        "3    False\n",
        "4    False\n",
        "dtype: bool"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Drop duplicates"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.drop_duplicates()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Drop duplicates in the first name column, but take the last obs in the duplicated set"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.drop_duplicates(['first_name'], take_last=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>first_name</th>\n",
        "      <th>last_name</th>\n",
        "      <th>age</th>\n",
        "      <th>preTestScore</th>\n",
        "      <th>postTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> Jason</td>\n",
        "      <td> Miller</td>\n",
        "      <td> 42</td>\n",
        "      <td>  4</td>\n",
        "      <td> 25</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  Tina</td>\n",
        "      <td>    Ali</td>\n",
        "      <td> 36</td>\n",
        "      <td> 31</td>\n",
        "      <td> 57</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  Jake</td>\n",
        "      <td> Milner</td>\n",
        "      <td> 24</td>\n",
        "      <td>  2</td>\n",
        "      <td> 62</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>   Amy</td>\n",
        "      <td>  Cooze</td>\n",
        "      <td> 73</td>\n",
        "      <td>  3</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>4 rows \u00d7 5 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "  first_name last_name  age  preTestScore  postTestScore\n",
        "1      Jason    Miller   42             4             25\n",
        "2       Tina       Ali   36            31             57\n",
        "3       Jake    Milner   24             2             62\n",
        "4        Amy     Cooze   73             3             70\n",
        "\n",
        "[4 rows x 5 columns]"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}