pandas_string_munging.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:c375ca43b9fa561b3b0288ec42d613c7b6a15f038f4d54498b2b709f86d722a2"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# String Munging In Dataframe\n",
      "\n",
      "This snippit was written by [Chris R. Albon](http://www.chrisralbon.com/) and is part of his collection of [well-documented Python snippits](https://github.com/chrisalbon/code_py). All code is written in Python 3 in iPython notebook and offered under the [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### import modules"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import numpy as np\n",
      "import re as re"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Create dataframe"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "raw_data = {'first_name': ['Jason', 'Molly', 'Tina', 'Jake', 'Amy'], \n",
      "        'last_name': ['Miller', 'Jacobson', 'Ali', 'Milner', 'Cooze'], \n",
      "        'email': ['jas203@gmail.com', 'momomolly@gmail.com', np.NAN, 'battler@milner.com', 'Ames1234@yahoo.com'], \n",
      "        'preTestScore': [4, 24, 31, 2, 3],\n",
      "        'postTestScore': [25, 94, 57, 62, 70]}\n",
      "df = pd.DataFrame(raw_data, columns = ['first_name', 'last_name', 'email', 'preTestScore', 'postTestScore'])\n",
      "df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>first_name</th>\n",
        "      <th>last_name</th>\n",
        "      <th>email</th>\n",
        "      <th>preTestScore</th>\n",
        "      <th>postTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> Jason</td>\n",
        "      <td>   Miller</td>\n",
        "      <td>    jas203@gmail.com</td>\n",
        "      <td>  4</td>\n",
        "      <td> 25</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> Molly</td>\n",
        "      <td> Jacobson</td>\n",
        "      <td> momomolly@gmail.com</td>\n",
        "      <td> 24</td>\n",
        "      <td> 94</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  Tina</td>\n",
        "      <td>      Ali</td>\n",
        "      <td>                 NaN</td>\n",
        "      <td> 31</td>\n",
        "      <td> 57</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  Jake</td>\n",
        "      <td>   Milner</td>\n",
        "      <td>  battler@milner.com</td>\n",
        "      <td>  2</td>\n",
        "      <td> 62</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>   Amy</td>\n",
        "      <td>    Cooze</td>\n",
        "      <td>  Ames1234@yahoo.com</td>\n",
        "      <td>  3</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>5 rows \u00d7 5 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "  first_name last_name                email  preTestScore  postTestScore\n",
        "0      Jason    Miller     jas203@gmail.com             4             25\n",
        "1      Molly  Jacobson  momomolly@gmail.com            24             94\n",
        "2       Tina       Ali                  NaN            31             57\n",
        "3       Jake    Milner   battler@milner.com             2             62\n",
        "4        Amy     Cooze   Ames1234@yahoo.com             3             70\n",
        "\n",
        "[5 rows x 5 columns]"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Which strings in the email column contains 'gmail'"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['email'].str.contains('gmail')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 22,
       "text": [
        "0     True\n",
        "1     True\n",
        "2      NaN\n",
        "3    False\n",
        "4    False\n",
        "Name: email, dtype: object"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Create a regular expression pattern that breaks apart emails"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pattern = '([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\\\.([A-Z]{2,4})'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 23
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Find everything in df.email that contains that pattern"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['email'].str.findall(pattern, flags=re.IGNORECASE)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "0       [(jas203, gmail, com)]\n",
        "1    [(momomolly, gmail, com)]\n",
        "2                          NaN\n",
        "3     [(battler, milner, com)]\n",
        "4     [(Ames1234, yahoo, com)]\n",
        "Name: email, dtype: object"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Create a pandas series containing the email elements"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "matches = df['email'].str.match(pattern, flags=re.IGNORECASE)\n",
      "matches"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 29,
       "text": [
        "0       (jas203, gmail, com)\n",
        "1    (momomolly, gmail, com)\n",
        "2                        NaN\n",
        "3     (battler, milner, com)\n",
        "4     (Ames1234, yahoo, com)\n",
        "Name: email, dtype: object"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Select the domains of the df.email"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "matches.str[1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 42,
       "text": [
        "0     gmail\n",
        "1     gmail\n",
        "2       NaN\n",
        "3    milner\n",
        "4     yahoo\n",
        "Name: email, dtype: object"
       ]
      }
     ],
     "prompt_number": 42
    }
   ],
   "metadata": {}
  }
 ]
}