Height_Weight.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Data\n",
    "\n",
    "From the [CDC Behavioral Risk Factor Surveillance System](https://www.cdc.gov/brfss/about/index.htm)\n",
    "\n",
    "[2017 Survey](https://www.cdc.gov/brfss/annual_data/annual_2017.html)\n",
    "\n",
    "The so-called [codebook (PDF)](https://www.cdc.gov/brfss/annual_data/2017/pdf/codebook17_llcp-v2-508.pdf) has all the details for each field.\n",
    "\n",
    "The data are in the form of a fixed width [ASCII file](https://www.cdc.gov/brfss/annual_data/2017/files/LLCP2017ASC.zip) which is a royal pain to read. Here's the [file layout](https://www.cdc.gov/brfss/annual_data/2017/llcp_varlayout_17_onecolumn.html) in tabular form.  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Starting Column</th>\n",
       "      <th>Variable Name</th>\n",
       "      <th>Field Length</th>\n",
       "      <th>PyStart</th>\n",
       "      <th>PyStop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>183</td>\n",
       "      <td>WEIGHT2</td>\n",
       "      <td>4</td>\n",
       "      <td>182</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>187</td>\n",
       "      <td>HEIGHT3</td>\n",
       "      <td>4</td>\n",
       "      <td>186</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>2034</td>\n",
       "      <td>HTIN4</td>\n",
       "      <td>3</td>\n",
       "      <td>2033</td>\n",
       "      <td>2036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>2037</td>\n",
       "      <td>HTM4</td>\n",
       "      <td>3</td>\n",
       "      <td>2036</td>\n",
       "      <td>2039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>2040</td>\n",
       "      <td>WTKG3</td>\n",
       "      <td>5</td>\n",
       "      <td>2039</td>\n",
       "      <td>2044</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Starting Column Variable Name  Field Length  PyStart  PyStop\n",
       "69               183       WEIGHT2             4      182     186\n",
       "70               187       HEIGHT3             4      186     190\n",
       "288             2034         HTIN4             3     2033    2036\n",
       "289             2037          HTM4             3     2036    2039\n",
       "290             2040         WTKG3             5     2039    2044"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FF = pd.read_csv('/home/vpoduri/DataFiles/field_format.csv')\n",
    "FF['PyStart'] = FF['Starting Column']-1\n",
    "FF['PyStop'] = FF['PyStart']+FF['Field Length']\n",
    "\n",
    "FF[FF['Variable Name'].str.contains('WEIGHT2|WTKG3|HEIGHT3|HTIN4|HTM4')]\n",
    "\n",
    "# WTKG3 = WEIGHT2 / 2.2046      (weight in kg)\n",
    "# HTM4 = HEIGHT3 * 0.0254       (height in m)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 414032 entries, 0 to 414031\n",
      "Data columns (total 4 columns):\n",
      "WEIGHT2    414032 non-null int64\n",
      "WTKG3      414032 non-null float64\n",
      "HTIN4      414032 non-null int64\n",
      "HTM4       414032 non-null float64\n",
      "dtypes: float64(2), int64(2)\n",
      "memory usage: 12.6 MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WEIGHT2</th>\n",
       "      <th>WTKG3</th>\n",
       "      <th>HTIN4</th>\n",
       "      <th>HTM4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>162</td>\n",
       "      <td>73.48</td>\n",
       "      <td>65</td>\n",
       "      <td>1.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>211</td>\n",
       "      <td>95.71</td>\n",
       "      <td>71</td>\n",
       "      <td>1.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>195</td>\n",
       "      <td>88.45</td>\n",
       "      <td>74</td>\n",
       "      <td>1.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>170</td>\n",
       "      <td>77.11</td>\n",
       "      <td>67</td>\n",
       "      <td>1.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>140</td>\n",
       "      <td>63.50</td>\n",
       "      <td>65</td>\n",
       "      <td>1.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   WEIGHT2  WTKG3  HTIN4  HTM4\n",
       "0      162  73.48     65  1.65\n",
       "1      211  95.71     71  1.80\n",
       "2      195  88.45     74  1.88\n",
       "3      170  77.11     67  1.70\n",
       "4      140  63.50     65  1.65"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read only the columns we are interested in\n",
    "# Weight in lb and kg and height in inches and m\n",
    "\n",
    "WH =pd.DataFrame([(line[182:186],line[2039:2044],line[2033:2036],line[2036:2039]) \n",
    "       for line in open('/home/vpoduri/DataFiles/BRFSS_2017')])\n",
    "\n",
    "WH.columns = ['WEIGHT2','WTKG3','HTIN4','HTM4'] # Weight(lb), Weight(kg), Height(in), Height(m)\n",
    "\n",
    "# Clean some data\n",
    "WH = WH.replace(r'^\\s*$|^777|^9',np.nan,regex=True)\n",
    "WH = WH.dropna()\n",
    "WH.reset_index(drop=True,inplace=True)\n",
    "\n",
    "# Transform to sensible numeric values\n",
    "WH = WH.transform(lambda x: x.astype(int))\n",
    "WH[['WTKG3','HTM4']] = WH[['WTKG3','HTM4']]/100\n",
    "\n",
    "\n",
    "WH.info()\n",
    "WH[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WEIGHT2</th>\n",
       "      <th>WTKG3</th>\n",
       "      <th>HTIN4</th>\n",
       "      <th>HTM4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>414032.000000</td>\n",
       "      <td>414032.000000</td>\n",
       "      <td>414032.00000</td>\n",
       "      <td>414032.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>180.109873</td>\n",
       "      <td>81.696180</td>\n",
       "      <td>66.93452</td>\n",
       "      <td>1.700209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>45.597974</td>\n",
       "      <td>20.683077</td>\n",
       "      <td>4.16689</td>\n",
       "      <td>0.106127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>22.680000</td>\n",
       "      <td>36.00000</td>\n",
       "      <td>0.910000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>150.000000</td>\n",
       "      <td>68.040000</td>\n",
       "      <td>64.00000</td>\n",
       "      <td>1.630000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>175.000000</td>\n",
       "      <td>79.380000</td>\n",
       "      <td>67.00000</td>\n",
       "      <td>1.700000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>203.000000</td>\n",
       "      <td>92.080000</td>\n",
       "      <td>70.00000</td>\n",
       "      <td>1.780000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>604.000000</td>\n",
       "      <td>273.970000</td>\n",
       "      <td>93.00000</td>\n",
       "      <td>2.360000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             WEIGHT2          WTKG3         HTIN4           HTM4\n",
       "count  414032.000000  414032.000000  414032.00000  414032.000000\n",
       "mean      180.109873      81.696180      66.93452       1.700209\n",
       "std        45.597974      20.683077       4.16689       0.106127\n",
       "min        50.000000      22.680000      36.00000       0.910000\n",
       "25%       150.000000      68.040000      64.00000       1.630000\n",
       "50%       175.000000      79.380000      67.00000       1.700000\n",
       "75%       203.000000      92.080000      70.00000       1.780000\n",
       "max       604.000000     273.970000      93.00000       2.360000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "WH.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "WH.to_pickle('/home/vpoduri/DataFiles/WH.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}