wrangle_report.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1dfd2cf2",
   "metadata": {},
   "source": [
    "## Data Wrangling Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "11a34200",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import packages that will  be used in the analysis\n",
    "%matplotlib inline \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import requests\n",
    "import io\n",
    "import json\n",
    "#I will use beautifulsoup library to load my data to my file\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "47e5ef72",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import all dataset to be used\n",
    "image_data=pd.read_csv(\"Data//image_prediction.csv\")\n",
    "twitter_enhanced = pd.read_csv(\"Data//twitter-archive-enhanced.csv\")\n",
    "tweet_api_data = pd.read_csv(\"Data//tweet_api_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4c446c98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2075 entries, 0 to 2074\n",
      "Data columns (total 12 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   tweet_id  2075 non-null   int64  \n",
      " 1   jpg_url   2075 non-null   object \n",
      " 2   img_num   2075 non-null   int64  \n",
      " 3   p1        2075 non-null   object \n",
      " 4   p1_conf   2075 non-null   float64\n",
      " 5   p1_dog    2075 non-null   bool   \n",
      " 6   p2        2075 non-null   object \n",
      " 7   p2_conf   2075 non-null   float64\n",
      " 8   p2_dog    2075 non-null   bool   \n",
      " 9   p3        2075 non-null   object \n",
      " 10  p3_conf   2075 non-null   float64\n",
      " 11  p3_dog    2075 non-null   bool   \n",
      "dtypes: bool(3), float64(3), int64(2), object(4)\n",
      "memory usage: 152.1+ KB\n"
     ]
    }
   ],
   "source": [
    "#chek the columns information on image predictions dataset\n",
    "image_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2a1184be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Data columns (total 17 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   tweet_id                    2356 non-null   int64  \n",
      " 1   in_reply_to_status_id       78 non-null     float64\n",
      " 2   in_reply_to_user_id         78 non-null     float64\n",
      " 3   timestamp                   2356 non-null   object \n",
      " 4   source                      2356 non-null   object \n",
      " 5   text                        2356 non-null   object \n",
      " 6   retweeted_status_id         181 non-null    float64\n",
      " 7   retweeted_status_user_id    181 non-null    float64\n",
      " 8   retweeted_status_timestamp  181 non-null    object \n",
      " 9   expanded_urls               2297 non-null   object \n",
      " 10  rating_numerator            2356 non-null   int64  \n",
      " 11  rating_denominator          2356 non-null   int64  \n",
      " 12  name                        2356 non-null   object \n",
      " 13  doggo                       2356 non-null   object \n",
      " 14  floofer                     2356 non-null   object \n",
      " 15  pupper                      2356 non-null   object \n",
      " 16  puppo                       2356 non-null   object \n",
      "dtypes: float64(4), int64(3), object(10)\n",
      "memory usage: 313.0+ KB\n"
     ]
    }
   ],
   "source": [
    "#chek the columns information on twitter archive dataset\n",
    "twitter_enhanced.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1c4f47ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2354 entries, 0 to 2353\n",
      "Data columns (total 32 columns):\n",
      " #   Column                         Non-Null Count  Dtype  \n",
      "---  ------                         --------------  -----  \n",
      " 0   Unnamed: 0                     2354 non-null   int64  \n",
      " 1   created_at                     2354 non-null   object \n",
      " 2   id                             2354 non-null   int64  \n",
      " 3   id_str                         2354 non-null   int64  \n",
      " 4   full_text                      2354 non-null   object \n",
      " 5   truncated                      2354 non-null   bool   \n",
      " 6   display_text_range             2354 non-null   object \n",
      " 7   entities                       2354 non-null   object \n",
      " 8   extended_entities              2073 non-null   object \n",
      " 9   source                         2354 non-null   object \n",
      " 10  in_reply_to_status_id          78 non-null     float64\n",
      " 11  in_reply_to_status_id_str      78 non-null     float64\n",
      " 12  in_reply_to_user_id            78 non-null     float64\n",
      " 13  in_reply_to_user_id_str        78 non-null     float64\n",
      " 14  in_reply_to_screen_name        78 non-null     object \n",
      " 15  user                           2354 non-null   object \n",
      " 16  geo                            0 non-null      float64\n",
      " 17  coordinates                    0 non-null      float64\n",
      " 18  place                          1 non-null      object \n",
      " 19  contributors                   0 non-null      float64\n",
      " 20  is_quote_status                2354 non-null   bool   \n",
      " 21  retweet_count                  2354 non-null   int64  \n",
      " 22  favorite_count                 2354 non-null   int64  \n",
      " 23  favorited                      2354 non-null   bool   \n",
      " 24  retweeted                      2354 non-null   bool   \n",
      " 25  possibly_sensitive             2211 non-null   float64\n",
      " 26  possibly_sensitive_appealable  2211 non-null   float64\n",
      " 27  lang                           2354 non-null   object \n",
      " 28  retweeted_status               179 non-null    object \n",
      " 29  quoted_status_id               29 non-null     float64\n",
      " 30  quoted_status_id_str           29 non-null     float64\n",
      " 31  quoted_status                  28 non-null     object \n",
      "dtypes: bool(4), float64(11), int64(5), object(12)\n",
      "memory usage: 524.3+ KB\n"
     ]
    }
   ],
   "source": [
    "#chek the columns information on twitter API dataset\n",
    "tweet_api_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f62279b9",
   "metadata": {},
   "source": [
    "From the above information we find out that **Image prediction dataset** does not have any null values whereas we have null values on user and twitter ids columns on **Enhancement archive dataset**\n",
    "We also have couple of columns on API twitter data with null values\n",
    "Most of columns we need for analysis have data and we are going to drop some columns that we dont need and have null values and this will result to us having cleaner datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9276dfb3",
   "metadata": {},
   "source": [
    "<a id='archive'></a>\n",
    "## Twitter Archive Data Wrangling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8133e5cd",
   "metadata": {},
   "source": [
    "Since this dataset have date column, we are going to format the date column to a format that we can work with in our analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8204f0ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2017-08-01 16:23:56 +0000\n",
       "1    2017-08-01 00:17:27 +0000\n",
       "Name: timestamp, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Le us start by looking at date formats on all datasets which need to be formatted to the correct format\n",
    "twitter_enhanced.timestamp[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "be8738ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2017-08-01\n",
       "1    2017-08-01\n",
       "Name: timestamp, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Convert the date column\n",
    "twitter_enhanced[\"timestamp\"]=pd.to_datetime(twitter_enhanced[\"timestamp\"])\n",
    "\n",
    "twitter_enhanced[\"timestamp\"] = twitter_enhanced[\"timestamp\"].dt.date\n",
    "#After converting now check the column\n",
    "twitter_enhanced.timestamp[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9ac0afff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Series name: timestamp\n",
      "Non-Null Count  Dtype \n",
      "--------------  ----- \n",
      "2356 non-null   object\n",
      "dtypes: object(1)\n",
      "memory usage: 18.5+ KB\n"
     ]
    }
   ],
   "source": [
    "#Let us confirm the column data type\n",
    "twitter_enhanced.timestamp.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "282f3a75",
   "metadata": {},
   "source": [
    "We have seen that the date has changed from timestamp to object\n",
    "We need to change the data type back to date format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eabf6d25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Series name: timestamp\n",
      "Non-Null Count  Dtype         \n",
      "--------------  -----         \n",
      "2356 non-null   datetime64[ns]\n",
      "dtypes: datetime64[ns](1)\n",
      "memory usage: 18.5 KB\n"
     ]
    }
   ],
   "source": [
    "#Convert the data type back to datetime\n",
    "twitter_enhanced[\"timestamp\"]=pd.to_datetime(twitter_enhanced[\"timestamp\"])\n",
    "#Confirm the date type again\n",
    "twitter_enhanced.timestamp.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95b09636",
   "metadata": {},
   "source": [
    "The date format has now changed to date format that is the correct format of the date column for analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c243b20",
   "metadata": {},
   "source": [
    "<a id='archive_clean'></a>\n",
    "### Twitter Archive Data Wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "112c3b91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Data columns (total 17 columns):\n",
      " #   Column                      Non-Null Count  Dtype         \n",
      "---  ------                      --------------  -----         \n",
      " 0   tweet_id                    2356 non-null   int64         \n",
      " 1   in_reply_to_status_id       78 non-null     float64       \n",
      " 2   in_reply_to_user_id         78 non-null     float64       \n",
      " 3   timestamp                   2356 non-null   datetime64[ns]\n",
      " 4   source                      2356 non-null   object        \n",
      " 5   text                        2356 non-null   object        \n",
      " 6   retweeted_status_id         181 non-null    float64       \n",
      " 7   retweeted_status_user_id    181 non-null    float64       \n",
      " 8   retweeted_status_timestamp  181 non-null    object        \n",
      " 9   expanded_urls               2297 non-null   object        \n",
      " 10  rating_numerator            2356 non-null   int64         \n",
      " 11  rating_denominator          2356 non-null   int64         \n",
      " 12  name                        2356 non-null   object        \n",
      " 13  doggo                       2356 non-null   object        \n",
      " 14  floofer                     2356 non-null   object        \n",
      " 15  pupper                      2356 non-null   object        \n",
      " 16  puppo                       2356 non-null   object        \n",
      "dtypes: datetime64[ns](1), float64(4), int64(3), object(9)\n",
      "memory usage: 313.0+ KB\n"
     ]
    }
   ],
   "source": [
    "#Make the copy of the data\n",
    "twitter_enhanced_v1=twitter_enhanced.copy()\n",
    "twitter_enhanced_v1.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fed0b6a",
   "metadata": {},
   "source": [
    "In our analysis we do not need retweet and inreply columns `in_reply_to_status_id`, `in_reply_to_user_id`, `retweeted_status_id`, `retweeted_status_user_id` and `retweeted_status_timestamp`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "478d6eb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Data columns (total 12 columns):\n",
      " #   Column              Non-Null Count  Dtype         \n",
      "---  ------              --------------  -----         \n",
      " 0   tweet_id            2356 non-null   int64         \n",
      " 1   timestamp           2356 non-null   datetime64[ns]\n",
      " 2   source              2356 non-null   object        \n",
      " 3   text                2356 non-null   object        \n",
      " 4   expanded_urls       2297 non-null   object        \n",
      " 5   rating_numerator    2356 non-null   int64         \n",
      " 6   rating_denominator  2356 non-null   int64         \n",
      " 7   name                2356 non-null   object        \n",
      " 8   doggo               2356 non-null   object        \n",
      " 9   floofer             2356 non-null   object        \n",
      " 10  pupper              2356 non-null   object        \n",
      " 11  puppo               2356 non-null   object        \n",
      "dtypes: datetime64[ns](1), int64(3), object(8)\n",
      "memory usage: 221.0+ KB\n"
     ]
    }
   ],
   "source": [
    "#Columns to delete\n",
    "delete_columns = ['in_reply_to_user_id', 'in_reply_to_status_id',  \n",
    "           'retweeted_status_id', 'retweeted_status_user_id', \n",
    "           'retweeted_status_timestamp']\n",
    "#Drop the columns\n",
    "twitter_enhanced_v1 = twitter_enhanced_v1.drop(delete_columns, axis=1)\n",
    "twitter_enhanced_v1.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "144c900d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0   2017-08-01\n",
       "1   2017-08-01\n",
       "Name: timestamp, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Le us start by looking at date formats on the dataset which need to be formatted to the correct format\n",
    "twitter_enhanced_v1.timestamp[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "327bfcd3",
   "metadata": {},
   "source": [
    "The date format in this dataset seem to be in the correct format so there is no need of reformating the column\n",
    "However when you look at the data we need to melt dog types column so that we can have dog types in one column and this will result to a tidy data that can easily be used in data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4e623986",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>Dog_type</th>\n",
       "      <th>type_available</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8470</th>\n",
       "      <td>699423671849451520</td>\n",
       "      <td>2016-02-16</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>\"Don't ever talk to me or my son again.\" ...bo...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/699423671...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>puppo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790</th>\n",
       "      <td>773922284943896577</td>\n",
       "      <td>2016-09-08</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Heinrich (pronounced \"Pat\"). He's a Bo...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/773922284...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Heinrich</td>\n",
       "      <td>doggo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8338</th>\n",
       "      <td>709449600415961088</td>\n",
       "      <td>2016-03-14</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Karma. She's just a head. Lost body durin...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/709449600...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>Karma</td>\n",
       "      <td>puppo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>585</th>\n",
       "      <td>800018252395122689</td>\n",
       "      <td>2016-11-19</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here's a doggo doin a struggle. 11/10 much det...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/800018252...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>doggo</td>\n",
       "      <td>doggo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3361</th>\n",
       "      <td>747651430853525504</td>\n",
       "      <td>2016-06-28</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Other pupper asked not to have his identity sh...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>floofer</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id  timestamp  \\\n",
       "8470  699423671849451520 2016-02-16   \n",
       "790   773922284943896577 2016-09-08   \n",
       "8338  709449600415961088 2016-03-14   \n",
       "585   800018252395122689 2016-11-19   \n",
       "3361  747651430853525504 2016-06-28   \n",
       "\n",
       "                                                 source  \\\n",
       "8470  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "790   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "8338  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "585   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "3361  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                   text  \\\n",
       "8470  \"Don't ever talk to me or my son again.\" ...bo...   \n",
       "790   This is Heinrich (pronounced \"Pat\"). He's a Bo...   \n",
       "8338  Meet Karma. She's just a head. Lost body durin...   \n",
       "585   Here's a doggo doin a struggle. 11/10 much det...   \n",
       "3361  Other pupper asked not to have his identity sh...   \n",
       "\n",
       "                                          expanded_urls  rating_numerator  \\\n",
       "8470  https://twitter.com/dog_rates/status/699423671...                10   \n",
       "790   https://twitter.com/dog_rates/status/773922284...                12   \n",
       "8338  https://twitter.com/dog_rates/status/709449600...                10   \n",
       "585   https://twitter.com/dog_rates/status/800018252...                11   \n",
       "3361                                                NaN                12   \n",
       "\n",
       "      rating_denominator      name Dog_type type_available  \n",
       "8470                  10      None    puppo           None  \n",
       "790                   10  Heinrich    doggo           None  \n",
       "8338                  10     Karma    puppo           None  \n",
       "585                   10      None    doggo          doggo  \n",
       "3361                  10      None  floofer           None  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#We need to unnest the data so that we can include dog types column as arrays in one column\n",
    "twitter_enhanced_v2= twitter_enhanced_v1.melt(id_vars=[\"tweet_id\", \"timestamp\",\"source\",\"text\",\"expanded_urls\",\n",
    "                                                       \"rating_numerator\",\"rating_denominator\",\"name\"], \n",
    "        var_name=\"Dog_type\", \n",
    "        value_name=\"type_available\")\n",
    "twitter_enhanced_v2.sample(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "935fa79f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12      2232\n",
       "11      1856\n",
       "10      1844\n",
       "13      1404\n",
       "9        632\n",
       "8        408\n",
       "7        220\n",
       "14       216\n",
       "5        148\n",
       "6        128\n",
       "3         76\n",
       "4         68\n",
       "2         36\n",
       "1         36\n",
       "75         8\n",
       "15         8\n",
       "420        8\n",
       "0          8\n",
       "80         4\n",
       "144        4\n",
       "17         4\n",
       "26         4\n",
       "20         4\n",
       "121        4\n",
       "143        4\n",
       "44         4\n",
       "60         4\n",
       "45         4\n",
       "50         4\n",
       "99         4\n",
       "204        4\n",
       "1776       4\n",
       "165        4\n",
       "666        4\n",
       "27         4\n",
       "182        4\n",
       "24         4\n",
       "960        4\n",
       "84         4\n",
       "88         4\n",
       "Name: rating_numerator, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check the numerator ratings\n",
    "twitter_enhanced_v2.rating_numerator.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e07299bc",
   "metadata": {},
   "source": [
    "From above we find out that there are some ratings that are more than 100 ratings for each column and investigate data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "de32b673",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>Dog_type</th>\n",
       "      <th>type_available</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>902</th>\n",
       "      <td>758467244762497024</td>\n",
       "      <td>2016-07-28</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Why does this never happen at my front door......</td>\n",
       "      <td>https://twitter.com/dog_rates/status/758467244...</td>\n",
       "      <td>165</td>\n",
       "      <td>150</td>\n",
       "      <td>None</td>\n",
       "      <td>doggo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1120</th>\n",
       "      <td>731156023742988288</td>\n",
       "      <td>2016-05-13</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Say hello to this unbelievably well behaved sq...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/731156023...</td>\n",
       "      <td>204</td>\n",
       "      <td>170</td>\n",
       "      <td>this</td>\n",
       "      <td>doggo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1634</th>\n",
       "      <td>684225744407494656</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Two sneaky puppers were not initially seen, mo...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/684225744...</td>\n",
       "      <td>143</td>\n",
       "      <td>130</td>\n",
       "      <td>None</td>\n",
       "      <td>doggo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1635</th>\n",
       "      <td>684222868335505415</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Someone help the girl is being mugged. Several...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/684222868...</td>\n",
       "      <td>121</td>\n",
       "      <td>110</td>\n",
       "      <td>None</td>\n",
       "      <td>doggo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1779</th>\n",
       "      <td>677716515794329600</td>\n",
       "      <td>2015-12-18</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>IT'S PUPPERGEDDON. Total of 144/120 ...I think...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/677716515...</td>\n",
       "      <td>144</td>\n",
       "      <td>120</td>\n",
       "      <td>None</td>\n",
       "      <td>doggo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id  timestamp  \\\n",
       "902   758467244762497024 2016-07-28   \n",
       "1120  731156023742988288 2016-05-13   \n",
       "1634  684225744407494656 2016-01-05   \n",
       "1635  684222868335505415 2016-01-05   \n",
       "1779  677716515794329600 2015-12-18   \n",
       "\n",
       "                                                 source  \\\n",
       "902   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1120  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1634  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1635  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1779  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                   text  \\\n",
       "902   Why does this never happen at my front door......   \n",
       "1120  Say hello to this unbelievably well behaved sq...   \n",
       "1634  Two sneaky puppers were not initially seen, mo...   \n",
       "1635  Someone help the girl is being mugged. Several...   \n",
       "1779  IT'S PUPPERGEDDON. Total of 144/120 ...I think...   \n",
       "\n",
       "                                          expanded_urls  rating_numerator  \\\n",
       "902   https://twitter.com/dog_rates/status/758467244...               165   \n",
       "1120  https://twitter.com/dog_rates/status/731156023...               204   \n",
       "1634  https://twitter.com/dog_rates/status/684225744...               143   \n",
       "1635  https://twitter.com/dog_rates/status/684222868...               121   \n",
       "1779  https://twitter.com/dog_rates/status/677716515...               144   \n",
       "\n",
       "      rating_denominator  name Dog_type type_available  \n",
       "902                  150  None    doggo           None  \n",
       "1120                 170  this    doggo           None  \n",
       "1634                 130  None    doggo           None  \n",
       "1635                 110  None    doggo           None  \n",
       "1779                 120  None    doggo           None  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_enhanced_v3=twitter_enhanced_v2.query(\"rating_denominator > 100\")\n",
    "twitter_enhanced_v3.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ac37e3d",
   "metadata": {},
   "source": [
    "From this we can see that we almost all of the tweets that have ratings more than 100 have no name of the dog. Also there is one tweet with ratings 204 that has dog name as `this` which I also consider as no name. `this` is not a correct name of the dog conclusing that all tweets with more than 100 ratings have no dog names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1efd599",
   "metadata": {},
   "source": [
    "<a id='image'></a>\n",
    "### Image Predictions Data Wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5b9c682b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2075 entries, 0 to 2074\n",
      "Data columns (total 12 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   tweet_id  2075 non-null   int64  \n",
      " 1   jpg_url   2075 non-null   object \n",
      " 2   img_num   2075 non-null   int64  \n",
      " 3   p1        2075 non-null   object \n",
      " 4   p1_conf   2075 non-null   float64\n",
      " 5   p1_dog    2075 non-null   bool   \n",
      " 6   p2        2075 non-null   object \n",
      " 7   p2_conf   2075 non-null   float64\n",
      " 8   p2_dog    2075 non-null   bool   \n",
      " 9   p3        2075 non-null   object \n",
      " 10  p3_conf   2075 non-null   float64\n",
      " 11  p3_dog    2075 non-null   bool   \n",
      "dtypes: bool(3), float64(3), int64(2), object(4)\n",
      "memory usage: 152.1+ KB\n"
     ]
    }
   ],
   "source": [
    "image_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f79719ec",
   "metadata": {},
   "source": [
    "We can see that there is no null value in any column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ea0a56ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p3_conf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.075000e+03</td>\n",
       "      <td>2075.000000</td>\n",
       "      <td>2075.000000</td>\n",
       "      <td>2.075000e+03</td>\n",
       "      <td>2.075000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.384514e+17</td>\n",
       "      <td>1.203855</td>\n",
       "      <td>0.594548</td>\n",
       "      <td>1.345886e-01</td>\n",
       "      <td>6.032417e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.785203e+16</td>\n",
       "      <td>0.561875</td>\n",
       "      <td>0.271174</td>\n",
       "      <td>1.006657e-01</td>\n",
       "      <td>5.090593e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>6.660209e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.044333</td>\n",
       "      <td>1.011300e-08</td>\n",
       "      <td>1.740170e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>6.764835e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.364412</td>\n",
       "      <td>5.388625e-02</td>\n",
       "      <td>1.622240e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.119988e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.588230</td>\n",
       "      <td>1.181810e-01</td>\n",
       "      <td>4.944380e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.932034e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.843855</td>\n",
       "      <td>1.955655e-01</td>\n",
       "      <td>9.180755e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.924206e+17</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.880140e-01</td>\n",
       "      <td>2.734190e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           tweet_id      img_num      p1_conf       p2_conf       p3_conf\n",
       "count  2.075000e+03  2075.000000  2075.000000  2.075000e+03  2.075000e+03\n",
       "mean   7.384514e+17     1.203855     0.594548  1.345886e-01  6.032417e-02\n",
       "std    6.785203e+16     0.561875     0.271174  1.006657e-01  5.090593e-02\n",
       "min    6.660209e+17     1.000000     0.044333  1.011300e-08  1.740170e-10\n",
       "25%    6.764835e+17     1.000000     0.364412  5.388625e-02  1.622240e-02\n",
       "50%    7.119988e+17     1.000000     0.588230  1.181810e-01  4.944380e-02\n",
       "75%    7.932034e+17     1.000000     0.843855  1.955655e-01  9.180755e-02\n",
       "max    8.924206e+17     4.000000     1.000000  4.880140e-01  2.734190e-01"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e1d206c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1531ebcb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adbd68b0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "180a5cf7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}