viral_predication.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import lib for crawling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-O3GgofcPH1g"
   },
   "outputs": [],
   "source": [
    "from newspaper import Article\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bZ-2U0k-PbJM"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup  \n",
    "import csv \n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0VYStTnuRcls"
   },
   "outputs": [],
   "source": [
    "req=requests.get(\"https://www.hindustantimes.com/india-news/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "U4NLdmoFRkTh",
    "outputId": "bc1d6eef-fea2-476a-df70-a2becdcec0b9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "req"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nQjh-W7DVuY8"
   },
   "outputs": [],
   "source": [
    "html=BeautifulSoup(req.content,'html5lib')\n",
    "data=html.findAll('div',attrs={'class':'media-left'})\n",
    "l=[]\n",
    "for i in data:\n",
    "    links = i.findAll('a')\n",
    "    for a in links:\n",
    "        l.append(a['href'])\n",
    "len(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 608
    },
    "colab_type": "code",
    "id": "HijPSYb-alnq",
    "outputId": "73c354ad-5e08-4182-df28-667168aa85b8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "usp6ckOPXMNs"
   },
   "outputs": [],
   "source": [
    "fetchData=[]\n",
    "for i in l:\n",
    "    article = Article(i, language=\"en\")\n",
    "    article.download() \n",
    "    article.parse() \n",
    "    article.nlp() \n",
    "    temp={}\n",
    "    temp['url']=i\n",
    "    temp['title']=article.title\n",
    "    temp['text']=article.text\n",
    "    temp['summary']=article.summary\n",
    "    temp['keywords']=article.keywords\n",
    " \n",
    "    fetchData.append(temp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking data how its going in Our CSV file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 511
    },
    "colab_type": "code",
    "id": "o1SvtsmJYi-M",
    "outputId": "04e4589b-ad98-4dc1-ea22-09df6d392b1c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>keywords</th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[33, group, 23, covid19, age, cases, rate, ave...</td>\n",
       "      <td>indiaUpdated: Apr 28, 2020 17:27 ISTRajasthan’...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:27 IST\\n\\nRa...</td>\n",
       "      <td>Covid-19: At 33%, Rajasthan’s recovery rate hi...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/covi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[delhi, doctors, 33, coronavirus, states, care...</td>\n",
       "      <td>indiaUpdated: Apr 28, 2020 17:22 ISTOver four ...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:22 IST\\n\\nOv...</td>\n",
       "      <td>33 doctors in over 4% Delhi health workers aff...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/33-d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[distribute, 30000, pregnant, uttarakhand, wcd...</td>\n",
       "      <td>indiaUpdated: Apr 28, 2020 17:18 ISTThe Uttara...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:18 IST\\n\\nTh...</td>\n",
       "      <td>Uttarakhand to distribute special kits to care...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/utta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[updates, covid19, dangerous, read, centre, 20...</td>\n",
       "      <td>News updates from Hindustan Times: Plasma ther...</td>\n",
       "      <td>News updates from Hindustan Times: Plasma ther...</td>\n",
       "      <td>News updates from Hindustan Times: Plasma ther...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/news...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[approved, covid19, coronavirus, dangerous, ce...</td>\n",
       "      <td>The government added further that plasma thera...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:24 IST\\n\\nTh...</td>\n",
       "      <td>Plasma therapy not approved treatment for Covi...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/plas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[defence, ventilators, saidthe, operations, ra...</td>\n",
       "      <td>Singh said such plans were necessary to ramp u...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:10 IST\\n\\nDe...</td>\n",
       "      <td>Rajnath Singh asks OFB, DPSUs to ramp up produ...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/rajn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[congress, surjewala, crore, modi, cong, debts...</td>\n",
       "      <td>“Waiving off bad loans of huge amounts for wil...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 16:24 IST\\n\\nTh...</td>\n",
       "      <td>Cong slams Centre, asks how it can afford to w...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/cong...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[card, states, consider, proof, sc, centre, as...</td>\n",
       "      <td>indiaUpdated: Apr 28, 2020 16:34 ISTThe Suprem...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 16:34 IST\\n\\nTh...</td>\n",
       "      <td>SC asks Centre to consider ‘one nation, one ra...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/sc-a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[jibe, killed, incident, sadhus, adityanath, c...</td>\n",
       "      <td>indiaUpdated: Apr 28, 2020 16:59 ISTMaharashtr...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 16:59 IST\\n\\nMa...</td>\n",
       "      <td>In Thackeray’s phone call after 2 sadhus kille...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/bula...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[directs, nagaland, days, labs, covid19, bench...</td>\n",
       "      <td>indiaUpdated: Apr 28, 2020 15:46 ISTThe Kohima...</td>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 15:46 IST\\n\\nTh...</td>\n",
       "      <td>High Court directs Nagaland to set up Covid-19...</td>\n",
       "      <td>https://www.hindustantimes.com/india-news/high...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            keywords  \\\n",
       "0  [33, group, 23, covid19, age, cases, rate, ave...   \n",
       "1  [delhi, doctors, 33, coronavirus, states, care...   \n",
       "2  [distribute, 30000, pregnant, uttarakhand, wcd...   \n",
       "3  [updates, covid19, dangerous, read, centre, 20...   \n",
       "4  [approved, covid19, coronavirus, dangerous, ce...   \n",
       "5  [defence, ventilators, saidthe, operations, ra...   \n",
       "6  [congress, surjewala, crore, modi, cong, debts...   \n",
       "7  [card, states, consider, proof, sc, centre, as...   \n",
       "8  [jibe, killed, incident, sadhus, adityanath, c...   \n",
       "9  [directs, nagaland, days, labs, covid19, bench...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  indiaUpdated: Apr 28, 2020 17:27 ISTRajasthan’...   \n",
       "1  indiaUpdated: Apr 28, 2020 17:22 ISTOver four ...   \n",
       "2  indiaUpdated: Apr 28, 2020 17:18 ISTThe Uttara...   \n",
       "3  News updates from Hindustan Times: Plasma ther...   \n",
       "4  The government added further that plasma thera...   \n",
       "5  Singh said such plans were necessary to ramp u...   \n",
       "6  “Waiving off bad loans of huge amounts for wil...   \n",
       "7  indiaUpdated: Apr 28, 2020 16:34 ISTThe Suprem...   \n",
       "8  indiaUpdated: Apr 28, 2020 16:59 ISTMaharashtr...   \n",
       "9  indiaUpdated: Apr 28, 2020 15:46 ISTThe Kohima...   \n",
       "\n",
       "                                                text  \\\n",
       "0  india\\n\\nUpdated: Apr 28, 2020 17:27 IST\\n\\nRa...   \n",
       "1  india\\n\\nUpdated: Apr 28, 2020 17:22 IST\\n\\nOv...   \n",
       "2  india\\n\\nUpdated: Apr 28, 2020 17:18 IST\\n\\nTh...   \n",
       "3  News updates from Hindustan Times: Plasma ther...   \n",
       "4  india\\n\\nUpdated: Apr 28, 2020 17:24 IST\\n\\nTh...   \n",
       "5  india\\n\\nUpdated: Apr 28, 2020 17:10 IST\\n\\nDe...   \n",
       "6  india\\n\\nUpdated: Apr 28, 2020 16:24 IST\\n\\nTh...   \n",
       "7  india\\n\\nUpdated: Apr 28, 2020 16:34 IST\\n\\nTh...   \n",
       "8  india\\n\\nUpdated: Apr 28, 2020 16:59 IST\\n\\nMa...   \n",
       "9  india\\n\\nUpdated: Apr 28, 2020 15:46 IST\\n\\nTh...   \n",
       "\n",
       "                                               title  \\\n",
       "0  Covid-19: At 33%, Rajasthan’s recovery rate hi...   \n",
       "1  33 doctors in over 4% Delhi health workers aff...   \n",
       "2  Uttarakhand to distribute special kits to care...   \n",
       "3  News updates from Hindustan Times: Plasma ther...   \n",
       "4  Plasma therapy not approved treatment for Covi...   \n",
       "5  Rajnath Singh asks OFB, DPSUs to ramp up produ...   \n",
       "6  Cong slams Centre, asks how it can afford to w...   \n",
       "7  SC asks Centre to consider ‘one nation, one ra...   \n",
       "8  In Thackeray’s phone call after 2 sadhus kille...   \n",
       "9  High Court directs Nagaland to set up Covid-19...   \n",
       "\n",
       "                                                 url  \n",
       "0  https://www.hindustantimes.com/india-news/covi...  \n",
       "1  https://www.hindustantimes.com/india-news/33-d...  \n",
       "2  https://www.hindustantimes.com/india-news/utta...  \n",
       "3  https://www.hindustantimes.com/india-news/news...  \n",
       "4  https://www.hindustantimes.com/india-news/plas...  \n",
       "5  https://www.hindustantimes.com/india-news/rajn...  \n",
       "6  https://www.hindustantimes.com/india-news/cong...  \n",
       "7  https://www.hindustantimes.com/india-news/sc-a...  \n",
       "8  https://www.hindustantimes.com/india-news/bula...  \n",
       "9  https://www.hindustantimes.com/india-news/high...  "
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=pd.DataFrame(fetchData)\n",
    "dataset.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G0j8lPrcZaBx"
   },
   "outputs": [],
   "source": [
    "dataset.to_csv('news_article_dataset.csv', index=False)  # saving that into csv file "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P6Qgi6P4DQZ1"
   },
   "source": [
    "### importing Online News Popularity from UIC DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QHwsPR_aDbkZ"
   },
   "outputs": [],
   "source": [
    "uic_data=pd.read_csv('OnlineNewsPopularity.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>timedelta</th>\n",
       "      <th>n_tokens_title</th>\n",
       "      <th>n_tokens_content</th>\n",
       "      <th>n_unique_tokens</th>\n",
       "      <th>n_non_stop_words</th>\n",
       "      <th>n_non_stop_unique_tokens</th>\n",
       "      <th>num_hrefs</th>\n",
       "      <th>num_self_hrefs</th>\n",
       "      <th>num_imgs</th>\n",
       "      <th>...</th>\n",
       "      <th>min_positive_polarity</th>\n",
       "      <th>max_positive_polarity</th>\n",
       "      <th>avg_negative_polarity</th>\n",
       "      <th>min_negative_polarity</th>\n",
       "      <th>max_negative_polarity</th>\n",
       "      <th>title_subjectivity</th>\n",
       "      <th>title_sentiment_polarity</th>\n",
       "      <th>abs_title_subjectivity</th>\n",
       "      <th>abs_title_sentiment_polarity</th>\n",
       "      <th>shares</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://mashable.com/2013/01/07/amazon-instant-...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>0.663594</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.815385</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.7</td>\n",
       "      <td>-0.350000</td>\n",
       "      <td>-0.600</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-0.187500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://mashable.com/2013/01/07/ap-samsung-spon...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>255.0</td>\n",
       "      <td>0.604743</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.791946</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.7</td>\n",
       "      <td>-0.118750</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://mashable.com/2013/01/07/apple-40-billio...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.575130</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.663866</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.466667</td>\n",
       "      <td>-0.800</td>\n",
       "      <td>-0.133333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://mashable.com/2013/01/07/astronaut-notre...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>531.0</td>\n",
       "      <td>0.503788</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.665635</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.8</td>\n",
       "      <td>-0.369697</td>\n",
       "      <td>-0.600</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://mashable.com/2013/01/07/att-u-verse-apps/</td>\n",
       "      <td>731.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1072.0</td>\n",
       "      <td>0.415646</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540890</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.220192</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>505</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 61 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url   timedelta  \\\n",
       "0  http://mashable.com/2013/01/07/amazon-instant-...       731.0   \n",
       "1  http://mashable.com/2013/01/07/ap-samsung-spon...       731.0   \n",
       "2  http://mashable.com/2013/01/07/apple-40-billio...       731.0   \n",
       "3  http://mashable.com/2013/01/07/astronaut-notre...       731.0   \n",
       "4   http://mashable.com/2013/01/07/att-u-verse-apps/       731.0   \n",
       "\n",
       "    n_tokens_title   n_tokens_content   n_unique_tokens   n_non_stop_words  \\\n",
       "0             12.0              219.0          0.663594                1.0   \n",
       "1              9.0              255.0          0.604743                1.0   \n",
       "2              9.0              211.0          0.575130                1.0   \n",
       "3              9.0              531.0          0.503788                1.0   \n",
       "4             13.0             1072.0          0.415646                1.0   \n",
       "\n",
       "    n_non_stop_unique_tokens   num_hrefs   num_self_hrefs   num_imgs  ...  \\\n",
       "0                   0.815385         4.0              2.0        1.0  ...   \n",
       "1                   0.791946         3.0              1.0        1.0  ...   \n",
       "2                   0.663866         3.0              1.0        1.0  ...   \n",
       "3                   0.665635         9.0              0.0        1.0  ...   \n",
       "4                   0.540890        19.0             19.0       20.0  ...   \n",
       "\n",
       "    min_positive_polarity   max_positive_polarity   avg_negative_polarity  \\\n",
       "0                0.100000                     0.7               -0.350000   \n",
       "1                0.033333                     0.7               -0.118750   \n",
       "2                0.100000                     1.0               -0.466667   \n",
       "3                0.136364                     0.8               -0.369697   \n",
       "4                0.033333                     1.0               -0.220192   \n",
       "\n",
       "    min_negative_polarity   max_negative_polarity   title_subjectivity  \\\n",
       "0                  -0.600               -0.200000             0.500000   \n",
       "1                  -0.125               -0.100000             0.000000   \n",
       "2                  -0.800               -0.133333             0.000000   \n",
       "3                  -0.600               -0.166667             0.000000   \n",
       "4                  -0.500               -0.050000             0.454545   \n",
       "\n",
       "    title_sentiment_polarity   abs_title_subjectivity  \\\n",
       "0                  -0.187500                 0.000000   \n",
       "1                   0.000000                 0.500000   \n",
       "2                   0.000000                 0.500000   \n",
       "3                   0.000000                 0.500000   \n",
       "4                   0.136364                 0.045455   \n",
       "\n",
       "    abs_title_sentiment_polarity   shares  \n",
       "0                       0.187500      593  \n",
       "1                       0.000000      711  \n",
       "2                       0.000000     1500  \n",
       "3                       0.000000     1200  \n",
       "4                       0.136364      505  \n",
       "\n",
       "[5 rows x 61 columns]"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uic_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "url\n",
      " timedelta\n",
      " n_tokens_title\n",
      " n_tokens_content\n",
      " n_unique_tokens\n",
      " n_non_stop_words\n",
      " n_non_stop_unique_tokens\n",
      " num_hrefs\n",
      " num_self_hrefs\n",
      " num_imgs\n",
      " num_videos\n",
      " average_token_length\n",
      " num_keywords\n",
      " data_channel_is_lifestyle\n",
      " data_channel_is_entertainment\n",
      " data_channel_is_bus\n",
      " data_channel_is_socmed\n",
      " data_channel_is_tech\n",
      " data_channel_is_world\n",
      " kw_min_min\n",
      " kw_max_min\n",
      " kw_avg_min\n",
      " kw_min_max\n",
      " kw_max_max\n",
      " kw_avg_max\n",
      " kw_min_avg\n",
      " kw_max_avg\n",
      " kw_avg_avg\n",
      " self_reference_min_shares\n",
      " self_reference_max_shares\n",
      " self_reference_avg_sharess\n",
      " weekday_is_monday\n",
      " weekday_is_tuesday\n",
      " weekday_is_wednesday\n",
      " weekday_is_thursday\n",
      " weekday_is_friday\n",
      " weekday_is_saturday\n",
      " weekday_is_sunday\n",
      " is_weekend\n",
      " LDA_00\n",
      " LDA_01\n",
      " LDA_02\n",
      " LDA_03\n",
      " LDA_04\n",
      " global_subjectivity\n",
      " global_sentiment_polarity\n",
      " global_rate_positive_words\n",
      " global_rate_negative_words\n",
      " rate_positive_words\n",
      " rate_negative_words\n",
      " avg_positive_polarity\n",
      " min_positive_polarity\n",
      " max_positive_polarity\n",
      " avg_negative_polarity\n",
      " min_negative_polarity\n",
      " max_negative_polarity\n",
      " title_subjectivity\n",
      " title_sentiment_polarity\n",
      " abs_title_subjectivity\n",
      " abs_title_sentiment_polarity\n",
      " shares\n"
     ]
    }
   ],
   "source": [
    "for t in uic_data.columns:\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## i will remove those features i can't calculate in mean time "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Attribute Information:\n",
    "0. url: URL of the article (non-predictive)\n",
    "1. timedelta: Days between the article publication and the dataset acquisition (non-predictive)\n",
    "2. n_tokens_title: Number of words in the title\n",
    "3. n_tokens_content: Number of words in the content\n",
    "4. n_unique_tokens: Rate of unique words in the content\n",
    "5. n_non_stop_words: Rate of non-stop words in the content\n",
    "6. n_non_stop_unique_tokens: Rate of unique non-stop words in the content\n",
    "7. num_hrefs: Number of links\n",
    "8. num_self_hrefs: Number of links to other articles published by Mashable\n",
    "9. num_imgs: Number of images\n",
    "10. num_videos: Number of videos\n",
    "11. average_token_length: Average length of the words in the content\n",
    "12. num_keywords: Number of keywords in the metadata\n",
    "13. data_channel_is_lifestyle: Is data channel 'Lifestyle'?\n",
    "14. data_channel_is_entertainment: Is data channel 'Entertainment'?\n",
    "15. data_channel_is_bus: Is data channel 'Business'?\n",
    "16. data_channel_is_socmed: Is data channel 'Social Media'?\n",
    "17. data_channel_is_tech: Is data channel 'Tech'?\n",
    "18. data_channel_is_world: Is data channel 'World'?\n",
    "19. kw_min_min: Worst keyword (min. shares)\n",
    "20. kw_max_min: Worst keyword (max. shares)\n",
    "21. kw_avg_min: Worst keyword (avg. shares)\n",
    "22. kw_min_max: Best keyword (min. shares)\n",
    "23. kw_max_max: Best keyword (max. shares)\n",
    "24. kw_avg_max: Best keyword (avg. shares)\n",
    "25. kw_min_avg: Avg. keyword (min. shares)\n",
    "26. kw_max_avg: Avg. keyword (max. shares)\n",
    "27. kw_avg_avg: Avg. keyword (avg. shares)\n",
    "28. self_reference_min_shares: Min. shares of referenced articles in Mashable\n",
    "29. self_reference_max_shares: Max. shares of referenced articles in Mashable\n",
    "30. self_reference_avg_sharess: Avg. shares of referenced articles in Mashable\n",
    "31. weekday_is_monday: Was the article published on a Monday?\n",
    "32. weekday_is_tuesday: Was the article published on a Tuesday?\n",
    "33. weekday_is_wednesday: Was the article published on a Wednesday?\n",
    "34. weekday_is_thursday: Was the article published on a Thursday?\n",
    "35. weekday_is_friday: Was the article published on a Friday?\n",
    "36. weekday_is_saturday: Was the article published on a Saturday?\n",
    "37. weekday_is_sunday: Was the article published on a Sunday?\n",
    "38. is_weekend: Was the article published on the weekend?\n",
    "39. LDA_00: Closeness to LDA topic 0\n",
    "40. LDA_01: Closeness to LDA topic 1\n",
    "41. LDA_02: Closeness to LDA topic 2\n",
    "42. LDA_03: Closeness to LDA topic 3\n",
    "43. LDA_04: Closeness to LDA topic 4\n",
    "44. global_subjectivity: Text subjectivity\n",
    "45. global_sentiment_polarity: Text sentiment polarity\n",
    "46. global_rate_positive_words: Rate of positive words in the content\n",
    "47. global_rate_negative_words: Rate of negative words in the content\n",
    "48. rate_positive_words: Rate of positive words among non-neutral tokens\n",
    "49. rate_negative_words: Rate of negative words among non-neutral tokens\n",
    "50. avg_positive_polarity: Avg. polarity of positive words\n",
    "51. min_positive_polarity: Min. polarity of positive words\n",
    "52. max_positive_polarity: Max. polarity of positive words\n",
    "53. avg_negative_polarity: Avg. polarity of negative words\n",
    "54. min_negative_polarity: Min. polarity of negative words\n",
    "55. max_negative_polarity: Max. polarity of negative words\n",
    "56. title_subjectivity: Title subjectivity\n",
    "57. title_sentiment_polarity: Title polarity\n",
    "58. abs_title_subjectivity: Absolute subjectivity level\n",
    "59. abs_title_sentiment_polarity: Absolute polarity level\n",
    "60. shares: Number of shares (target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from textblob import TextBlob "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ajksbdjkbd', 'akjsd', 'aojsd', 'oajsd', 'ojd', 'asodjs']"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def tokenize_filtered(text):\n",
    "    text_tokens=text.translate(str.maketrans('', '', string.punctuation))\n",
    "    text_tokens = word_tokenize(text_tokens)\n",
    "    tokens_without_sw =[word for word in text_tokens if not word in stopwords.words()]\n",
    "    return tokens_without_sw\n",
    "\n",
    "tokenize_filtered('ajksbdjkbd akjsd aojsd . oajsd ojd s; asodjs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rate_unique(text):\n",
    "    text=tokenize_filtered(text)\n",
    "    no_order = list(set(text))\n",
    "    rate_unique=len(no_order)/len(text)\n",
    "    return rate_unique\n",
    "def rate_nonstop(text):\n",
    "    filter_set=tokenize_filtered(text)\n",
    "    text=word_tokenize(text)\n",
    "    rate_nonstop=len(filter_set)/len(text)\n",
    "    no_order = list(set(filter_set))\n",
    "    rate_unique_nonstop=len(no_order)/len(text)\n",
    "    return rate_nonstop,rate_unique_nonstop\n",
    "def avg_token(text):\n",
    "    text=tokenize_filtered(text)\n",
    "    lent=[]\n",
    "    for i in text:\n",
    "        lent.append(len(i))\n",
    "    return np.average(lent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_words=[]\n",
    "neg_words=[]\n",
    "def polar(words):\n",
    "    all_tokens=tokenize_filtered(words)\n",
    "    for i in all_tokens:\n",
    "        analysis=TextBlob(i)\n",
    "        polarity=analysis.sentiment.polarity\n",
    "        if polarity>0:\n",
    "            pos_words.append(i)\n",
    "        if polarity<0:\n",
    "            neg_words.append(i)\n",
    "    return pos_words,neg_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rates(words):\n",
    "    words=polar(words)\n",
    "    pos=words[0]\n",
    "    neg=words[1]\n",
    "    all_words=words\n",
    "    global_rate_positive_words=(len(pos)/len(all_words))/100\n",
    "    global_rate_negative_words=(len(neg)/len(all_words))/100\n",
    "    pol_pos=[]\n",
    "    pol_neg=[]\n",
    "    for i in pos:\n",
    "        analysis=TextBlob(i)\n",
    "        pol_pos.append(analysis.sentiment.polarity)\n",
    "        avg_positive_polarity=analysis.sentiment.polarity\n",
    "    for j in neg:\n",
    "        analysis2=TextBlob(j)\n",
    "        pol_neg.append(analysis2.sentiment.polarity)\n",
    "        avg_negative_polarity=analysis2.sentiment.polarity\n",
    "    min_positive_polarity=min(pol_pos)\n",
    "    max_positive_polarity=max(pol_pos)\n",
    "    min_negative_polarity=min(pol_neg)\n",
    "    max_negative_polarity=max(pol_neg)\n",
    "    avg_positive_polarity=np.average(pol_pos)\n",
    "    avg_negative_polarity=np.average(pol_neg)\n",
    "    return global_rate_positive_words,global_rate_negative_words,avg_positive_polarity,min_positive_polarity,max_positive_polarity,avg_negative_polarity,min_negative_polarity,max_negative_polarity\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.hindustantimes.com/india-news/covid-19-at-33-rajasthan-s-recovery-rate-higher-than-national-average-of-23/story-hamBYhRto5FTabjT2rAN2J.html'"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "ename": "ArticleException",
     "evalue": "Article `download()` failed with HTTPSConnectionPool(host='www.hindustantimes.com', port=443): Max retries exceeded with url: /india-news/uttarakhand-woman-admitted-in-aiims-rishikesh-tests-positive-for-covid-19-tally-increases-to-52/story-rNQ4t9Gi88o7opQLC4RlxJ.html (Caused by SSLError(SSLError(\"bad handshake: SysCallError(10054, 'WSAECONNRESET')\"))) on URL https://www.hindustantimes.com/india-news/uttarakhand-woman-admitted-in-aiims-rishikesh-tests-positive-for-covid-19-tally-increases-to-52/story-rNQ4t9Gi88o7opQLC4RlxJ.html",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mArticleException\u001b[0m                          Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-241-d1a07fe6c0de>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0marticle\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mArticle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"en\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0marticle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0marticle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m     \u001b[0manalysis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTextBlob\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marticle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mpolarity\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0manalysis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msentiment\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpolarity\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\newspaper\\article.py\u001b[0m in \u001b[0;36mparse\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    189\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    190\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 191\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mthrow_if_not_downloaded_verbose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    192\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    193\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdoc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_parser\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromstring\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhtml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\newspaper\\article.py\u001b[0m in \u001b[0;36mthrow_if_not_downloaded_verbose\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    530\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload_state\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mArticleDownloadState\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFAILED_RESPONSE\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    531\u001b[0m             raise ArticleException('Article `download()` failed with %s on URL %s' %\n\u001b[1;32m--> 532\u001b[1;33m                   (self.download_exception_msg, self.url))\n\u001b[0m\u001b[0;32m    533\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    534\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mthrow_if_not_parsed_verbose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mArticleException\u001b[0m: Article `download()` failed with HTTPSConnectionPool(host='www.hindustantimes.com', port=443): Max retries exceeded with url: /india-news/uttarakhand-woman-admitted-in-aiims-rishikesh-tests-positive-for-covid-19-tally-increases-to-52/story-rNQ4t9Gi88o7opQLC4RlxJ.html (Caused by SSLError(SSLError(\"bad handshake: SysCallError(10054, 'WSAECONNRESET')\"))) on URL https://www.hindustantimes.com/india-news/uttarakhand-woman-admitted-in-aiims-rishikesh-tests-positive-for-covid-19-tally-increases-to-52/story-rNQ4t9Gi88o7opQLC4RlxJ.html"
     ]
    }
   ],
   "source": [
    "dataset_extract=[]\n",
    "\n",
    "for i in l:\n",
    "    article = Article(i, language=\"en\")\n",
    "    article.download() \n",
    "    article.parse()\n",
    "    analysis=TextBlob(article.text)\n",
    "    polarity=analysis.sentiment.polarity\n",
    "    title_analysis=TextBlob(article.title)\n",
    "    extract={}\n",
    "    extract['text']=article.text\n",
    "    extract['n_tokens_title']=len(tokenize_filtered(article.title))\n",
    "    extract['n_tokens_content']=len(tokenize_filtered(article.text))\n",
    "    extract['n_unique_tokens']=rate_unique(article.text)\n",
    "    extract['n_non_stop_words']=rate_nonstop(article.text)[0]\n",
    "    extract['n_non_stop_unique_tokens']=rate_nonstop(article.text)[1]\n",
    "    extract['num_hrefs']=article.html.count(\"https://www.hindustantimes.com/\")\n",
    "    extract['num_imgs']=len(article.images)\n",
    "    extract['num_videos']=len(article.movies)\n",
    "    extract['average_token_length']=avg_token(article.text)\n",
    "    extract['num_keywords']=len(article.keywords)\n",
    "\n",
    "    if \"life-style\" in article.url or article.keywords:\n",
    "        extract['data_channel_is_lifestyle']=1\n",
    "    else:\n",
    "        extract['data_channel_is_lifestyle']=0\n",
    "    if \"etimes\" in article.url or article.keywords:\n",
    "        extract['data_channel_is_entertainment']=1\n",
    "    else:\n",
    "        extract['data_channel_is_entertainment']=0\n",
    "    if \"business\" in article.url or article.keywords:\n",
    "        extract['data_channel_is_bus']=1\n",
    "    else:\n",
    "        extract['data_channel_is_bus']=0\n",
    "    if \"twitter\" or \"facebook\" or \"social media\" in article.text.lower() or article.url or article.keywords:\n",
    "        data_channel_is_socmed=1\n",
    "        data_channel_is_tech=0\n",
    "        data_channel_is_world=0\n",
    "    else:\n",
    "        data_channel_is_socmed=0\n",
    "    if (\"technology\" or \"tech\") in article.url or article.keywords:\n",
    "        data_channel_is_tech=1\n",
    "        data_channel_is_socmed=0\n",
    "        data_channel_is_world=0\n",
    "    else:\n",
    "        data_channel_is_tech=0\n",
    "    if \"world\" in article.url or article.keywords:\n",
    "        data_channel_is_world=1\n",
    "        data_channel_is_tech=0\n",
    "        data_channel_is_socmed=0\n",
    "    else:\n",
    "        data_channel_is_world=0\n",
    "        \n",
    "    extract['data_channel_is_socmed']=data_channel_is_socmed\n",
    "    extract['data_channel_is_tech']=data_channel_is_tech\n",
    "    extract['data_channel_is_world']=data_channel_is_world   \n",
    "    extract['global_subjectivity']=analysis.sentiment.subjectivity\n",
    "    extract['global_sentiment_polarity']=analysis.sentiment.polarity\n",
    "    extract['global_rate_positive_words']=rates(article.text)[0]\n",
    "    extract['global_rate_negative_words']=rates(article.text)[1]\n",
    "    extract['avg_positive_polarity']=rates(article.text)[2]\n",
    "    extract['min_positive_polarity']=rates(article.text)[3]\n",
    "    extract['max_positive_polarity']=rates(article.text)[4]\n",
    "    extract['avg_negative_polarity']=rates(article.text)[5]\n",
    "    extract['min_negative_polarity']=rates(article.text)[6]\n",
    "    extract['max_negative_polarity']=rates(article.text)[7]    \n",
    "    extract['title_subjectivity']=title_analysis.sentiment.subjectivity\n",
    "    extract['title_sentiment_polarity']=title_analysis.sentiment.polarity\n",
    "    dataset_extract.append(extract)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_token_length</th>\n",
       "      <th>avg_negative_polarity</th>\n",
       "      <th>avg_positive_polarity</th>\n",
       "      <th>data_channel_is_bus</th>\n",
       "      <th>data_channel_is_entertainment</th>\n",
       "      <th>data_channel_is_lifestyle</th>\n",
       "      <th>data_channel_is_socmed</th>\n",
       "      <th>data_channel_is_tech</th>\n",
       "      <th>data_channel_is_world</th>\n",
       "      <th>global_rate_negative_words</th>\n",
       "      <th>...</th>\n",
       "      <th>n_non_stop_words</th>\n",
       "      <th>n_tokens_content</th>\n",
       "      <th>n_tokens_title</th>\n",
       "      <th>n_unique_tokens</th>\n",
       "      <th>num_hrefs</th>\n",
       "      <th>num_imgs</th>\n",
       "      <th>num_keywords</th>\n",
       "      <th>num_videos</th>\n",
       "      <th>title_sentiment_polarity</th>\n",
       "      <th>title_subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.640483</td>\n",
       "      <td>-0.264130</td>\n",
       "      <td>0.318819</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10.55</td>\n",
       "      <td>...</td>\n",
       "      <td>0.531300</td>\n",
       "      <td>331</td>\n",
       "      <td>11</td>\n",
       "      <td>0.543807</td>\n",
       "      <td>244</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.450000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.369650</td>\n",
       "      <td>-0.262816</td>\n",
       "      <td>0.319222</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10.70</td>\n",
       "      <td>...</td>\n",
       "      <td>0.610451</td>\n",
       "      <td>257</td>\n",
       "      <td>12</td>\n",
       "      <td>0.712062</td>\n",
       "      <td>248</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.633803</td>\n",
       "      <td>-0.262401</td>\n",
       "      <td>0.319618</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10.88</td>\n",
       "      <td>...</td>\n",
       "      <td>0.559055</td>\n",
       "      <td>142</td>\n",
       "      <td>7</td>\n",
       "      <td>0.584507</td>\n",
       "      <td>245</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.345238</td>\n",
       "      <td>0.535714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.731235</td>\n",
       "      <td>-0.266206</td>\n",
       "      <td>0.321633</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10.99</td>\n",
       "      <td>...</td>\n",
       "      <td>0.591691</td>\n",
       "      <td>413</td>\n",
       "      <td>14</td>\n",
       "      <td>0.704600</td>\n",
       "      <td>261</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.477509</td>\n",
       "      <td>-0.267776</td>\n",
       "      <td>0.322605</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.33</td>\n",
       "      <td>...</td>\n",
       "      <td>0.572277</td>\n",
       "      <td>289</td>\n",
       "      <td>8</td>\n",
       "      <td>0.605536</td>\n",
       "      <td>248</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>0.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.455645</td>\n",
       "      <td>-0.267470</td>\n",
       "      <td>0.321246</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.38</td>\n",
       "      <td>...</td>\n",
       "      <td>0.604878</td>\n",
       "      <td>248</td>\n",
       "      <td>8</td>\n",
       "      <td>0.721774</td>\n",
       "      <td>245</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5.774336</td>\n",
       "      <td>-0.272956</td>\n",
       "      <td>0.320885</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.54</td>\n",
       "      <td>...</td>\n",
       "      <td>0.588542</td>\n",
       "      <td>226</td>\n",
       "      <td>12</td>\n",
       "      <td>0.632743</td>\n",
       "      <td>246</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.383333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>6.018779</td>\n",
       "      <td>-0.273551</td>\n",
       "      <td>0.321444</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.89</td>\n",
       "      <td>...</td>\n",
       "      <td>0.566489</td>\n",
       "      <td>213</td>\n",
       "      <td>10</td>\n",
       "      <td>0.647887</td>\n",
       "      <td>248</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5.858491</td>\n",
       "      <td>-0.274525</td>\n",
       "      <td>0.321661</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12.06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.553525</td>\n",
       "      <td>212</td>\n",
       "      <td>11</td>\n",
       "      <td>0.617925</td>\n",
       "      <td>249</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5.941606</td>\n",
       "      <td>-0.274540</td>\n",
       "      <td>0.321654</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12.14</td>\n",
       "      <td>...</td>\n",
       "      <td>0.600877</td>\n",
       "      <td>274</td>\n",
       "      <td>10</td>\n",
       "      <td>0.605839</td>\n",
       "      <td>244</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.160000</td>\n",
       "      <td>0.540000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5.718062</td>\n",
       "      <td>-0.272398</td>\n",
       "      <td>0.321281</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12.25</td>\n",
       "      <td>...</td>\n",
       "      <td>0.527907</td>\n",
       "      <td>227</td>\n",
       "      <td>8</td>\n",
       "      <td>0.682819</td>\n",
       "      <td>245</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>6.123596</td>\n",
       "      <td>-0.268587</td>\n",
       "      <td>0.320665</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12.51</td>\n",
       "      <td>...</td>\n",
       "      <td>0.575679</td>\n",
       "      <td>445</td>\n",
       "      <td>12</td>\n",
       "      <td>0.629213</td>\n",
       "      <td>244</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    average_token_length  avg_negative_polarity  avg_positive_polarity  \\\n",
       "0               5.640483              -0.264130               0.318819   \n",
       "1               6.369650              -0.262816               0.319222   \n",
       "2               5.633803              -0.262401               0.319618   \n",
       "3               5.731235              -0.266206               0.321633   \n",
       "4               6.477509              -0.267776               0.322605   \n",
       "5               6.455645              -0.267470               0.321246   \n",
       "6               5.774336              -0.272956               0.320885   \n",
       "7               6.018779              -0.273551               0.321444   \n",
       "8               5.858491              -0.274525               0.321661   \n",
       "9               5.941606              -0.274540               0.321654   \n",
       "10              5.718062              -0.272398               0.321281   \n",
       "11              6.123596              -0.268587               0.320665   \n",
       "\n",
       "    data_channel_is_bus  data_channel_is_entertainment  \\\n",
       "0                     0                              0   \n",
       "1                     0                              0   \n",
       "2                     0                              0   \n",
       "3                     0                              0   \n",
       "4                     0                              0   \n",
       "5                     0                              0   \n",
       "6                     0                              0   \n",
       "7                     0                              0   \n",
       "8                     0                              0   \n",
       "9                     0                              0   \n",
       "10                    0                              0   \n",
       "11                    0                              0   \n",
       "\n",
       "    data_channel_is_lifestyle  data_channel_is_socmed  data_channel_is_tech  \\\n",
       "0                           0                       1                     0   \n",
       "1                           0                       1                     0   \n",
       "2                           0                       1                     0   \n",
       "3                           0                       1                     0   \n",
       "4                           0                       1                     0   \n",
       "5                           0                       1                     0   \n",
       "6                           0                       1                     0   \n",
       "7                           0                       1                     0   \n",
       "8                           0                       1                     0   \n",
       "9                           0                       1                     0   \n",
       "10                          0                       1                     0   \n",
       "11                          0                       1                     0   \n",
       "\n",
       "    data_channel_is_world  global_rate_negative_words  ...  n_non_stop_words  \\\n",
       "0                       0                       10.55  ...          0.531300   \n",
       "1                       0                       10.70  ...          0.610451   \n",
       "2                       0                       10.88  ...          0.559055   \n",
       "3                       0                       10.99  ...          0.591691   \n",
       "4                       0                       11.33  ...          0.572277   \n",
       "5                       0                       11.38  ...          0.604878   \n",
       "6                       0                       11.54  ...          0.588542   \n",
       "7                       0                       11.89  ...          0.566489   \n",
       "8                       0                       12.06  ...          0.553525   \n",
       "9                       0                       12.14  ...          0.600877   \n",
       "10                      0                       12.25  ...          0.527907   \n",
       "11                      0                       12.51  ...          0.575679   \n",
       "\n",
       "    n_tokens_content  n_tokens_title  n_unique_tokens  num_hrefs  num_imgs  \\\n",
       "0                331              11         0.543807        244        23   \n",
       "1                257              12         0.712062        248        23   \n",
       "2                142               7         0.584507        245        23   \n",
       "3                413              14         0.704600        261        23   \n",
       "4                289               8         0.605536        248        23   \n",
       "5                248               8         0.721774        245        23   \n",
       "6                226              12         0.632743        246        23   \n",
       "7                213              10         0.647887        248        23   \n",
       "8                212              11         0.617925        249        23   \n",
       "9                274              10         0.605839        244        23   \n",
       "10               227               8         0.682819        245        23   \n",
       "11               445              12         0.629213        244        23   \n",
       "\n",
       "    num_keywords  num_videos  title_sentiment_polarity  title_subjectivity  \n",
       "0              0           0                  0.050000            0.450000  \n",
       "1              0           0                  0.000000            0.000000  \n",
       "2              0           0                  0.345238            0.535714  \n",
       "3              0           0                 -0.050000            0.900000  \n",
       "4              0           0                 -0.600000            0.900000  \n",
       "5              0           0                  0.000000            0.000000  \n",
       "6              0           0                 -0.200000            0.383333  \n",
       "7              0           0                  0.000000            0.000000  \n",
       "8              0           0                 -0.200000            0.000000  \n",
       "9              0           0                  0.160000            0.540000  \n",
       "10             0           0                 -0.050000            0.200000  \n",
       "11             0           0                  0.000000            0.000000  \n",
       "\n",
       "[12 rows x 28 columns]"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data=pd.DataFrame(dataset_extract)\n",
    "final_data_text=final_data\n",
    "final_data=final_data.drop(['text'],axis=1)\n",
    "final_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "from that https://www.kaggle.com/thehapyone/online-news-popularity-a-classification-problem\n",
    "after result was Random Forest is quite good at predicating on UIC data  so i will use Random forest \n",
    "\n",
    "also the here https://archive.ics.uci.edu/ml/datasets/Online+News+Popularity\n",
    "\n",
    "they said \"The estimated relative performance values were estimated by the authors using a Random Forest classifier and a rolling windows as assessment method. See their article for more details on how the relative performance values were set.\"\n",
    "so we will move forward with random forest "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>timedelta</th>\n",
       "      <th>n_tokens_title</th>\n",
       "      <th>n_tokens_content</th>\n",
       "      <th>n_unique_tokens</th>\n",
       "      <th>n_non_stop_words</th>\n",
       "      <th>n_non_stop_unique_tokens</th>\n",
       "      <th>num_hrefs</th>\n",
       "      <th>num_self_hrefs</th>\n",
       "      <th>num_imgs</th>\n",
       "      <th>...</th>\n",
       "      <th>min_positive_polarity</th>\n",
       "      <th>max_positive_polarity</th>\n",
       "      <th>avg_negative_polarity</th>\n",
       "      <th>min_negative_polarity</th>\n",
       "      <th>max_negative_polarity</th>\n",
       "      <th>title_subjectivity</th>\n",
       "      <th>title_sentiment_polarity</th>\n",
       "      <th>abs_title_subjectivity</th>\n",
       "      <th>abs_title_sentiment_polarity</th>\n",
       "      <th>shares</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://mashable.com/2013/01/07/amazon-instant-...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>0.663594</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.815385</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.7</td>\n",
       "      <td>-0.35</td>\n",
       "      <td>-0.6</td>\n",
       "      <td>-0.2</td>\n",
       "      <td>0.5</td>\n",
       "      <td>-0.1875</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1875</td>\n",
       "      <td>593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 61 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url   timedelta  \\\n",
       "0  http://mashable.com/2013/01/07/amazon-instant-...       731.0   \n",
       "\n",
       "    n_tokens_title   n_tokens_content   n_unique_tokens   n_non_stop_words  \\\n",
       "0             12.0              219.0          0.663594                1.0   \n",
       "\n",
       "    n_non_stop_unique_tokens   num_hrefs   num_self_hrefs   num_imgs  ...  \\\n",
       "0                   0.815385         4.0              2.0        1.0  ...   \n",
       "\n",
       "    min_positive_polarity   max_positive_polarity   avg_negative_polarity  \\\n",
       "0                     0.1                     0.7                   -0.35   \n",
       "\n",
       "    min_negative_polarity   max_negative_polarity   title_subjectivity  \\\n",
       "0                    -0.6                    -0.2                  0.5   \n",
       "\n",
       "    title_sentiment_polarity   abs_title_subjectivity  \\\n",
       "0                    -0.1875                      0.0   \n",
       "\n",
       "    abs_title_sentiment_polarity   shares  \n",
       "0                         0.1875      593  \n",
       "\n",
       "[1 rows x 61 columns]"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uic_data.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As UIC data is collected to predict number of shares for post so i will train my model \n",
    "in which no of shares will be predicated value \n",
    "also in virality we always measure how much a post get shared in between so let's continue "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(data):\n",
    "    \"\"\"Clean the column names by stripping and lowercase.\"\"\"\n",
    "    clean_col_map = {x: x.lower().strip() for x in list(data)}\n",
    "    return data.rename(index=str, columns=clean_col_map)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "uic_data1=clean(uic_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set, test_set = train_test_split(uic_data1, test_size=0.20, random_state=42)\n",
    "\n",
    "x_train = train_set.drop(['weekday_is_monday','weekday_is_tuesday','weekday_is_wednesday','weekday_is_thursday','weekday_is_friday','weekday_is_saturday','weekday_is_sunday','is_weekend','url','shares', 'timedelta', 'lda_00','lda_01','lda_02','lda_03','lda_04','num_self_hrefs', 'kw_min_min', 'kw_max_min', 'kw_avg_min','kw_min_max','kw_max_max','kw_avg_max','kw_min_avg','kw_max_avg','kw_avg_avg','self_reference_min_shares','self_reference_max_shares','self_reference_avg_sharess','rate_positive_words','rate_negative_words','abs_title_subjectivity','abs_title_sentiment_polarity'],axis=1)\n",
    "y_train = train_set['shares']\n",
    "\n",
    "x_test = train_set.drop(['weekday_is_monday','weekday_is_tuesday','weekday_is_wednesday','weekday_is_thursday','weekday_is_friday','weekday_is_saturday','weekday_is_sunday','is_weekend','url','shares', 'timedelta', 'lda_00','lda_01','lda_02','lda_03','lda_04','num_self_hrefs', 'kw_min_min', 'kw_max_min', 'kw_avg_min','kw_min_max','kw_max_max','kw_avg_max','kw_min_avg','kw_max_avg','kw_avg_avg','self_reference_min_shares','self_reference_max_shares','self_reference_avg_sharess','rate_positive_words','rate_negative_words','abs_title_subjectivity','abs_title_sentiment_polarity'], axis=1)\n",
    "y_test = test_set['shares']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ASUS\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=None,\n",
       "           oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model= RandomForestRegressor(random_state=42)\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_score=model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1300 1154.3\n"
     ]
    }
   ],
   "source": [
    "print(y_test[1], y_score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting For Own Crawled Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>Virality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:27 IST\\n\\nRa...</td>\n",
       "      <td>6810.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:22 IST\\n\\nOv...</td>\n",
       "      <td>7970.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:18 IST\\n\\nTh...</td>\n",
       "      <td>6820.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>News updates from Hindustan Times: Plasma ther...</td>\n",
       "      <td>6968.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:24 IST\\n\\nTh...</td>\n",
       "      <td>8188.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 17:10 IST\\n\\nDe...</td>\n",
       "      <td>7970.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 16:24 IST\\n\\nTh...</td>\n",
       "      <td>6700.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 16:34 IST\\n\\nTh...</td>\n",
       "      <td>7970.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 16:59 IST\\n\\nMa...</td>\n",
       "      <td>8080.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 15:46 IST\\n\\nTh...</td>\n",
       "      <td>7300.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>india\\n\\nUpdated: Apr 28, 2020 15:59 IST\\n\\nRa...</td>\n",
       "      <td>7970.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Another scientist writes to govt, warns about ...</td>\n",
       "      <td>6700.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  Virality\n",
       "0   india\\n\\nUpdated: Apr 28, 2020 17:27 IST\\n\\nRa...    6810.6\n",
       "1   india\\n\\nUpdated: Apr 28, 2020 17:22 IST\\n\\nOv...    7970.6\n",
       "2   india\\n\\nUpdated: Apr 28, 2020 17:18 IST\\n\\nTh...    6820.5\n",
       "3   News updates from Hindustan Times: Plasma ther...    6968.7\n",
       "4   india\\n\\nUpdated: Apr 28, 2020 17:24 IST\\n\\nTh...    8188.7\n",
       "5   india\\n\\nUpdated: Apr 28, 2020 17:10 IST\\n\\nDe...    7970.6\n",
       "6   india\\n\\nUpdated: Apr 28, 2020 16:24 IST\\n\\nTh...    6700.6\n",
       "7   india\\n\\nUpdated: Apr 28, 2020 16:34 IST\\n\\nTh...    7970.6\n",
       "8   india\\n\\nUpdated: Apr 28, 2020 16:59 IST\\n\\nMa...    8080.6\n",
       "9   india\\n\\nUpdated: Apr 28, 2020 15:46 IST\\n\\nTh...    7300.6\n",
       "10  india\\n\\nUpdated: Apr 28, 2020 15:59 IST\\n\\nRa...    7970.6\n",
       "11  Another scientist writes to govt, warns about ...    6700.6"
      ]
     },
     "execution_count": 255,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_test=pd.DataFrame(model.predict(final_data),final_data_text[\"text\"])\n",
    "final_test.reset_index(level=0, inplace=True)\n",
    "final_test= final_test.rename(index=str, columns={\"index\": \"News\", 0: \"Virality\"})\n",
    "final_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "viral predication.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}