CountVectorizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CountVectorizer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyM0dGGKEnViFf30UC3+vBY/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yashika51/Understanding-Count-Vectorizer/blob/master/CountVectorizer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wMkTRLbKudt3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#importing necessary libraries\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1PoRqZwL-cF4",
        "colab_type": "text"
      },
      "source": [
        "### Small Example"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNkSJvxBDsv4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# checking with smaller list of senetences\n",
        "text=[\"Some Mobile networks send 'stay at home' text\",\"Some customers have already received the message, while others are set to get it later in the day\",]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u3NDv0uYMkvV",
        "colab_type": "code",
        "outputId": "63d0060f-a83a-41a0-e526-93249d937d45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "print(text)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[\"Some Mobile networks send 'stay at home' text\", 'Some customers have already received the message, while others are set to get it later in the day']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kRPL1j8WMm-o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#initializing count vectorizer, lowercase=True turns text to lowercase if not already, this argument is optional and by default it is true\n",
        "cv=CountVectorizer(text,lowercase=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "37kLVdcaN3tK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#fit transform on our text\n",
        "c=cv.fit_transform(text)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cBg3kWMRN4Sf",
        "colab_type": "code",
        "outputId": "22a473bf-e8c9-4a47-9b94-abf20e32d540",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 418
        }
      },
      "source": [
        "#Checking the vocabulary, note these are not number of occurences but position in the sparse vector\n",
        "cv.vocabulary_\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'already': 0,\n",
              " 'are': 1,\n",
              " 'at': 2,\n",
              " 'customers': 3,\n",
              " 'day': 4,\n",
              " 'get': 5,\n",
              " 'have': 6,\n",
              " 'home': 7,\n",
              " 'in': 8,\n",
              " 'it': 9,\n",
              " 'later': 10,\n",
              " 'message': 11,\n",
              " 'mobile': 12,\n",
              " 'networks': 13,\n",
              " 'others': 14,\n",
              " 'received': 15,\n",
              " 'send': 16,\n",
              " 'set': 17,\n",
              " 'some': 18,\n",
              " 'stay': 19,\n",
              " 'text': 20,\n",
              " 'the': 21,\n",
              " 'to': 22,\n",
              " 'while': 23}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E9oXJ8fv_Inf",
        "colab_type": "text"
      },
      "source": [
        "### Now playing with more set of sentences\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l8HV4u6vRxoC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "document=[\"devastating social and economic consequences of COVID-19\",\n",
        "          \"investment and initiatives already ongoing around the world to expedite deployment of innovative COVID-19\",\n",
        "          \"We commit to the shared aim of equitable global access to innovative tools for COVID-19 for all\",\n",
        "          \"We ask the global community and political leaders to support this landmark collaboration, and for donors\",\n",
        "           \"In the fight against COVID-19, no one should be left behind\"\n",
        "]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2H68lJCMXGM4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Initializing\n",
        "cv_doc=CountVectorizer(document)\n",
        "\n",
        "#Fitting on our document\n",
        "cv_vector=cv_doc.fit_transform(document)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eqallg8AXUrb",
        "colab_type": "code",
        "outputId": "ce4a2c67-86c1-4fc0-9390-e64ea08b82cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 803
        }
      },
      "source": [
        "#checking the vocabulary\n",
        "cv_doc.vocabulary_"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'19': 0,\n",
              " 'access': 1,\n",
              " 'against': 2,\n",
              " 'aim': 3,\n",
              " 'all': 4,\n",
              " 'already': 5,\n",
              " 'and': 6,\n",
              " 'around': 7,\n",
              " 'ask': 8,\n",
              " 'be': 9,\n",
              " 'behind': 10,\n",
              " 'collaboration': 11,\n",
              " 'commit': 12,\n",
              " 'community': 13,\n",
              " 'consequences': 14,\n",
              " 'covid': 15,\n",
              " 'deployment': 16,\n",
              " 'devastating': 17,\n",
              " 'donors': 18,\n",
              " 'economic': 19,\n",
              " 'equitable': 20,\n",
              " 'expedite': 21,\n",
              " 'fight': 22,\n",
              " 'for': 23,\n",
              " 'global': 24,\n",
              " 'in': 25,\n",
              " 'initiatives': 26,\n",
              " 'innovative': 27,\n",
              " 'investment': 28,\n",
              " 'landmark': 29,\n",
              " 'leaders': 30,\n",
              " 'left': 31,\n",
              " 'no': 32,\n",
              " 'of': 33,\n",
              " 'one': 34,\n",
              " 'ongoing': 35,\n",
              " 'political': 36,\n",
              " 'shared': 37,\n",
              " 'should': 38,\n",
              " 'social': 39,\n",
              " 'support': 40,\n",
              " 'the': 41,\n",
              " 'this': 42,\n",
              " 'to': 43,\n",
              " 'tools': 44,\n",
              " 'we': 45,\n",
              " 'world': 46}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wsCkRrP7Xf23",
        "colab_type": "code",
        "outputId": "bd28d8c5-8229-4861-ce86-cbff993ef9c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "# let's check the length of our vocabulary\n",
        "len(cv_doc.get_feature_names()) "
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "47"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JrEDn-FPABdx",
        "colab_type": "code",
        "outputId": "0de16c7f-8e80-4b85-e2aa-5038133eddff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 803
        }
      },
      "source": [
        "#In case you are wondering what get_feature_names would return :)\n",
        "cv_doc.get_feature_names()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['19',\n",
              " 'access',\n",
              " 'against',\n",
              " 'aim',\n",
              " 'all',\n",
              " 'already',\n",
              " 'and',\n",
              " 'around',\n",
              " 'ask',\n",
              " 'be',\n",
              " 'behind',\n",
              " 'collaboration',\n",
              " 'commit',\n",
              " 'community',\n",
              " 'consequences',\n",
              " 'covid',\n",
              " 'deployment',\n",
              " 'devastating',\n",
              " 'donors',\n",
              " 'economic',\n",
              " 'equitable',\n",
              " 'expedite',\n",
              " 'fight',\n",
              " 'for',\n",
              " 'global',\n",
              " 'in',\n",
              " 'initiatives',\n",
              " 'innovative',\n",
              " 'investment',\n",
              " 'landmark',\n",
              " 'leaders',\n",
              " 'left',\n",
              " 'no',\n",
              " 'of',\n",
              " 'one',\n",
              " 'ongoing',\n",
              " 'political',\n",
              " 'shared',\n",
              " 'should',\n",
              " 'social',\n",
              " 'support',\n",
              " 'the',\n",
              " 'this',\n",
              " 'to',\n",
              " 'tools',\n",
              " 'we',\n",
              " 'world']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SKmDsLMMARCi",
        "colab_type": "code",
        "outputId": "04b4bfae-5c4b-48bc-89a6-288538da4a4c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#checking the shape of the count vector, (5,47) means 5 rows(sentences) and 47 columns(unique words)\n",
        "cv_vector.shape"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 47)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jKet3XueAiL_",
        "colab_type": "code",
        "outputId": "44abfcb0-2849-4975-e01e-5c0b061e0a69",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#check if any stop_words present, empty set tells no stop words are present\n",
        "cv_doc.stop_words_"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "set()"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GwLVtLz7A_1M",
        "colab_type": "text"
      },
      "source": [
        "# Let's see some more examples experimenting with multiple parameters"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xOwLJ3r0DOaF",
        "colab_type": "text"
      },
      "source": [
        "## Count Vectorizer with predefined and default lits of stop_words\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bio8SWFlDdAS",
        "colab_type": "code",
        "outputId": "31a2b75e-32ec-4ec5-8e91-b6076072608b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "cv1=CountVectorizer(document,stop_words='english')\n",
        "#Lets test cv1 on our doc\n",
        "cv1_doc=cv1.fit_transform(document)\n",
        "#after removing stop_words now number of unique words reduced from 47 to 30 and shape returned is (5,30)\n",
        "cv1_doc.shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 30)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jVzBE45YA9M_",
        "colab_type": "text"
      },
      "source": [
        "### Count Vectorizer with custom stop words, you can specify any word here that is unwanted in your corpus"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jnpNn9vI9Kbz",
        "colab_type": "code",
        "outputId": "8c6c6b2d-84f7-468d-c2d2-19e070f3b337",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#passing the list of stop_words\n",
        "cv2=CountVectorizer(document,stop_words=['the','we','should','this','to'])\n",
        "#checking the stop_words\n",
        "cv2.stop_words"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['the', 'we', 'should', 'this', 'to']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZRDfU13OB0Fh",
        "colab_type": "code",
        "outputId": "fa8d0d18-258f-4f25-8a16-5dfda83922e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Lets test cv1 on our doc\n",
        "cv2_doc=cv1.fit_transform(document)\n",
        "\n",
        "#after removing stop_words now number of unique words reduced from 47 to 42 and shape returned is (5,42)\n",
        "cv2_doc.shape"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 30)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k74sNi2pCRRC",
        "colab_type": "code",
        "outputId": "844ed5af-e810-4a0c-9ff4-e93d18a9300c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Are there any stop_words internally used by Count vectorizer?\n",
        "cv2.stop_words"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['the', 'we', 'should', 'this', 'to']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "exdbjQQvCfyC",
        "colab_type": "text"
      },
      "source": [
        "## What does min_df do?\n",
        "\n",
        "min_df considers words that are only present in minimum of 2 documents. We can also pass a proportion instead of absolute number.\n",
        "\n",
        "\n",
        "For example, min_df=0.25 ignores words that are present in less than 25% of the document"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5qrzHKmf-dLN",
        "colab_type": "code",
        "outputId": "66544ca3-b45f-4b2b-8263-3b7a107fafbb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 636
        }
      },
      "source": [
        "#new initialization with min_df=2\n",
        "cv3=CountVectorizer(document, min_df=2)\n",
        "cv3_doc=cv3.fit_transform(document)\n",
        "# you will see a lot of words here\n",
        "cv3.stop_words_"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'access',\n",
              " 'against',\n",
              " 'aim',\n",
              " 'all',\n",
              " 'already',\n",
              " 'around',\n",
              " 'ask',\n",
              " 'be',\n",
              " 'behind',\n",
              " 'collaboration',\n",
              " 'commit',\n",
              " 'community',\n",
              " 'consequences',\n",
              " 'deployment',\n",
              " 'devastating',\n",
              " 'donors',\n",
              " 'economic',\n",
              " 'equitable',\n",
              " 'expedite',\n",
              " 'fight',\n",
              " 'in',\n",
              " 'initiatives',\n",
              " 'investment',\n",
              " 'landmark',\n",
              " 'leaders',\n",
              " 'left',\n",
              " 'no',\n",
              " 'one',\n",
              " 'ongoing',\n",
              " 'political',\n",
              " 'shared',\n",
              " 'should',\n",
              " 'social',\n",
              " 'support',\n",
              " 'this',\n",
              " 'tools',\n",
              " 'world'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cl8au4Vq_r_A",
        "colab_type": "code",
        "outputId": "57537fa6-d535-45c7-c396-048e62d272ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "#We can see a lot of thw words are removed as they are present in less than 2 documents, this is the vocabulary we have now\n",
        "cv3.vocabulary_"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'19': 0,\n",
              " 'and': 1,\n",
              " 'covid': 2,\n",
              " 'for': 3,\n",
              " 'global': 4,\n",
              " 'innovative': 5,\n",
              " 'of': 6,\n",
              " 'the': 7,\n",
              " 'to': 8,\n",
              " 'we': 9}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ejt3iPeYU1Y",
        "colab_type": "code",
        "outputId": "01cf4311-2fd8-40d3-8657-212a99f224fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Woah! only 10 out of 47 left\n",
        "cv3_doc.shape"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 10)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HObQDKkLFd0l",
        "colab_type": "text"
      },
      "source": [
        "## Now it's turn of max_df \n",
        "\n",
        "As you might have guessed it's opposite to min_df and considers words based on their presence in maximum n number of documents specified.\n",
        "\n",
        "\n",
        "Let's test the proportion instead of absolute number here. If words are present in more than 50% of the document they are ignored."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yFaxcVJiY0SH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cv4=CountVectorizer(document,max_df=0.50)\n",
        "cv4_doc=cv4.fit_transform(document)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aTEGj1gYGmZi",
        "colab_type": "code",
        "outputId": "2880fcd9-1eff-4ca6-d44b-ad81ed7bc1d5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 702
        }
      },
      "source": [
        "cv4.vocabulary_"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'access': 0,\n",
              " 'against': 1,\n",
              " 'aim': 2,\n",
              " 'all': 3,\n",
              " 'already': 4,\n",
              " 'around': 5,\n",
              " 'ask': 6,\n",
              " 'be': 7,\n",
              " 'behind': 8,\n",
              " 'collaboration': 9,\n",
              " 'commit': 10,\n",
              " 'community': 11,\n",
              " 'consequences': 12,\n",
              " 'deployment': 13,\n",
              " 'devastating': 14,\n",
              " 'donors': 15,\n",
              " 'economic': 16,\n",
              " 'equitable': 17,\n",
              " 'expedite': 18,\n",
              " 'fight': 19,\n",
              " 'for': 20,\n",
              " 'global': 21,\n",
              " 'in': 22,\n",
              " 'initiatives': 23,\n",
              " 'innovative': 24,\n",
              " 'investment': 25,\n",
              " 'landmark': 26,\n",
              " 'leaders': 27,\n",
              " 'left': 28,\n",
              " 'no': 29,\n",
              " 'one': 30,\n",
              " 'ongoing': 31,\n",
              " 'political': 32,\n",
              " 'shared': 33,\n",
              " 'should': 34,\n",
              " 'social': 35,\n",
              " 'support': 36,\n",
              " 'this': 37,\n",
              " 'tools': 38,\n",
              " 'we': 39,\n",
              " 'world': 40}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d2VvUnYXY4pF",
        "colab_type": "code",
        "outputId": "9a3fad26-190f-46c1-814a-cb3f415f065b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#these are the only words that crossed the limit of presence in 50% of document\n",
        "cv4.stop_words_"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'19', 'and', 'covid', 'of', 'the', 'to'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zppmIcJhGZ16",
        "colab_type": "code",
        "outputId": "146f1a31-57cb-42cf-ef54-c478e0143b96",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "cv4_doc.shape"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 41)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nc3xh0RsG65I",
        "colab_type": "text"
      },
      "source": [
        "# Now the bigger picture, let's do some custom preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ZxCcnoLHMLh",
        "colab_type": "text"
      },
      "source": [
        "## Starting with Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EMguJD4BGiI2",
        "colab_type": "code",
        "outputId": "26779dbb-9ed9-4897-b561-cd2ed40a80a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "#defining custom tokenizer, we can tokenize the document easily with libraries like nltk\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk.tokenize import word_tokenize\n",
        "def tok(text):\n",
        "  tokens=word_tokenize(text)\n",
        "  return tokens\n",
        "tok(str(document))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[',\n",
              " \"'devastating\",\n",
              " 'social',\n",
              " 'and',\n",
              " 'economic',\n",
              " 'consequences',\n",
              " 'of',\n",
              " 'COVID-19',\n",
              " \"'\",\n",
              " ',',\n",
              " \"'investment\",\n",
              " 'and',\n",
              " 'initiatives',\n",
              " 'already',\n",
              " 'ongoing',\n",
              " 'around',\n",
              " 'the',\n",
              " 'world',\n",
              " 'to',\n",
              " 'expedite',\n",
              " 'deployment',\n",
              " 'of',\n",
              " 'innovative',\n",
              " 'COVID-19',\n",
              " \"'\",\n",
              " ',',\n",
              " \"'We\",\n",
              " 'commit',\n",
              " 'to',\n",
              " 'the',\n",
              " 'shared',\n",
              " 'aim',\n",
              " 'of',\n",
              " 'equitable',\n",
              " 'global',\n",
              " 'access',\n",
              " 'to',\n",
              " 'innovative',\n",
              " 'tools',\n",
              " 'for',\n",
              " 'COVID-19',\n",
              " 'for',\n",
              " 'all',\n",
              " \"'\",\n",
              " ',',\n",
              " \"'We\",\n",
              " 'ask',\n",
              " 'the',\n",
              " 'global',\n",
              " 'community',\n",
              " 'and',\n",
              " 'political',\n",
              " 'leaders',\n",
              " 'to',\n",
              " 'support',\n",
              " 'this',\n",
              " 'landmark',\n",
              " 'collaboration',\n",
              " ',',\n",
              " 'and',\n",
              " 'for',\n",
              " 'donors',\n",
              " \"'\",\n",
              " ',',\n",
              " \"'In\",\n",
              " 'the',\n",
              " 'fight',\n",
              " 'against',\n",
              " 'COVID-19',\n",
              " ',',\n",
              " 'no',\n",
              " 'one',\n",
              " 'should',\n",
              " 'be',\n",
              " 'left',\n",
              " 'behind',\n",
              " \"'\",\n",
              " ']']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IxVbvY8oHp8A",
        "colab_type": "code",
        "outputId": "707b8bcb-fb36-44ba-ecb8-76188f9e2edf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 803
        }
      },
      "source": [
        "cv5=CountVectorizer(document,tokenizer=tok)\n",
        "cv5.fit_transform(document)\n",
        "cv5.vocabulary_"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{',': 0,\n",
              " 'access': 1,\n",
              " 'against': 2,\n",
              " 'aim': 3,\n",
              " 'all': 4,\n",
              " 'already': 5,\n",
              " 'and': 6,\n",
              " 'around': 7,\n",
              " 'ask': 8,\n",
              " 'be': 9,\n",
              " 'behind': 10,\n",
              " 'collaboration': 11,\n",
              " 'commit': 12,\n",
              " 'community': 13,\n",
              " 'consequences': 14,\n",
              " 'covid-19': 15,\n",
              " 'deployment': 16,\n",
              " 'devastating': 17,\n",
              " 'donors': 18,\n",
              " 'economic': 19,\n",
              " 'equitable': 20,\n",
              " 'expedite': 21,\n",
              " 'fight': 22,\n",
              " 'for': 23,\n",
              " 'global': 24,\n",
              " 'in': 25,\n",
              " 'initiatives': 26,\n",
              " 'innovative': 27,\n",
              " 'investment': 28,\n",
              " 'landmark': 29,\n",
              " 'leaders': 30,\n",
              " 'left': 31,\n",
              " 'no': 32,\n",
              " 'of': 33,\n",
              " 'one': 34,\n",
              " 'ongoing': 35,\n",
              " 'political': 36,\n",
              " 'shared': 37,\n",
              " 'should': 38,\n",
              " 'social': 39,\n",
              " 'support': 40,\n",
              " 'the': 41,\n",
              " 'this': 42,\n",
              " 'to': 43,\n",
              " 'tools': 44,\n",
              " 'we': 45,\n",
              " 'world': 46}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vE0Xwah_MPVq",
        "colab_type": "text"
      },
      "source": [
        "## Combinations makes more sense with n-gram\n",
        "\n",
        "Let's check the vocabulary full of unigrams(single words) and bigrams(combinations of two words)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqKVpVWkKqFW",
        "colab_type": "code",
        "outputId": "c47babb0-c4e3-413d-b1f4-3be9b1c2c209",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "cv6=CountVectorizer(document, ngram_range=(1,2))\n",
        "cv6.fit_transform(document)\n",
        "cv6.vocabulary_"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'19': 0,\n",
              " '19 for': 1,\n",
              " '19 no': 2,\n",
              " 'access': 3,\n",
              " 'access to': 4,\n",
              " 'against': 5,\n",
              " 'against covid': 6,\n",
              " 'aim': 7,\n",
              " 'aim of': 8,\n",
              " 'all': 9,\n",
              " 'already': 10,\n",
              " 'already ongoing': 11,\n",
              " 'and': 12,\n",
              " 'and economic': 13,\n",
              " 'and for': 14,\n",
              " 'and initiatives': 15,\n",
              " 'and political': 16,\n",
              " 'around': 17,\n",
              " 'around the': 18,\n",
              " 'ask': 19,\n",
              " 'ask the': 20,\n",
              " 'be': 21,\n",
              " 'be left': 22,\n",
              " 'behind': 23,\n",
              " 'collaboration': 24,\n",
              " 'collaboration and': 25,\n",
              " 'commit': 26,\n",
              " 'commit to': 27,\n",
              " 'community': 28,\n",
              " 'community and': 29,\n",
              " 'consequences': 30,\n",
              " 'consequences of': 31,\n",
              " 'covid': 32,\n",
              " 'covid 19': 33,\n",
              " 'deployment': 34,\n",
              " 'deployment of': 35,\n",
              " 'devastating': 36,\n",
              " 'devastating social': 37,\n",
              " 'donors': 38,\n",
              " 'economic': 39,\n",
              " 'economic consequences': 40,\n",
              " 'equitable': 41,\n",
              " 'equitable global': 42,\n",
              " 'expedite': 43,\n",
              " 'expedite deployment': 44,\n",
              " 'fight': 45,\n",
              " 'fight against': 46,\n",
              " 'for': 47,\n",
              " 'for all': 48,\n",
              " 'for covid': 49,\n",
              " 'for donors': 50,\n",
              " 'global': 51,\n",
              " 'global access': 52,\n",
              " 'global community': 53,\n",
              " 'in': 54,\n",
              " 'in the': 55,\n",
              " 'initiatives': 56,\n",
              " 'initiatives already': 57,\n",
              " 'innovative': 58,\n",
              " 'innovative covid': 59,\n",
              " 'innovative tools': 60,\n",
              " 'investment': 61,\n",
              " 'investment and': 62,\n",
              " 'landmark': 63,\n",
              " 'landmark collaboration': 64,\n",
              " 'leaders': 65,\n",
              " 'leaders to': 66,\n",
              " 'left': 67,\n",
              " 'left behind': 68,\n",
              " 'no': 69,\n",
              " 'no one': 70,\n",
              " 'of': 71,\n",
              " 'of covid': 72,\n",
              " 'of equitable': 73,\n",
              " 'of innovative': 74,\n",
              " 'one': 75,\n",
              " 'one should': 76,\n",
              " 'ongoing': 77,\n",
              " 'ongoing around': 78,\n",
              " 'political': 79,\n",
              " 'political leaders': 80,\n",
              " 'shared': 81,\n",
              " 'shared aim': 82,\n",
              " 'should': 83,\n",
              " 'should be': 84,\n",
              " 'social': 85,\n",
              " 'social and': 86,\n",
              " 'support': 87,\n",
              " 'support this': 88,\n",
              " 'the': 89,\n",
              " 'the fight': 90,\n",
              " 'the global': 91,\n",
              " 'the shared': 92,\n",
              " 'the world': 93,\n",
              " 'this': 94,\n",
              " 'this landmark': 95,\n",
              " 'to': 96,\n",
              " 'to expedite': 97,\n",
              " 'to innovative': 98,\n",
              " 'to support': 99,\n",
              " 'to the': 100,\n",
              " 'tools': 101,\n",
              " 'tools for': 102,\n",
              " 'we': 103,\n",
              " 'we ask': 104,\n",
              " 'we commit': 105,\n",
              " 'world': 106,\n",
              " 'world to': 107}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O7oLVUn0M_Ds",
        "colab_type": "text"
      },
      "source": [
        "## Limiting Vocabulary size\n",
        "\n",
        "We can mention the maximum vocabulary size we intend to keep."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FW_dwSfKMbEH",
        "colab_type": "code",
        "outputId": "a7815b26-cb88-4217-d716-c177525e0950",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 351
        }
      },
      "source": [
        "cv7=CountVectorizer(document,max_features=20)\n",
        "cv7_doc=cv7.fit_transform(document)\n",
        "cv7.vocabulary_\n"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'19': 0,\n",
              " 'aim': 1,\n",
              " 'and': 2,\n",
              " 'around': 3,\n",
              " 'covid': 4,\n",
              " 'for': 5,\n",
              " 'global': 6,\n",
              " 'innovative': 7,\n",
              " 'investment': 8,\n",
              " 'landmark': 9,\n",
              " 'leaders': 10,\n",
              " 'left': 11,\n",
              " 'no': 12,\n",
              " 'of': 13,\n",
              " 'one': 14,\n",
              " 'shared': 15,\n",
              " 'should': 16,\n",
              " 'the': 17,\n",
              " 'to': 18,\n",
              " 'we': 19}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2iaBT1peNOfE",
        "colab_type": "code",
        "outputId": "45c33de4-98dc-4d95-fabe-d8f3b797c9cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#only 20 unique words as we demanded with max_features\n",
        "cv7_doc.shape"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 20)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jK98_qfqNfYI",
        "colab_type": "text"
      },
      "source": [
        "## Custom Preprocessing\n",
        "\n",
        "\n",
        "Although our data is clean in this notebook, the real world data is very messy and in case you want to clean that along with Count Vectorizer you can pass your custom preprocessor as an argument to Count Vectorizer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P0yuspD0NVd2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import regex as re\n",
        "def custom_preprocessor(text):\n",
        "  #lowering the text case\n",
        "  text=text.lower() \n",
        "  # remove special chars\n",
        "  text=re.sub(\"\\\\W\",\" \",text)\n",
        "  return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TIwdI-HbOjSI",
        "colab_type": "code",
        "outputId": "9f8b28f2-a08b-42ce-bfdb-558701fe9cbd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "cv8=CountVectorizer(document,preprocessor=custom_preprocessor)\n",
        "cv8.fit_transform(document)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<5x47 sparse matrix of type '<class 'numpy.int64'>'\n",
              "\twith 66 stored elements in Compressed Sparse Row format>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GgXT3VR-OsBJ",
        "colab_type": "code",
        "outputId": "cde99116-f82a-4cda-a2f1-0560487ac942",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 803
        }
      },
      "source": [
        "cv8.vocabulary_"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'19': 0,\n",
              " 'access': 1,\n",
              " 'against': 2,\n",
              " 'aim': 3,\n",
              " 'all': 4,\n",
              " 'already': 5,\n",
              " 'and': 6,\n",
              " 'around': 7,\n",
              " 'ask': 8,\n",
              " 'be': 9,\n",
              " 'behind': 10,\n",
              " 'collaboration': 11,\n",
              " 'commit': 12,\n",
              " 'community': 13,\n",
              " 'consequences': 14,\n",
              " 'covid': 15,\n",
              " 'deployment': 16,\n",
              " 'devastating': 17,\n",
              " 'donors': 18,\n",
              " 'economic': 19,\n",
              " 'equitable': 20,\n",
              " 'expedite': 21,\n",
              " 'fight': 22,\n",
              " 'for': 23,\n",
              " 'global': 24,\n",
              " 'in': 25,\n",
              " 'initiatives': 26,\n",
              " 'innovative': 27,\n",
              " 'investment': 28,\n",
              " 'landmark': 29,\n",
              " 'leaders': 30,\n",
              " 'left': 31,\n",
              " 'no': 32,\n",
              " 'of': 33,\n",
              " 'one': 34,\n",
              " 'ongoing': 35,\n",
              " 'political': 36,\n",
              " 'shared': 37,\n",
              " 'should': 38,\n",
              " 'social': 39,\n",
              " 'support': 40,\n",
              " 'the': 41,\n",
              " 'this': 42,\n",
              " 'to': 43,\n",
              " 'tools': 44,\n",
              " 'we': 45,\n",
              " 'world': 46}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i-AtK3cpOw1V",
        "colab_type": "text"
      },
      "source": [
        "# Before you go, let's combine most of the things together"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EgNijQpqOt3i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cv_final=CountVectorizer(document,preprocessor=custom_preprocessor,tokenizer=tok,max_features=45,ngram_range=(1,1),stop_words='english',min_df=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GU0xV-vnPOzs",
        "colab_type": "code",
        "outputId": "0c247de7-da17-44ab-9fc5-bf86b78355df",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 518
        }
      },
      "source": [
        "cv_final_doc=cv_final.fit_transform(document)\n",
        "cv_final.vocabulary_"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'19': 0,\n",
              " 'access': 1,\n",
              " 'aim': 2,\n",
              " 'ask': 3,\n",
              " 'collaboration': 4,\n",
              " 'commit': 5,\n",
              " 'community': 6,\n",
              " 'consequences': 7,\n",
              " 'covid': 8,\n",
              " 'deployment': 9,\n",
              " 'devastating': 10,\n",
              " 'donors': 11,\n",
              " 'economic': 12,\n",
              " 'equitable': 13,\n",
              " 'expedite': 14,\n",
              " 'fight': 15,\n",
              " 'global': 16,\n",
              " 'initiatives': 17,\n",
              " 'innovative': 18,\n",
              " 'investment': 19,\n",
              " 'landmark': 20,\n",
              " 'leaders': 21,\n",
              " 'left': 22,\n",
              " 'ongoing': 23,\n",
              " 'political': 24,\n",
              " 'shared': 25,\n",
              " 'social': 26,\n",
              " 'support': 27,\n",
              " 'tools': 28,\n",
              " 'world': 29}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xjoNImQYPTRP",
        "colab_type": "code",
        "outputId": "9ada66fd-a7c2-4acd-a294-beb731c0f735",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "cv_final_doc.shape"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 30)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bBpCO45ePZTp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}