Spam_Slayer_Input_Processor_Updated.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.2"
    },
    "colab": {
      "name": "Spam_Slayer_Input_Processor_Updated.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "7X0fcLfpnFWZ",
        "colab_type": "code",
        "outputId": "d4f8c5f5-3fd6-4269-8ab0-ae4f2262cea0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159
        }
      },
      "source": [
        "#!git clone https://github.com/ChenyuHeidiZhang/Spam-Slayer.git"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'Spam-Slayer'...\n",
            "remote: Enumerating objects: 98, done.\u001b[K\n",
            "remote: Counting objects: 100% (98/98), done.\u001b[K\n",
            "remote: Compressing objects: 100% (69/69), done.\u001b[K\n",
            "remote: Total 1788 (delta 43), reused 80 (delta 29), pack-reused 1690\u001b[K\n",
            "Receiving objects: 100% (1788/1788), 105.89 MiB | 37.24 MiB/s, done.\n",
            "Resolving deltas: 100% (55/55), done.\n",
            "Checking out files: 100% (3279/3279), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZYDH0wFlje0J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import print_function\n",
        "import os\n",
        "import io\n",
        "import re\n",
        "import matplotlib.pyplot as plt\n",
        "import gensim\n",
        "from six.moves import cPickle as pickle\n",
        "import numpy as np\n",
        "import scipy.stats as stats"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgmKXluGm7iR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import zipfile\n",
        "with zipfile.ZipFile('/content/Spam-Slayer/Amazon reviews data.zip', 'r') as zip_ref:\n",
        "    zip_ref.extractall('.')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXilQ6Aoje0N",
        "colab_type": "code",
        "outputId": "51e811bc-a2cd-495e-d26a-f9be35be6b83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "truthful_pos = '/content/Spam-Slayer/Data/op_spam_v1.4/positive_polarity/truthful_from_TripAdvisor/'\n",
        "truthful_neg = '/content/Spam-Slayer/Data/op_spam_v1.4/negative_polarity/truthful_from_Web/'\n",
        "\n",
        "deceptive_pos = '/content/Spam-Slayer/Data/op_spam_v1.4/positive_polarity/deceptive_from_MTurk/'\n",
        "deceptive_neg = '/content/Spam-Slayer/Data/op_spam_v1.4/negative_polarity/deceptive_from_MTurk/'\n",
        "\n",
        "amazon_truthful = '/content/Amazon reviews data/t'\n",
        "amazon_deceptive = '/content/Amazon reviews data/d'\n",
        "\n",
        "\n",
        "truthful_reviews_link = []\n",
        "\n",
        "for fold in os.listdir(truthful_pos):\n",
        "    foldLink = os.path.join(truthful_pos, fold)\n",
        "    if os.path.isdir(foldLink):\n",
        "        for f in os.listdir(foldLink):\n",
        "            fileLink = os.path.join(foldLink, f)\n",
        "            truthful_reviews_link.append(fileLink)\n",
        "\n",
        "for fold in os.listdir(truthful_neg):\n",
        "    foldLink = os.path.join(truthful_neg, fold)\n",
        "    if os.path.isdir(foldLink):\n",
        "        for f in os.listdir(foldLink):\n",
        "            fileLink = os.path.join(foldLink, f)\n",
        "            truthful_reviews_link.append(fileLink)\n",
        "\n",
        "for filename in os.listdir(amazon_truthful):\n",
        "    fileLink = os.path.join(amazon_truthful, filename)\n",
        "    truthful_reviews_link.append(fileLink)\n",
        "\n",
        "deceptive_reviews_link = []\n",
        "\n",
        "for fold in os.listdir(deceptive_pos):\n",
        "    foldLink = os.path.join(deceptive_pos, fold)\n",
        "    if os.path.isdir(foldLink):\n",
        "        for f in os.listdir(foldLink):\n",
        "            fileLink = os.path.join(foldLink, f)\n",
        "            deceptive_reviews_link.append(fileLink)\n",
        "\n",
        "for fold in os.listdir(deceptive_neg):\n",
        "    foldLink = os.path.join(deceptive_neg, fold)\n",
        "    if os.path.isdir(foldLink):\n",
        "        for f in os.listdir(foldLink):\n",
        "            fileLink = os.path.join(foldLink, f)\n",
        "            deceptive_reviews_link.append(fileLink)\n",
        "\n",
        "for filename in os.listdir(amazon_deceptive):\n",
        "    fileLink = os.path.join(amazon_deceptive, filename)\n",
        "    deceptive_reviews_link.append(fileLink)\n",
        "        \n",
        "print('Number of truthfuls reviews ', len(truthful_reviews_link))\n",
        "print('Number of deceptives reviews ', len(deceptive_reviews_link))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of truthfuls reviews  11300\n",
            "Number of deceptives reviews  11300\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJZ7fVVcje0R",
        "colab_type": "code",
        "outputId": "2b7529dd-b433-4c3a-f4ec-41a153a272e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "def clean_str(string):\n",
        "    \"\"\"\n",
        "    Tokenization/string cleaning for all datasets except for SST.\n",
        "    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py\n",
        "    \"\"\"\n",
        "    string = re.sub(r\"[^A-Za-z0-9(),!?\\'\\`]\", \" \", string)\n",
        "    string = re.sub(r\"\\'s\", \" \\'s\", string)\n",
        "    string = re.sub(r\"\\'ve\", \" \\'ve\", string)\n",
        "    string = re.sub(r\"n\\'t\", \" n\\'t\", string)\n",
        "    string = re.sub(r\"\\'re\", \" \\'re\", string)\n",
        "    string = re.sub(r\"\\'d\", \" \\'d\", string)\n",
        "    string = re.sub(r\"\\'ll\", \" \\'ll\", string)\n",
        "    string = re.sub(r\",\", \" , \", string)\n",
        "    string = re.sub(r\"!\", \" ! \", string)\n",
        "    string = re.sub(r\"\\(\", \" \\( \", string)\n",
        "    string = re.sub(r\"\\)\", \" \\) \", string)\n",
        "    string = re.sub(r\"\\?\", \" \\? \", string)\n",
        "    string = re.sub(r\"\\s{2,}\", \" \", string)\n",
        "    return string.strip().lower()\n",
        "\n",
        "def handleFile(filePath):\n",
        "    with open(filePath, \"r\", encoding = \"ISO-8859-1\") as f:\n",
        "        lines=f.readlines()\n",
        "        file_voc = []\n",
        "        file_numWords = 0\n",
        "        for line in lines:\n",
        "            cleanedLine = clean_str(line)\n",
        "            cleanedLine = cleanedLine.strip()\n",
        "            cleanedLine = cleanedLine.lower()\n",
        "            words = cleanedLine.split(' ')\n",
        "            file_numWords = file_numWords + len(words)\n",
        "            file_voc.extend(words)\n",
        "    return file_voc, file_numWords\n",
        "\n",
        "\n",
        "allFilesLinks = truthful_reviews_link + deceptive_reviews_link\n",
        "vocabulary = []\n",
        "numWords = []\n",
        "for fileLink in allFilesLinks:\n",
        "    file_voc, file_numWords = handleFile(fileLink)\n",
        "    vocabulary.extend(file_voc)\n",
        "    numWords.append(file_numWords)\n",
        "\n",
        "vocabulary = set(vocabulary)\n",
        "vocabulary = list(vocabulary)\n",
        "\n",
        "print('The total number of files is ', len(numWords))\n",
        "print('The total number of words in the files is ', sum(numWords))\n",
        "print('Vocabulary size is ', len(vocabulary))\n",
        "print('The average number of words in the files is', sum(numWords)/len(numWords))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The total number of files is  22600\n",
            "The total number of words in the files is  1815773\n",
            "Vocabulary size is  37475\n",
            "The average number of words in the files is 80.34393805309735\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "7TkZZ79Pje0T",
        "colab_type": "code",
        "outputId": "8849fe00-aa91-4c24-a932-96fd2a2a139a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        }
      },
      "source": [
        "\"\"\"Visualize the data in histogram format\"\"\"\n",
        "%matplotlib inline\n",
        "plt.grid(linestyle='-.',alpha=0.3)\n",
        "plt.hist(numWords, 50, color=\"black\", alpha=0.8)\n",
        "plt.xlabel('Sequence Length')\n",
        "plt.ylabel('Frequency')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuUI+dZ5/Hv05Jat1ZLrR4IXmey\n4xDjrGNCsDu2s0AIZLEn4eIsGHAOZzMJXpyzMRAWWHACi51ADmRhCZhLCMEGO7B2jNkl3uCsM7kY\nznJixzOJ4yvGE+ficRyy6oukVqnVLfWzf+jtjtTS9NSopdar8vM5R2dKb5Vevb9WVz+jUuktUVWM\nMcaYYZga9wCMMcZEhxUVY4wxQ2NFxRhjzNBYUTHGGDM0VlSMMcYMjRUVY4wxQ2NFxRhjzNBYUTHG\nGDM0VlSMMcYMTXzcA9hvBw4c0EOHDo17GMYYM1GOHz9eUtVvON12z7micujQIY4dO3bGj1tdXWVm\nZmYEI9o/lsEfUchhGfywXxlE5EthtrPDXyGtra2Newh7Zhn8EYUclsEPvmWwomKMMWZorKgYY4wZ\nGisqxhhjhsaKSkiZTGbcQ9gzy+CPKOSwDH7wLYMVlZASicS4h7BnlsEfUchhGfzgWwYrKiE1Go1x\nD2HPLIM/opDDMvjBtwxWVEJKp9PjHsKeWQZ/RCGHZfCDbxmsqIS0vLw87iHsmWXwRxRyWAY/+Jbh\nOfeN+r1YWFjo2z7IN/SNMSaK7J2KMcaYobGiYowxZmisqIQkIuMewp5ZBn9EIYdl8INvGayohDQ/\nPz/uIeyZZfBHFHJYBj/4lmFkRUVEbhaRr4nII33W/YKIqIgccPdFRG4UkRMi8pCIXNix7RERedLd\njnS0XyQiD7vH3CgjLtflcnmU3e8Ly+CPKOSwDH7wLcMo36n8BXB4Z6OIHAQuA77c0fwa4Fx3uwZ4\nr9u2CFwPXAJcDFwvInPuMe8FfqrjcT3PNUy5XG6U3e8Ly+CPKOSwDH7wLcPIioqq/gOw1GfVe4Bf\nArSj7QrgVm27DyiIyFnA5cBRVV1S1WXgKHDYrZtV1ftUVYFbgdeNKgtArVYbZff7wjL4Iwo5LIMf\nfMuwr5+piMgVwDOq+rkdq84Gnu64f9K17dZ+sk/7yPg2FcIgLIM/opDDMvjBtwz79uVHEckAb6d9\n6Gtficg1tA+rcfDgQUqlUtf6VCpFJpNhdXWVXC7H4uJiTx+bm5sUi0Wq1SrZbJbZ2dntdaVSiVQq\nRTqdJggCstksS0u9b9KKxSK1Wo1MJkO9Xu+5Yls6nSaZTLK2tkYmk+nbx/z8PNVqlZmZGYIg6Okj\nk8mQSCRoNBqk0+mub9uWy2VEhPn5ecrlMrlcjlqt1vNLmc1micfjrK+vk0wmWVlZ6Vo/NTVFsVik\nXC4zOztLtVplfX29p49YLEaz2SSRSPQc993qY2VlhUKhQLlcZmNjo2ubmZkZRITNzU3i8Tjlcrmr\nn1gsxtzc3K595HI5VBVVJRaLUalUutbH43EKhcJ2HysrKzSbza5tZmdnabVaiAgiQrVa7VqfSCTI\n5/PbfSwvL9Nqtbq2yefzNJtNpqamUNWen8fOPpaWltjc3OzpY2Njg3g8TqvV6vkf6vT0NLlcjkql\nQj6f79tHoVCg0WgwPT1Ns9ns6SOZTJLNZqlWq+TzeRYXF2kfDPi6ubk56vU6zWaTIAgIgqBrfZj9\nqVgsEgQBqVSKRqNBvV7v6WM/9idVpVKpDLQ/AV7sT+VyeeD9qVOY/SkM2fkLM0wicgj4sKpeICLf\nCnwc2PoNfD7wFdqflbwDuFdVb3OPewJ41dZNVd/s2t8H3Otun1TVF7v213dut5uFhQUd5BvwpVKJ\nw4f7f2wzKd+oL5VKHDhwYNzD2JMoZIBo5LAMftivDCJyXFX7TyvSYd8Of6nqw6r6jap6SFUP0T5k\ndaGqfhW4C3iDOwvsUqCsqs8C9wCXicic+4D+MuAet64iIpe6s77eAHxov7IYY4zpb5SnFN8GfAo4\nT0ROisjVu2x+N/AUcAJ4P/AWAFVdAn4deMDd3unacNv8mXvM54GPjCLHlmw2O8ru94Vl8EcUclgG\nP/iWYWSfqajq60+z/lDHsgLXnmK7m4Gb+7QfAy7Y2yjDi8cnf+5Ny+CPKOSwDH7wLYN9oz6knR+c\nTSLL4I8o5LAMfvAtgxWVkJLJ5LiHsGeWwR9RyGEZ/OBbBisqIe08DXASWQZ/RCGHZfCDbxmsqBhj\njBkaKyrGGGOGxoqKMcaYobGiEtLU1OT/qCyDP6KQwzL4wbcMfo3GY8VicdxD2DPL4I8o5LAMfvAt\ngxWVkHy7EM4gLIM/opDDMvjBtwxWVELqnJV4UlkGf0Qhh2Xwg28ZrKiEtHO680lkGfwRhRyWwQ++\nZbCiEpJvUyEMwjL4Iwo5LIMffMtgRcUYY8zQWFExxhgzNFZUjDHGDI0VlZB8uxDOICyDP6KQwzL4\nwbcMVlRCisVi4x7CnlkGf0Qhh2Xwg28ZrKiE1Gw2xz2EPbMM/ohCDsvgB98yWFEJKZFIjHsIe2YZ\n/BGFHJbBD75lsKISkm9TIQzCMvgjCjksgx98yzCyoiIiN4vI10TkkY623xaRfxKRh0Tkf4lIoWPd\n20TkhIg8ISKXd7Qfdm0nROS6jvZzROR+1/5BEZkeVRZjjDHhjPKdyl8Ah3e0HQUuUNWXAv8MvA1A\nRM4HrgJe4h7zxyISE5EY8EfAa4Dzgde7bQHeDbxHVV8ELANXjzCLMcaYEEZWVFT1H4ClHW0fVdWt\nT5XuA57vlq8AblfVhqp+ATgBXOxuJ1T1KVVdB24HrhARAb4XuNM9/hbgdaPKYowxJpz4GJ/7J4EP\nuuWzaReZLSddG8DTO9ovAeaBlY4C1bl9DxG5BrgG4ODBg5RKpa71qVSKTCbD6uoquVyOxcXFvv0U\ni0Wq1SrZbLZrZtBSqUQqlSKdThMEAdlslqWlpb6Pr9VqZDIZ6vU6a2trXevT6TTJZJK1tTUymUzf\nPubn56lWq8zMzBAEQU8fmUyGRCJBo9EgnU6zvLy8va5arSIizM/PUy6XyeVy1Go1Go1GVx/ZbJZ4\nPM76+jrJZJKVlZWu9VNTUxSLRcrlMrOzs1Sr1Z75h7LZLLFYjGazSSKR6Dnuu9XHysoKhUKBcrnM\nxsZG1zYzMzOICJubm8TjccrlctfkebFYjLm5uV37yOVyqCqqSiwWo1KpdK2Px+MUCoXtPlZWVnrO\nppmdnaXVaiEiiEjPBH6JRIJ8Pr/dx/LyMq1Wq2ubfD5Ps9lkamoKVT1tH0tLS2xubvb0sbGxQTwe\np9VqUavVutZPT0+Ty+WoVCrk8/m+fRQKBRqNBtPT0zSbzZ4+kskk2WyWarVKPp9ncXERVe3aZm5u\njnq9zubmJkEQEARB1/ow+1OxWCQIAlKpFI1Gg3q93tPHfuxPIkKlUhlof9p6/Lj3p2q1OvD+1CnM\n/hSG7PyFGSYROQR8WFUv2NH+K8AC8MOqqiLyh8B9qvqXbv1NwEfc5odV9T+69v9Au6jc4LZ/kWs/\nCHxk5/P0s7CwoMeOHRsoz8LCQt/2QfszxphJISLHVbX/H8EO+372l4i8EfgB4Cf06xXtGeBgx2bP\nd22nal8ECiIS39E+Mjv/dzGJLIM/opDDMvjBtwz7WlRE5DDwS8APqWrne+a7gKtEJCki5wDnAp8G\nHgDOdWd6TdP+MP8uV4w+CVzpHn8E+NAox14oFE6/kecsgz+ikMMy+MG3DKM8pfg24FPAeSJyUkSu\nBv4QyAFHReRBEfkTAFV9FLgDeAz4P8C1qtpyn5n8NHAP8Dhwh9sW4JeBnxeRE7Q/Y7lpVFnAv3PB\nB2EZ/BGFHJbBD75lGNkH9ar6+j7Np/zDr6rvAt7Vp/1u4O4+7U/RPjtsXwzygZVvLIM/opDDMvjB\ntwz2jXpjjDFDY0XFGGPM0FhRMcYYMzRWVEKamZkZ9xD2zDL4Iwo5LIMffMtgRSWk9swwk80y+CMK\nOSyDH3zLYEUlpJ3TXUwiy+CPKOSwDH7wLYMVlZDi8XFOkzYclsEfUchhGfzgWwYrKiH59gWjQVgG\nf0Qhh2Xwg28ZrKgYY4wZGisqxhhjhsaKijHGmKGxohJSLBYb9xD2zDL4Iwo5LIMffMtgRSWkubm5\ncQ9hzyyDP6KQwzL4wbcMVlRC8u1COIOwDP6IQg7L4AffMlhRCcm3C+EMwjL4Iwo5LIMffMtgRSUk\n384FH4Rl8EcUclgGP/iWwYpKSL5dCGcQlsEfUchhGfzgWwYrKsYYY4bGiooxxpihGVlREZGbReRr\nIvJIR1tRRI6KyJPu3znXLiJyo4icEJGHROTCjscccds/KSJHOtovEpGH3WNuFN/mfzbGmOegUb5T\n+Qvg8I6264CPq+q5wMfdfYDXAOe62zXAe6FdhIDrgUuAi4HrtwqR2+anOh6387mGKpfLjbL7fWEZ\n/BGFHJbBD75lGFlRUdV/AJZ2NF8B3OKWbwFe19F+q7bdBxRE5CzgcuCoqi6p6jJwFDjs1s2q6n2q\nqsCtHX2NKs8ou98XlsEfUchhGfzgW4b9/kzlear6rFv+KvA8t3w28HTHdidd227tJ/u0j4xvL9wg\nLIM/opDDMvjBtwxju7qLqqqI7MtPQ0SuoX1YjYMHD1IqlbrWp1IpMpkMq6ur5HI5FhcXe/qYmZmh\nWCxSrVbJZrPMzs5uryuVSqRSKdLpNEEQkM1mWVra+SYNisUitVqNTCZDvV5nbW2ta306nSaZTLK2\ntkYmk+nbx/z8PNVqlZmZGYIg6Okjk8mQSCRoNBqk02mWl5e31zWbTYIgYH5+nnK5TC6Xo1ar0Wg0\nuvrIZrPE43HW19dJJpM939idmpqiWCxSLpeZnZ2lWq2yvr7e00csFqPZbJJIJHrOpd/qY2VlhUKh\nQLlc7jk1cmZmBhFhc3OTeDxOuVym2WxSq9WA9pxHc3Nzu/aRy+VQVVSVWCxGpVLpWh+PxykUCtt9\nrKys0Gw2u7aZnZ2l1WohIogI1Wq1a30ikSCfz2/3sby8TKvV6tomn8/TbDaZmppCVSmXy9s5+vWx\ntLTUc0W/fD7PxsYG8XicVqvV9XiA6elpcrkclUqFfD7ft49CoUCj0WB6errrZ7klmUySzWapVqvk\n83kWFxd7/mjNzc1Rr9cREYIgIAiCrvVh9qdisUgQBKRSKRqNBvV6vaeP/diftn5eg+xP0L6U77j3\np2azSb1eH2h/6hRmfwpDRlnlROQQ8GFVvcDdfwJ4lao+6w5h3auq54nI+9zybZ3bbd1U9c2u/X3A\nve72SVV9sWt/fed2u1lYWNBjx46dcZZSqcThw/0/thmkv3EolUocOHBg3MPYkyhkgGjksAx+2K8M\nInJcVRdOt91+H/66C9g6g+sI8KGO9je4s8AuBcruMNk9wGUiMuc+oL8MuMetq4jIpe6srzd09GWM\nMWZMRnb4S0Ruo/1O44CInKR9FtdvAXeIyNXAl4Afc5vfDbwWOAEEwJsAVHVJRH4deMBt905V3XoP\n+xbaZ5ilgY+4mzHGmDEaWVFR1defYtWr+2yrwLWn6Odm4OY+7ceAC/YyRmOMMcNl36gPKR4f2zkN\nQ2MZ/BGFHJbBD75lsKISkm/TSw/CMvgjCjksgx98y2BFJSTfLoQzCMvgjyjksAx+8C1DqKIiIt86\n6oH4zrf/DQzCMvgjCjksgx98yxD2ncofi8inReQtIpIf6Yg85dv/BgZhGfwRhRyWwQ++ZQhVVFT1\nu4CfAA4Cx0Xkf4jI9410ZJ7Z+S3rSWQZ/BGFHJbBD75lCP2Ziqo+Cfwq8MvAdwM3isg/icgPj2pw\nxhhjJkvYz1ReKiLvAR4Hvhf4QVX9N275PSMcnzHGmAkS9gTnPwD+DHi7qm7P/KaqXxGRXx3JyIwx\nxkycsEXl+4G6qrYARGQKSKlqoKofGNnoPNI5K/Gksgz+iEIOy+AH3zKE/UzlY7Tn2NqScW3PGTun\nMp9ElsEfUchhGfzgW4awRSWlqqtbd9xyZjRD8lN7MuTJZhn8EYUclsEPvmUIW1RqInLh1h0RuQio\n77J95Pj2wg3CMvgjCjksgx98yxD2M5WfA/5aRL4CCPBNwI+PbFQe2nm1v0lUrVZJJpPjHsaeRCED\nRCOHZfCDbxlCFRVVfUBEXgyc55qeUNUzv86kMcaYSDuTOZNfDhxyj7lQRFDVW0cyKmOMMRMpVFER\nkQ8A3ww8CGydaqCAFRVjjDHbwr5TWQDOd1dofE5KJBLjHsKeWQZ/RCGHZfCDbxnCnv31CO0P55+z\n8vnJn5zZMvgjCjksgx98yxC2qBwAHhORe0Tkrq3boE8qIv9ZRB4VkUdE5DYRSYnIOSJyv4icEJEP\nisi02zbp7p9w6w919PM21/6EiFw+6HjC8G166UFYBn9EIYdl8INvGcIe/rphWE8oImcDP0v7cFpd\nRO4ArgJeC7xHVW8XkT8Brgbe6/5dVtUXichVwLuBHxeR893jXgL8K+BjIvItW1PJDJtvF8IZhGXw\nRxRyWAY/+JYh7PVU/h74IpBwyw8An9nD88aBtIjEaX8z/1naMx7f6dbfArzOLV/h7uPWv1ra3/a5\nArhdVRuq+gXgBHDxHsa0q+Xl5VF1vW8sgz+ikMMy+MG3DGGnvv8p2n/Q3+eazgb+dpAnVNVngN8B\nvky7mJSB48CKqm5dbeake46t53raPbbptp/vbO/zmKHzbX6dQVgGf0Qhh2Xwg28Zwh7+upb2u4D7\noX3BLhH5xkGeUETmaL/LOAdYAf4aODxIX2fwnNcA1wAcPHiQUqnUtT6VSpHJZFhdXSWXy7G4uNjT\nx+bmJsVikWq1Sjab7ZoZtFQqkUqlSKfTBEFANptlaWmpp49isUitViOTyVCv11lbW+tan06nSSaT\nrK2tkclk+vYxPz9PtVplZmaGIAh6+shkMiQSCRqNBul0uut/MeVyGRFhfn6ecrlMLpejVqvRaDS6\n+shms8TjcdbX10kmkz3HbKempigWi5TLZWZnZ6lWq6yvr/f0EYvFaDabJBIJyuVy3z5WVlYoFAqU\ny2U2Nrq/TzszM4OIsLm5STwep1wud/UTi8WYm5vbtY9cLoeqoqrEYjEqlUrX+ng8TqFQ2O5jZWWl\n50p6s7OztFotRAQR6ZldIZFIkM/nt/tYXl7u2dHz+TzNZpOpqSlUtefnsbOPpaUlNjc3e/rY2Ngg\nHo/TarWo1Wpd66enp8nlclQqFfL5fN8+CoUCjUaD6elpms1mTx/JZJJsNku1WiWfz7O4uMjOkz7n\n5uao1+s0m02CICAIgq71YfanYrFIEASkUikajQb1er2nj/3Yn1SVSqUy0P4EeLE/lcvlgfenTmH2\npzAkzFnCInK/ql4iIp9V1W93h60+o6ovPeMnFPlR4LCqXu3uvwF4BfCjwDepalNEXgHcoKqXi8g9\nbvlT7nm/CnwDcB2Aqv6m62d7u92ef2FhQY8dO3amw6ZUKnH4cP/aN0h/41AqlThw4MC4h7EnUcgA\n0chhGfywXxlE5LiqLpxuu7Bnf/29iLyd9ucg30f73cX/HnBsXwYuFZGM+2zk1cBjwCeBK902R4AP\nueW73H3c+k+478vcBVzlzg47BzgX+PSAYzLGGDMEYQ9/XUf7LKyHgTcDd9O+EuQZU9X7ReRO2h/0\nN4HPAn8K/B1wu4j8hmu7yT3kJuADInICWKJ9xheq+qg7c+wx18+1ozrzC/w7F3wQlsEfUchhGfzg\nW4awE0puAu93tz1T1euB63c0P0Wfs7dUdY32obF+/bwLeNcwxnQ6O4+zT6Kt47GTLAoZIBo5LIMf\nfMsQdu6vL9Ce66uLqr5w6CPy1NRU2COF/rIM/ohCDsvgB98ynMncX1tStN85FIc/HH9FYdozy+CP\nKOSwDH7wLUPYLz8udtyeUdXfA75/xGPzyurq6uk38pxl8EcUclgGP/iWIezhrws77k7RfudyJtdi\nMcYY8xwQtjD8947lJu0pW35s6KMxxhgz0cKe/fU9ox6IMcaYyRf28NfP77ZeVX93OMPxl0+n7A3K\nMvgjCjksgx98y3AmZ3+9nPa32AF+kPa3158cxaB85NsXjAZhGfwRhRyWwQ++ZQh7gvPzgQtV9RdU\n9ReAi4AXqOo7VPUdoxueP3y7EM4gLIM/opDDMvjBtwxhi8rzgM5pM9dd23OGbxfCGYRl8EcUclgG\nP/iWIWxRuRX4tIjcICI30J4C/5bdHxIt/abNnjSWwR9RyGEZ/OBbhrBnf71LRD4CfJdrepOqfnZ0\nw/LPzutSTCLL4I8o5LAMfvAtw5lMGpMBKqr6+8BJN928McYYsy3s5YSvB34ZeJtrSgB/OapBGWOM\nmUxh36n8e+CHgBqAqn4FyI1qUMYYYyZT2KKy7q62qAAikh3dkPzk27ngg7AM/ohCDsvgB98yhC0q\nd4jI+4CCiPwU8DGGdMGuSbGxsTHuIeyZZfBHFHJYBj/4liHs2V+/465NXwHOA35NVY+OdGSeiccn\nf1Jmy+CPKOSwDH7wLcNpRyMiMeBjblLJ51Qh6dRqtcY9hD2zDP6IQg7L4AffMpz28JeqtoBNERna\ngTsRKYjInSLyTyLyuIi8QkSKInJURJ50/865bUVEbhSREyLyUOe1XUTkiNv+SRE5Mqzx9VOr1UbZ\n/b6wDP6IQg7L4AffMoR937QKPCwiR3FngAGo6s8O+Ly/D/wfVb1SRKZpfwfm7cDHVfW3ROQ64Dra\npzG/BjjX3S4B3gtcIiJF4Hrak10qcFxE7lLV5QHHZIwxZo/CFpX/6W575t7xvBJ4I4CqrgPrInIF\n8Cq32S3AvbSLyhXAre7ss/vcu5yz3LZHVXXJ9XsUOAzcNoxxGmOMOXO7FhUReYGqfllVhznP1znA\n/wP+XES+DTgOvBV4nqo+67b5Kl+fsPJs4OmOx590badqN8YYMyane6fyt8CFACLyN6r6I0N6zguB\nn1HV+0Xk92kf6tqmqioiOoTnAkBErgGuATh48CClUqlrfSqVIpPJsLq6Si6XY3FxsXfQ8TjFYpFq\ntUo2m2V2dnZ7XalUIpVKkU6nCYKAbDbbd5K3YrFIrVYjk8lQr9dZW1vrWp9Op0kmk6ytrZHJZPr2\nMT8/T7VaZWZmhiAIevrIZDIkEgkajQbpdJrl5a8fDQyCgMXFRebn5ymXy+RyOWq1Go1Go6uPbDZL\nPB5nfX2dZDLZM7X21NQUxWKRcrnM7Ows1WqV9fX1nj5isRjNZpNEIkG5XO7bx8rKCoVCgXK53HNq\n5MzMDCLC5uYm8XiccrlMEATbr18sFmNubm7XPnK5HKqKqhKLxahUKl3r4/E4hUJhu4+VlRWazWbX\nNrOzs7RaLUQEEaFarXatTyQS5PP57T6Wl5d7PjzN5/M0m02mpqZQ1a4c/fpYWlrqmdMpn8+zsbFB\nPB6n1Wr1HEufnp4ml8tRqVTI5/N9+ygUCjQaDaanp2k2mz19JJNJstks1WqVfD7P4uIi7YMEXzc3\nN0e9XmdqaoogCAiCoGt9mP2pWCwSBAGpVIpGo0G9Xu/pYz/2p0QiQaVSGWh/AhCRse9PQRCwtLQ0\n0P7UKcz+FIbs/IXpWinyWVX99p3LeyEi3wTcp6qH3P3vol1UXgS8SlWfdYe37lXV89z3Y+5V1dvc\n9k/QPvT1Krf9m11713ansrCwoMeOHTvjcasqL3/5y/uuG6S/cVBVRGTcw9iTKGSAaOSwDH7Yrwwi\nclxVF0633enO/tJTLA9MVb8KPC0i57mmVwOP0b6q5NYZXEeAD7nlu4A3uLPALgXK7jDZPcBlIjLn\nzhS7zLWNxM7/4U4iy+CPKOSwDH7wLcPpDn99m4hUAAHSbhl3X1V19tQP3dXPAH/lzvx6CngT7QJ3\nh4hcDXwJ+DG37d3Aa4ETQOC2RVWXROTXgQfcdu/c+tB+FHybCmEQlsEfUchhGfzgW4Zdi4qqxkbx\npKr6IO1TgXd6dZ9tFbj2FP3cDNw83NH159uFcAaxddx1kkUhA0Qjh2Xwg28ZzuR6Ks9pvl0IZxCW\nwR9RyGEZ/OBbBisqxhhjhsaKijHGmKGxomKMMWZorKiEVCgUxj2EPbMM/ohCDsvgB98yWFEJaee3\nZCeRZfBHFHJYBj/4lsGKSkjT09PjHsKeWQZ/RCGHZfCDbxmsqIS0cz6oSWQZ/BGFHJbBD75lsKIS\nkm8XwhmEZfBHFHJYBj/4lsGKijHGmKGxomKMMWZorKgYY4wZGisqISWTyXEPYc8sgz+ikMMy+MG3\nDFZUQspms+Mewp5ZBn9EIYdl8INvGU53PRXj7LyEbKeFhf4XQ/PtipBbl4idZFHIANHIYRn84FsG\ne6cSkk8v2qAsgz+ikMMy+MG3DFZUQlpcXBz3EPbMMvgjCjksgx98y2BFJaT2BSgnm2XwRxRyWAY/\n+JbBiooxxpihsaJijDFmaMZWVEQkJiKfFZEPu/vniMj9InJCRD4oItOuPenun3DrD3X08TbX/oSI\nXD6eJMYYY7aM853KW4HHO+6/G3iPqr4IWAaudu1XA8uu/T1uO0TkfOAq4CXAYeCPRSQ2qsHOzc2N\nqut9Yxn8EYUclsEPvmUYS1ERkecD3w/8mbsvwPcCd7pNbgFe55avcPdx61/ttr8CuF1VG6r6BeAE\ncPGoxlyv10fV9b6xDP6IQg7L4AffMozry4+/B/wSkHP354EVVd26MMBJ4Gy3fDbwNICqNkWk7LY/\nG7ivo8/Ox3QRkWuAawAOHjxIqVTqWp9KpchkMqyurpLL5fqeopfL5SgWi1SrVbLZLLOzs13rq9Uq\n1WqVQqHA4uJi3+cpFovUajUymQz1ep21tbWu9el0mmQyydraGplMhqWlpZ5xzM/PU61WmZmZIQiC\nnj4ymQyJRIJGo0E6nWZ5eXl7XavVotFoMD8/T7lcJpfLUavVeq4cl81micfjrK+vk0wmWVlZ6Vo/\nNTVFsVikXC4zOztLtVplfX29p49YLEaz2SSRSFAul/v2sbKyQqFQoFwus7Gx0bXNzMwMIsLm5ibx\neJxyuUyr1drOHIvFmJub27WPXC6HqqKqxGIxKpVK1/p4PE6hUNjuY2Vlpef6FLOzs7RaLUQEEen5\nImwikSCfz2/3sby8TKvV6tqEweWeAAAQ90lEQVQmn8/TbDaZmppCVanVal2v3c4+lpaW2Nzc7Olj\nY2ODeDxOq9XqmfJ8enqaXC5HpVIhn8/37aNQKNBoNJienqbZbPb0kUwmyWaz21+oW1xc7Dm7aG5u\njnq9ztTUFEEQEARB1/ow+1OxWCQIAlKpFI1Go+cPYyqVIp1OEwQB2Wy2774wjP1pdnaWSqUy0P4E\nICJj359arRbr6+sD7U+dwuxPYch+n44mIj8AvFZV3yIirwJ+EXgjcJ87xIWIHAQ+oqoXiMgjwGFV\nPenWfR64BLjBPeYvXftN7jF3souFhQUd5JvuQRDwyle+8owe49s36oMgIJPJjHsYexKFDBCNHJbB\nD/uVQUSOq2r/6UM6jOPw13cAPyQiXwRup33Y6/eBgohsvXN6PvCMW34GOAjg1ueBxc72Po8Zup3/\nG5tElsEfUchhGfzgW4Z9Lyqq+jZVfb6qHqL9QfsnVPUngE8CV7rNjgAfcst3ufu49Z/Q9turu4Cr\n3Nlh5wDnAp/epxjGGGP68GlCyV8GbheR3wA+C9zk2m8CPiAiJ4Al2oUIVX1URO4AHgOawLWq2urt\n1hhjzH4Za1FR1XuBe93yU/Q5e0tV14AfPcXj3wW8a3QjNMYYcybsG/UhpVKpcQ9hzyyDP6KQwzL4\nwbcMVlRCmvQzRMAy+CQKOSyDH3zLYEUlpNXV1XEPYc8sgz+ikMMy+MG3DFZUQsrlcqffyHOWwR9R\nyGEZ/OBbBisqIfl2IZxBWAZ/RCGHZfCDbxmsqBhjjBkaKyrGGGOGxoqKMcaYobGiYowxZmisqIRU\nLBbHPYQ9swz+iEIOy+AH3zJYUQnJt5lAB2EZ/BGFHJbBD75lsKISkm9TIQzCMvgjCjksgx98y2BF\nJaSdV3ObRJbBH1HIYRn84FsGKyoh+XYd6EFYBn9EIYdl8INvGayoGGOMGRorKsYYY4bGiooxxpih\nsaISkm9nWAzCMvgjCjksgx98y2BFJaR0Oj3uIeyZZfBHFHJYBj/4lmHfi4qIHBSRT4rIYyLyqIi8\n1bUXReSoiDzp/p1z7SIiN4rICRF5SEQu7OjriNv+SRE5Mspx+/YFo0FYBn9EIYdl8INvGcbxTqUJ\n/IKqng9cClwrIucD1wEfV9VzgY+7+wCvAc51t2uA90K7CAHXA5cAFwPXbxWiUchms6Pqet9YBn9E\nIYdl8INvGfa9qKjqs6r6GbdcBR4HzgauAG5xm90CvM4tXwHcqm33AQUROQu4HDiqqkuqugwcBQ6P\natxLS0uj6nrfWAZ/RCGHZfCDbxnG+pmKiBwCvh24H3ieqj7rVn0VeJ5bPht4uuNhJ13bqdqNMcaM\nSXxcTywiM8DfAD+nqhUR2V6nqioiOsTnuob2oTMOHjxIqVTqWp9KpchkMqyurpLL5fpennNzc5Ni\nsUi1WiWbzTI7O9u1vlqtUq1WKRQKLC4u9n2eYrFIrVYjk8lQr9dZW1vrWp9Op0kmk6ytrZHJZPr+\nD2R+fp5qtcrMzAxBEPT0kclkSCQSNBoN0uk0y8vL2+vK5TIiwvz8POVymVwuR61W65nmIZvNEo/H\nWV9fJ5lMsrKy0rV+amqKYrFIuVxmdnaWarXK+vp6Tx+xWIxms0kikaBcLvftY2VlhUKhQLlcZmNj\no2ubmZkZRITNzU3i8Tjlcrmrn1gsxtzc3K595HI5VBVVJRaLUalUutbH43EKhcJ2HysrKzSbza5t\nZmdnabVaiAgiQrVa7VqfSCTI5/PbfSwvL9Nqtbq2yefzNJtNpqamUNWen8fOPpaWltjc3OzpY2Nj\ng3g8TqvVolarda2fnp4ml8tRqVTI5/N9+ygUCjQaDaanp2k2mz19JJNJstks1WqVfD7P4uIiqt27\n4tzcHPV6nWazSRAEPcf0w+xPxWKRIAhIpVI0Go2eb4WnUinS6TRBEJDNZvvuC8PYn1SVSqUy0P4E\neLE/lcvlgfenTmH2pzBk5y/MfhCRBPBh4B5V/V3X9gTwKlV91h3euldVzxOR97nl2zq327qp6ptd\ne9d2p7KwsKDHjh074zGXSiUOHz6zo2uDPM8olUolDhw4MO5h7EkUMkA0clgGP+xXBhE5rqoLp9tu\nHGd/CXAT8PhWQXHuArbO4DoCfKij/Q3uLLBLgbI7THYPcJmIzLkP6C9zbcYYY8ZkHIe/vgP4D8DD\nIvKga3s78FvAHSJyNfAl4MfcuruB1wIngAB4E4CqLonIrwMPuO3eqaoj+8TKtwvhDMIy+CMKOSyD\nH3zLsO9FRVX/LyCnWP3qPtsrcO0p+roZuHl4ozu1nceeJ1GtViOXy417GHsShQwQjRyWwQ++ZRjb\nB/WTJpPJnPFjFhb6H34c12ctg2TwTRQyQDRyWAY/+JbBpmkJybdrFgzCMvgjCjksgx98y2BFJaSd\npxpOIsvgjyjksAx+8C2DFRVjjDFDY0XFGGPM0FhRMcYYMzRWVELy7ZoFg7AM/ohCDsvgB98yWFEJ\nKZlMjnsIe2YZ/BGFHJbBD75lsKISkm9nWAzCMvgjCjksgx98y2BFJSTfvmA0CMvgjyjksAx+8C2D\nFZWQfLsQziAsgz+ikMMy+MG3DFZUjDHGDI0VFWOMMUNjE0qOgW8TTRpjzLDYOxVjjDFDY0UlpPn5\n+XEPYc8sgz+ikMMy+MG3DFZUQqpWq+Mewp5ZBn9EIYdl8INvGayohDQzMzPuIeyZZfBHFHJYBj/4\nlsE+qA8pCIKRP8eoP8APgsC7X8AzFYUMEI0clsEPvmWY+HcqInJYRJ4QkRMict2onse3qRAGYRn8\nEYUclsEPvmWY6HcqIhID/gj4PuAk8ICI3KWqj413ZMNlpyAbYybFRBcV4GLghKo+BSAitwNXAJEq\nKqdyqmIDVnCMMeMx6UXlbODpjvsngUvGNBav9Cs4L3jBC/jyl788lP6taBlj+pn0ohKKiFwDXOPu\nrorIEwN0cwAoDW9U++/48eNDyyAiw+hmEBP/OjhRyGEZ/LBfGf51mI0mvag8AxzsuP9819ZFVf8U\n+NO9PJGIHFPVUx9vmgCWwR9RyGEZ/OBbhkk/++sB4FwROUdEpoGrgLvGPCZjjHnOmuh3KqraFJGf\nBu4BYsDNqvromIdljDHPWRNdVABU9W7g7n14qj0dPvOEZfBHFHJYBj94lUFUddxjMMYYExGT/pmK\nMcYYj1hRCWG/poIZBhH5oog8LCIPisgx11YUkaMi8qT7d861i4jc6HI9JCIXjmnMN4vI10TkkY62\nMx6ziBxx2z8pIkc8yHCDiDzjXosHReS1Heve5jI8ISKXd7SP7XdNRA6KyCdF5DEReVRE3uraJ+a1\n2CXDxLwWIpISkU+LyOdchne49nNE5H43ng+6k5MQkaS7f8KtP3S6bCOlqnbb5Ub7BIDPAy8EpoHP\nAeePe1y7jPeLwIEdbf8NuM4tXwe82y2/FvgIIMClwP1jGvMrgQuBRwYdM1AEnnL/zrnluTFnuAH4\nxT7bnu9+j5LAOe73Kzbu3zXgLOBCt5wD/tmNdWJei10yTMxr4X6eM245Adzvfr53AFe59j8B/pNb\nfgvwJ275KuCDu2Ub9fjtncrpbU8Fo6rrwNZUMJPkCuAWt3wL8LqO9lu17T6gICJn7ffgVPUfgKUd\nzWc65suBo6q6pKrLwFHg8OhH33aKDKdyBXC7qjZU9QvACdq/Z2P9XVPVZ1X1M265CjxOe9aKiXkt\ndslwKt69Fu7nueruJtxNge8F7nTtO1+HrdfnTuDVIiKcOttIWVE5vX5Twez2SzpuCnxURI5LeyYB\ngOep6rNu+avA89yyz9nOdMy+Zvlpd2jo5q3DRkxABncI5dtp/y95Il+LHRlggl4LEYmJyIPA12gX\n5c8DK6ra7DOe7bG69WVgnjFlsKISPd+pqhcCrwGuFZFXdq7U9vviiTrlbxLH7LwX+GbgZcCzwH8f\n73DCEZEZ4G+An1PVSue6SXkt+mSYqNdCVVuq+jLas4RcDLx4zEMKzYrK6YWaCsYXqvqM+/drwP+i\n/Qv5L1uHtdy/X3Ob+5ztTMfsXRZV/Rf3x2ETeD9fP/TgbQYRSdD+Y/xXqvo/XfNEvRb9MkziawGg\nqivAJ4FX0D68uPXdws7xbI/Vrc8Di4wpgxWV05uYqWBEJCsiua1l4DLgEdrj3ToD5wjwIbd8F/AG\ndxbPpUC54zDHuJ3pmO8BLhOROXdo4zLXNjY7Pp/697RfC2hnuMqdtXMOcC7wacb8u+aOw98EPK6q\nv9uxamJei1NlmKTXQkS+QUQKbjlN+3pRj9MuLle6zXa+Dluvz5XAJ9w7ylNlG61RnwkQhRvts1z+\nmfZxzV8Z93h2GecLaZ/t8Tng0a2x0j6++nHgSeBjQNG1C+2LnH0eeBhYGNO4b6N9SGKD9nHfqwcZ\nM/CTtD+MPAG8yYMMH3BjfIj2Dn5Wx/a/4jI8AbzGh9814DtpH9p6CHjQ3V47Sa/FLhkm5rUAXgp8\n1o31EeDXXPsLaReFE8BfA0nXnnL3T7j1LzxdtlHe7Bv1xhhjhsYOfxljjBkaKyrGGGOGxoqKMcaY\nobGiYowxZmisqBhjjBkaKyom8kTkV9xsrw+5GWovGfeY9kJE/kJErjz9lgP3/7Ids/jeICK/OKrn\nM9Ey8Vd+NGY3IvIK4Adoz1zbEJEDtGedNaf2MmCB/bmiqokYe6diou4soKSqDQBVLanqVwBE5CIR\n+Xs3+eY9HVORXOSuZfE5EfltcddIEZE3isgfbnUsIh8WkVe55ctE5FMi8hkR+Ws399TW9W3e4dof\nFpEXu/YZEflz1/aQiPzIbv2EISL/RUQecP1tXYPjkIg8LiLvd+/WPuq+pY2IvLzj3dtvi8gj7tvj\n7wR+3LX/uOv+fBG5V0SeEpGfHfjVMJFnRcVE3UeBgyLyzyLyxyLy3bA9P9QfAFeq6kXAzcC73GP+\nHPgZVf22ME/g3v38KvDvtD2Z5zHg5zs2Kbn29wJbh5H+K+1pTb5VVV8KfCJEP7uN4TLa03BcTPud\nxkXy9clEzwX+SFVfAqwAP9KR883anriwBaDtad5/jfY1OV6mqh90276Y9pT2FwPXu5+fMT3s8JeJ\nNFVdFZGLgO8Cvgf4oLSv4ncMuAA42p4uihjwrJtzqaDt66NAe3qP15zmaS6lfUGkf3R9TQOf6li/\nNTHjceCH3fK/oz2f1NY4l0XkB07Tz24uc7fPuvsztIvJl4EvqOqDHWM45HLmVHWr//9B+zDhqfyd\ne7fXEJGv0Z7+/mTIsZnnECsqJvJUtQXcC9wrIg/TnnzvOPCoqr6ic9utifxOoUn3u/vU1sNoX5Tq\n9ad4XMP922L3fe50/exGgN9U1fd1NbavKdLoaGoB6QH639mH/e0wfdnhLxNpInKeiJzb0fQy4Eu0\nJ9j7BvdBPiKSEJGXaHuq8RUR+U63/U90PPaLwMtEZEpEDvL16dPvA75DRF7k+sqKyLecZmhHgWs7\nxjk3YD9b7gF+suOznLNF5BtPtbHLWe04E+6qjtVV2pfiNeaMWVExUTcD3CIij4nIQ7jrlbvPDq4E\n3i0in6M9m+2/dY95E/BH0r7ynnT09Y/AF4DHgBuBrcvW/j/gjcBt7jk+xekvqvQbwJz7cPxzwPec\nYT/vE5GT7vYpVf0o7UNYn3Lvxu7k9IXhauD9LmeW9hUDoT3F+vk7Pqg3JhSbpdiYXbjDRx9W1QvG\nPJShE5EZdddCd58znaWqbx3zsMyEs+Oixjx3fb+IvI3234Ev0X6XZMye2DsVY4wxQ2OfqRhjjBka\nKyrGGGOGxoqKMcaYobGiYowxZmisqBhjjBkaKyrGGGOG5v8DOJNe5BD3T/8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FyspMqKVje0W",
        "colab_type": "code",
        "outputId": "e08783b0-6328-429b-a0e1-c595508fccad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        }
      },
      "source": [
        "word2vec_path = \"https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\"\n",
        "w2v_model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)\n",
        "wordsVectors = []\n",
        "notFoundwords = []\n",
        "for word in vocabulary:\n",
        "    try:\n",
        "        vector = w2v_model[word]\n",
        "        wordsVectors.append(vector)\n",
        "    except Exception as e:\n",
        "        notFoundwords.append(word)\n",
        "        wordsVectors.append(np.random.uniform(-0.25,0.25,300))  \n",
        "\n",
        "del w2v_model\n",
        "wordsVectors = np.asarray(wordsVectors)\n",
        "\n",
        "print('The number of missing words is ', len(notFoundwords))\n",
        "\n",
        "\"\"\"Save\"\"\"\n",
        "pickle_file = os.path.join('./', 'save.pickle')\n",
        "\n",
        "try:\n",
        "    f = open(pickle_file, 'wb')\n",
        "    save = {\n",
        "        'wordsVectors': wordsVectors,\n",
        "        'vocabulary': vocabulary,\n",
        "        'notFoundwords': notFoundwords\n",
        "    }\n",
        "\n",
        "    pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
        "    f.close()\n",
        "except Exception as e:\n",
        "    print('Unable to save data to', pickle_file, ':', e)\n",
        "    raise\n",
        "\n",
        "statinfo = os.stat(pickle_file)\n",
        "print('Compressed pickle size:', statinfo.st_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "The number of missing words is  8887\n",
            "Compressed pickle size: 90362180\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PJxOZFsyje0Z",
        "colab_type": "code",
        "outputId": "c0f1d39b-fc62-4fbc-dd29-daedfdef773f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "MAX_SEQ_LENGTH = 160\n",
        "def convertFileToIndexArray(filePath):\n",
        "    doc = np.zeros(MAX_SEQ_LENGTH, dtype='int32')\n",
        "    with open(filePath, \"r\", encoding = \"ISO-8859-1\") as f:\n",
        "        lines=f.readlines()\n",
        "        indexCounter = 0\n",
        "        for line in lines:\n",
        "            cleanedLine = clean_str(line)\n",
        "            cleanedLine = cleanedLine.strip()\n",
        "            cleanedLine = cleanedLine.lower()\n",
        "            words = cleanedLine.split(' ')\n",
        "            for word in words:\n",
        "                doc[indexCounter] = vocabulary.index(word)\n",
        "                indexCounter = indexCounter + 1\n",
        "                if (indexCounter >= MAX_SEQ_LENGTH):\n",
        "                    break\n",
        "            if (indexCounter >= MAX_SEQ_LENGTH):\n",
        "                break\n",
        "    return doc\n",
        "\n",
        "totalFiles = len(truthful_reviews_link) + len(deceptive_reviews_link)\n",
        "idsMatrix = np.ndarray(shape=(totalFiles, MAX_SEQ_LENGTH), dtype='int32')\n",
        "labels = np.ndarray(shape=(totalFiles, 2), dtype='int32')\n",
        "\n",
        "counter = 0\n",
        "for filePath in truthful_reviews_link:\n",
        "    idsMatrix[counter] = convertFileToIndexArray(filePath)\n",
        "    counter = counter + 1\n",
        "\n",
        "for filePath in deceptive_reviews_link:\n",
        "    idsMatrix[counter] = convertFileToIndexArray(filePath)\n",
        "    counter = counter + 1\n",
        "    \n",
        "labels[0:len(truthful_reviews_link)] = np.array([1, 0])\n",
        "labels[len(truthful_reviews_link):totalFiles] = np.array([0, 1])\n",
        "\n",
        "print('The shape of the ids matrix is ', idsMatrix.shape)\n",
        "print('The shape of the labels is ', labels.shape)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The shape of the ids matrix is  (22600, 160)\n",
            "The shape of the labels is  (22600, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gk5QL17Bje0c",
        "colab_type": "code",
        "outputId": "08bda904-fbd1-4d37-ae3f-39253dc68816",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "\"\"\"\n",
        "Create a training set, a validation set and a test set after mixing the data\n",
        "80% for the training set\n",
        "10% for the validation set\n",
        "10% for the test set\n",
        "\"\"\"\n",
        "size = idsMatrix.shape[0]\n",
        "testSize = int(size * 0.1)\n",
        "shuffledIndex = np.random.permutation(size)\n",
        "testIndexes = shuffledIndex[0:testSize]\n",
        "validationIndexes = shuffledIndex[testSize:2*testSize]\n",
        "trainIndexes = shuffledIndex[2*testSize:size]\n",
        "\n",
        "test_data = idsMatrix[testIndexes]\n",
        "test_labels = labels[testIndexes]\n",
        "\n",
        "validation_data = idsMatrix[validationIndexes]\n",
        "validation_labels = labels[validationIndexes]\n",
        "\n",
        "train_data = idsMatrix[trainIndexes]\n",
        "train_labels = labels[trainIndexes]\n",
        "\n",
        "print('train data shape ', train_data.shape)\n",
        "print('train labels shape ', train_labels.shape)\n",
        "print('validation data shape ', validation_data.shape)\n",
        "print('validation labels shape ', validation_labels.shape)\n",
        "print('test data shape ', test_data.shape)\n",
        "print('test labels shape ', test_labels.shape)\n",
        "\n",
        "pickle_file = os.path.join('./', 'data_saved.pickle')\n",
        "\n",
        "try:\n",
        "    f = open(pickle_file, 'wb')\n",
        "    save = {\n",
        "        'train_data': train_data,\n",
        "        'train_labels': train_labels,\n",
        "        'validation_data': validation_data,\n",
        "        'validation_labels': validation_labels,\n",
        "        'test_data': test_data,\n",
        "        'test_labels': test_labels\n",
        "    }\n",
        "    \n",
        "    pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
        "    f.close()\n",
        "except Exception as e:\n",
        "    print('Unable to save data to', pickle_file, ':', e)\n",
        "    raise\n",
        "\n",
        "statinfo = os.stat(pickle_file)\n",
        "print('Compressed pickle size:', statinfo.st_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train data shape  (18080, 160)\n",
            "train labels shape  (18080, 2)\n",
            "validation data shape  (2260, 160)\n",
            "validation labels shape  (2260, 2)\n",
            "test data shape  (2260, 160)\n",
            "test labels shape  (2260, 2)\n",
            "Compressed pickle size: 14645267\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}