bert-english-language-learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-output": true,
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2022-10-30T20:32:47.085084Z",
     "iopub.status.busy": "2022-10-30T20:32:47.084576Z",
     "iopub.status.idle": "2022-10-30T20:35:05.708674Z",
     "shell.execute_reply": "2022-10-30T20:35:05.706603Z",
     "shell.execute_reply.started": "2022-10-30T20:32:47.084969Z"
    },
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow-text\n",
      "  Downloading tensorflow_text-2.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.9 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m38.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting tensorflow<2.11,>=2.10.0\n",
      "  Downloading tensorflow-2.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (578.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m578.0/578.0 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tensorflow-hub>=0.8.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow-text) (0.12.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (3.3.0)\n",
      "Collecting tensorflow-io-gcs-filesystem>=0.23.1\n",
      "  Downloading tensorflow_io_gcs_filesystem-0.27.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m57.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (59.8.0)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (1.15.0)\n",
      "Collecting keras<2.11,>=2.10.0\n",
      "  Downloading keras-2.10.0-py2.py3-none-any.whl (1.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m57.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (3.7.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (1.6.3)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (0.2.0)\n",
      "Requirement already satisfied: tensorboard<2.11,>=2.10 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (2.10.1)\n",
      "Requirement already satisfied: numpy>=1.20 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (1.21.6)\n",
      "Collecting tensorflow-estimator<2.11,>=2.10.0\n",
      "  Downloading tensorflow_estimator-2.10.0-py2.py3-none-any.whl (438 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m438.7/438.7 kB\u001b[0m \u001b[31m30.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (21.3)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (0.4.0)\n",
      "Collecting flatbuffers>=2.0\n",
      "  Downloading flatbuffers-22.10.26-py2.py3-none-any.whl (26 kB)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (1.43.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (1.1.2)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (1.1.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (1.12.1)\n",
      "Collecting absl-py>=1.0.0\n",
      "  Downloading absl_py-1.3.0-py3-none-any.whl (124 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.6/124.6 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting libclang>=13.0.0\n",
      "  Downloading libclang-14.0.6-py2.py3-none-manylinux2010_x86_64.whl (14.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m61.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (4.4.0)\n",
      "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.11,>=2.10.0->tensorflow-text) (3.19.4)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.7/site-packages (from astunparse>=1.6.0->tensorflow<2.11,>=2.10.0->tensorflow-text) (0.37.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (0.6.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (1.8.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (2.2.2)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (0.4.6)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (3.3.7)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (2.28.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (1.35.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->tensorflow<2.11,>=2.10.0->tensorflow-text) (3.0.9)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (0.2.7)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (4.2.4)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (4.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (4.13.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (2022.9.24)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (2.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (1.26.12)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.7/site-packages (from werkzeug>=1.0.1->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (2.1.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (3.8.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow<2.11,>=2.10.0->tensorflow-text) (3.2.0)\n",
      "Installing collected packages: libclang, keras, flatbuffers, tensorflow-io-gcs-filesystem, tensorflow-estimator, absl-py, tensorflow, tensorflow-text\n",
      "  Attempting uninstall: keras\n",
      "    Found existing installation: keras 2.6.0\n",
      "    Uninstalling keras-2.6.0:\n",
      "      Successfully uninstalled keras-2.6.0\n",
      "  Attempting uninstall: flatbuffers\n",
      "    Found existing installation: flatbuffers 1.12\n",
      "    Uninstalling flatbuffers-1.12:\n",
      "      Successfully uninstalled flatbuffers-1.12\n",
      "  Attempting uninstall: tensorflow-estimator\n",
      "    Found existing installation: tensorflow-estimator 2.6.0\n",
      "    Uninstalling tensorflow-estimator-2.6.0:\n",
      "      Successfully uninstalled tensorflow-estimator-2.6.0\n",
      "  Attempting uninstall: absl-py\n",
      "    Found existing installation: absl-py 0.15.0\n",
      "    Uninstalling absl-py-0.15.0:\n",
      "      Successfully uninstalled absl-py-0.15.0\n",
      "  Attempting uninstall: tensorflow\n",
      "    Found existing installation: tensorflow 2.6.4\n",
      "    Uninstalling tensorflow-2.6.4:\n",
      "      Successfully uninstalled tensorflow-2.6.4\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "tfx-bsl 1.9.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-transform 1.9.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-transform 1.9.0 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<2.10,>=1.15.5, but you have tensorflow 2.10.0 which is incompatible.\n",
      "tensorflow-io 0.21.0 requires tensorflow<2.7.0,>=2.6.0, but you have tensorflow 2.10.0 which is incompatible.\n",
      "tensorflow-io 0.21.0 requires tensorflow-io-gcs-filesystem==0.21.0, but you have tensorflow-io-gcs-filesystem 0.27.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed absl-py-1.3.0 flatbuffers-22.10.26 keras-2.10.0 libclang-14.0.6 tensorflow-2.10.0 tensorflow-estimator-2.10.0 tensorflow-io-gcs-filesystem-0.27.0 tensorflow-text-2.10.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting tf-models-official\n",
      "  Downloading tf_models_official-2.10.0-py2.py3-none-any.whl (2.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tensorflow-text~=2.10.0 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (2.10.0)\n",
      "Requirement already satisfied: Cython in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (0.29.32)\n",
      "Requirement already satisfied: matplotlib in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (3.5.3)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (1.7.3)\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (0.1.97)\n",
      "Collecting gin-config\n",
      "  Downloading gin_config-0.5.0-py3-none-any.whl (61 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.3/61.3 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: oauth2client in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (4.1.3)\n",
      "Requirement already satisfied: psutil>=5.4.3 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (5.9.1)\n",
      "Collecting seqeval\n",
      "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (1.15.0)\n",
      "Collecting tensorflow-model-optimization>=0.4.1\n",
      "  Downloading tensorflow_model_optimization-0.7.3-py2.py3-none-any.whl (238 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.9/238.9 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting sacrebleu==2.2.0\n",
      "  Downloading sacrebleu-2.2.0-py3-none-any.whl (116 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.6/116.6 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pyyaml<6.0,>=5.1\n",
      "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m636.6/636.6 kB\u001b[0m \u001b[31m36.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pandas>=0.22.0 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (1.3.5)\n",
      "Requirement already satisfied: tensorflow-hub>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (0.12.0)\n",
      "Collecting opencv-python-headless==4.5.2.52\n",
      "  Downloading opencv_python_headless-4.5.2.52-cp37-cp37m-manylinux2014_x86_64.whl (38.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.2/38.2 MB\u001b[0m \u001b[31m31.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting tf-slim>=1.1.0\n",
      "  Downloading tf_slim-1.1.0-py2.py3-none-any.whl (352 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m352.1/352.1 kB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pycocotools\n",
      "  Downloading pycocotools-2.0.5.tar.gz (24 kB)\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.20 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (1.21.6)\n",
      "Collecting py-cpuinfo>=3.3.0\n",
      "  Downloading py_cpuinfo-9.0.0-py3-none-any.whl (22 kB)\n",
      "Requirement already satisfied: tensorflow~=2.10.0 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (2.10.0)\n",
      "Requirement already satisfied: Pillow in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (9.1.1)\n",
      "Requirement already satisfied: google-api-python-client>=1.6.7 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (1.12.11)\n",
      "Requirement already satisfied: tensorflow-addons in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (0.14.0)\n",
      "Requirement already satisfied: tensorflow-datasets in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (4.3.0)\n",
      "Collecting immutabledict\n",
      "  Downloading immutabledict-2.2.2-py3-none-any.whl (4.0 kB)\n",
      "Requirement already satisfied: kaggle>=1.3.9 in /opt/conda/lib/python3.7/site-packages (from tf-models-official) (1.5.12)\n",
      "Requirement already satisfied: regex in /opt/conda/lib/python3.7/site-packages (from sacrebleu==2.2.0->tf-models-official) (2021.11.10)\n",
      "Requirement already satisfied: lxml in /opt/conda/lib/python3.7/site-packages (from sacrebleu==2.2.0->tf-models-official) (4.9.1)\n",
      "Requirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.7/site-packages (from sacrebleu==2.2.0->tf-models-official) (0.9.0)\n",
      "Requirement already satisfied: portalocker in /opt/conda/lib/python3.7/site-packages (from sacrebleu==2.2.0->tf-models-official) (2.6.0)\n",
      "Requirement already satisfied: colorama in /opt/conda/lib/python3.7/site-packages (from sacrebleu==2.2.0->tf-models-official) (0.4.5)\n",
      "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (3.0.1)\n",
      "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (0.1.0)\n",
      "Requirement already satisfied: google-api-core<3dev,>=1.21.0 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (1.33.2)\n",
      "Requirement already satisfied: httplib2<1dev,>=0.15.0 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (0.20.4)\n",
      "Requirement already satisfied: google-auth<3dev,>=1.16.0 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (1.35.0)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from kaggle>=1.3.9->tf-models-official) (2.28.1)\n",
      "Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.7/site-packages (from kaggle>=1.3.9->tf-models-official) (2.8.2)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from kaggle>=1.3.9->tf-models-official) (4.64.0)\n",
      "Requirement already satisfied: urllib3 in /opt/conda/lib/python3.7/site-packages (from kaggle>=1.3.9->tf-models-official) (1.26.12)\n",
      "Requirement already satisfied: python-slugify in /opt/conda/lib/python3.7/site-packages (from kaggle>=1.3.9->tf-models-official) (6.1.2)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.7/site-packages (from kaggle>=1.3.9->tf-models-official) (2022.9.24)\n",
      "Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.22.0->tf-models-official) (2022.1)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (59.8.0)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (0.4.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (3.7.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (1.1.2)\n",
      "Requirement already satisfied: flatbuffers>=2.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (22.10.26)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (1.43.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.11,>=2.10.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (2.10.0)\n",
      "Requirement already satisfied: keras<2.11,>=2.10.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (2.10.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (0.2.0)\n",
      "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (3.19.4)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (4.4.0)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (0.27.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (1.3.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (3.3.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (14.0.6)\n",
      "Requirement already satisfied: tensorboard<2.11,>=2.10 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (2.10.1)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (1.12.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (1.1.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (1.6.3)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from tensorflow~=2.10.0->tf-models-official) (21.3)\n",
      "Requirement already satisfied: dm-tree~=0.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow-model-optimization>=0.4.1->tf-models-official) (0.1.7)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->tf-models-official) (3.0.9)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib->tf-models-official) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->tf-models-official) (4.33.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->tf-models-official) (1.4.3)\n",
      "Requirement already satisfied: rsa>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from oauth2client->tf-models-official) (4.8)\n",
      "Requirement already satisfied: pyasn1-modules>=0.0.5 in /opt/conda/lib/python3.7/site-packages (from oauth2client->tf-models-official) (0.2.7)\n",
      "Requirement already satisfied: pyasn1>=0.1.7 in /opt/conda/lib/python3.7/site-packages (from oauth2client->tf-models-official) (0.4.8)\n",
      "Requirement already satisfied: scikit-learn>=0.21.3 in /opt/conda/lib/python3.7/site-packages (from seqeval->tf-models-official) (1.0.2)\n",
      "Requirement already satisfied: typeguard>=2.7 in /opt/conda/lib/python3.7/site-packages (from tensorflow-addons->tf-models-official) (2.13.3)\n",
      "Requirement already satisfied: dill in /opt/conda/lib/python3.7/site-packages (from tensorflow-datasets->tf-models-official) (0.3.5.1)\n",
      "Requirement already satisfied: promise in /opt/conda/lib/python3.7/site-packages (from tensorflow-datasets->tf-models-official) (2.3)\n",
      "Requirement already satisfied: tensorflow-metadata in /opt/conda/lib/python3.7/site-packages (from tensorflow-datasets->tf-models-official) (1.9.0)\n",
      "Requirement already satisfied: importlib-resources in /opt/conda/lib/python3.7/site-packages (from tensorflow-datasets->tf-models-official) (5.8.0)\n",
      "Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from tensorflow-datasets->tf-models-official) (0.18.2)\n",
      "Requirement already satisfied: attrs>=18.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow-datasets->tf-models-official) (21.4.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.7/site-packages (from astunparse>=1.6.0->tensorflow~=2.10.0->tf-models-official) (0.37.1)\n",
      "Collecting protobuf<3.20,>=3.9.2\n",
      "  Downloading protobuf-3.19.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m37.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core<3dev,>=1.21.0->google-api-python-client>=1.6.7->tf-models-official) (1.56.3)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3dev,>=1.16.0->google-api-python-client>=1.6.7->tf-models-official) (4.2.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->kaggle>=1.3.9->tf-models-official) (3.3)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->kaggle>=1.3.9->tf-models-official) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn>=0.21.3->seqeval->tf-models-official) (1.0.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn>=0.21.3->seqeval->tf-models-official) (3.1.0)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (2.2.2)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (0.6.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (0.4.6)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (3.3.7)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (1.8.1)\n",
      "Requirement already satisfied: zipp>=3.1.0 in /opt/conda/lib/python3.7/site-packages (from importlib-resources->tensorflow-datasets->tf-models-official) (3.8.0)\n",
      "Requirement already satisfied: text-unidecode>=1.3 in /opt/conda/lib/python3.7/site-packages (from python-slugify->kaggle>=1.3.9->tf-models-official) (1.3)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (4.13.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.7/site-packages (from werkzeug>=1.0.1->tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (2.1.1)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow~=2.10.0->tf-models-official) (3.2.0)\n",
      "Building wheels for collected packages: pycocotools, seqeval\n",
      "  Building wheel for pycocotools (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pycocotools: filename=pycocotools-2.0.5-cp37-cp37m-linux_x86_64.whl size=373766 sha256=9e115bed2f53fba8267c8929de5ecc0f2d0c2ac15b804a5e63b50d08c7865b55\n",
      "  Stored in directory: /root/.cache/pip/wheels/85/c4/f0/7128093a134f590e4383fd60cb484960878721d98b9a515317\n",
      "  Building wheel for seqeval (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16180 sha256=2779abaa30ec2daa6eb731ef12bafd8f0d112be34bca43dc212e8206c45046bc\n",
      "  Stored in directory: /root/.cache/pip/wheels/05/96/ee/7cac4e74f3b19e3158dce26a20a1c86b3533c43ec72a549fd7\n",
      "Successfully built pycocotools seqeval\n",
      "Installing collected packages: py-cpuinfo, gin-config, tf-slim, tensorflow-model-optimization, sacrebleu, pyyaml, protobuf, opencv-python-headless, immutabledict, seqeval, pycocotools, tf-models-official\n",
      "  Attempting uninstall: pyyaml\n",
      "    Found existing installation: PyYAML 6.0\n",
      "    Uninstalling PyYAML-6.0:\n",
      "      Successfully uninstalled PyYAML-6.0\n",
      "  Attempting uninstall: protobuf\n",
      "    Found existing installation: protobuf 3.19.4\n",
      "    Uninstalling protobuf-3.19.4:\n",
      "      Successfully uninstalled protobuf-3.19.4\n",
      "  Attempting uninstall: opencv-python-headless\n",
      "    Found existing installation: opencv-python-headless 4.5.4.60\n",
      "    Uninstalling opencv-python-headless-4.5.4.60:\n",
      "      Successfully uninstalled opencv-python-headless-4.5.4.60\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "beatrix-jupyterlab 3.1.7 requires google-cloud-bigquery-storage, which is not installed.\n",
      "tfx-bsl 1.9.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-transform 1.9.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-transform 1.9.0 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<2.10,>=1.15.5, but you have tensorflow 2.10.0 which is incompatible.\n",
      "tensorflow-io 0.21.0 requires tensorflow<2.7.0,>=2.6.0, but you have tensorflow 2.10.0 which is incompatible.\n",
      "tensorflow-io 0.21.0 requires tensorflow-io-gcs-filesystem==0.21.0, but you have tensorflow-io-gcs-filesystem 0.27.0 which is incompatible.\n",
      "pandas-profiling 3.1.0 requires markupsafe~=2.0.1, but you have markupsafe 2.1.1 which is incompatible.\n",
      "nnabla 1.31.0 requires protobuf<=3.19.4; platform_system != \"Windows\", but you have protobuf 3.19.6 which is incompatible.\n",
      "grpcio-status 1.47.0 requires grpcio>=1.47.0, but you have grpcio 1.43.0 which is incompatible.\n",
      "gcsfs 2022.5.0 requires fsspec==2022.5.0, but you have fsspec 2022.8.2 which is incompatible.\n",
      "apache-beam 2.40.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.5.1 which is incompatible.\n",
      "apache-beam 2.40.0 requires pyarrow<8.0.0,>=0.15.1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "allennlp 2.10.0 requires protobuf==3.20.0, but you have protobuf 3.19.6 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed gin-config-0.5.0 immutabledict-2.2.2 opencv-python-headless-4.5.2.52 protobuf-3.19.6 py-cpuinfo-9.0.0 pycocotools-2.0.5 pyyaml-5.4.1 sacrebleu-2.2.0 seqeval-1.2.2 tensorflow-model-optimization-0.7.3 tf-models-official-2.10.0 tf-slim-1.1.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install  tensorflow-text \n",
    "!pip install tf-models-official  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-10-30T20:36:22.520546Z",
     "iopub.status.busy": "2022-10-30T20:36:22.520001Z",
     "iopub.status.idle": "2022-10-30T20:36:28.767870Z",
     "shell.execute_reply": "2022-10-30T20:36:28.766403Z",
     "shell.execute_reply.started": "2022-10-30T20:36:22.520491Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-30 20:36:24.073936: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-10-30 20:36:24.256990: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/lib/x86_64-linux-gnu:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:\n",
      "2022-10-30 20:36:24.257045: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "2022-10-30 20:36:24.297661: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2022-10-30 20:36:25.432562: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/lib/x86_64-linux-gnu:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:\n",
      "2022-10-30 20:36:25.432791: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/lib/x86_64-linux-gnu:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:\n",
      "2022-10-30 20:36:25.432807: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow_addons/utils/ensure_tf_install.py:67: UserWarning: Tensorflow Addons supports using Python ops for all Tensorflow versions above or equal to 2.4.0 and strictly below 2.7.0 (nightly versions are not supported). \n",
      " The versions of TensorFlow you are currently using is 2.10.0 and is not supported. \n",
      "Some things might work, some things might not.\n",
      "If you were to encounter a bug, do not file an issue.\n",
      "If you want to make sure you're using a tested and supported configuration, either change the TensorFlow version or the TensorFlow Addons's version. \n",
      "You can find the compatibility matrix in TensorFlow Addon's readme:\n",
      "https://github.com/tensorflow/addons\n",
      "  UserWarning,\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import nltk \n",
    "import shutil\n",
    "\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text\n",
    "from official.nlp import optimization  # to create AdamW optimizer\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.get_logger().setLevel('ERROR')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:02.450954Z",
     "iopub.status.busy": "2022-10-30T20:37:02.450537Z",
     "iopub.status.idle": "2022-10-30T20:37:02.593687Z",
     "shell.execute_reply": "2022-10-30T20:37:02.592406Z",
     "shell.execute_reply.started": "2022-10-30T20:37:02.450922Z"
    }
   },
   "outputs": [],
   "source": [
    "path = \"../input/feedback-prize-english-language-learning/\"\n",
    "train = pd.read_csv(path+\"train.csv\")\n",
    "test = pd.read_csv(path+\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:08.883598Z",
     "iopub.status.busy": "2022-10-30T20:37:08.882997Z",
     "iopub.status.idle": "2022-10-30T20:37:08.923223Z",
     "shell.execute_reply": "2022-10-30T20:37:08.921949Z",
     "shell.execute_reply.started": "2022-10-30T20:37:08.883527Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>full_text</th>\n",
       "      <th>cohesion</th>\n",
       "      <th>syntax</th>\n",
       "      <th>vocabulary</th>\n",
       "      <th>phraseology</th>\n",
       "      <th>grammar</th>\n",
       "      <th>conventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0016926B079C</td>\n",
       "      <td>I think that students would benefit from learn...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0022683E9EA5</td>\n",
       "      <td>When a problem is a change you have to let it ...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00299B378633</td>\n",
       "      <td>Dear, Principal\\n\\nIf u change the school poli...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>003885A45F42</td>\n",
       "      <td>The best time in life is when you become yours...</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0049B1DF5CCC</td>\n",
       "      <td>Small act of kindness can impact in other peop...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        text_id                                          full_text  cohesion  \\\n",
       "0  0016926B079C  I think that students would benefit from learn...       3.5   \n",
       "1  0022683E9EA5  When a problem is a change you have to let it ...       2.5   \n",
       "2  00299B378633  Dear, Principal\\n\\nIf u change the school poli...       3.0   \n",
       "3  003885A45F42  The best time in life is when you become yours...       4.5   \n",
       "4  0049B1DF5CCC  Small act of kindness can impact in other peop...       2.5   \n",
       "\n",
       "   syntax  vocabulary  phraseology  grammar  conventions  \n",
       "0     3.5         3.0          3.0      4.0          3.0  \n",
       "1     2.5         3.0          2.0      2.0          2.5  \n",
       "2     3.5         3.0          3.0      3.0          2.5  \n",
       "3     4.5         4.5          4.5      4.0          5.0  \n",
       "4     3.0         3.0          3.0      2.5          2.5  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:09.805821Z",
     "iopub.status.busy": "2022-10-30T20:37:09.805100Z",
     "iopub.status.idle": "2022-10-30T20:37:09.834416Z",
     "shell.execute_reply": "2022-10-30T20:37:09.833075Z",
     "shell.execute_reply.started": "2022-10-30T20:37:09.805775Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3911 entries, 0 to 3910\n",
      "Data columns (total 8 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   text_id      3911 non-null   object \n",
      " 1   full_text    3911 non-null   object \n",
      " 2   cohesion     3911 non-null   float64\n",
      " 3   syntax       3911 non-null   float64\n",
      " 4   vocabulary   3911 non-null   float64\n",
      " 5   phraseology  3911 non-null   float64\n",
      " 6   grammar      3911 non-null   float64\n",
      " 7   conventions  3911 non-null   float64\n",
      "dtypes: float64(6), object(2)\n",
      "memory usage: 244.6+ KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No null values "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MODEL "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:16.795207Z",
     "iopub.status.busy": "2022-10-30T20:37:16.794577Z",
     "iopub.status.idle": "2022-10-30T20:37:16.814798Z",
     "shell.execute_reply": "2022-10-30T20:37:16.813296Z",
     "shell.execute_reply.started": "2022-10-30T20:37:16.795153Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BERT model selected           : https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1\n",
      "Preprocess model auto-selected: https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\n"
     ]
    }
   ],
   "source": [
    "# Choose a BERT model to fine-tune\n",
    "bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' \n",
    "\n",
    "map_name_to_handle = {\n",
    "    'bert_en_uncased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',\n",
    "    'bert_en_cased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',\n",
    "    'bert_multi_cased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',\n",
    "    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',\n",
    "    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',\n",
    "    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',\n",
    "    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',\n",
    "    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',\n",
    "    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',\n",
    "    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',\n",
    "    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',\n",
    "    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',\n",
    "    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',\n",
    "    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',\n",
    "    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',\n",
    "    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',\n",
    "    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',\n",
    "    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',\n",
    "    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',\n",
    "    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',\n",
    "    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',\n",
    "    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',\n",
    "    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',\n",
    "    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',\n",
    "    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',\n",
    "    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',\n",
    "    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',\n",
    "    'albert_en_base':\n",
    "        'https://tfhub.dev/tensorflow/albert_en_base/2',\n",
    "    'electra_small':\n",
    "        'https://tfhub.dev/google/electra_small/2',\n",
    "    'electra_base':\n",
    "        'https://tfhub.dev/google/electra_base/2',\n",
    "    'experts_pubmed':\n",
    "        'https://tfhub.dev/google/experts/bert/pubmed/2',\n",
    "    'experts_wiki_books':\n",
    "        'https://tfhub.dev/google/experts/bert/wiki_books/2',\n",
    "    'talking-heads_base':\n",
    "        'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',\n",
    "}\n",
    "\n",
    "## Text inputs need to be transformed to numeric token ids and arranged in several Tensors \n",
    "##before being input to BERT. TensorFlow Hub provides a matching preprocessing model for \n",
    "## each of the BERT models \n",
    "\n",
    "map_model_to_preprocess = {\n",
    "    'bert_en_uncased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'bert_en_cased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'bert_multi_cased_L-12_H-768_A-12':\n",
    "        'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3',\n",
    "    'albert_en_base':\n",
    "        'https://tfhub.dev/tensorflow/albert_en_preprocess/3',\n",
    "    'electra_small':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'electra_base':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'experts_pubmed':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'experts_wiki_books':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "    'talking-heads_base':\n",
    "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
    "}\n",
    "\n",
    "tfhub_handle_encoder = map_name_to_handle[bert_model_name]\n",
    "tfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]\n",
    "\n",
    "print(f'BERT model selected           : {tfhub_handle_encoder}')\n",
    "print(f'Preprocess model auto-selected: {tfhub_handle_preprocess}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:18.187152Z",
     "iopub.status.busy": "2022-10-30T20:37:18.186752Z",
     "iopub.status.idle": "2022-10-30T20:37:20.690720Z",
     "shell.execute_reply": "2022-10-30T20:37:20.689189Z",
     "shell.execute_reply.started": "2022-10-30T20:37:18.187121Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-30 20:37:18.553902: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/lib/x86_64-linux-gnu:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:\n",
      "2022-10-30 20:37:18.553980: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2022-10-30 20:37:18.554028: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (83d12de7ce4e): /proc/driver/nvidia/version does not exist\n",
      "2022-10-30 20:37:18.554331: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:20.693174Z",
     "iopub.status.busy": "2022-10-30T20:37:20.692760Z",
     "iopub.status.idle": "2022-10-30T20:37:20.924599Z",
     "shell.execute_reply": "2022-10-30T20:37:20.923290Z",
     "shell.execute_reply.started": "2022-10-30T20:37:20.693138Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keys       : ['input_word_ids', 'input_mask', 'input_type_ids']\n",
      "Shape      : (1, 128)\n",
      "Word Ids   : [ 101 2023 2003 2107 2019 6429 3185  999  102    0    0    0]\n",
      "Input Mask : [1 1 1 1 1 1 1 1 1 0 0 0]\n",
      "Type Ids   : [0 0 0 0 0 0 0 0 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "## Let's try the preprocessing model on some text and see the output:\n",
    "\n",
    "text_test = ['this is such an amazing movie!']\n",
    "text_preprocessed = bert_preprocess_model(text_test)\n",
    "\n",
    "print(f'Keys       : {list(text_preprocessed.keys())}')\n",
    "print(f'Shape      : {text_preprocessed[\"input_word_ids\"].shape}')\n",
    "print(f'Word Ids   : {text_preprocessed[\"input_word_ids\"][0, :12]}')\n",
    "print(f'Input Mask : {text_preprocessed[\"input_mask\"][0, :12]}')\n",
    "print(f'Type Ids   : {text_preprocessed[\"input_type_ids\"][0, :12]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:25.394049Z",
     "iopub.status.busy": "2022-10-30T20:37:25.393615Z",
     "iopub.status.idle": "2022-10-30T20:37:32.793815Z",
     "shell.execute_reply": "2022-10-30T20:37:32.792644Z",
     "shell.execute_reply.started": "2022-10-30T20:37:25.394015Z"
    }
   },
   "outputs": [],
   "source": [
    "#bert_model = hub.KerasLayer(tfhub_handle_encoder)\n",
    "\n",
    "def build_classifier_model():\n",
    "    text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "    preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')\n",
    "    \n",
    "    encoder_inputs = preprocessing_layer(text_input)\n",
    "    encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')\n",
    "    outputs = encoder(encoder_inputs)\n",
    "    net = outputs['pooled_output']\n",
    "    net = tf.keras.layers.Dropout(0.1)(net)\n",
    "    \n",
    "    #net = tf.keras.layers.Dense(6, activation=None, name='classifier')(net)\n",
    "    #Y1 output\n",
    "    #y1_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y1_output\")(net)\n",
    "    y1_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y1_outputclass\")(net)\n",
    "\n",
    "    #Y2 output\n",
    "    #y2_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y2_output\")(net)\n",
    "    y2_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y2_outputclass\")(net)\n",
    "\n",
    "    #Y3 output\n",
    "    #y3_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y3_output\")(net)\n",
    "    y3_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y3_outputclass\")(net)\n",
    "\n",
    "\n",
    "    #Y4 output\n",
    "    #y4_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y4_output\")(net)\n",
    "    y4_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y4_outputclass\")(net)\n",
    "\n",
    "    #Y5 output\n",
    "    #y5_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y5_output\")(net)\n",
    "    y5_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y5_outputclass\")(net)\n",
    "\n",
    "\n",
    "    #Y6 output\n",
    "    #y6_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y6_output\")(net)\n",
    "    y6_outputclass = tf.keras.layers.Dense(units = 1, activation = \"softmax\", name = \"y6_outputclass\")(net)\n",
    "\n",
    "    return tf.keras.Model(text_input, outputs = [y1_outputclass, y2_outputclass,y3_outputclass, y4_outputclass, y5_outputclass, y6_outputclass])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:32.796030Z",
     "iopub.status.busy": "2022-10-30T20:37:32.795630Z",
     "iopub.status.idle": "2022-10-30T20:37:42.669431Z",
     "shell.execute_reply": "2022-10-30T20:37:42.668074Z",
     "shell.execute_reply.started": "2022-10-30T20:37:32.795987Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[[0.1945738 ]]\n",
      "\n",
      " [[0.61917245]]\n",
      "\n",
      " [[0.24192055]]\n",
      "\n",
      " [[0.56271565]]\n",
      "\n",
      " [[0.5455041 ]]\n",
      "\n",
      " [[0.7310586 ]]], shape=(6, 1, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "classifier_model = build_classifier_model()\n",
    "bert_raw_result = classifier_model(tf.constant(text_test))\n",
    "print(tf.sigmoid(bert_raw_result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:42.671400Z",
     "iopub.status.busy": "2022-10-30T20:37:42.670755Z",
     "iopub.status.idle": "2022-10-30T20:37:44.077613Z",
     "shell.execute_reply": "2022-10-30T20:37:44.076217Z",
     "shell.execute_reply.started": "2022-10-30T20:37:42.671366Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABkMAAAHBCAYAAADacMxXAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzde5RWdb0/8M8zzDDMcBnACLyQkpaUi7DMFG9gINARQwkYRLxUKC1OKiKWnTz+WuGyVeCtxLA6S0/reHTQlBUi3smTXFZqqGUKpscSgeQiyMjFAb6/P1o8x6cZdGacmSf2vF5r7bWc7/7svT/f/WxnzfO8efbOpZRSAAAAAAAAZNPdJcXuAAAAAAAAoDUJQwAAAAAAgEwThgAAAAAAAJkmDAEAAAAAADKttNgNAABAsVx//fWxbNmyYrcBbWL69OkxaNCgYrcBAABF4ZshAAC0W8uWLYvly5cXu402s3z58nY1X/7PPffcE6+//nqx2wAAgKLxzRAAANq1448/Pu6+++5it9Emxo0bFxHRbubL/8nlcsVuAQAAiso3QwAAAAAAgEwThgAAAAAAAJkmDAEAAAAAADJNGAIAAAAAAGSaMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmCUMAAAAAAIBME4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAACwT126dIlcLlewzJ49u9htNUuW5gIAADSNMAQAANin2traWLFiRUREjB49OlJKMWPGjCJ31TxZmgsAANA0whAAAGiiLl26xEknndRujrs/cY4AAICGCEMAAAAAAIBME4YAAAAAAACZJgwBAIBGmj17duRyuXjnnXdiyZIl+Ydwl5aWFtStX78+LrnkkjjssMOiY8eO0atXrxgzZkw8++yz+ZqTTjqp4EHekyZNioiIYcOGFYxv3ry50cdtS/Pnzy/o87XXXovq6uro3r17HHDAATFq1Kh45ZVX8vV755DL5eKQQw6Jp556KoYOHRpdu3aNysrKOPXUU2PJkiX5+muuuSZf/97bXj344IP58Y985CP19t+S52jXrl1RU1MTp512WvTp0ycqKipiwIABcdNNN8WePXsiImLz5s31Hsp+zTXX5Ld/7/jYsWPz+27MNfKP53jlypUxfvz4OOCAA/JjGzZsaPb8AACgPRGGAABAI82YMSNSStG5c+c48cQTI6UUKaXYtWtXvmbt2rVx7LHHxrx58+KWW26JTZs2xW9+85vYtGlTDBo0KJYtWxYREU8++WQ8++yz0blz5xg4cGDceuutERGxcOHCOO644+LOO++MlFJ07969Ucdta2eeeWaklGL06NERETFt2rSYNm1avPHGG1FTUxOPP/54nH322fn6vXMYOHBgbN68OS699NK45pprYt26dfE///M/sWnTpvjiF78YTzzxREREXHXVVfk5v9fIkSMjpRTHHHNMwXhrnKMHH3wwJkyYEF/84hfjxRdfjNdffz0uuuiimD59enz729+OiIju3btHSilGjBgRJSUl8ec//zmuuuqqiIgoLS2NlFIMGjQo7rjjjrjnnnsiovHXyD+e4ylTpsTUqVPj9ddfj+XLl0eHDh2aPTcAAGhvhCEAANCCvvOd78Rf/vKXuP766+Nf/uVfokuXLnHUUUfFXXfdFSmluPjii/O1AwcOjNtuuy2ee+65OO+88yKlFFOmTImhQ4fGhAkTijiLpps8eXIMGjQoOnfuHMOGDYvTTz89nnrqqQa/ufDOO+/ELbfckq///Oc/H//1X/8V7777blx66aVF6H7fhgwZEt/5zneiR48e8ZGPfCQuvvjimDhxYtx0003x9ttv5+umT58ee/bsieuvv75g+yVLlsRf//rXGDduXH6sKdfIe33729+OIUOGRGVlZRx33HGxa9eugm/HAAAA+yYMAQCAFjR//vwoKSmJUaNGFYz36dMnjjrqqHjmmWdi9erV+fFx48bFd7/73bj33nvjpJNOio0bN8bMmTPbuu0P7dhjjy34uW/fvhERsWbNmnq1nTt3jqOPPrpgbMCAAXHQQQfFc889F2vXrm29Rptg1KhRsXjx4nrjAwcOjLq6unjhhRfyY8OHD48BAwbE7bffHhs3bsyPz5o1Ky6++OIoKyvLjzX1GtnrC1/4QktMCwAA2iVhCAAAtJCdO3fGli1bYs+ePVFVVVXvWRK///3vIyLi5ZdfLthu5syZcdxxx8XSpUtj3LhxUVKy//2ZXlVVVfBzx44dIyLyz9Z4r+7duze4j49+9KMREfHmm2+2cHfNs2XLlrj66qtjwIAB0aNHj/zreMUVV0RExLZt2wrqp02bFtu2bYtbbrklIiJWrVoVjz/+eFx00UX5muZeIxFR75ZhAABA4+1/77IAAKDIcrlcg+Pl5eXRvXv3KC0tjbq6uvxzK/5xOfXUUwu2+81vfhNbtmyJAQMGxNSpU+O5555r0nH3Nxs3boyUUr3xvSHI3lAkIqKkpCTefffderWbN29ucN8teY7OOOOMmDlzZlx44YWxatWq2LNnT6SU4oYbboiIqDeHc845J3r37h0333xz7Ny5M6677ro4//zzo0ePHvma5l4jAADAhyMMAQCAJqqsrCz4gP7II4+Mn/3sZxERMWbMmNi1a1csWbKk3nY//OEP42Mf+1jBQ73/93//N77+9a/Hr371q/j1r38dFRUVMXr06Fi/fn2Tjrs/2bFjRzz11FMFY3/4wx9izZo1MXDgwDjwwAPz4wceeGC88cYbBbXr1q2Lv/71rw3uuyXOUWlpabzwwguxZMmS6NOnT1xyySXRq1evfNCyffv2BrcrLy+PqVOnxptvvhnXXXdd3HHHHQ0+A6Wp1wgAAPDhCUMAAKCJPve5z8WqVavi9ddfj2XLlsWrr74aJ598ckRE/OAHP4jDDz88vva1r8WiRYtiy5YtsWnTprj11lvj+9//fsyePTtKS0sjIqK2tjbOPPPMuPHGG+PTn/50HHbYYXHPPffEmjVrYuzYsVFXV9fo4+5Pqqqq4t/+7d9i2bJl8c4778TTTz8dkyZNio4dO8ZNN91UUDt8+PBYs2ZN3HzzzVFbWxuvvPJKXHrppQXfHnmvljpHHTp0iCFDhsS6deti1qxZsWHDhti+fXssXrw45s6du8/tpk6dGhUVFXHVVVfFsGHD4ogjjqhX05RrBAAAaCEJAADaqbFjx6axY8c2ebuXXnopnXzyyalz586pb9++ac6cOQXrN27cmKZPn54+/vGPp7KystSrV680fPjw9Mgjj+Rr/vVf/zVFRH75wx/+kNavX18wFhFp5syZjT5ua8y3c+fO9XqaNWtWWrZsWb3x7373uymlVG/89NNPz+9v4MCB6eCDD05/+tOf0ogRI1LXrl1TRUVFGjx4cHryySfrHX/z5s1p8uTJ6cADD0wVFRXppJNOSk899VQ65phj8vv/9re/3ahz1NBc9rW8+OKLaf369WnKlCmpb9++qaysLPXu3TtdcMEF6corr8zXHXPMMfV6vvDCC1NEpCeeeGKf57Ux10hD57i5b+EiItXU1DRrWwAAyIB5uZQauFkvAAC0A+PGjYuIiLvvvrvInbSNf4b5Hn300bFhw4ZYvXp10XpobbfddlvMmTMnnn766WK3kpfL5aKmpibGjx9f7FYAAKAY7nabLAAAgBY0d+7cmD59erHbAAAA3kMYAgAA8CH84he/iLPOOitqa2tj7ty58dZbb/kGBgAA/JMRhgAAAK1u9uzZkcvl4rnnnos33ngjcrlcXHXVVcVuq8XMnz8/evToET/96U/jrrvu8gB0AAD4J+MvdAAAoNXNmDEjZsyYUew2WsXkyZNj8uTJxW4DAAB4H74ZAgAAAAAAZJowBAAAAAAAyDRhCAAAAAAAkGnCEAAAAAAAINOEIQAAAAAAQKYJQwAAAAAAgEwThgAAAAAAAJkmDAEAAAAAADJNGAIAAAAAAGSaMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmlRa7AQAAKKbly5fHuHHjit1Gm1i+fHlERLuZLwAAwF7CEAAA2q1BgwYVu4U2dfzxxxe7hVi/fn28+OKLccoppxS7lXZl7Nix0bdv32K3AQAARZNLKaViNwEAALQP8+bNi+rq6vA2BAAAaEN3e2YIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmSYMAQAAAAAAMk0YAgAAAAAAZJowBAAAAAAAyDRhCAAAAAAAkGnCEAAAAAAAINOEIQAAAAAAQKYJQwAAAAAAgEwThgAAAAAAAJkmDAEAAAAAADJNGAIAAAAAAGSaMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmCUMAAAAAAIBME4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmSYMAQAAAAAAMk0YAgAAAAAAZJowBAAAAAAAyDRhCAAAAAAAkGmlxW4AAADIptWrV8f5558fu3fvzo9t2LAhSktLY8iQIQW1Rx55ZNx6661t3CEAANBeCEMAAIBWccghh8Rrr70Wr776ar11TzzxRMHPJ598clu1BQAAtENukwUAALSa8847L8rKyj6wbsKECW3QDQAA0F4JQwAAgFZzzjnnRF1d3fvWfPrTn46jjjqqjToCAADaI2EIAADQao444oj4zGc+E7lcrsH1ZWVlcf7557dxVwAAQHsjDAEAAFrVeeedFx06dGhw3a5du2L8+PFt3BEAANDeCEMAAIBWdfbZZ8eePXvqjedyuTjuuOPisMMOa/umAACAdkUYAgAAtKqDDjooTjjhhCgpKXz70aFDhzjvvPOK1BUAANCeCEMAAIBWd+6559YbSynFV77ylSJ0AwAAtDfCEAAAoNWNGzeu4JshHTp0iGHDhsVHP/rRInYFAAC0F8IQAACg1fXo0SOGDx+ef5B6SikmTZpU5K4AAID2QhgCAAC0iUmTJuUfpF5aWhpf/vKXi9wRAADQXghDAACANvHlL385ysvL8//drVu3IncEAAC0F6XFbgAAgP3L6tWrY+nSpcVug/3U5z73uVi6dGn069cv5s2bV+x22E+NHz++2C0AALCfyaWUUrGbAABg/zFv3ryorq4udhtAO+ZtLAAATXS322QBANAsKSWLpcnLu+++G9/61reavF1ERE1NTdH7txR3qampKfJvPgAA9lfCEAAAoM2UlZXF9773vWK3AQAAtDPCEAAAoE1VVFQUuwUAAKCdEYYAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmSYMAQAAAAAAMk0YAgAAAAAAZJowBAAAyJwuXbpELpcrWGbPnp1f379//4J1J510UhG7bbwPmhcAANAwYQgAABC1tbXxiU98IkaNGlXsVlpEbW1trFixIiIiRo8eHSmlmDFjRn794sWL4+ijj44LLrgg6urq4sknnyxWq03yQfMCAAAaJgwBAAAipRR79uyJPXv2FLuVVvfSSy/FCSecEKNGjYrbbrstSktLi90SAADQyvzVDwAARNeuXeOVV14pdhutbsmSJTFmzJiYOXNmXHTRRcVuBwAAaCPCEAAAoF24995746KLLorbb789M7cDAwAAGsdtsgAAaFWzZ8/OP+j5kEMOiaeeeiqGDh0aXbt2jcrKyjj11FNjyZIl+fr58+cXPBx65cqVMX78+DjggAPyYxs2bIiIiPXr18cll1wShx12WHTs2DF69eoVY8aMiWeffbZNjr9x48aYPn16HH744dGxY8fo0aNHfOlLX4rFixfXOw/vrS0vL49DDjkkhg0bFrfffnts3749X9eYOUVE7Ny5M66++uro379/VFZWRs+ePeOMM86IX//617F79+4m1f3jnHfs2NHg+GuvvRbV1dXRvXv3OOCAA2LUqFENfpvkpZdeijPPPDOqqqqisrIyvvCFL8T9998fw4YNy+9r8uTJjb+IWsDNN98cU6dOjQceeOB9g5DGnP/GXiO7du2KmpqaOO2006JPnz5RUVERAwYMiJtuuqne7cga+3o2VWN62Lx5c72Hsl9zzTX57d87Pnbs2FY5VwAA0OoSAAA0QU1NTWrOn5EDBw5MnTt3ToMGDUpLly5NtbW16amnnkqf+cxnUseOHdNvfvObgvrRo0eniEiDBw9OixcvTu+8805avnx56tChQ1q/fn1as2ZNOvTQQ1Pv3r3TwoUL09atW9Mf//jHNHjw4NSpU6e0dOnSVj3+2rVrU79+/VLv3r3TggUL0pYtW9LKlSvTmDFjUi6XSz//+c/z+9pb26dPn7RgwYL09ttvp3Xr1qWZM2emiEg33HBDSik1aU6TJ09OVVVV6eGHH07btm1L69atSzNmzEgRkRYvXtzkuvfOefv27Q2Ojx49On/uHnnkkVRRUZGOPfbYgtqXX345de/ePR188MHp4Ycfzs9h2LBhqVevXqm8vLxxF8w/iIhUU1PTpG1WrFiRIiJ16dIlRUS6/PLL37e+qdfUB10jCxYsSBGRrr322rRp06a0fv369OMf/ziVlJSkGTNmFOyrKa/T3nmNHj36A89BU3oYMWJEKikpSX/+85/r7WfQoEHpjjvuaLVz1VjN/f0DAEC7N89fkQAANMmHCUMiIq1YsaJg/Pnnn08RkQYOHFgwvvfD0wceeKDB/Z1//vkpIgo+oE3p78FDeXl5OuaYY1r1+BdccEGKiHTnnXcWjO/YsSMddNBBqaKiIq1bt66gtqEP80eOHJkPQ5oyp379+qUTTjih3v4++clPFnx43ti69855X2HIggULCsbHjh2bIqLgw+xx48aliEj33HNPQe2bb76ZKisrixKGHHnkkalbt24pItKsWbP2Wd/Ua+qDrpEFCxakIUOG1BufNGlSKisrS1u2bMmPNeV1amoY0tgeHnrooRQRaerUqQW1Tz75ZDr44IPTu+++mx9r6XPVWMIQAACaaZ7bZAEA0GY6d+4cRx99dMHYgAED4qCDDornnnsu1q5dW2+bL3zhCw3ua/78+VFSUlLvlkd9+vSJo446Kp555plYvXp1qx3/vvvui4iI008/vWC8vLw8hg4dGtu3b4+HHnqooPZLX/pSvf0sWrQopk2b1uQ5jRw5MpYuXRoXXXRRLF++PH8rpZUrV8aQIUPy2za2rjGOPfbYgp/79u0bERFr1qzJjz344IMRETFixIiC2l69ekX//v2bdLyW0r9//3jwwQeja9euccUVV8T111/fYF1zrqmIfV8jo0aNavCWaQMHDoy6urp44YUX8mMt+To1t4fhw4fHgAED4vbbb4+NGzfmx2fNmhUXX3xxlJWV5cda+lwBAEBrE4YAANBmunfv3uD4Rz/60YiIePPNN+ut69y5c72xnTt3xpYtW2LPnj1RVVVV73kHv//97yMi4uWXX27V43fq1Cm6du1ab33v3r0jImLdunUfWNvcOc2ZMyd++ctfxquvvhpDhw6Nbt26xciRI/PBy16NrWuMqqqqgp87duwYEZF/9sTOnTtj69at0alTp+jSpUu97Xv06NHkY7aUQYMGxaJFi6JLly5x+eWXx4033liwvrnXVETD10hExJYtW+Lqq6+OAQMGRI8ePfL7uuKKKyIiYtu2bfnalnydmttDRMS0adNi27Ztccstt0RExKpVq+Lxxx+Piy66KF/TGucKAABamzAEAIA2s3Hjxkgp1RvfG0LsDSU+SHl5eXTv3j1KS0ujrq4uUkoNLqeeemqrHb+qqip27NgRW7durbf+b3/7W0T8/V/Jf1Btc+eUy+Xi3HPPjUcffTQ2b94c8+fPj5RSjBkzpuCbD42tawnl5eXRtWvX2LFjR9TW1tZb31DY1JZOPPHEeOCBB6Jz585x2WWXxU9+8pP8uuZeU+/njDPOiJkzZ8aFF14Yq1atij179kRKKW644YaIiIJrsbVep6b0EBFxzjnnRO/evePmm2+OnTt3xnXXXRfnn39+QZDVGucKAABamzAEAIA2s2PHjnjqqacKxv7whz/EmjVrYuDAgXHggQc2el9jxoyJXbt2xZIlS+qt++EPfxgf+9jHYteuXa12/LPOOisiIhYuXFgwvnPnznjssceioqIif6uovbUPPPBAvf189rOfjcsuu6zJc+revXu89NJLERFRVlYWp512WsyfPz9yuVxBT42tayl7bwW293ZZe61bty5WrVrV4sdrqpNPPjkWLlwYlZWVcckll8ScOXPy65pzTe3L7t27Y8mSJdGnT5+45JJLolevXpHL5SIiYvv27fXqW/p1Ki0tjRdeeKFJPUT8PeiYOnVqvPnmm3HdddfFHXfcEZdeemm9upY8VwAA0BaEIQAAtJmqqqr4t3/7t1i2bFm888478fTTT8ekSZOiY8eOcdNNNzVpXz/4wQ/i8MMPj6997WuxaNGi2LJlS2zatCluvfXW+P73vx+zZ8+O0tLSVj1+v379Ytq0aXH//ffH1q1bY9WqVTFx4sRYu3Zt3HTTTfnbZe2tveyyy2LhwoWxdevWWL16dUydOjXWrl2bD0OaOqdvfOMb8fzzz8fOnTvjzTffjB/96EeRUoovfvGLBb02tq4lXHvttdGzZ8+YNm1aPPLII1FbWxt//OMf46tf/Wr06dOnxY/XHIMHD477778/Kioq4pvf/Gb+llDNuab2pUOHDjFkyJBYt25dzJo1KzZs2BDbt2+PxYsXx9y5cxvcpqVfp+b0EBExderUqKioiKuuuiqGDRsWRxxxRL2aljxXAADQJlr3Ae0AAGRNTU1Nas6fkQMHDkwHH3xw+tOf/pRGjBiRunbtmioqKtLgwYPTk08+ma9btmxZioh6S0M2btyYpk+fnj7+8Y+nsrKy1KtXrzR8+PD0yCOPtMnxN2zYkKZNm5b69euXysrKUlVVVRoxYkR67LHHPrD2wAMPTBMmTEirVq1q1pyeffbZNGXKlPSpT30qVVZWpp49e6bjjz8+/fznP0979uxpUt19991Xb77nnHNOg+fiu9/9bkop1Rs//fTT88dcuXJlOvPMM1O3bt1SZWVlOuGEE9ITTzyRhgwZkiorKxs8lx8kIlJNTU2j6zt37lyvx1mzZhXUPProo6mioiK/fubMmY06/429RtavX5+mTJmS+vbtm8rKylLv3r3TBRdckK688sr8Nsccc0xKqfGvZ0Pz2tfy4osvNqmH97rwwgtTRKQnnnhin+e4Jc9VYzX39w8AAO3evFxKDdw0GQAA9mHevHlRXV3d4LM33s/RRx8dGzZsiNWrV7dSZ//cxyeif//+sX379vjLX/7S5G1zuVzU1NTE+PHjW6Ez/tFtt90Wc+bMiaeffrrYrRRo7u8fAADavbvdJgsAAGgx69ati549e0ZdXV3B+GuvvRavvPJKq9yai5Y3d+7cmD59erHbAACAFiMMAQAAWtRbb70VU6ZMiddffz22bdsWv/vd76K6ujq6desW//7v/17s9mjAL37xizjrrLOitrY25s6dG2+99ZZv4QAAkCnCEAAAWtXs2bMjl8vFc889F2+88Ubkcrm46qqr2s3x25s+ffrEo48+Gps3b45TTjklevToEV/+8pfjE5/4RPzud7+Lj3/848VukX2YP39+9OjRI37605/GXXfd5QHoAABkimeGAADQJO7ZTzF4ZggRfv8AANBsnhkCAAAAAABkmzAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmSYMAQAAAAAAMk0YAgAAAAAAZJowBAAAAAAAyDRhCAAAAAAAkGnCEAAAAAAAINOEIQAAAAAAQKaVFrsBAAD2T/PmzSt2C7Qzy5YtK3YLFJlrAACA5sqllFKxmwAAYP8xb968qK6uLnYbQDvmbSwAAE10tzAEAABoM3vDNG9DAACANnS3Z4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmSYMAQAAAAAAMk0YAgAAAAAAZJowBAAAAAAAyDRhCAAAAAAAkGnCEAAAAAAAINOEIQAAAAAAQKYJQwAAAAAAgEwThgAAAAAAAJkmDAEAAAAAADJNGAIAAAAAAGSaMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmCUMAAAAAAIBME4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmVZa7AYAAIBsWr9+fdx3330FY08//XRERPzsZz8rGO/SpUtMnDixzXoDAADal1xKKRW7CQAAIHt27twZvXr1infeeSc6dOgQEREppUgpRUnJ/31Jva6uLs4777z4z//8z2K1CgAAZNvdbpMFAAC0ivLy8hg3blyUlpZGXV1d1NXVxa5du2L37t35n+vq6iIifCsEAABoVcIQAACg1UycODHefffd963p3r17DB06tI06AgAA2iNhCAAA0GpOPfXU6NWr1z7Xl5WVxaRJk6K01OMMAQCA1iMMAQAAWk1JSUlMnDgxOnbs2OD6urq6OPvss9u4KwAAoL0RhgAAAK3q7LPP3uetsg488MAYNGhQG3cEAAC0N8IQAACgVR133HFx6KGH1hsvKyuL888/P3K5XBG6AgAA2hNhCAAA0OrOPffcKCsrKxhziywAAKCtCEMAAIBWd84550RdXV3B2BFHHBGf+cxnitQRAADQnghDAACAVte/f//49Kc/nb8lVllZWXz1q18tclcAAEB7IQwBAADaxHnnnRcdOnSIiL/fImv8+PFF7ggAAGgvhCEAAECbmDBhQuzevTsiIo455pg44ogjiqf24IoAACAASURBVNwRAADQXghDAACANnHooYfGscceGxF//5YIAABAW8mllFKxmwAA4J/fvHnzorq6uthtAOR5OwsAQCPdXVrsDgAA2L/U1NQUuwX2Y2+//XbccsstceWVVzZr++rq6pg2bVoMGjSohTtjf7Js2bK48cYbi90GAAD7EWEIAABN4qHXfFiDBw+OT3ziE83atrq6OgYNGuQ6RBgCAECTeGYIAADQppobhAAAADSXMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmCUMAAAAAAIBME4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAABoNV26dIlcLtfgUllZGQMHDozrr78+du/e3ejt/nF5+umnG7Vdp06d4jOf+UzMmTMnUkr5bY4++uhGHyuXy8U111zTZuev2O66666C87c/auiamD17dn59//79C9addNJJRey28T5oXgAAQCFhCAAAraa2tjZWrFgRERGjR4+OlFKklOLtt9+OBx98MCIiLr/88rjiiisatd0/LlVVVY3abufOnbF8+fLo1q1bfPOb34xvf/vbBdvdfffdBfudMmVKREQsWrSoYLy6urrlT9I/sQkTJkRKKYYOHVrsVpqtoWtixowZ+fWLFy+Oo48+Oi644IKoq6uLJ598slitNskHzQsAACgkDAEAoM117do1TjnllJg7d25ERNx6661RV1fXasfr2LFjHH300XHnnXdGSUlJ3HDDDbFp06ZWOx77h5deeilOOOGEGDVqVNx2221RWlpa7JYAAIBW4q99AACK5sgjj4yIiG3btsWWLVviIx/5SJO237x5c5Pq+/btGwceeGC88cYb8dxzz8Wpp54azz77bKO3v+uuu5p0PP55LVmyJMaMGRMzZ86Miy66qNjtAAAArcw3QwAAKJqVK1dGRESvXr2aFIScdNJJcfvttzfrmHufF7K/PgODD+/ee++N0aNHx3/8x38IQgAAoJ0QhgAA0OZqa2vjt7/9bXzjG9+IysrK/O2yWttf//rXWLt2bXTr1i2OOuqoVj/e+vXr45JLLonDDjssOnbsGL169YoxY8YUfBtl/vz5BQ/Bfu2116K6ujq6d+8eBxxwQIwaNSpeeeWVevveuHFjTJ8+PQ4//PAoLy+PQw45JIYNGxa33357bN++vcG6jh07Ro8ePeJLX/pSLF68uN4+X3rppTjzzDOjqqoqOnfuHCeffPL7PkOjOfNbuXJljB8/Pg444ID82IYNG5p7ipvs5ptvjqlTp8YDDzwQo0aN2mddS85t165dUVNTE6eddlr06dMnKioqYsCAAXHTTTfFnj17Co67c+fOuPrqq6N///5RWVkZPXv2jDPOOCN+/etfx+7du5s978b0sHnz5noPZb/mmmvy2793fOzYsa1yrgAAoNUkAABohJqamtScPx9XrFiRIqLB5cgjj0y/+tWvmrxdRKTbbrvtfbcbPXp0fuzdd99NK1asSCeeeGLq2LFj+uUvf/m+PU+ZMiVFRFq0aFGT57vXmjVr0qGHHpp69+6dFi5cmLZu3Zr++Mc/psGDB6dOnTqlpUuXFtSPHj063/fSpUtTbW1teuSRR1JFRUU69thjC2rXrl2b+vXrl/r06ZMWLFiQ3n777bRu3bo0c+bMFBHphhtuKKjr3bt3WrBgQdqyZUtauXJlGjNmTMrlcunnP/95fp8vv/xy6t69ezr44IPTww8/nLZu3Zqef/75NHz48HTYYYel8vLyFpnf4MGD0+LFi9M777yTli9fnjp06JDWr1/f6PMaEammpqbR9Sn93zXRpUuXFBHp8ssvf9/6lp7bggULUkSka6+9Nm3atCmtX78+/fjHP04lJSVpxowZBfuaPHlyqqqqSg8//HDatm1bWrduXZoxY0aKiLR48eIG5/Xea31fmtLDiBEjUklJSfrzn/9cbz+DBg1Kd9xxR6udq8Zq7u8jAADarXn+egQAoFE+bBjy3g9s6+rq0quvvpr+3//7fymXy6UxY8akd9999wO32+vEE0/8wDCkoeWss85q8APef9QSYcj555+fIqLgg+OU/h5QlJeXp2OOOaZgfO+HxAsWLCgYHzt2bIqIgg+KL7jggn2GAiNHjsyHIXvr7rzzzoKaHTt2pIMOOihVVFSkdevWpZRSGjduXIqIdM899xTUvvHGG6m8vLxeGNLc+T3wwAP1em6KDxOGHHnkkalbt24pItKsWbP2Wd/Sc1uwYEEaMmRIvfFJkyalsrKytGXLlvxYv3790gknnFCv9pOf/OSHDkMa28NDDz2UIiJNnTq1oPbJJ59MBx98cMH/q8W6DoQhAAA00Ty3yQIAoM2VlpZGv3794nvf+15MnDgx7r333vjxj3/coscYPXp0pJQipRSrV6+O6urquO++++JnP/tZix5nX+bPnx8lJSX1bsXUp0+fOOqoo+KZZ56J1atX19vu2GOPLfi5b9++ERGxZs2a/Nh9990XERFf+tKX6m2/aNGimDZtWkHd6aefXlBTXl4eQ4cOje3bt8dDDz0UEREPPvhgRESMGDGioPaggw6KT37yky02vy984Qv1xtpK//7948EHH4yuXbvGFVdcEddff32DdS09t1GjRjV4W7KBAwdGXV1dvPDCC/mxkSNHxtKlS+Oiiy6K5cuX52+NtXLlyhgyZEhjp/qhehg+fHgMGDAgbr/99ti4cWN+fNasWXHxxRdHWVlZfmx/vA4AAGifhCEAABTVKaecEhERjz32WKO3efLJJ+OCCy5odP3BBx8ct99+exx++OExa9asePrpp5vaZpPs3LkztmzZEnv27Imqqqp6z2H4/e9/HxERL7/8cr1tq6qqCn7u2LFjRET+uQ57992pU6fo2rXrB/awr7revXtHRMS6deti586dsXXr1ujUqVN06dKlXu1HP/rRFptf586d99lzWxg0aFAsWrQounTpEpdffnnceOONBetbY25btmyJq6++OgYMGBA9evTI7+uKK66IiIht27bla+fMmRO//OUv49VXX42hQ4dGt27dYuTIkflgq7ma0kNExLRp02Lbtm1xyy23RETEqlWr4vHHHy944Pz+fB0AAND+CEMAACiqlFJE1P8wtqV16tQprr322kgpxZVXXtmqxyovL4/u3btHaWlp1NXV5b+h8o/Lqaee2qx9V1VVxY4dO2Lr1q3Nrvvb3/4WEX//F/zl5eXRtWvX2LFjR9TW1tar3bRpU5vNry2ceOKJ8cADD0Tnzp3jsssui5/85Cf5da0xtzPOOCNmzpwZF154YaxatSr27NkTKaW44YYbIuL//h+IiMjlcnHuuefGo48+Gps3b4758+dHSinGjBmzz2+ytHQPERHnnHNO9O7dO26++ebYuXNnXHfddXH++edHjx49WvVcAQBAaxGGAABQVL/97W8jov7toRrj85//fNx1112Nrh83blx89rOfjcceeyweeeSRJh+vKcaMGRO7du2KJUuW1Fv3wx/+MD72sY/Frl27mrXvs846KyIiHnjggXrrPvvZz8Zll11WULdw4cKCmp07d8Zjjz0WFRUV+dti7b3l1t7bZe21YcOGWLlyZb3jtOb82sLJJ58cCxcujMrKyrjkkktizpw5+XUtObfdu3fHkiVLok+fPnHJJZdEr169IpfLRUTE9u3b69V37949XnrppYiIKCsri9NOOy3mz58fuVyu3uvYGKWlpfHCCy80qYeIvwcdU6dOjTfffDOuu+66uOOOO+LSSy+tV7e/XwcAALQfwhAAANrcrl274rXXXovvfe978d///d9x8MEHx/Tp01v9uLlcLq655pqIiLjyyivr/Wv4lvSDH/wgDj/88Pja174WixYtii1btsSmTZvi1ltvje9///sxe/bsKC0tbfa++/XrF5dddlksXLgwtm7dGqtXr46pU6fG2rVr82HI3rpp06bF/fffH1u3bo1Vq1bFxIkTY+3atXHTTTflb5d17bXXRs+ePWPatGnxyCOPRG1tbfzpT3+KSZMmNXjrrNacX1sZPHhw3H///VFRURHf/OY387eEasm5dejQIYYMGRLr1q2LWbNmxYYNG2L79u2xePHimDt3boPbfOMb34jnn38+du7cGW+++Wb86Ec/ipRSfPGLX2zWPJvTQ0TE1KlTo6KiIq666qoYNmxYHHHEEfVqsnAdAADQTrTyE9oBAMiImpqa1NQ/Hzt37pwiot6Sy+VS165d08CBA9O3vvWt9Le//a1R2zW03Hnnne+7XXV1db2+TjrppPz6E088MT9+2223NXiMrVu3NvFs/d3GjRvT9OnT08c//vFUVlaWevXqlYYPH54eeeSRfM2yZcvqHe+73/1uSinVGz/99NPz223YsCFNmzYt9evXL5WVlaUDDzwwTZgwIa1ataqgh3+sq6qqSiNGjEiPPfZYvX5XrlyZzjzzzNStW7dUUVGRjj322HT//fenoUOH5nv4+te//qHn92HehkREqqmpaXR9Q9fErFmzCmoeffTRVFFRkV8/c+bMFp3b+vXr05QpU1Lfvn1TWVlZ6t27d7rgggvSlVdemd/mmGOOSSml9Oyzz6YpU6akT33qU6mysjL17NkzHX/88ennP/952rNnz/vOa1/Liy++2KQe3uvCCy9MEZGeeOKJfZ7jYlwHzfl9BABAuzYvl1Ir/nM4AAAyY968eVFdXd2q36aAD5LL5aKmpibGjx9f7Fbahdtuuy3mzJkTTz/9dLFbKeD3EQAATXS322QBAADQoLlz57bJLewAAKC1CUMAAACIiIhf/OIXcdZZZ0VtbW3MnTs33nrrLd/CAQAgE4QhAADQBLlc7gOX733ve8VuE5pt/vz50aNHj/jpT38ad911lwegAwCQCf6qBQCAJvCMArJs8uTJMXny5GK3AQAALc43QwAAAAAAgEwThgAAAAAAAJkmDAEAAAAAADJNGAIAAAAAAGSaMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmCUMAAAAAAIBME4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADItNJiNwAAwP4ll8sVuwXauerq6qiuri52GwAAwH5EGAIAQKOccMIJUVNTU+w22M8tW7YsbrzxRtcSAADQpnIppVTsJgAAgPZh3rx5UV1dHd6GAAAAbehuzwwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmSYMAQAAAAAAMk0YAgAAAAAAZJowBAAAAAAAyDRhCAAAAAAAkGnCEAAAAAAAINOEIQAAAAAAQKYJQwAAAAAAgEwThgAAAAAAAJkmDAEAAAAAADJNGAIAAAAAAGSaMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmCUMAAAAAAIBME4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAAAAAABApglDAAAAAACATBOGAAAAAAAAmVZa7AYAAIBsqquri9ra2oKxd955JyIi3nrrrYLxXC4X3bt3b7PeAACA9kUYAgAAtIqNGzfGIYccErt37663rmfPngU/DxkyJBYvXtxWrQEAAO2M22QBAACtok+fPnHKKadEScn7v+3I5XJx9tlnt1FXAABAeyQMAQAAWs25554buVzufWtKSkriK1/5Sht1BAAAtEfCEAAAoNV85StfiQ4dOuxzfYcOHWLkyJFxwAEHtGFXAABAeyMMAQAAWk23bt1i5MiRUVra8OMKU0oxadKkNu4KAABob4QhAABAq5o0aVKDD1GPiOjYsWOMGjWqjTsCAADaG2EIAADQqs4444yorKysN15aWhpnnXVWdOnSpQhdAQAA7YkwBAAAaFWdOnWKMWPGRFlZWcH4rl274pxzzilSVwAAQHsiDAEAAFrdxIkTo66urmCsW7ducdpppxWpIwAAoD0RhgAAAK1u2LBh0bNnz/zPZWVlMWHChOjYsWMRuwIAANoLYQgAANDqSktLY8KECflbZdXV1cXEiROL3BUAANBeCEMAAIA2cfbZZ+dvldW7d+84+eSTi9wRAADQXghDAACANnHiiSfGQQcdFBER5557bpSUeDsCAAC0jdJiNwAAAG1h3LhxxW6BiOjatWtERKxYscJr8k9g0KBBMX369GK3AQAArU4YAgBAu3DPPffE8ccfH4ccckixW2m3li9fHnV1ddG1a9fo0aNHsdtp95YvX17sFgAAoM0IQwAAaDcuu+yyGD9+fLHbaLf2fhNk3LhxXod/Ar6ZAwBAe+ImvQAAQJsShAAAAG1NGAIAAAAAAGSaMAQAAAAAAMg0YQgAAAAAAJBpwhAAAAAAACDThCEAAAAAAECmCUMAAAAAAIBME4YAAAAAAACZJgwBAAAAAAAyTRgCAAAAAABkmjAEAAAAAADINGEIAAAAAACQacIQAAAAAAAg04QhAADQSHfddVfkcrnI5XLRqVOnYrfTLnTp0iV/zvcuJSUl0aNHjxg4cGBMnTo1nnnmmWK3CQAA/JMThgAAQCNNmDAhUkoxdOjQYrfSbtTW1saKFSv+f3v3HRfVlfYB/DcwQ0fEBkgREEElCgRrgNhDyiaWkGR3xRRj2WiMJtHEFLMmml1fNdWYTYybGNsbNfummGISS9yADSJWBKRaAAEFpA0wzHn/cOcuwwDO4MAdht/387kfcbhz73OnPNxznnvPAQBMnjwZQgjU19cjLS0Nb7zxBtLS0jBs2DA88cQTqK6uljlaIiIiIiKyVCyGEBERERHRTbm4uCA6OlruMAAAtra28PDwwOTJk7F//3688MIL2LRpE/70pz9BCCF3eB3Gkt4TIiIiIiJLx2IIERERERF1aqtWrcLIkSPx7bff4osvvpA7HCIiIiIiskAshhARERERUaemUCjw9NNPAwA+/PBDmaMhIiIiIiJLxGIIEREREVEL0tLSMGXKFLi5ucHZ2RkxMTFISEgwWO/rr7/Wm+A7PT0dDz/8MHr27Ck9VlJSAgC4evUqnnvuOfTv3x92dnZwd3fHPffcgwMHDkjbW7t2rfQ8Hx8fJCUlYcKECXB1dYWTkxPGjRuHxMREgziM2fbKlSulbTceYmnPnj3S47169TKIpaqqComJidI6SqXSLK+xueiO5ciRI6ivr+d7QkREREREelgMISIiIiJqRmZmJkaPHo3k5GR8+eWXuHLlCj788EOsWLECWVlZeutOmTIFQghMnjwZADB37lzMmzcPFy9exJEjR2BrawsAKCwsxPDhw7F9+3a89957KCkpwdGjR+Hk5IQJEyZg48aNAIDFixdDCIGwsDCUlZVh4cKFWLlyJQoLC/Hvf/8b165dw/jx43Hw4EEpBmO3/eqrr0IIAWdnZ71juPvuuyGEQGRkpN7julicnZ0RFRUFIQSEENBoNOZ9wW+Rp6cnAECj0aCkpITvCRERERER6WExhIiIiIioGS+//DLKysrw3nvvYdKkSXBxccGQIUPw2WefoaCgoNXnvvjiixg7diycnJwwcuRIaDQa9OrVCy+99BJycnLw7rvv4g9/+AO6deuG4OBgbN++HV5eXnjmmWdw5coVvW1VVVXhww8/xOjRo+Hs7Ixhw4Zh69atqKurw8KFC6X12rJta3KzidP5nhARERERdW0shhARERERNWPPnj0AgNjYWL3H+/bti+Dg4FafO2LEiGYf/+qrrwAA9913n97j9vb2mDBhAmpqavDTTz/p/c7Z2Rnh4eF6jw0ZMgR9+/bFyZMnpcJMW7ZtTXSvg0ql0htSSofvCRERERFR18ZiCBERERFRE7W1taioqICDgwNcXFwMft+nT59Wn990uCPdNsvLy+Hg4ABXV1eD33t4eAC4MbRSY927d292H7oYioqK2rxta6Kby2X06NFQqVQGv+d7QkRERETUtbEYQkRERETUhL29PVxdXaFWq1FZWWnw+2vXrrVpm25ublCr1aioqDD4vW64JN3cFzpXr15tdgiooqIiADc64NuybRsbG9TV1RmsW1ZW1mz8CoWipUOTnVarxfr16wEA8+fPN/p5fE+IiIiIiLoOFkOIiIiIiJpxzz33APjvcFk6JSUlSE9Pb9M2p06dCgD4/vvv9R6vra3Fvn374OjoaDAsl1qtRlJSkt5jp0+fRn5+PsLCwuDl5dWmbXt5eeHy5ct66xYWFuLChQvNxu7k5KTXUR8SEoINGzbc9Jg7wksvvYRjx45h6tSpeOihh0x6Lt8TIiIiIqKugcUQIiIiIqJm/O1vf0OPHj2waNEi/PLLL6isrERqairi4+ObHTrLGH//+98REBCARYsW4bvvvkNFRQUyMjLw5z//GQUFBXjvvfek4ZN03Nzc8PLLL+Pw4cOoqqpCcnIy4uPjYWdnh/fee6/N277rrruQn5+PDz74AJWVlcjKysLChQtbHALs9ttvR0ZGBi5evIjDhw8jOzsbMTExbXodbpVWq0VRURG++eYbTJgwAatXr8bMmTOxbds2k++W4HtCRERERNRFCCIiIiKiLgCA2LFjh0nPSU9PF1OmTBHdunUTjo6OYvjw4eK7774TEyZMEAAEAPHkk0+Kw4cPS/9vvDSnpKRELFq0SAQEBAiVSiXc3NxEbGys2Ldvn8G6YWFhwtvbW6SmporY2Fjh6uoqHB0dxZgxY0RCQsItbbusrEzMmjVLeHl5CUdHRxEdHS2SkpJEZGSkFP+LL74orZ+WliZiYmKEs7Oz8PX1FevXrzfptRRCiLi4OBEXF2fSc5ydnQ1eV4VCIdzc3MSQIUPEU089JX7//XeD5/E9ubm2vB9ERERERJ3UToUQzQx2S0RERERkZRQKBXbs2IGHH35Y7lCMFh4ejpKSEly6dEnuUMxCN4TVrl27ZI6k7azpPbGG94OIiIiIyEi7OEwWERERERERERERERFZNRZDiIiIiIiIiIiIiIjIqrEYQkRERERkYdauXQuFQoGTJ0/i8uXLUCgUePXVV+UOq0vje0JERERE1Lkp5Q6AiIiIiIj0LV68GIsXL5Y7DGqE7wkRERERUefGO0OIiIiIiIiIiIiIiMiqsRhCRERERERERERERERWjcNkEREREZHVKikpQV5eHnJzc+UOhcgiFRcX48cff4S/vz/8/f3h6Ogod0hERERERO2CxRAiIiIi6rQaFzuaWyorKwEANja8IZqoORkZGbj33nul/3t4eEiFEX9/f/Tr10/v/yyWEBEREVFnxWIIEREREVms0tJS5Ofno6CgANnZ2XpLZmYmysvLpXXd3d0RGBiIwMBA3HfffQgMDISXlxf69u2LQYMGwdnZWcYjIbJMUVFR2LJlC/Lz8w2+Y3v37pW+fzqNv2e675fu/8HBwXB1dZXxaIiIiIiIWsZiCBERERHJpq3FjokTJ2LOnDl6xQ4nJycZj4So83JwcJC+W81Rq9XNFksSExNZLCEiIiKiToPFECIiIiJqNyx2EHV+LJYQERERkTVgMYSIiIiI2ozFDiJisYSIiIiIOgMWQ4iIiIioRSx2ENGtYrGEiIiIiCwBiyFEREREXRiLHUQkNxZLiIiIiKgjsBhCREREZMVY7CCizo7FEiIiIiIyBxZDiIiIiDoxFjuIqKtjsYSIiIiIjMFiCBEREZEFY7GDiOjWsFhCRERERACgEEIIuYMgIiIi6qquXr2KnJwc5OXlITc3V1p0j1VWVgIAbGxs0LdvX/j7+xss/fr1g5+fH+zs7GQ+GsvV0NAApVIJf39/+Pj4wNPTU+6QZFNQUAAvLy9Z9n3kyBEAwKhRo2TZv051dTXq6urQvXt3WeOQU3p6Oi5fvoxRo0bh+++/lzsci3b9+nW9/Nx0KS0tldb18PAwyM+6nwMCAuDg4CDjkRARERF1abtYDCEiIiJqR7W1tbh8+bLBFcc3u7Oj6ZXHvLPDNNXV1Th69CgSEhKQmJiIQ4cOoaKiAiqVCsHBwRg0aJDcIXa42tpaJCcno7CwEGPHjkXPnj3lDkk2p06dQmZmJkJDQxEcHAyFQiF3SB1KCIGDBw/i2rVr0Gq18PPzQ0xMDKKiohAdHY3Q0FDY2NjIHWan0dKdJdnZ2a3eWdI01wcGBsLR0VHGIyEiIiKyaiyGEBEREd2K2tpa5OXlIScnR7qjQ7fk5uaiuLhYWtfDwwMBAQEICAiQrhLW/cw7O25NUVERjh49isTERCQkJCA5ORm1tbXw8vJCdHS01MkbERHRJTt5f/75ZzzxxBNQKpXYvHkzxowZI3dIshJC4P3338eLL76I22+/HVu2bEH//v3lDqvDVVdX4/jx49L3JjExEaWlpXB1dcXIkSOl701UVBQ76W+B7s6Sxn8fGv+d0N0BqFQq4ePjo/f3ofHi5eXV5Qp3RERERGbEYggRERHRzZSWljZ7xW92djby8vLQ0NAA4Ma49I3Hj2+8cCx588rPz9frwD1+/DiEENJ8KVFRUYiJiUFAQIDcocqqpqYGS5cuxbp16xAXF4ePP/4Y7u7ucodlMc6cOYPp06cjNzcXa9aswZw5c+QOSVYNDQ1IS0uTvlu//fYbcnNzoVQqERYWJhVHxo0bh169eskdrtVo/Dem6RxRaWlpqKqqAgDY2dnBx8en2bsHAwMDERAQwGIJERERUctYDCEiIiJqrtih65A6d+4cqqurAQD29vbw9vZucYiTvn37ynwk1qlpB+3Bgwdx4cIFqFQqDB06VOqgnTBhAnr06CF3uBYjOTkZ8fHxKCwsxAcffID4+Hi5Q7JIarUay5cvx5o1a/Dggw/io48+4ueokaaFx5SUFGi1WgQGBurdORIaGip3qFarmSR1mAAAIABJREFUtYL8hQsXoNFoALRekA8KCoKbm5vMR0JEREQkKxZDiIiIyPrV1NToXWnbeMnIyEBFRQUAQKVSoVevXi12Jvn7+3fJIZY6WlVVFVJSUqQO2ISEBJSVlaFbt24YMWKE1AEbHR3NyYibodVqsW7dOrzwwguIjo7G559/Dh8fH7nDsnh79+7F448/DhsbG3z++ecYN26c3CFZpIqKCr35eBISEqBWq+Hp6Ylhw4ZJxZERI0Zw6L8OUF9fj4sXLxrcUaJbcnNzodVqAejPV9L4jpLAwEAMHDgQzs7OMh8NERERUbtiMYSIiIg6v9YmKc/OzkZpaam0bkuT1wYGBsLPzw9KpVLGI+maCgsLkZSUJHWsJiUloa6ujvN9tEFeXh4effRRHD16FK+//jqWLFnC18wEZWVlmDdvHr744gssWLAAq1evhr29vdxhWTSNRoOTJ09KxZH9+/fj6tWrcHZ2Rnh4uPQdvvPOO3lnggz495GIiIhIwmIIERERWT6NRoOLFy8iOzvbYOLZnJwcFBYWSuv26tVLb8LZxhPR9uvXjx2bFiA7O1vvqvJz587BxsYGISEhUsfpmDFj0K9fP7lD7VR27dqFuXPnwsvLC9u2bUN4eLjcIXVamzdvxtNPPw0/Pz9s27YNYWFhcofUqTT9jqempsLW1lbvOz5u3Dj4+vrKHWqXV1paajC5e+P/19TUALhx56Sfn5/e31XdPCUBAQHo06ePzEdCREREdFMshhAREZFlKCkpQU5OjlTw0F21mpOTozcmuqurq0GRo/Hi4uIi85FQY02vGv/1119RXFxscNV4TEwMunfvLne4nVJ5eTnmz5+P7du3Y/bs2XjnnXfg5OQkd1idXm5uLh599FEkJSVh+fLlvMvmFhQUFCA5OVkqjhw7dgz19fUG844MHjyYE4BbmMLCwmaLJDk5Obh48SLq6+sBAC4uLlJxpLl/OaQhERERWQAWQ4iIiKhj1NXV4dKlS80O05GZmYny8nJp3daG6ggICGBnmQWrrKzEkSNHpOJHYmIiampq4OHhgeHDh3M+ATPbt28fHn/8cWg0Gvzzn//EvffeK3dIVqWhoQFr167Fa6+9hjvvvBObNm2Ct7e33GF1ei3liT59+mDEiBFSnhg+fDjv5rNwrU3unpOTA113Q2t/1/v16wdbW1uZj4SIiIi6ABZDiIiIyHxa6xTJy8tDQ0MDgJY7Rby8vBAYGAhHR0eZj4SMVVBQIE1ynpiYiJSUFGi1Wl7x3c7UajWWL1+ONWvWYOrUqfj444/Rs2dPucOyWklJSYiPj0dRURHWr1+PP//5z3KHZFVauoPMyckJERERenMHubu7yx0uGam1+UoyMjJQUVEB4MYQXL6+vs2eF/Tv3593DRIREZG5sBhCRERExlOr1cjPz2+2YyM9PR2VlZUAADs7O/j4+DTbsREUFMRJdDux5uYCUCqVCAsLkzorx4wZw/Hj29HZs2cRHx+P7OxsrFmzBnPmzJE7pC6hpqYGS5cuxbp16xAXF4cNGzawk7YdGTO30J133gl/f3+5Q6U2MvUCCi8vL/Tt21fvnGLQoEEcFpCIiIiMxWIIERER/ZdGo8GFCxeQn5+PgoICDnnRxdXX1+PUqVNSh+T+/ftx9epVuLi4ICwsTK9DkgWu9ieEwPvvv48XX3wRERER2LJlC4KCguQOq8vZs2cPZs6cCTs7O2zevBl33nmn3CF1CVeuXMGxY8ek4khSUhLq6urg5eWld+dIREQE53axAq0NrZmVlYWysjJpXQ6tSUREREZiMYSIiKirae1KzMYTlTs4OBhcgalbQkJCOFG5Fbp+/TqOHTumdzW2Wq2Gl5cXIiMjpQ7HkSNHQqVSyR1ul1JYWIiZM2fil19+wSuvvIJly5ax4CijoqIizJo1Cz/88AMWL16MN954g3PgdLCqqiqkpKRIuSohIQFlZWXo1q0bRowYIRVHoqOjOXm3FdKdyzR38ca5c+dQXV0NALC3t4e3t3ez5zLBwcFwdXWV+UiIiIioA7EYQkREZG3UajVycnL07uZo/K9uKKvGY3QHBAQY/NurVy+Zj4TaW35+vtSR2Np8H6GhoXKH2qV9+eWXmDt3Lvr06YOtW7ciMjJS7pDoPzZv3ox58+Zh0KBB2Lp1K0JCQuQOqctqaGhAWlqalNMOHjyICxcuQKVSYejQoVJOGz9+POfXsXINDQ3SXSVNz4FycnJw5coVaV1PT88Wz4N8fHx4lxEREZF1YTGEiIioM2rt7o7c3FxotVoArQ8d4efnB6VSKfORUEdp2lH422+/ITc312C+j3HjxrEQZiGuX7+OJUuWYMOGDZgxYwb+8Y9/wNnZWe6wqIm0tDTEx8cjNTUVf//73/HMM89wWB4L0bTge/z4cQghWPDt4swxsTvvKiEiIuqUWAwhIiKyRPX19cjLy0NWVpbeGNm6n3UNdTs7O/Tr1w/9+/eXGuiNf+ZQVl1XdXU1jh8/rtcRWFpaCldXV4wcOVKvI9DR0VHucKmJw4cPY8aMGaioqMDGjRtx//33yx0StUKj0WDlypVYuXIlJk6ciM8++wxeXl5yh0VNNB0K8LfffkNtbS3nHSE9hYWFBnOU6H7Oz8+X1vP09Gzx/IvffyIiIovEYggREZFc1Go1srKykJqa2urcHby7g4xRVFSEo0ePSsWP5ORkdvJ1QuxU79yOHDmCGTNmoLy8HBs3bsQDDzwgd0jUiqZF40OHDuHatWtwcXHBqFGjWDQmA61N7J6WloaqqioArc9VMnDgQN7lR0REJA8WQ4iIiNpTa8NZZWdnS+s1V/AYPHgwQkND4e7uLuMRkKXSDf+yd+9eJCQk4Ny5c7CxsUFISIjUeRcTE4OAgAC5QyUjpaWlYfr06Th37hyHW+rEmg5v9uGHH/IuvU6i6XCCCQkJyMnJ4XCCZLTWzvtycnKg635p7UKXgIAA5n4iIqL2wWIIERHRrWht3Olz586huroaQOtXCA4aNAhOTk4yHwlZsqYddL/++isuXrwoTQw8ceJEREVFISoqCj169JA7XDKREAKffPIJnnvuOQwePBhbt25FcHCw3GHRLfrXv/6FuXPnolu3btiyZQuioqLkDonaoOm8IykpKdBqtQbzjgwePJgd2NQqtVqN/Pz8Zs8ZU1NTUVNTAwBwcHBA3759mz1nDA0NhYODg8xHQkRE1GmxGEJERHQzvMqPOlpVVRVSUlL0JjsvLy9Ht27dMGLECKkDLiYmBvb29nKHS7egqKgITz75JH788UcsXrwYb7zxBuzs7OQOi8yksLAQTz75JH7++We88sorWLZsGWxtbeUOi25BRUUFjh49KhVHEhISoFar4enpiWHDhknFkREjRvC7TCYpLS3F2bNnDYZPzcrKQllZGQBAqVTCz8+v2XPN/v37o3v37jIfBRERkUVjMYSIiKi18Z/T09NRWVkJ4MZk5T4+Ps02QENCQjgMCrVZYWEhkpKSpI61pKQk1NXVGcz3cfvtt7OoZkV+/PFHzJw5Ew4ODti8eTNiYmLkDonage7On2effRZDhgzB1q1bERQUJHdYZCYajQYnT56UiiMHDhxASUkJnJ2dER4eLuXwO++8E25ubnKHS51Uaxfm5OXloaGhAUDrF+b069ePxVgiIurqWAwhIqKuoaSkBFlZWcjKypKustM1Ii9fvizd3dGnTx/p6rrGV9oFBgaib9++7Igms8jOzta7qjg1NRW2trZ6832MGTMG/fr1kztUagc1NTVYunQp3n//fcyYMQPr16+Hq6ur3GFRO0tNTcX06dORlZWFtWvXYs6cOXKHRO3EmBw/duxY+Pn5yR0qWYHa2lrk5uYanN/qfm48ZKu/v7/BOa5u4fBbRETUBbAYQkRE1qO5q+bOnj2Ls2fPSsMLqFQq+Pr6NnvFXFBQEK/aJLMz9qrhmJgYDm/RBRw7dgzx8fEoLi7Ghx9+iD/96U9yh0QdqLa2Fn/961+xZs0aTJkyBRs2bEDPnj3lDovaWUFBAZKTk6XiyLFjx1BfX8+7/6hDtHZXSW5uLrRaLYDm7yoZPHgwbrvtNp6fEBGRtWAxhIiIOg+NRoO8vDzpDo/MzEy9n9VqNQDAyckJQUFB0pVujX/28/PjEAHUrpqOJ5+YmIiamhp4eHhg+PDhHE++i9JoNHjrrbewbNkyjBkzBps2bYK3t7fcYZFM9u/fj8ceewz19fX49NNPce+998odEnWgyspKnDhxQiqO/Pvf/8b169fRp08fjBgxQvo7MXz4cM4LRe2qurq62XPqrKwsXLhwQRp+q1evXnrn1I3/7dOnj8xHQUREZDQWQ4iIyLK0Nn9HamoqampqALQ+JrK/vz9sbGxkPhLqKvLz86UOrcTERKSkpECr1SIwMFC62jcqKgqDBw/mFb9dVG5uLmbMmIHk5GQsX74cS5YsYY4ilJeXY/78+di+fTtmz56Nd955B05OTnKHRTJoegfhr7/+iuLiYjg5OSEiIkLv7hF3d3e5w6Uuor6+HhcvXmz2nDwtLQ1VVVUAbgy/5e3tLd1JEhoaynlKiIjIUrEYQkREHU+tViM/Px9nz55FampqmyaBDAwMlPkoqKvKzs7G3r17pU6r7OxsKJVKhIWFSZ1VY8eORe/eveUOlSzA5s2bMX/+fPj7+2Pbtm0YOnSo3CGRhdm1axfmzp0LLy8vbN26FREREXKHRBag6bwj586dg42Njd68I3feeSf8/f3lDpW6qMbDbzU+p8/MzER5eTkAwM7ODj4+Ps2exw8ePBiOjo4yHwUREXUxLIYQEVH7aG184pycHGnCcnd3d4OryAIDAxEcHMwJhUl29fX1OHXqlNQhtW/fPly7dg0uLi4ICwtDdHQ0Jk6ciDvuuINXdJOe4uJizJkzB9988w0WLFiA1atXc7gbalFeXh4ee+wxHDp0CC+//DJee+013j1Eeq5cuYJjx45JxZGkpCTU1dUZzDsSERHBzw7Jju0AIiKyUCyGEBFR2xlzRVhrE5YPGjSIHchkUa5fv45jx47pXY2rVqvh5eWFyMhIqcNp5MiRUKlUcodLFuqXX37B448/DqVSic8//xxjx46VOyTqBIQQeP/99/HCCy8gKioKn3/+OXx9feUOiyxUVVUVUlJSpL9VCQkJKCsrg6urK0aOHCkVR6Kjo+Hg4CB3uEQStVqNrKwsg7vDjZnQPTAwEKGhofDy8pL5KIiIqJNiMYSIiFrWlrGCm44XzLGCyZIZO99HaGio3KFSJ6BWq7F8+XKsWbMGDz74ID766CP06NFD7rCokzl9+jSmT5+OCxcu4IMPPkB8fLzcIVEn0NDQgLS0NL1J2fPy8gyGcRw/fjx69uwpd7hEzWpt7sCzZ89CrVYD4NyBRETUZiyGEBF1dS01Os6ePYuMjAxoNBoArTc6AgICODE0WTx2FFF7On36NOLj45GXl4d169ZhxowZcodEnZharcaLL76IdevWIS4uDh9//DEnziaTNS34Hz9+HEIIFvypU9JoNLhw4UKzhZL09HRUVlYCMLxIq/GFWrxIi4ioy2MxhIioK6ipqcH58+eRmZkp/atbLl26JK3n7e2N/v37IygoCP3795eWoKAgdO/eXcYjIDJddXU1jh8/rtcRVFpayiFEyKwaD200bNgwbNmyBYGBgXKHRVbi559/xhNPPMEh18gsjBkKcuLEiZx3hDodIQQuXbqErKwsacnMzJR+bjyhe0BAAAYMGIABAwYgKChI+tfPz4+FEiIi68diCBGRtaipqTEoduh+1hU8FAoFfH199U78dUv//v3h6Ogo81EQtV3TyWWTk5NRW1vLyWWp3Vy4cAGPPvqoNOn1smXL2JFCZldcXIzZs2fj22+/xYIFC7B69WrY29vLHRZZgfr6epw6dUoqjuzbtw/Xrl2Di4sLRo0apXf3CM8RqTMrLi6WCiPnz5/Xu0istLQUwI1CSWBgoEGRhIUSIiKrwmIIEVFn0nRIq8aTljc34WDjuTsCAwMxcOBAODs7y3wUROaRnZ2td3XruXPnYGNjg5CQEKnzJiYmBgEBAXKHSlZo165dmDt3Ljw9PbFt2zZERETIHRJZuc2bN2P+/Pnw9/fHtm3bMHToULlDIiuj1Wpx7tw5vUnZc3JyDIaTHDt2LHr37i13uERmUVpa2uIcJQUFBQAAlUoFX19fDr1FRNT5sRhCRGRpbrXgERISAhcXF5mPgsi8NBoN0tPTkZiYiL179+LgwYMoKiqCk5MTIiIipOJHVFQUJ6ymdlVeXo6nn34a27Ztw+zZs/HOO+/AyclJ7rCoi8jJycGjjz6K5ORkLF++HEuWLOGdbtSums47kpKSAq1WazDvyODBgzl/HFmdlgolZ86cQWFhIQAWSoiIOhkWQ4iI5MCCB1HrKisrceLECakD5rfffkN5eTm6deuGESNGSB0wMTExHC6GOsz+/fvx2GOPob6+Hv/85z9x3333yR0SdUEajQZvvfUWli1bhrFjx+Kzzz6Dt7e33GFRF1FRUYGjR49KxZHExETU1NTAw8MDw4cPl4ojI0aMgJ2dndzhErUbFkqIiDolFkOIiNpLawWPvLw8NDQ0AGDBgwgACgsLkZSUJBU/jh07hvr6eoP5Pm6//XZeeUodrr6+Hm+++SZWrFiBKVOmYMOGDejZs6fcYVEXd+zYMcTHx6O0tBSffPIJpkyZIndI1AVpNBqcPHlSKo4cOHAAJSUlcHZ2Rnh4uN6wld27d5c7XKIO0Vyh5OzZszh79izKysoA3JijxMfHh4USIqKOxWIIEdGtMLXg0bTowYIHdVVN5/tITU2Fra2t3nwfY8eOhZ+fn9yhkpXLyMhAcHBwi79PTU1FfHw8MjMzsXbtWsyZM6cDoyNqXXV1NV566SW8//77mDFjBj788MNWzytu9nknMgf+jSdqWUuFkjNnzqC8vByAYaGkcfuRhRIiolvCYggR0c3casEjODgYrq6uMh8FkXx41ShZqi+//BJz587F6dOn0bdvX73fCSHwySef4Nlnn8WQIUOwdetWBAUFyRQpUeu++uorzJkzBy4uLti8eTNiYmIM1tm6dSuee+45nDlzBn369JEhSuqqePcnkXFYKCEiancshhARATcKHllZWcjIyEBmZiYyMzNx/vx5ZGZm4uLFi9IcHt7e3ggKCsKAAQMQFBSk9zMn0CW6geOJU2dw8eJFhIaGorKyEuPGjcPevXulTrjCwkI8+eST+Pnnn/H8889jxYoVUKlUMkdM1LorV65g1qxZ2LNnj8HnNi8vD6GhoaiursakSZOwZ88edjqTbG42L9jEiRMRFRWF4cOHc14wov8oKipCZmZms+3V69evAwAcHBykNmrjduqAAQPg6+sr8xEQEVkEFkOIqGspLS3Vu7NDd7VNRkYGNBoNAN7hQWSq/Px8qUMjMTERKSkp0Gq1CAwMlK72jIqKwuDBg9n5RhZBq9Vi3LhxOHz4MOrr62FjY4O3334bCxcuxL/+9S/MnTsX3bp1w5YtWxAVFSV3uERG093R9Nxzz2Hw4MHSHU1jxozB0aNHpc/7u+++iwULFsgdLhGAG3eQpqenIzExEXv37sXBgwdRVFQEJycnRERESOcRUVFR6NGjh9zhElkcY+4osbe3h7e3t8E8laGhofDy8pL5CIiIOgyLIURkfUpKSpCRkYH09HScP38eGRkZ0r9qtRrAjYJHcHAwgoODERISggEDBiA4OBgDBgyAs7OzzEdAZD7Z2dkIDAw02/a0Wi3OnTsnFT8SEhKQk5MDpVKJsLAwqfgxduxY9O7d22z7JTKnN998E6+99pp01x8AqFQqPPHEE9iwYQPmzp2Lt956i38PqNPSzXVz/vx5xMfHY8OGDQaf96SkJISFhckYJVHLms47cu7cOdjY2OjNOxITE4OAgACz7bOmpgZlZWXsGCarcuXKFYN2cXp6OrKyslBbWwsA6NWrFwYMGICQkBCpTaz719HRUeYjICIyKxZDiKhzqq2txeXLlw3u8jhz5gwKCwsB6I+n2vQKGHN2DhNZomvXruH555/Hd999h6KiojbfkVFfX49Tp05JHRL79u3DtWvX4OLiglGjRund+cHGEnUGycnJGDVqlDTfk45SqUTfvn3x9ttv48EHH5QpOiLzqaurw9KlS7Fu3Trp7lcdpVKJgIAAnDx5krmbOoUrV67g2LFjUnEkOTkZtbW1BvOOREREwMbGpk37OHDgAKZMmYLVq1djzpw5vJuVrF5+fr5eW1p3R0l6errevJhN29KDBw9GSEgIlEqlzEdARGQyFkOIqHl1dXWyj+Wv0Whw4cKFZicuz83Nla5w9PLyMjg5Cw0N5QRy1GV98cUXePrpp1FWVoaGhgakpaUhJCTEqOdev34dx44d07saU61Ww8vLC5GRkYiOjsbEiRNvqbOBSC5VVVUYMmQILl68aNA5DNzoIH722WexevVqGaIjMi+1Wo3w8HBkZWW1+Hn/y1/+gnXr1skQHdGtqa6uxvHjx/WG6SwtLYWrqytGjhwpFUeio6Ph4OBg1DZXrFiB5cuXQwiBkSNH4rPPPsPAgQPb+UiILE9dXR0uXbpk0AbPzs5GTk4OhBBQqVTw9fVtdnjpgIAAWYuJltCXQUQWi8UQItJXX1+PjRs3YtWqVcjMzOyQCWN1Y5w2N5dH42GtmrvDY9CgQZy4nOg/8vPzMW/ePHzzzTdQKBQQQsDW1hYff/wxnnzyyRaf07gj4fjx4xBCGMz3ERoa2sFHQ2R+jz32GLZv395sx7COQqHA3r17MX78+A6MjMj85s+fjw0bNtz08/7NN9/g/vvv78DIiMxPd/GH7pzm3//+N/Ly8gyG8Rw/fjx69uzZ7DYmTJiAAwcOSB29Wq0Wixcvxuuvv86J3In+o6ysDFlZWQZ3k5w+fVqayN3e3h79+/c3GJlhyJAh8PDwaPcYv/zyS7z77rtYtWoVoqOj231/RNSpsBhCRDdotVrs2LEDL7/8MvLy8iCEQHp6OoKDg82y/aaTuukKHxkZGaioqAAAODg4SJO4Nb7CZOjQoejWrZtZ4iCyRlqtFhs3bsSzzz6L+vp61NfXS79TKpWIj4/HZ599ZpaOAqLOaseOHfjjH/940/VsbGzQp08fnDt3Dt27d++AyIjM78cff8R9992HmzX1FAoFunfvjtTUVHh6enZQdEQdo+kFHykpKdBqtc1e8NHQ0AA3NzdUVVXpbcPW1ha+vr749NNPMW7cOJmOhKhzaOkix9TUVNTU1AD470WOTS90HDhwoNnmanvzzTfx6quvAgBiY2OxevVqDB061CzbJqJOj8UQIgL27t2L559/HqdPn4ZCoZCGn9q9ezf+8Ic/GL0d3e20TU9+dAsAvdtpm97lIffttESd0enTpzFz5kwcP35cb3Lcxjw8PDBkyBAcOXIElZWV6NGjh14nwLBhw3jFI1m1vLw83Hbbbaiqqmq1c1ipVEKr1UIIgddffx3Lli3rwCiJzEOj0WDUqFH4/fffoVKpoNFoWv3cq1QqjB07Fj/99BPPw8iqlZSU4NChQ1JxJDk5GXV1dfDx8UFoaCh++umnZp9na2sLrVaLWbNmYe3atbxIi8hETYe/blwwaW34a12fwcCBA00a/vrRRx/F9u3b0dDQAKVSiYaGBkybNg2rVq1CUFBQex0mEXUOLIYQdWVHjhzBCy+8gN9++w22trZ6k8na2dnhb3/7G55//nmD5zWdaK09T2SIqHlqtRqrVq3Cm2++CQCtDoMCAHFxcZg0aRKioqIwePBgdnhRl6HVajFmzBgcPXpU764pHaVSCY1GA1dXV9x111246667cN9998Hb21uGaInMJzs7G7t378Y333yDhIQEaDQaqFQq1NXVGaxrY2ODt99+GwsXLpQhUiJ51NTUICkpCb/99hv+7//+D6dOnWr1fEqpVKJ79+74xz/+gbi4uA6MlMh6tccFlREREThx4oTeY0qlEkIIzJw5E6+//jq8vLw67BiJyKKwGELUFaWmpmLZsmX46quvYGtr2+xJv0qlwgMPPIDY2FhkZGTg/PnzSE9PR3Z2ttSI7t27N0JCQhAcHIzg4GAMGDBA+pdXmRO1n99++w1PPPEEcnNz9YqYLVEoFNi1axcefPDBDoiOyLLoJsTVFet1V/gCQFhYGB544AHcc889GD58OIv1ZLUqKiqwd+9e7NmzB9999x3y8/OhUqnQ0NAgfR9UKhWSk5M5lAh1SQ899BC++uqrm55X2djYQKvV4p577sGGDRvg4+PTQRESdT3Xrl1DRkaG3nL+/HlkZGSguroaAODm5ib1Qej6JmbNmmUw5J2OSqWCjY0NFi1ahKVLl3JIVKKux7AYcunSJRw6dEiugIiondxxxx3QarVYsWIFPv30U9jY2Nz0SnJvb2+UlJTcdPKznTt3dsQhEHV5VVVV2Lx5M3799VdpgnRjKJVKxMbG4tFHHzV6X76+vhg9enRbQ23V4cOHcfHixXbZNlFjGRkZ+Otf/yp19jo7OyMiIgIREREICwuDq6urzBFal4cffrhdtsv2ifldunQJJ06cwPHjx5GWliZ1AHt5eWH16tWws7OTOULqCu644452KyaY2j6ZPXu2NPmzMRQKBezt7TFjxgxMmDCBd9wSdYDG7ZPS0tJm7yY5c+YMamtrb7otlUoFBwcHvPLKK3jmmWdw4sQJtk+IrFAz7ZNdEE3s2LFDAODChYuVLbGxsUKpVAqVSmX0c9zd3ZumiGbJfWxcuHAx/xIXF2fU978t4uLiZD8+Lly4mH9pL2yfcOFincuOHTvaLW/IfWxcuHAx/2JM+2Tfvn0mbdPW1lb06dNH3H777bIfHxcuXMy/NGOnEi0QHD2LLMDOnTvxyCOP8PPYRkIIrFu3DgsXLpQmBLSxsTH6+aWlpSgrKzPq1tEdO3a02xWhRKZQKBRW/3m8fv0JedRqAAAYlElEQVQ6Ll26hEuXLiE/Px8XL15Efn4+Lly4gNzcXBQUFKC0tFTvOba2tigvL4ezs/NNt//QQw+1V+iSuLg47Nq1q933Q11XQUEBHBwc4O7u3up6us87P49tpztfa288H+wY58+fh7+/P1QqldyhWCS2T8yjI+6kMPZ8cMuWLQZ3zyqVSvTq1Qu+vr7w9/eHt7c3fH190bdvX/j4+MDb2xt9+/bl0MBklK7QPmlvxrZPsrKyDOZDvZmioiIUFRXB1dUVe/fuxYgRI9oaJpFZsH1y61prn7RYDCGizk+hUOCZZ57BwoULsXTpUri6uiIlJQVJSUm4cOEChBBQKpWwsbFpdjJN4MYQIzwZILIs3bp1w+DBgzF48OAW16mtrUVBQYFUNNEVSIwphhBZA06MSdQ2AwYMkDsEog7l6emJ9evXS8UOb29veHh4cOgrok4oIyOjxWKISqWCRqOBEAJ2dnYIDg7G6NGjER4eji+//BJubm7s+yDqAlgMIeoiIiIi9K5EqaiowKlTp3Dy5EmcOHECx44dw7lz51BXVwcbGxuoVCrU1tYiPT2dJwREnZC9vT38/f3h7+8vdyhEREREFmvSpEmYNGmS3GEQkRno+jRUKhXq6+sB3JhkPTw8HCNGjEB4eDjCwsIQEhICpfK/XaIHDhyQK2Qi6mAshhB1Ua6uroiKikJUVJT0mEajQXp6Ok6cOCFNqllUVCRjlEREREREREREN6fRaDBt2jTcfvvtCA8PR3h4OLy9veUOi4gsCIshRCRRKpUIDQ1FaGgopk+fLnc4RERERERERERG2bNnj9whEJGFM34mZSIiIiIiIiIiIiIiok6IxRAiIiIiIiIiIiIiIrJqLIYQEREREREREREREZFVYzGEiIiIiIiIiIiIiIisGoshRERERERERERERERk1VgMISIiIiIiIiIiIiIiq8ZiCBERERERERERERERWTUWQ4iIiIiIiIiIiIiIyKqxGEJERERERERERERERFZN1mLIDz/8gODgYCiVSjnDsHpr166FQqGAQqGAj4+P3OG0OxcXF+l4dYuNjQ3c3d0RFhaGefPm4ffff5c7TDJSaWkpPvroI4wfPx49evSAo6MjBgwYgOnTp+PkyZNyh2eVmDOYMzo7IQQSExMxf/58BAcHw97eHn369EF0dDS2bt0KIYTcIVod5g3mDWvzwAMPQKFQYOXKlXKHYpWYM5gzOrvo6GiD91S3LFq0SO7wrA5zBnOGtaivr8c777yDyMhIuLq6ok+fPrjnnnuwe/dutlHMjHmDeaMlshRDsrKy8MADD+Cll17ClStX5AihS1m8eDGEEAgLC5M7lA5RWVmJlJQUAMDkyZMhhEB9fT3S0tLwxhtvIC0tDcOGDcMTTzyB6upqmaOlm1myZAkWLFiAyZMnIzU1FVevXsWnn36KEydOIDIyEl9//bXcIVod5gzmjM4uPT0d0dHRyMjIwJdffony8nIcOXIEfn5+mDFjBpYsWSJ3iFaHeYN5w5ps3rwZu3fvljsMq8acwZxBZArmDOYMa1BVVYXx48dj06ZNeOedd1BUVITk5GS4uLjggQcewNmzZ+UO0aowbzBvtESWYsiyZctwxx134Pfff4erq6scIZjExcUF0dHRXXb/1sDW1hYeHh6YPHky9u/fjxdeeAGbNm3Cn/70J1bfO4GZM2di4cKF8PT0hJOTE2JiYrB9+3Y0NDTghRdekDs8A3J/Z+XevzVgzuj8lEoldu7ciaFDh8LBwQGBgYHYtGkTevbsiQ8++AC1tbVyh6hH7u+t3Pu3Bswb1iE/Px+LFi3CjBkz5A6lVXJ/Z+XevzVgzuj8kpKSIIQwWN599125QzMg93dW7v1bA+aMzm/JkiU4deoUfv75Z9x5551wdHSEn58fNm3aBHt7e7nDMyD391bu/VsD5o3myVIM+ec//4mlS5dyeCySzapVqzBy5Eh8++23+OKLL+QOh1qxceNGfPzxxwaPh4WFwdHREVlZWV06iVPHYM7oXAYOHIj6+nq4u7vrPW5nZwdfX1/U1tZCrVbLFB11FcwbndPs2bPx0EMP4a677pI7FOpimDOIyBTMGZ3LlStXsGHDBkyfPh0eHh56v3N2doZarcZtt90mU3TUVTBv3CBLMcTR0VGO3RJJFAoFnn76aQDAhx9+KHM01BZVVVWoqanBbbfdBoVCIXc4ZOWYM6xDWVkZzp8/j4iICLi5uckdDlk55o3O59NPP8XZs2exdu1auUOhLog5g4hMwZzRuXz77bdoaGjgnQ4kK+aNG26pGFJWVmYwOYtukkGNRqP3eFxcnFkCbsnVq1fx3HPPoX///rCzs4O7uzvuueceHDhwQFpn5cqVUjyNE9CePXukx3v16iU9rptsp6qqComJidI6ujtamk7Gk5SUhAkTJsDV1RVOTk4YN24cEhMT223/zR27vb09fHx8MHHiRGzatAk1NTWtvm4ajQY7duzApEmT4OnpCUdHRwwZMgTvvfcetFqt3rq1tbV47bXXMHDgQDg5OaFHjx64//77paRu6npy070HR44cQX19vfR4cXExnnnmGfj7+8POzg69e/fGtGnTcOLECWmdr7/+Wu/znZubi0ceeQTdu3dHz5498Yc//AFZWVl6+zPldTEmhs7InDlj165dAIBXXnmlTbEwZzBnmIo5Qx7myBvXr19HYmIiHnjgAXh6emLz5s1tioV5g3nDVMwbHa+tOePSpUt4/vnn8emnn5ptGF/mDOYMUzFndLxbOc/YsmULwsPD4ezsDDc3N2ko37ZizmDOMBVzhjzakjeOHz8OAHB3d8fzzz8PX19f2NnZoV+/fnjmmWdw7dq1NsXCvMG8YSrmDQCiiR07dohmHm5VbGyssLGxEZmZmQa/Gz16tNi2bVuLz/X29ha2trYm7a+pgoICERAQIDw8PMTu3btFeXm5SE9PF9OmTRMKhUJ88skneus7OzuLqKgog+1ERkaKnj17Gjze0vo6YWFhwtnZWYwePVocOnRIVFZWiqSkJDF06FBhZ2cnfv3113bbv+7YPT09xe7du8X169dFYWGhWLFihQAg3nnnHb04vb299Z6/e/duAUD87W9/E9euXRPFxcXi/fffFzY2NmLx4sV6686aNUu4ubmJn3/+WVRXV4vCwkKxePFiAUAcOHDA5PWM0ZbPoxBCpKSkCABi8uTJLa5TU1MjAAgAIj8/XwghRH5+vujXr5/w8PAQ33//vaioqBBnzpwRY8aMEQ4ODuLQoUN625g8ebK0H917/8svvwhHR0cxfPhwvXWNfV1MjcEYAMSOHTtMfl57bf9WcoYQQhQWFgoPDw8xa9Ysk2MVgjnDmnOGEG37vDNn6IuLixNxcXEmP689t9/WvKH7bAMQY8eOFadOnWpTzMwb1ps32vp5Z974r7aer7Xn9k3NGbGxsWLevHnS/7ds2SIAiBUrVrQpZuYM680ZbJ+wfSKEEFFRUWLGjBni999/F5WVlSItLU3MmDFDABALFiwwOV7mDOvNGUKwfcL2yX9fZ09PTzF9+nSRlZUlSktLxeeffy6cnZ1FcHCwKCsrM2n/zBvWmzfYPmnX9slOsxRDfvrpJwFArwEhhBAJCQnC29tb1NXVtfhccxRDHn/8cQFA/O///q/e42q1WvTt21c4OjqKwsJC6fH2SAAAREpKit7jp06dEgBEWFiYUdtry/51x97cH9a7777bqAQwduxYg+fGx8cLlUolysvLpccCAgLEHXfcYbBucHCw3gfY2PWM0Z6NjerqaoME8NhjjwkABie7BQUFwt7eXkRGRuo9rksAu3fv1ns8Li5OABDFxcXSY8a+LqbGYAxLa2zcSs4oKSkR4eHh4pFHHhEajaZN8TJnWG/OEKL9GhtdKWdYYmPjVvJGbW2tOHfunPjLX/4ibG1txRtvvGFyzMwb1ps32rOx0VXyhiUWQ0zJGRs2bBCBgYGisrJSeuxWiyHMGdabM9g+uXkMxrCm9kljI0aMEADEkSNHTIqXOcN6c4YQbJ/cLAZjdPb2SWxsrAAgAgICRH19vd76K1euFADEsmXLTNo/84b15g22T24ew820ezFECCGGDBkinJycRElJifTY5MmTxapVq1p9njmKIW5ubgKAuH79usHvdFdnfP7559Jj7VUNbU7fvn31PmDm3n9rx95cnE0TQEvWrFkjAOhV3p566ikBQMyePVscPny4xY5oY9czRns2NrKysgQAoVKppD9Sbm5uwsbGRi/x6dx+++0CgLh48aL0mC4BNP4DI4QQzz77rAAgTp48KT1m7OtiagzGsLTGhhBtyxmVlZUiMjJS/PnPf76lzxVzhvXmDCHar7HRlXKGJTY2hGj7uUZjU6dOFQDEL7/8YtK+mTesN2+0Z2Ojq+QNSyyGCGFczsjLyxNubm4GVy/eajGEOcN6cwbbJzePwRjW0j5pavXq1QKAeOWVV0zaN3OG9eYMIdg+uVkMxujs7ZNp06ZJr11TJ0+eFADEiBEjTNo384b15g22T24ew820Vgwx2wTqixYtQnV1tTQBS0ZGBvbv3485c+aYaxfNqq2tRXl5ORwcHJod39fDwwMAUFhY2K5xdO/evdnH+/TpAwAoKioy+z5vduzGKC8vx2uvvYYhQ4bA3d1dGvdtyZIlAIDq6mpp3fXr12Pz5s3Izs7GhAkT0K1bN9x999346quv9LZp7HpyS0hIAACMHj0aKpVKej21Wi3c3NwMxoDUjfF4/vx5g201nYjXzs4OAPTGGjTmdbmVGDobU3OGRqPBQw89BG9vb3z++eewtbVt036ZM5gz2oo5Q37mONe4//77AQDfffed0c9h3mDeaCvmDXkZkzN2796N8vJyjB07Vu91mDFjBgBg2bJl0mOZmZlG7Zc5gzmjrZgz5GWO8wwvLy8Apn2/mDOYM9qKOUN+xuYNf39/AEDPnj0NtqH7jhUXFxu9X+YN5o22Yt64xQnUG5s+fTo8PDzwwQcfoLa2Fm+99RYee+wxuLu7m2sXzbK3t4ebmxvUajUqKioMfn/lyhUAgKenp/SYjY0N6urqDNYtKytrdh8KheKmcVy9ehVCCIPHdV98XSIw5/5vduzGuP/++7FixQrMnj0bGRkZ0Gq1EELgnXfeAQC9Y9I1DPfu3YuysjJ8/fXXEEJg2rRpePvtt01eT05arRbr168HAMyfPx/Ajdeze/fuUCqVqK+vhxCi2WXcuHFt2qcxr0t7x2BJTM0Zc+fORW1tLXbu3Kk3cVZQUBCOHDli9H6ZM5gz2oI5wzKY41zD3t4eAEyapJB5g3mjLZg35GdMzpg/f36zx79lyxYAwIoVK6THgoKCjNovcwZzRlswZ8jPHOcZ+fn5APS/XzfDnMGc0RbMGZbB2Lyhm7S6oKDAYBu675iugGEM5g3mjbZg3rjBbMUQe3t7zJs3D0VFRXjrrbewbds2LFy40Fybb9XUqVMBAN9//73e47W1tdi3bx8cHR0RGxsrPe7l5YXLly/rrVtYWIgLFy40u30nJye9L2xISAg2bNigt45arUZSUpLeY6dPn0Z+fj7CwsKkK0TMvX/dsf/www8Gz4uIiMCzzz7b7DYBoKGhAYmJifD09MQzzzyD3r17S8mmpqbGYP3u3bsjLS0NAKBSqTBp0iR8/fXXUCgUeq+9sevJ6aWXXsKxY8cwdepUPPTQQ9Lj06ZNg0ajQWJiosFz/ud//gd+fn7QaDRt2qexr0t7xmBJTMkZy5cvx9mzZ/HNN99IHZm3gjmDOcNUzBmWwdi8sXjxYsTHxze7jR9//BEAMHz4cJP2zbzBvGEq5g35sX3CnGHqenJizpCfsTlj48aNiIyMNHhcCIGdO3cC+O+dqMZizmDOMBVzhmUwNm/ce++98Pb2xp49e6BWq/V+t3v3bgDAlClTTNo38wbzhqmYN/5DNHErY/4WFxcLR0dHoVAoWh2frDFzzBlSUFAgAgIChIeHh9i9e7e4fv26SE9PF9OmTRMKhUJs2LBBb/2nn35aABDr1q0TFRUVIjMzUzz88MPC29u72XHq7r77buHm5iYuXLggDh06JJRKpUhNTZV+HxYWJtzc3MSECRPEoUOHRGVlpUhKShJDhw4VdnZ2BmMQm3P/umP38vIS3333nbh+/bq4ePGieOqpp4SHh4fIy8vTi7PpOHnjx48XAMTq1atFcXGxqK6uFvv37xd+fn4GY6q7ubmJMWPGiJMnTwq1Wi2uXLkili9fLgCIlStXmryeMcw1Jm9DQ4O4cuWK+Prrr6VjnjlzpqiurtZ73pUrV0T//v1FYGCg+OGHH0RZWZm4evWq+Oijj4STk5PBOJ+6cfJqamr0Hn/xxRcFoD+RlLGvi6kxGAMWOCavEMbljM8++0wAaHU5fPiwSftlzrDenCGEecbk7eo5w1LH5BXCuLzx/PPPC4VCIV5//XWRk5Mj1Gq1yMnJES+88IIAICIjIw3ey5th3rDevGGuMXm7ct6w1DlDhGhb+0SIW58zhDnDenMG2ydsn3zyyScCuDFp8vnz50VNTY1IS0sT06dPFwDEggULTN4vc4b15gwh2D5h++SGH3/8USiVSjF58mSRkZEhSktLxebNm4Wzs7MYOXIk2ycm7N/a8wbbJ+3aPjHfBOo6s2fPFgDEwYMHW1xn9+7dLXZqfvLJJ23ab0lJiVi0aJEICAgQKpVKuLm5idjYWLFv3z6DdcvKysSsWbOEl5eXcHR0FNHR0SIpKUlERkZKcbz44ovS+mlpaSImJkY4OzsLX19fsX79er3t6b5YqampIjY2Vri6ugpHR0cxZswYkZCQ0O77b3rsXl5e4o9//KPIyMgQQvx3AqDGi25Ct+LiYjF37lzh6+srVCqV8PDwEI8//rhYunSptG5kZKQQQogTJ06IuXPnikGDBgknJyfRo0cPMWrUKPHJJ58IrVYrxWPsesZoy+fR2dnZ4HgVCoVwc3MTQ4YMEU899ZT4/fffW3z+1atXxXPPPScCAwOFSqUSvXv3FnfddZdeMjx8+HCLr2nTx++77z6TXxdjYjCFpTY2hLh5zrjvvvvMXgwRgjnDWnOGEKZ/HpkzDFlyY0OIm+eN8vJysXHjRhEbGyv8/f2FnZ2dcHFxEZGRkeLvf/+7yQ0NHeYN68wbbfk8Mm/os+RiiBDGtU905s6d2+y5RmxsrMn7Zc6wzpzB9gnbJ2q1WuzatUtMnTpV9O/fX9jb2ws3NzcxduxYsX379jbHzJxhnTlDCLZP2D75r0OHDonY2Fjh5uYm7OzsxMCBA8Xy5cvZPmHe0MP2Sbu2T3Yq/hOwZOfOnXjkkUeaHfPNGJ999hnWr1+P5OTkNj2/MwoPD0dJSQkuXbokdyhW51Y/j3SDQqHAjh078PDDD1vc9pkzyNza+/PeFehumd21a5dFbp95g8ypvT/vXUF7n6+xfWI65oz2w/aJebB9YlmYM9oX2ye3ju0Ty8O80X7YPrl1rZyv7TLbnCE6H330EZ577jlzb5aIrBRzBhGZinmDiEzBnEFEpmDOICJTMW8QdR63XAzZuHEjpk6disrKSnz00UcoLS1ltZuIWsScQUSmYt4gIlMwZxCRKZgziMhUzBtEnZdZ7gz5+uuv4e7ujn/84x/44osvoFQqzbFZKBSKmy7Lly83y77aYu3atVAoFDh58iQuX74MhUKBV199VbZ4iDoL5gzmDCJTMW8wbxCZgjmDOYPIFMwZzBlEpmLeYN6gzumWv6mzZs3CrFmzzBGLAUsfh3Xx4sVYvHix3GEQdSrMGcwZRKZi3mDeIDIFcwZzBpEpmDOYM4hMxbzBvEGdl9nnDCEiIiIiIiIiIiIiIrIkLIYQEREREREREREREZFVYzGEiIiIiIiIiIiIiIisGoshRERERERERERERERk1VgMISIiIiIiIiIiIiIiq8ZiCBERERERERERERERWTUWQ4iIiIiIiIiIiIiIyKqxGEJERERERERERERERFaNxRAiIiIiIiIiIiIiIrJqLIYQEREREREREREREZFVYzGEiIiIiIiIiIiIiIisGoshRERERERERERERERk1VgMISIiIiIiIiIiIiIiq6Zs6Rc7d+7syDiImnX48GEA/Dx2Brr3isgS8PN4ay5dugQfH5923wdzO1mCS5cuAeC5xq3oqJzL94gsAdsnnQfPB8mS8PN4a9g+oa6E7ZNb12rOFU3s2LFDAODChYuVLTt27Gj6dTcbuY+NCxcu5l/i4uLaLWfExcXJfnxcuHAx/9Je2D7hwsU6F7ZPuHDhYsrC9gkXLlxMXZqxU/GfEwUiIiIiIiIiIiIiIiJrtItzhhARERERERERERERkVVjMYSIiIiIiIiIiIiIiKwaiyFERERERERERERERGTVWAwhIiIiIiIiIiIiIiKr9v9bSfx21Ud2sAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.keras.utils.plot_model(classifier_model, show_shapes=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:37:44.081029Z",
     "iopub.status.busy": "2022-10-30T20:37:44.080636Z",
     "iopub.status.idle": "2022-10-30T20:37:44.137234Z",
     "shell.execute_reply": "2022-10-30T20:37:44.136290Z",
     "shell.execute_reply.started": "2022-10-30T20:37:44.080995Z"
    }
   },
   "outputs": [],
   "source": [
    "AUTOTUNE = tf.data.AUTOTUNE\n",
    "batch_size = 128\n",
    "seed = 42\n",
    "\n",
    "raw_train_ds = tf.data.TextLineDataset(\n",
    "   '../input/feedback-prize-english-language-learning/train.csv',)\n",
    "\n",
    "train_ds = raw_train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
    "\n",
    "val_ds = tf.data.TextLineDataset(\n",
    "    '../input/feedback-prize-english-language-learning/train.csv',)\n",
    "\n",
    "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
    "\n",
    "test_ds = tf.data.TextLineDataset(\n",
    "    '../input/feedback-prize-english-language-learning/test.csv')\n",
    "\n",
    "test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:38:07.162528Z",
     "iopub.status.busy": "2022-10-30T20:38:07.162052Z",
     "iopub.status.idle": "2022-10-30T20:38:07.193553Z",
     "shell.execute_reply": "2022-10-30T20:38:07.192678Z",
     "shell.execute_reply.started": "2022-10-30T20:38:07.162490Z"
    }
   },
   "outputs": [],
   "source": [
    "loss = tf.keras.losses.mean_absolute_error\n",
    "\n",
    "metrics = tf.metrics.Accuracy()\n",
    "\n",
    "epochs = 2\n",
    "steps_per_epoch = tf.data.experimental.cardinality(train_ds).numpy()\n",
    "num_train_steps = steps_per_epoch * epochs\n",
    "num_warmup_steps = int(0.1*num_train_steps)\n",
    "\n",
    "init_lr = 3e-5\n",
    "\"\"\"optimizer = optimization.create_optimizer(init_lr=init_lr,\n",
    "                                          num_train_steps=num_train_steps,\n",
    "                                          num_warmup_steps=num_warmup_steps,\n",
    "                                          \n",
    "                                          optimizer_type='adamw')\"\"\"\n",
    "\n",
    "classifier_model.compile(loss = 'mean_absolute_error', optimizer = 'adam', metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-30T20:38:13.953125Z",
     "iopub.status.busy": "2022-10-30T20:38:13.952476Z",
     "iopub.status.idle": "2022-10-30T20:57:04.625769Z",
     "shell.execute_reply": "2022-10-30T20:57:04.624305Z",
     "shell.execute_reply.started": "2022-10-30T20:38:13.953087Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1\n",
      "Epoch 1/2\n",
      "123/123 [==============================] - 566s 5s/step - loss: 5.6263 - y1_outputclass_loss: 0.7120 - y2_outputclass_loss: 0.7013 - y3_outputclass_loss: 0.6948 - y4_outputclass_loss: 0.6979 - y5_outputclass_loss: 0.7166 - y6_outputclass_loss: 2.1036 - y1_outputclass_accuracy: 0.0024 - y2_outputclass_accuracy: 0.0023 - y3_outputclass_accuracy: 0.0024 - y4_outputclass_accuracy: 0.0024 - y5_outputclass_accuracy: 0.0024 - y6_outputclass_accuracy: 0.0024\n",
      "Epoch 2/2\n",
      "123/123 [==============================] - 565s 5s/step - loss: 5.2695 - y1_outputclass_loss: 0.6397 - y2_outputclass_loss: 0.6323 - y3_outputclass_loss: 0.6293 - y4_outputclass_loss: 0.6392 - y5_outputclass_loss: 0.6254 - y6_outputclass_loss: 2.1036 - y1_outputclass_accuracy: 0.0024 - y2_outputclass_accuracy: 0.0024 - y3_outputclass_accuracy: 0.0024 - y4_outputclass_accuracy: 0.0024 - y5_outputclass_accuracy: 0.0024 - y6_outputclass_accuracy: 0.0024\n"
     ]
    }
   ],
   "source": [
    "X =train['full_text'] \n",
    "y = train.drop(columns=['full_text','text_id'])\n",
    "\n",
    "print(f'Training model with {tfhub_handle_encoder}')\n",
    "history = classifier_model.fit(x=X, y =y, epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-10-28T20:06:27.299003Z",
     "iopub.status.idle": "2022-10-28T20:06:27.299789Z",
     "shell.execute_reply": "2022-10-28T20:06:27.299581Z",
     "shell.execute_reply.started": "2022-10-28T20:06:27.299556Z"
    }
   },
   "outputs": [],
   "source": [
    "ypred =classifier_model.predict([test['full_text']])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}