bert_wongnai.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KJ8wl-4hJ9SE"
   },
   "source": [
    "# Set up"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NtPQrB_UKD68"
   },
   "source": [
    "### > Set GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 306
    },
    "colab_type": "code",
    "id": "t8NQKLo7JGDh",
    "outputId": "a3857d62-1c72-47b9-dba9-d4c298c3bc37"
   },
   "outputs": [],
   "source": [
    "# BERT requires high computation and memory so using free powerful GPU on Colab is going to save a lot of time and money\n",
    "# To enable GPU accelerator: Runtime > change runtime type > under 'hardware accelerator', select GPU > SAVE\n",
    "# Then Colab will reconnect and use this code to check if GPU is enabled. \n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LCi8HRYpKLU_"
   },
   "source": [
    "### > Install and import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 938
    },
    "colab_type": "code",
    "id": "K5ENji7H_h4H",
    "outputId": "819c265a-692d-4082-a06a-100f23a33272",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Since this version of BERT was released before tensorflow 2, \n",
    "# so make sure to use tensorflow-gpu 1.x (took so much longer if run on CPU)\n",
    "!pip install tensorflow-gpu==1.15\n",
    "\n",
    "# Install sentencepiece >> used for tokenizing Thai senetences\n",
    "!pip install sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 173
    },
    "colab_type": "code",
    "id": "NnSKPFWCKOK7",
    "outputId": "cd848976-e235-4bb4-a12c-7fe56b73b544"
   },
   "outputs": [],
   "source": [
    "!pip install gdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Chivm5mIKecX"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KYdknG_bbVhY"
   },
   "source": [
    "# Download required files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jqUZBeVvlIYk"
   },
   "source": [
    "### > Thai Pretrained BERT model - bert_base_th (ThAIKeras)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "bDXjbjZ9dZME",
    "outputId": "41af1085-2ddb-4edf-f8a8-765befbdc92e"
   },
   "outputs": [],
   "source": [
    "!gdown --id 1J3uuXZr_Se_XIFHj7zlTJ-C9wzI9W_ot # use only id in the link https://drive.google.com/uc?id=1J3uuXZr_Se_XIFHj7zlTJ-C9wzI9W_ot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "id": "EQCyOgj5mGKp",
    "outputId": "db69098f-b4fa-4470-b385-a2bbb5483cde"
   },
   "outputs": [],
   "source": [
    "!unzip bert_base_th.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ghw7DcVYlSPm"
   },
   "source": [
    "### > th_wiki_bpe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "no4UzBQXk3HG",
    "outputId": "2d926b25-8ead-4d90-a3fb-8e3cf527ba57"
   },
   "outputs": [],
   "source": [
    "!gdown --id 1F7pCgt3vPlarI9RxKtOZUrC_67KMNQ1W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "etKZMi_2mJ9Q",
    "outputId": "bcd31d26-b23d-4e04-b235-83257b58abab"
   },
   "outputs": [],
   "source": [
    "!mkdir th_wiki_bpe\n",
    "!unzip th_wiki_bpe.zip -d /content/th_wiki_bpe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_Y_HH_LLltdK"
   },
   "source": [
    "### > Wongnai dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "id": "C1ajeb7vLZ5g",
    "outputId": "2bd21256-9f96-475e-e4ed-8381d26cb29b"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/wongnai/wongnai-corpus.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Tlouf4TuA0aZ"
   },
   "outputs": [],
   "source": [
    "os.chdir('wongnai-corpus/review')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "colab_type": "code",
    "id": "mUYDi-LUytaw",
    "outputId": "78062cc6-12b4-4f6d-cc09-4b9749719f2c"
   },
   "outputs": [],
   "source": [
    "!unzip review_dataset.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_5SVtMt6BI31"
   },
   "outputs": [],
   "source": [
    "os.chdir('../..')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xgD4DocxlwSt"
   },
   "source": [
    "### > BERT classifier finetuner modified for Thai\n",
    "https://github.com/KongpolC/bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "1R3-RYV1poNO",
    "outputId": "304767cd-9ba6-44d1-b9c1-c6adc6671601"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/KongpolC/bert.git"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cOthNCVNq-xe"
   },
   "source": [
    "Now you should have these folders in the directory\n",
    "```\n",
    ".\n",
    "|-- bert\n",
    "|-- bert_base_th\n",
    "|-- th_wiki_bpe\n",
    "|-- wongnai-corpus\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EYONKY7-0N3i"
   },
   "source": [
    "# Split \"w_review_train.csv\" into train and test set (90:10)\n",
    "- We have to do so because the given test file from Wongnai does not have label in it.\n",
    "- Change column names to proper names ie. 'review', 'rating'.\n",
    "- Split with stratification to balance each class in train and test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "81okISLG3mGI"
   },
   "outputs": [],
   "source": [
    "def create_data(file_path, test_size=0.1):\n",
    "  '''\n",
    "  Split wongnai training data into train and test set and save them in new folder called \"wongnai_data\"\n",
    "\n",
    "  Arguments:\n",
    "  file_path -- data file to be split.\n",
    "  test_size -- proportion of the test split. Float between 0-1.\n",
    "\n",
    "  Returns:\n",
    "  train -- DataFrame of training data.\n",
    "  test -- DataFrame of test data.\n",
    "  '''\n",
    "\n",
    "  # Preprocess data\n",
    "  data = pd.read_csv(file_path, sep=';')\n",
    "  data = data.append({data.columns[0]:data.columns[0], data.columns[1]:data.columns[1]}, ignore_index=True)\n",
    "  data.columns = ['review', 'rating']\n",
    "\n",
    "  # Split data with stratification to avoid class imbalance\n",
    "  x = data[['review']]\n",
    "  y = data[['rating']]\n",
    "  x_train, x_test, y_train, y_test = train_test_split(x, y, stratify=y, test_size=test_size, random_state=42)\n",
    "\n",
    "  train = x_train.join(y_train)\n",
    "  test = x_test.join(y_test)\n",
    "\n",
    "  # Save to csv files\n",
    "  try:\n",
    "    os.mkdir('wongnai_data')\n",
    "  except:\n",
    "    print('Folder already exists, proceed to saving.')\n",
    "  train.to_csv('wongnai_data/train.csv', index=False) # , header=False)\n",
    "  test.to_csv('wongnai_data/test.csv', index=False) # , header=False)\n",
    "\n",
    "  return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nnanNSMOCaH-"
   },
   "outputs": [],
   "source": [
    "# Run create_data() and save it in new \"wongnai_data\" folder\n",
    "train, test = create_data('/content/wongnai-corpus/review/w_review_train.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mleGELgynTDL"
   },
   "source": [
    "# Verify if all classes in train and test sets are balanced\n",
    "The percentages of every class in train and test set must match.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    },
    "colab_type": "code",
    "id": "u_swZokpnY0V",
    "outputId": "e7830a99-36b1-45ca-b220-17c84530bc90"
   },
   "outputs": [],
   "source": [
    "print('Percentage of 1 star review:\\nTrain:', len(train[train['rating']==1])/len(train), '\\nTest:', len(test[test['rating']==1])/len(test))\n",
    "print('\\nPercentage of 2 star review:\\nTrain:', len(train[train['rating']==2])/len(train), '\\nTest:', len(test[test['rating']==2])/len(test))\n",
    "print('\\nPercentage of 3 star review:\\nTrain:', len(train[train['rating']==3])/len(train), '\\nTest:', len(test[test['rating']==3])/len(test))\n",
    "print('\\nPercentage of 4 star review:\\nTrain:', len(train[train['rating']==4])/len(train), '\\nTest:', len(test[test['rating']==4])/len(test))\n",
    "print('\\nPercentage of 5 star review:\\nTrain:', len(train[train['rating']==5])/len(train), '\\nTest:', len(test[test['rating']==5])/len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "2Y1nHFr1zlBI",
    "outputId": "416cef8e-9658-4598-98fb-c88578f7db7a"
   },
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "TW3khQhrq4sx",
    "outputId": "617b3f51-f2b7-4522-a4c1-3a7ab428403c"
   },
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9JE9rePvz0GJ"
   },
   "source": [
    "# Finetune the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VOiJx7EYaoC7"
   },
   "outputs": [],
   "source": [
    "# Create 'model' directory to store the finetuned model\n",
    "os.mkdir('model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5n4D1WoV6U3w"
   },
   "outputs": [],
   "source": [
    "# Declare path to parse when finetuning\n",
    "os.environ['BPE_DIR'] = '/content/th_wiki_bpe'\n",
    "os.environ['DATA_DIR'] = '/content/wongnai_data'\n",
    "os.environ['OUTPUT_DIR'] = '/content/model'\n",
    "os.environ['BERT_BASE_DIR'] = '/content/bert_base_th'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "guyyQfAxBfbL"
   },
   "source": [
    "If the dataset is larger or your GPU memory is less that it raises OOM >> decrease \"train_batch_size\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UQQ5vJ8Q-kXM"
   },
   "outputs": [],
   "source": [
    "# Run finetuning\n",
    "!python bert/run_classifier.py \\\n",
    "  --task_name=wongnai \\\n",
    "  --do_train=true \\\n",
    "  --do_predict=true \\\n",
    "  --data_dir=$DATA_DIR \\\n",
    "  --vocab_file=$BPE_DIR/th.wiki.bpe.op25000.vocab \\\n",
    "  --bert_config_file=$BERT_BASE_DIR/bert_config.json \\\n",
    "  --init_checkpoint=$BERT_BASE_DIR/model.ckpt \\\n",
    "  --max_seq_length=128 \\\n",
    "  --train_batch_size=16 \\\n",
    "  --learning_rate=5e-5 \\\n",
    "  --num_train_epochs=2.0 \\\n",
    "  --output_dir=$OUTPUT_DIR \\\n",
    "  --spm_file=$BPE_DIR/th.wiki.bpe.op25000.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Colab runtime often disconnect and all created files will be lost. It's a good idea to  download prediction results and the latest model automatically once training is completed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 493
    },
    "colab_type": "code",
    "id": "lcfHa61049PH",
    "outputId": "dc989bab-cf14-44d2-9adb-6810c53df9c7"
   },
   "outputs": [],
   "source": [
    "# Load model and result in case colab reconnect and lost the files\n",
    "from google.colab import files\n",
    "\n",
    "# Download test results\n",
    "files.download('model/test_results.tsv')\n",
    "\n",
    "# Download model with highest index\n",
    "dir = [i for i in os.listdir('model') if 'model.ckpt-' in i]\n",
    "step = 0\n",
    "for name in dir:\n",
    "  new_step = int(name.split('.')[1][5:])\n",
    "  if new_step > step:\n",
    "    step = new_step\n",
    "for name in dir:\n",
    "  if 'model.ckpt-' + str(step) in name:\n",
    "    print('downloading ' + name)\n",
    "    files.download('model/' + name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AUJgEmIyipD_"
   },
   "source": [
    "# Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IDQw9Co6Gizo"
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_true, y_pred, classes, destination=None,\n",
    "                          normalize=False,\n",
    "                          title=None,\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if not title:\n",
    "        if normalize:\n",
    "            title = 'Normalized confusion matrix'\n",
    "        else:\n",
    "            title = 'Confusion matrix, without normalization'\n",
    "\n",
    "    # Compute confusion matrix\n",
    "    #cm = np.array([[ 11, 2 , 40 ,  2], [  0  , 0 ,  0  , 0], [ 37  , 8 ,114  , 4], [ 11 ,  0  ,33 , 1]])\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    # Only use the labels that appear in the data\n",
    "    #classes = classes[unique_labels(y_true, y_pred)]\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    ax.figure.colorbar(im, ax=ax)\n",
    "    # We want to show all ticks...\n",
    "    ax.set(xticks=np.arange(cm.shape[1]),\n",
    "           yticks=np.arange(cm.shape[0]),\n",
    "           # ... and label them with the respective list entries\n",
    "           xticklabels=classes, yticklabels=classes,\n",
    "           title=title,\n",
    "           ylabel='True label',\n",
    "           xlabel='Predicted label')\n",
    "\n",
    "    # Rotate the tick labels and set their alignment.\n",
    "    plt.setp(ax.get_xticklabels(), rotation=0, ha=\"right\",\n",
    "             rotation_mode=\"anchor\")\n",
    "\n",
    "    # Loop over data dimensions and create text annotations.\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i in range(cm.shape[0]):\n",
    "        for j in range(cm.shape[1]):\n",
    "            ax.text(j, i, format(cm[i, j], fmt),\n",
    "                    ha=\"center\", va=\"center\",\n",
    "                    color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "    fig.tight_layout()\n",
    "    #plt.show()\n",
    "    try:\n",
    "        plt.savefig(destination)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xnjkPeu8E2o3"
   },
   "outputs": [],
   "source": [
    "def compare_answer(row):\n",
    "  '''\n",
    "  Compares 2 columns of actual and predicted class. \n",
    "  \n",
    "  Returns:\n",
    "  -- 1 if they are the same\n",
    "  -- 0 otherwise.\n",
    "  '''\n",
    "  \n",
    "  if int(row['prediction']) == row['actual']:\n",
    "    return 1\n",
    "  else:\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0e1viod5iq9W"
   },
   "outputs": [],
   "source": [
    "def benchmark(actual_path, pred_path):\n",
    "  '''\n",
    "  Calculates model accuracy and confusion matrix\n",
    "\n",
    "  Arguments:\n",
    "  actual_path -- path of test file which contains actual labels.\n",
    "  pred_path -- path of prediction file which contains predicted labels.\n",
    "\n",
    "  Returns:\n",
    "  accuracy -- accuracy of the prediction\n",
    "  confusion matrix -- save as image file\n",
    "  '''\n",
    "\n",
    "  # Read files to dataframes\n",
    "  actual = pd.read_csv(actual_path, sep=',')\n",
    "  pred = pd.read_csv(pred_path, sep='\\t')\n",
    "\n",
    "  # Create result dataframe\n",
    "  result = pred.copy()\n",
    "  result['prediction'] = result.idxmax(axis=1).astype(int)\n",
    "  result['actual'] = actual['rating']\n",
    "  result['correct'] = result.apply(lambda row: compare_answer(row), axis=1)\n",
    "\n",
    "  # Calculate accuracy\n",
    "  accuracy = sum(result['correct'])/len(result)\n",
    "\n",
    "  # Confusion Matrix\n",
    "  confusion_matrix = plot_confusion_matrix(result['actual'], result['prediction'], classes=[\"1\",\"2\",\"3\",\"4\",\"5\"], destination=None, normalize=False, title='Confusion Matrix')\n",
    "\n",
    "  return accuracy, result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 399
    },
    "colab_type": "code",
    "id": "MemX_MuZHzlw",
    "outputId": "25d6b662-6539-41a5-eda9-74123494b1b9"
   },
   "outputs": [],
   "source": [
    "# Benchmark accuracy\n",
    "actual_path = 'wongnai_data/test.csv'\n",
    "pred_path = 'model/test_results.tsv'\n",
    "accuracy, result = benchmark(actual_path, pred_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "HF7aCFQbyGhX",
    "outputId": "f7f0691a-5496-46f7-b789-2f1e9725c744"
   },
   "outputs": [],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "NqAOmGT7z8eR",
    "outputId": "b931fac2-b7b8-42c2-ee3b-5efc5d6e3179"
   },
   "outputs": [],
   "source": [
    "result.head()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "bert_wongnai.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}