Test System.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "\n",
    "from IPython.display import clear_output\n",
    "\n",
    "from bot import BertGoggles\n",
    "\n",
    "from logistic import Logistic\n",
    "\n",
    "import pickle as pkl\n",
    "import numpy as np\n",
    "\n",
    "import sklearn as sk\n",
    "from sklearn import linear_model\n",
    "from sklearn import preprocessing as pre\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from squad_test import compute_exact, compute_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## To run with CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.config.experimental.set_visible_devices(devices=[], device_type='GPU')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Index Dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INDEX = 'Index'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_MODEL = 'pretrained/model'\n",
    "BERT_MODEL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CHKPT Dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_CHKPT = 'fine-tuned/checkpoint'\n",
    "BERT_CHKPT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAXLEN = 350"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOGISTIC = 'log_1.pkl'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_goggles = BertGoggles(BERT_MODEL, BERT_CHKPT, MAXLEN, INDEX, logistic_dir=LOGISTIC, top_n=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('squad_val_questions.pkl', 'rb') as file:\n",
    "    questions = pkl.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keep last half of questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = questions[len(questions)//2:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove impossible questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_questions = []\n",
    "\n",
    "for q in questions:\n",
    "    if not q['label']:\n",
    "        new_questions.append(q)\n",
    "        \n",
    "questions = new_questions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anserini Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_anserini_results(anserini, collection, top_n=10):\n",
    "    results = []\n",
    "    not_found = []\n",
    "    times = []\n",
    "    positions = []\n",
    "    \n",
    "    wait_iter = 50\n",
    "\n",
    "    for i, c in enumerate(collection):\n",
    "\n",
    "        question = c['question']\n",
    "\n",
    "        correct_id = c['id']\n",
    "        \n",
    "        start = time.time()\n",
    "        hits = anserini.search(question, top_n=top_n)\n",
    "        times.append(time.time() - start)\n",
    "        \n",
    "        found = False\n",
    "        position = -1\n",
    "        for j, hit in enumerate(hits):\n",
    "            if int(hit.id) == int(correct_id):\n",
    "                found = int(hit.id) == int(correct_id)\n",
    "                position = j\n",
    "                \n",
    "        positions.append(position)\n",
    "\n",
    "        if found:\n",
    "            results.append(hits)\n",
    "        else:\n",
    "            results.append(hits)\n",
    "            not_found.append(i)\n",
    "            \n",
    "        if (i + 1) % wait_iter == 0:\n",
    "            from IPython.display import clear_output\n",
    "            clear_output(wait=True)\n",
    "            print('{0:.2f}%'.format(round(i / len(collection), 4) * 100))\n",
    "            \n",
    "            avg = np.mean(times)\n",
    "            time_left = (len(collection) - i) * avg / 60 \n",
    "            \n",
    "            print('Time remaining: {0:.2f} mins'.format(time_left))\n",
    "            \n",
    "    return results, not_found, positions, times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_results, ans_not_found, ans_positions, ans_times = \\\n",
    "    get_anserini_results(bert_goggles.answerini, questions, top_n=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bert_results(bert, anserini_results, collection):\n",
    "    results = []\n",
    "    not_found = []\n",
    "    times = []\n",
    "    positions = []\n",
    "    \n",
    "    wait_iter = 5\n",
    "\n",
    "    for i, (c, r) in enumerate(zip(collection, anserini_results)):\n",
    "\n",
    "        question = c['question']\n",
    "\n",
    "        correct_id = c['id']\n",
    "        \n",
    "        start = time.time()\n",
    "        hits = bert.search(question, r)\n",
    "        times.append(time.time() - start)\n",
    "        \n",
    "        found = False\n",
    "        position = -1\n",
    "        dict_hits = []\n",
    "        for j, hit in enumerate(hits):\n",
    "            if int(hit.id) == int(correct_id):\n",
    "                found = True\n",
    "                position = j\n",
    "            dict_hits.append(hit._asdict())\n",
    "            \n",
    "        if not found:\n",
    "            not_found.append(i)\n",
    "            \n",
    "        print(len(not_found))\n",
    "                \n",
    "        positions.append(position)\n",
    "\n",
    "        results.append(dict_hits)\n",
    "            \n",
    "        if (i + 1) % wait_iter == 0:\n",
    "            from IPython.display import clear_output\n",
    "            clear_output(wait=True)\n",
    "            print('{0:.2f}%'.format(round(i / len(collection), 4) * 100))\n",
    "            \n",
    "            avg = np.mean(times)\n",
    "            time_left = (len(collection) - i) * avg / 60 \n",
    "            \n",
    "            print('Time remaining: {0:.2f} mins'.format(time_left))\n",
    "            \n",
    "    return results, not_found, positions, times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_results = get_bert_results(bert_goggles.bert_model, ans_results, questions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save BERT results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open('bert_results.pkl', 'wb') as file:\n",
    "    pkl.dump(BERT_results, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load BERT results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('bert_results.pkl', 'rb') as file:\n",
    "    bert_results, bert_not_found, bert_positions, bert_times = pkl.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT Without Impossible Questions Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EM = []\n",
    "F1 = []\n",
    "\n",
    "for br, p, q in zip(bert_results, bert_positions, questions):\n",
    "    \n",
    "    if p == -1:\n",
    "        F1.append(0)\n",
    "        EM.append(0)\n",
    "        continue\n",
    "    \n",
    "    result = br[p]\n",
    "    \n",
    "    golds = q['answers']\n",
    "    answer = result['text']\n",
    "    em = 0\n",
    "    f1 = 0\n",
    "    \n",
    "    for gold in golds:\n",
    "        em = max(em, compute_exact(gold, answer))\n",
    "        new_f1, _ = compute_f1(gold, answer)\n",
    "        f1 = max(f1, new_f1)\n",
    "    \n",
    "    F1.append(f1)\n",
    "    EM.append(em)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F1 Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "round(np.mean(F1), 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EM Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "round(np.mean(EM), 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# System Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rScore(correct_index):\n",
    "    \n",
    "    score = 0\n",
    "    \n",
    "    if correct_index < 10 and correct_index >= 5:\n",
    "        score = 0.1\n",
    "    elif correct_index < 5:\n",
    "        score = 1 - 0.1 * correct_index\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EM and F1 Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EM = []\n",
    "F1 = []\n",
    "\n",
    "for p, q, b in zip(bert_positions, questions, bert_results):\n",
    "\n",
    "    if p == -1:\n",
    "        continue\n",
    "\n",
    "    predicted_answer = b[p]['text']\n",
    "    answers = q['answers']\n",
    "\n",
    "    f1 = 0\n",
    "    em = 0\n",
    "\n",
    "    for ans in answers:\n",
    "        new_f1, _ = compute_f1(ans, predicted_answer)\n",
    "        f1 = max(f1, new_f1)\n",
    "        em = max(em, compute_exact(ans, predicted_answer))\n",
    "\n",
    "    F1.append(rf1)\n",
    "    EM.append(rem)\n",
    "\n",
    "EM = np.asarray(EM)\n",
    "F1 = np.asarray(F1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### rScores for Anserini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_positions = np.asarray(bert_positions)\n",
    "\n",
    "ans_rScores = []\n",
    "\n",
    "bert_positions_present = bert_positions[bert_positions != -1]\n",
    "\n",
    "for pos in bert_positions_present:\n",
    "    ans_rScores.append(rScore(pos))\n",
    "\n",
    "np.sum(ans_rScores) / len(bert_positions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_input = []\n",
    "\n",
    "for i, (bert, ans) in enumerate(zip(bert_results, ans_results)):\n",
    "\n",
    "    if i in ans_not_found:\n",
    "        continue\n",
    "\n",
    "    content_score = []\n",
    "    for b, a in zip(bert, ans):\n",
    "        content_score.append([a.score, b['score'], b['null_score']])\n",
    "\n",
    "    log_input.append(content_score)\n",
    "    \n",
    "log_input = np.asarray(log_input)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_sorted = []\n",
    "\n",
    "for s in log_input:\n",
    "    \n",
    "    log_sorted.append(np.argsort(bert_goggles.logistic.score(s)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_ranks = []\n",
    "bert_positions_present = bert_positions[bert_positions != -1]\n",
    "\n",
    "for bp, ls in zip(bert_positions_present, log_sorted):\n",
    "    log_ranks.append(np.where(ls == bp)[0][0])\n",
    "\n",
    "log_ranks = np.asarray(log_ranks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Improve Positions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(bert_positions_present - log_ranks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### System rScore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "system_rScore = []\n",
    "\n",
    "for pos in log_ranks:\n",
    "    system_rScore.append(rScore(pos))\n",
    "\n",
    "np.sum(system_rScore) / len(bert_positions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### System REM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(np.asarray(EM) * np.asarray(system_rScore)) / len(bert_positions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### System RF1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(np.asarray(F1) * np.asarray(system_rScore)) / len(bert_positions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### System Latency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean(bert_times) + np.mean(ans_times)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "goggles",
   "language": "python",
   "name": "goggles"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}