test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f7047707-008f-451f-bede-c88593a50a27",
   "metadata": {},
   "source": [
    "## Test with a small data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "edf455b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "resume = 'data/checkpoints/0906_214036/model_best.pth'\n",
    "resume = 'data/checkpoints/0906_214036/checkpoint.pth'\n",
    "resume = 'data/checkpoints/test_checkpoint/checkpoint.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6bc2b063",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import torch\n",
    "import argparse\n",
    "from tqdm import tqdm\n",
    "from tabulate import tabulate\n",
    "\n",
    "from utils.unique import unique\n",
    "import model.loss as module_loss\n",
    "import model.model as module_arch\n",
    "import model.metric as module_metric\n",
    "from parse_config import ConfigParser\n",
    "import data_loader.data_loaders as module_data\n",
    "\n",
    "PAD = '<pad>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a3767500-7409-4c14-ba53-b9251180a03a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load:: config, sample data, and checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7614cbad",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading checkpoint: data/checkpoints/test_checkpoint/checkpoint.pth ...\n"
     ]
    }
   ],
   "source": [
    "args = argparse.ArgumentParser(description='PyTorch Template')\n",
    "args.add_argument('-c', '--config', default=None, type=str, help='config file path (default: None)')\n",
    "args.add_argument('-r', '--resume', default=f\"{resume}\", type=str, help='path to latest checkpoint (default: None)')\n",
    "args.add_argument('-d', '--device', default=None, type=str, help='indices of GPUs to enable (default: all)')\n",
    "args.add_argument('-f', '--file', default=None, type=str, help='Error')\n",
    "\n",
    "config = ConfigParser.from_args(args)\n",
    "logger = config.get_logger('test')\n",
    "\n",
    "data_loader = config.init_obj('data_loader', module_data)\n",
    "test_data_loader = data_loader.get_test()\n",
    "\n",
    "# build model architecturea\n",
    "model = config.init_obj('arch', module_arch)\n",
    "# logger.info(model)\n",
    "\n",
    "# get function handles of loss and metrics\n",
    "criterion = getattr(module_loss, config['loss'])\n",
    "metric_fns = [getattr(module_metric, met) for met in config['metrics']]\n",
    "\n",
    "logger.info('Loading checkpoint: {} ...'.format(config.resume))\n",
    "checkpoint = torch.load(config.resume)\n",
    "state_dict = checkpoint['state_dict']\n",
    "if config['n_gpu'] > 1:\n",
    "    model = torch.nn.DataParallel(model)\n",
    "model.load_state_dict(state_dict)\n",
    "layers_train = config._config['trainer']['layers_train']\n",
    "\n",
    "# prepare model for testing\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model = model.to(device)\n",
    "model.eval()\n",
    "\n",
    "total_loss = 0.0\n",
    "total_metrics = torch.zeros(len(metric_fns))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2e7ede0-364e-4ea2-ab70-0e966055f333",
   "metadata": {},
   "source": [
    "# Making a prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3bf27ba5-acbd-4241-8cdb-b2437e5abe8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s>|เมื่อ|วันที่||29||ก|.|ค|.||55||สํานักข่าว||บีบีซี||รายงาน||ว่า||มหาวิทยาลั|ย||ออก||ซ์||ฟอร์ด||ของ||อังกฤษ||ได้||แก้ไข||กฎ|การ||แต่งกาย||ภายใน||มหาวิทยาลั|ย|หลัง||ชมรม|เพื่อ|ความ|หลากหลาย||ทางเพศ||ย|ื่นคําร้อง||ว่า||กฎ||ที่|มี||อยู่||เดิม|ไม่||เป็นธรรม||กับ|กลุ่ม||นศ||ข้าม||เพศ||trans|g|ender||ใน||มหาวิทยาลั|ย|</s> \n",
      "\n",
      "[2, 5]         day            วันที่29\n",
      "[2, 12]        date           วันที่29ก.ค.55\n",
      "[3, 5]         cardinal       29\n",
      "[5, 10]        month          ก.ค.\n",
      "[10, 12]       year           55\n",
      "[12, 16]       media          สํานักข่าวบีบีซี\n",
      "[14, 16]       media          บีบีซี\n",
      "[20, 29]       org_edu        มหาวิทยาลัยออกซ์ฟอร์ด\n",
      "[23, 29]       org_edu        ออกซ์ฟอร์ด\n",
      "[31, 33]       country        อังกฤษ\n",
      "[81, 85]       norp_others    transgender\n"
     ]
    }
   ],
   "source": [
    "from utils.prediction import predict, get_dict_prediction, show\n",
    "\n",
    "text = \"\"\"\n",
    "เมื่อวันที่29ก.ค.55 สำนักข่าวบีบีซีรายงานว่า มหาวิทยาลัยออกซ์ฟอร์ดของอังกฤษได้แก้ไขกฎการแต่งกายภายในมหาวิทยาลัย หลังชมรมเพื่อความหลากหลายทางเพศยื่นคำร้องว่ากฎที่มีอยู่เดิมไม่เป็นธรรมกับกลุ่มนศข้ามเพศtransgenderใน\n",
    "มหาวิทยาลัย\n",
    "\"\"\"\n",
    "\n",
    "tokens, out = predict(model, text, data_loader, config)\n",
    "tokens = [tk for tk in tokens if tk!=PAD]\n",
    "\n",
    "print(\"|\".join(tokens), \"\\n\")\n",
    "[show(x) for x in out];"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9e4159b-77d1-40a9-924e-dda58a460333",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Test with a small data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9ce78f40",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]/home/weerayut/anaconda3/envs/workspace/lib/python3.6/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/weerayut/anaconda3/envs/workspace/lib/python3.6/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "5it [00:34,  6.92s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "results = []\n",
    "with torch.no_grad():\n",
    "    for batch_idx, instance in tqdm(enumerate(test_data_loader)):\n",
    "        input_ids = torch.tensor(instance['input_ids']).to(device)\n",
    "        attention_mask = torch.tensor(instance['attention_mask']).to(device)\n",
    "        batch_size = input_ids.shape[0]\n",
    "        output = model(input_ids, attention_mask)\n",
    "        \n",
    "        loss = 0\n",
    "        nested_lm_conll_ids = {l:None for l in range(len(layers_train))}\n",
    "        for index, layer in enumerate(layers_train):\n",
    "            temp_nested_lm_conll_ids = torch.tensor(instance['nested_lm_conll_ids'][layer])\n",
    "            temp_nested_lm_conll_ids = temp_nested_lm_conll_ids.to(device)\n",
    "            nested_lm_conll_ids[index]=temp_nested_lm_conll_ids\n",
    "            loss+=criterion(output[index], temp_nested_lm_conll_ids)\n",
    "            \n",
    "        total_loss += loss.item() * batch_size\n",
    "        predictions = {x:[] for x in range(batch_size)}\n",
    "        lm_entities = {x:[] for x in range(batch_size)}\n",
    "        for sent_ids in range(batch_size):\n",
    "            for layer in range(len(output)):\n",
    "                predictions[sent_ids].append(output[layer][sent_ids].argmax(axis=0))\n",
    "                lm_entities[sent_ids].append(nested_lm_conll_ids[layer][sent_ids])\n",
    "                \n",
    "        for sent_ids in range(batch_size):\n",
    "            tokens = instance['lm_tokens'][sent_ids]\n",
    "            tokens = [w for w in tokens if w!=PAD]\n",
    "            preds = []\n",
    "            for index in range(len(layers_train)):\n",
    "                preds+=get_dict_prediction(\n",
    "                        tokens, \n",
    "                        predictions[sent_ids][index], \n",
    "                        attention_mask[sent_ids], \n",
    "                        data_loader.ids2tag)\n",
    "            entities_labels = []\n",
    "            \n",
    "            for index in range(len(layers_train)):\n",
    "                entities_labels+=get_dict_prediction(\n",
    "                    tokens, \n",
    "                    lm_entities[sent_ids][index], \n",
    "                    attention_mask[sent_ids], \n",
    "                    data_loader.ids2tag)\n",
    "            results.append({\n",
    "                'sentence_id': instance['sentence_id'][sent_ids],\n",
    "                'tokens': tokens,\n",
    "                'entities': entities_labels,\n",
    "                'predictions':preds})\n",
    "            for i, metric in enumerate(metric_fns):\n",
    "                total_metrics[i] += metric( \n",
    "                        output, nested_lm_conll_ids, attention_mask, \n",
    "                        data_loader.boundary_type, info=False, ids2tag=data_loader.ids2tag\n",
    "            ) * batch_size     "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2114f1b2-8ece-46d3-82ec-1d7fb36e65b6",
   "metadata": {},
   "source": [
    "## Results "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7a03f91d-57d1-4389-ac13-eb6b6be2fb09",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " <s>||ความคืบหน้|า|หลัง||ศาลปกครอง||กลาง|มี||คําสั่ง|ไม่|รับ||คําฟ้อง|และ|ไม่||คุ้มครอง||ชั่วคราว||ใน||คดี||ที่||ผู้ตรวจการ|แผ่นดิน||ยื่นฟ้อง||ต่อ||ศาลปกครอง||ว่า||_||สํานักงาน||คณะกรรมการ||กิจการ||กระจาย|เสียง||_||กิจการ||โทรทัศน์|และ||กิจการ||โทรคมนาคม||แห่งชาติ||_||กสทช|.||_||จัด||การประมูล||คลื่น|ความถี่||_||ย่าน||_||2.1||<unk>|z||_|พร้อมทั้ง||ออก||ประกาศ||_|เรื่อง||_||หลักเกณฑ์|และ||วิธี|การ|อนุญาต||ให้|ใช้||คลื่น|ความถี่||ดังกล่าว|โดย|ไม่|ชอบ|ด้วยกฎหมาย||_||ด้วย||เหตุผล||ว่า||_||ผู้ตรวจการ|แผ่นดิน||_|ไม่|มี||สิทธิ|และ||หน้าที่||_||เสมือน|หนึ่ง|ผู้||ฟ้องคดี||_|เมื่อ|วันที่||_||3||_||ธ|.|ค|.||55||_|ล่าสุด||_||3||_||ม|.|ค|.||56||_||ผู้ตรวจการ|แผ่นดิน||ได้||ยื่น||อุทธรณ์||คําสั่ง|ไม่|รับ||คําฟ้อง||ของ||ศาลปกครอง||กลาง|แล้ว|</s> \n",
      "\n",
      "Predictions\n",
      "['SID:0, Index[5, 9]', 'X', 'ศาลปกครองกลาง', 'goverment']\n",
      "['SID:0, Index[35, 37]', '/', 'ศาลปกครอง', 'goverment']\n",
      "['SID:0, Index[41, 63]', '/', 'สํานักงานคณะกรรมการกิจการกระจายเสียง_กิจการโทรทัศน์และกิจการโทรคมนาคมแห่งชาติ', 'goverment']\n",
      "['SID:0, Index[65, 68]', '/', 'กสทช.', 'goverment']\n",
      "['SID:0, Index[152, 166]', '/', 'วันที่_3_ธ.ค.55', 'date']\n",
      "['SID:0, Index[171, 182]', '/', '3_ม.ค.56', 'date']\n",
      "['SID:0, Index[201, 205]', '/', 'ศาลปกครองกลาง', 'goverment']\n",
      "['SID:0, Index[152, 157]', '/', 'วันที่_3', 'day']\n",
      "['SID:0, Index[159, 164]', '/', 'ธ.ค.', 'month']\n",
      "['SID:0, Index[164, 166]', '/', '55', 'year']\n",
      "['SID:0, Index[171, 173]', '/', '3', 'day']\n",
      "['SID:0, Index[175, 180]', '/', 'ม.ค.', 'month']\n",
      "['SID:0, Index[180, 182]', '/', '56', 'year']\n",
      "['SID:0, Index[155, 157]', '/', '3', 'cardinal']\n",
      "Not answer\n",
      "['SID:0, Index[5, 9]', 'X', 'ศาลปกครองกลาง', 'org_other']\n",
      "['SID:0, Index[28, 31]', 'X', 'ผู้ตรวจการแผ่นดิน', 'role']\n",
      "['SID:0, Index[41, 68]', 'X', 'สํานักงานคณะกรรมการกิจการกระจายเสียง_กิจการโทรทัศน์และกิจการโทรคมนาคมแห่งชาติ_กสทช.', 'goverment']\n",
      "['SID:0, Index[83, 88]', 'X', '2.1<unk>z', 'quantity']\n",
      "['SID:0, Index[129, 132]', 'X', 'ผู้ตรวจการแผ่นดิน', 'role']\n",
      "['SID:0, Index[184, 187]', 'X', 'ผู้ตรวจการแผ่นดิน', 'role']\n",
      "['SID:0, Index[83, 85]', 'X', '2.1', 'cardinal']\n",
      "['SID:0, Index[85, 88]', 'X', '<unk>z', 'unit']\n",
      "['SID:0, Index[43, 63]', 'X', 'คณะกรรมการกิจการกระจายเสียง_กิจการโทรทัศน์และกิจการโทรคมนาคมแห่งชาติ', 'role']\n",
      "\n",
      " <s>|โดย|เว็บไซต์||เนชั่น||ทัน|ข่าว||_|[||1||]||รายงาน||ว่า||_|รัก||ษ||เก|ช|า||_||แฉ|่||ฉาย||_||โฆษก||สํานักงาน||ผู้ตรวจการ|แผ่นดิน||_||แถลง||ว่า||_||ผู้ตรวจการ|แผ่นดิน||ได้||ยื่น||อุทธรณ์||คําสั่ง|ไม่|รับ||คําฟ้อง||ของ||ศาลปกครอง||กลาง|กรณี|ขอให้||เพิกถอน|การ||จัด||ประมูล||คลื่น|ความถี่||_||3||<unk>||_||ต่อ||ศาลปกครอง||สูงสุด|แล้ว||_|โดย||ยื่น||ทาง|ไปรษณีย์||ไป|เมื่อ|วันที่||_||28||_||ธ|.|ค|.||_||2555||_|และ||ตรวจสอบ|แล้ว||ว่า||คําร้อง||อุทธรณ์||ส่ง||ถึง||สํานักงาน||ศาลปกครอง|แล้ว|เมื่อ|ช่วง||บ่าย||ของ|วันที่||_||2||_||ม|.|ค|.||_||2556||_||ที่ผ่านมา||_|ทั้งนี้||_|การ||ยื่น||อุทธรณ์||ดังกล่าว|เป็นการ||ยื่น||ใน||ประเด็น||เดิม|โดย|ผู้|ตรวจ||ฯ|</s> \n",
      "\n",
      "Predictions\n",
      "['SID:1, Index[3, 8]', '/', 'เนชั่นทันข่าว', 'media']\n",
      "['SID:1, Index[11, 13]', '/', '1', 'cardinal']\n",
      "['SID:1, Index[37, 44]', '/', 'โฆษกสํานักงานผู้ตรวจการแผ่นดิน', 'role']\n",
      "['SID:1, Index[95, 99]', 'X', 'ศาลปกครองสูงสุด', 'facility_other']\n",
      "['SID:1, Index[111, 127]', '/', 'วันที่_28_ธ.ค._2555', 'date']\n",
      "['SID:1, Index[154, 170]', '/', 'วันที่_2_ม.ค._2556', 'date']\n",
      "['SID:1, Index[3, 5]', '/', 'เนชั่น', 'media']\n",
      "['SID:1, Index[30, 35]', '/', 'แฉ่ฉาย', 'last']\n",
      "['SID:1, Index[39, 44]', '/', 'สํานักงานผู้ตรวจการแผ่นดิน', 'goverment']\n",
      "['SID:1, Index[111, 116]', '/', 'วันที่_28', 'day']\n",
      "['SID:1, Index[118, 123]', '/', 'ธ.ค.', 'month']\n",
      "['SID:1, Index[125, 127]', '/', '2555', 'year']\n",
      "['SID:1, Index[154, 159]', '/', 'วันที่_2', 'day']\n",
      "['SID:1, Index[161, 166]', '/', 'ม.ค.', 'month']\n",
      "['SID:1, Index[168, 170]', '/', '2556', 'year']\n",
      "['SID:1, Index[114, 116]', 'X', '28', 'cardinal']\n",
      "['SID:1, Index[157, 159]', '/', '2', 'cardinal']\n",
      "Not answer\n",
      "['SID:1, Index[21, 35]', 'X', 'รักษเกชา_แฉ่ฉาย', 'person']\n",
      "['SID:1, Index[52, 55]', 'X', 'ผู้ตรวจการแผ่นดิน', 'role']\n",
      "['SID:1, Index[69, 73]', 'X', 'ศาลปกครองกลาง', 'goverment']\n",
      "['SID:1, Index[87, 91]', 'X', '3<unk>', 'quantity']\n",
      "['SID:1, Index[95, 99]', 'X', 'ศาลปกครองสูงสุด', 'goverment']\n",
      "['SID:1, Index[143, 147]', 'X', 'สํานักงานศาลปกครอง', 'goverment']\n",
      "['SID:1, Index[21, 28]', 'X', 'รักษเกชา', 'firstname']\n",
      "['SID:1, Index[87, 89]', 'X', '3', 'cardinal']\n",
      "['SID:1, Index[145, 147]', 'X', 'ศาลปกครอง', 'facility_other']\n"
     ]
    }
   ],
   "source": [
    "for ids, item in enumerate(results[:2]):\n",
    "    print('\\n',\"|\".join(item[\"tokens\"]),'\\n')\n",
    "    sid = item['sentence_id']\n",
    "    entities=item['entities']\n",
    "    predictions=item['predictions']\n",
    "    \n",
    "    print(\"Predictions\")\n",
    "    for pred in predictions:\n",
    "        tag = \"/\" if pred in entities else \"X\"\n",
    "        out = [f\"SID:{sid}, Index{pred['span']}\", tag, ''.join(pred['text']), pred['entity_type']]\n",
    "        print(out)\n",
    "    \n",
    "    print(\"Not answer\")\n",
    "    for en in entities:\n",
    "        if en not in predictions:\n",
    "            out = [f\"SID:{sid}, Index{en['span']}\", \"X\", \"\".join(en['text']), en['entity_type']]\n",
    "            print(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7ec1ca29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculate F1-score based on.\n",
      "labels_true: 1139\n",
      "num_labels: 1670\n",
      "predictions_true: 1139\n",
      "num_predictions: 1294\n",
      "\n",
      "<<< Results Evaluations >>>\n",
      "\n",
      "-------------  ---------  -------  -------  ----------------  ---------------  ----------\n",
      "group/n.class  precision  recall   f1       predictions_true  num_predictions  num_labels\n",
      "group 0: 20    89.7507    80.0     84.5953  972               1083             1215\n",
      "group 1: 19    81.8182    53.1496  64.4391  135               165              254\n",
      "group 2: 65    69.5652    15.9204  25.9109  32                46               201\n",
      "-------------  ---------  -------  -------  ----------------  ---------------  ----------\n",
      "----------------  ---------  -------  --------  ----------  ---------  ----------\n",
      "tag               precision  recall   f1-score  preds_true  num_preds  num_labels\n",
      "total             88.0216    68.2036  76.8556   1139        1294       1670\n",
      "cardinal          88.8158    78.9474  83.5913   135         152        171\n",
      "country           94.5652    87.8788  91.0995   87          92         99\n",
      "person            92.6829    87.3563  89.9408   76          82         87\n",
      "unit              94.7368    84.7059  89.441    72          76         85\n",
      "goverment         83.3333    64.1026  72.4638   50          60         78\n",
      "role              87.0968    36.4865  51.4286   27          31         74\n",
      "firstname         80.0       83.3333  81.6327   60          75         72\n",
      "month             100.0      98.2759  99.1304   57          57         58\n",
      "province          98.1818    93.1034  95.5752   54          55         58\n",
      "last              90.566     84.2105  87.2727   48          53         57\n",
      "date              96.0       90.566   93.2039   48          50         53\n",
      "title             91.0714    98.0769  94.4444   51          56         52\n",
      "day               89.0909    94.2308  91.5888   49          55         52\n",
      "quantity          94.7368    78.2609  85.7143   36          38         46\n",
      "year              95.6522    100.0    97.7778   44          46         44\n",
      "district          91.1765    81.5789  86.1111   31          34         38\n",
      "loc_others        69.2308    29.0323  40.9091   9           13         31\n",
      "orgcorp           87.5       70.0     77.7778   21          24         30\n",
      "facility_other    36.3636    13.3333  19.5122   4           11         30\n",
      "media             89.6552    89.6552  89.6552   26          29         29\n",
      "org_other         34.7826    32.0     33.3333   8           23         25\n",
      "org_edu           91.6667    55.0     68.75     11          12         20\n",
      "norp_political    60.0       15.0     24.0      3           5          20\n",
      "disease           100.0      57.8947  73.3333   11          11         19\n",
      "product_food      0          0        0         0           1          18\n",
      "time              80.0       94.1176  86.4865   16          20         17\n",
      "duration          100.0      70.5882  82.7586   12          12         17\n",
      "rel               90.0       60.0     72.0      9           10         15\n",
      "city              72.2222    100.0    83.871    13          18         13\n",
      "nationality       100.0      15.3846  26.6667   2           2          13\n",
      "event_others      25.0       7.6923   11.7647   1           4          13\n",
      "nickname          100.0      8.3333   15.3846   1           1          12\n",
      "mult              88.8889    88.8889  88.8889   8           9          9\n",
      "continent         100.0      33.3333  50.0      3           3          9\n",
      "state             100.0      22.2222  36.3636   2           2          9\n",
      "norp_others       0          0        0         0           0          9\n",
      "money             100.0      100.0    100.0     8           8          8\n",
      "sub_district      72.7273    100.0    84.2105   8           11         8\n",
      "roadname          100.0      57.1429  72.7273   4           4          7\n",
      "vehicle           0          0        0         0           0          7\n",
      "org_political     66.6667    66.6667  66.6667   4           6          6\n",
      "tv_show           100.0      50.0     66.6667   3           3          6\n",
      "weapon            0          0        0         0           1          6\n",
      "song              100.0      60.0     75.0      3           3          5\n",
      "restaurant        100.0      40.0     57.1429   2           2          5\n",
      "hotel             50.0       20.0     28.5714   1           2          5\n",
      "fund              0          0        0         0           0          5\n",
      "animal_species    0          0        0         0           0          5\n",
      "island            100.0      75.0     85.7143   3           3          4\n",
      "bridge            100.0      75.0     85.7143   3           3          4\n",
      "museum            0          0        0         0           0          4\n",
      "natural_disaster  0          0        0         0           3          4\n",
      "stock_exchange    0          0        0         0           0          4\n",
      "station           0          0        0         0           0          4\n",
      "book              0          0        0         0           2          4\n",
      "sports_team       0          0        0         0           0          4\n",
      "space             0          0        0         0           0          4\n",
      "jargon            100.0      100.0    100.0     3           3          3\n",
      "percent           100.0      66.6667  80.0      2           2          3\n",
      "film              0          0        0         0           0          3\n",
      "product_drug      0          0        0         0           0          3\n",
      "address           0          0        0         0           1          3\n",
      "distance          0          0        0         0           1          3\n",
      "language          100.0      100.0    100.0     2           2          2\n",
      "airport           100.0      100.0    100.0     2           2          2\n",
      "law               100.0      100.0    100.0     2           2          2\n",
      "psudoname         100.0      50.0     66.6667   1           1          2\n",
      "river             100.0      50.0     66.6667   1           1          2\n",
      "building          100.0      50.0     66.6667   1           1          2\n",
      "war               0          0        0         0           0          2\n",
      "longtitude        0          0        0         0           0          2\n",
      "sciname           0          0        0         0           2          2\n",
      "mountian          0          0        0         0           0          2\n",
      "food_ingredient   0          0        0         0           0          2\n",
      "band              0          0        0         0           0          2\n",
      "religion          0          0        0         0           0          2\n",
      "namemod           0          0        0         0           0          2\n",
      "latitude          0          0        0         0           0          2\n",
      "periodic          0          0        0         0           1          2\n",
      "season            0          0        0         0           0          2\n",
      "temperature       0          0        0         0           0          2\n",
      "award             0          0        0         0           0          2\n",
      "org_religious     0          0        0         0           0          2\n",
      "hospital          100.0      100.0    100.0     1           1          1\n",
      "ocean             0          0        0         0           0          1\n",
      "concert           0          0        0         0           0          1\n",
      "sports_event      0          0        0         0           0          1\n",
      "army              0          0        0         0           0          1\n",
      "god               0          0        0         0           0          1\n",
      "speed             0          0        0         0           0          1\n",
      "weight            0          0        0         0           0          1\n",
      "game              0          0        0         0           0          1\n",
      "nicknametitle     0          0        0         0           0          1\n",
      "middle            0          0        0         0           0          1\n",
      "index             0          0        0         0           0          1\n",
      "energy            0          0        0         0           1          1\n",
      "port              0          0        0         0           0          1\n",
      "postcode          0          0        0         0           0          1\n",
      "animate           0          0        0         0           0          1\n",
      "stadium           0          0        0         0           0          1\n",
      "woa               0          0        0         0           0          1\n",
      "soi               0          0        0         0           0          1\n",
      "fold              0          0        0         0           0          1\n",
      "electronics       0          0        0         0           0          1\n",
      "----------------  ---------  -------  --------  ----------  ---------  ----------\n"
     ]
    }
   ],
   "source": [
    "## Can input both BIESO and BIO\n",
    "from model.eval import ClassEvaluator\n",
    "results_eval, conll_results = ClassEvaluator()(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f842504b-3412-4d27-b327-326ba8b24ebf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77d9b1a7-d186-4820-9dbd-3691b14b24b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:workspace] *",
   "language": "python",
   "name": "conda-env-workspace-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}