compute_acc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import json\n",
    "\n",
    "split = 'test'\n",
    "results = json.load(open('saved_model/%s.json'%split))\n",
    "dataset = json.load(open('/home/liqing/Desktop/VizWiz_new/data/Annotations/%s.json'%split))\n",
    "img2gt = {x['image']:x['answers'] for x in dataset}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8000\n",
      "other : 0.5885\n",
      "number : 0.0175\n",
      "unanswerable : 0.345125\n",
      "yes/no : 0.048875\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "img2ans_type = {}\n",
    "for one_data in dataset:\n",
    "    ans_counter = Counter([x['answer'] for x in one_data['answers']])\n",
    "    ans = ans_counter.most_common(1)[0][0]\n",
    "    if ans == 'yes' or ans == 'no':\n",
    "        ans_type = 'yes/no'\n",
    "    elif ans == 'unanswerable' or ans == 'unsuitable':\n",
    "        ans_type = 'unanswerable'\n",
    "    elif ans.isdigit():\n",
    "        ans_type = 'number'\n",
    "    else:\n",
    "        ans_type = 'other'\n",
    "    img2ans_type[one_data['image']] = ans_type\n",
    "    \n",
    "all_ans = img2ans_type.values()\n",
    "print len(all_ans)\n",
    "for ans_type in set(all_ans):\n",
    "    print ans_type, ':', all_ans.count(ans_type)*1.0/len(all_ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.47804166666666664\n",
      "other : 0.28412630982724435\n",
      "number : 0.20952380952380953\n",
      "unanswerable : 0.8014004587709767\n",
      "yes/no : 0.6257459505541346\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "img2acc = {}\n",
    "for pred in results:\n",
    "    img = pred['image']\n",
    "    pred_ans = pred['answer']\n",
    "    gt_ans = img2gt[img]\n",
    "    gt_ans = [x['answer'] for x in gt_ans]\n",
    "    gt_ans = [x.lower() for x in gt_ans]\n",
    "    cur_acc = np.minimum(1.0, gt_ans.count(pred_ans)/3.0)\n",
    "    img2acc[img] = cur_acc\n",
    "\n",
    "print 'Accuracy :', np.mean(img2acc.values())\n",
    "for ans_type in set(all_ans):\n",
    "    acc_per_type = np.mean([acc for img, acc in img2acc.items() if img2ans_type[img] == ans_type])\n",
    "    print ans_type, ':', acc_per_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer\n",
    "from pycocoevalcap.bleu.bleu import Bleu\n",
    "from pycocoevalcap.meteor.meteor import Meteor\n",
    "from pycocoevalcap.rouge.rouge import Rouge\n",
    "from pycocoevalcap.cider.cider import Cider\n",
    "\n",
    "class COCOEvalCap:\n",
    "    def __init__(self,images,gts,res):\n",
    "        self.evalImgs = []\n",
    "        self.eval = {}\n",
    "        self.imgToEval = {}\n",
    "        self.params = {'image_id': images}\n",
    "        self.gts = gts\n",
    "        self.res = res\n",
    "\n",
    "    def evaluate(self):\n",
    "        imgIds = self.params['image_id']\n",
    "        gts = self.gts\n",
    "        res = self.res\n",
    "\n",
    "        # =================================================\n",
    "        # Set up scorers\n",
    "        # =================================================\n",
    "        print 'tokenization...'\n",
    "        tokenizer = PTBTokenizer()\n",
    "        gts  = tokenizer.tokenize(gts)\n",
    "        res = tokenizer.tokenize(res)\n",
    "\n",
    "        # =================================================\n",
    "        # Set up scorers\n",
    "        # =================================================\n",
    "        print 'setting up scorers...'\n",
    "        scorers = [\n",
    "            (Bleu(4), [\"Bleu_1\", \"Bleu_2\", \"Bleu_3\", \"Bleu_4\"]),\n",
    "            (Meteor(),\"METEOR\"),\n",
    "            (Rouge(), \"ROUGE_L\"),\n",
    "            (Cider(), \"CIDEr\")\n",
    "        ]\n",
    "\n",
    "        # =================================================\n",
    "        # Compute scores\n",
    "        # =================================================\n",
    "        eval = {}\n",
    "        for scorer, method in scorers:\n",
    "            print 'computing %s score...'%(scorer.method())\n",
    "            assert(set(gts.keys()) == set(res.keys()))\n",
    "            score, scores = scorer.compute_score(gts, res)\n",
    "            if type(method) == list:\n",
    "                for sc, scs, m in zip(score, scores, method):\n",
    "                    self.setEval(sc, m)\n",
    "                    self.setImgToEvalImgs(scs, imgIds, m)\n",
    "                    print \"%s: %0.3f\"%(m, sc)\n",
    "            else:\n",
    "                self.setEval(score, method)\n",
    "                self.setImgToEvalImgs(scores, imgIds, method)\n",
    "                print \"%s: %0.3f\"%(method, score)\n",
    "        self.setEvalImgs()\n",
    "\n",
    "    def setEval(self, score, method):\n",
    "        self.eval[method] = score\n",
    "\n",
    "    def setImgToEvalImgs(self, scores, imgIds, method):\n",
    "        for imgId, score in zip(imgIds, scores):\n",
    "            if not imgId in self.imgToEval:\n",
    "                self.imgToEval[imgId] = {}\n",
    "                self.imgToEval[imgId][\"image_id\"] = imgId\n",
    "            self.imgToEval[imgId][method] = score\n",
    "\n",
    "    def setEvalImgs(self):\n",
    "        self.evalImgs = [eval for imgId, eval in self.imgToEval.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenization...\n",
      "setting up scorers...\n",
      "computing Bleu score...\n",
      "{'reflen': 8873, 'guess': [8484, 484, 75, 9], 'testlen': 8484, 'correct': [5163, 236, 29, 5]}\n",
      "ratio: 0.956159134453\n",
      "Bleu_1: 0.581\n",
      "Bleu_2: 0.520\n",
      "Bleu_3: 0.464\n",
      "Bleu_4: 0.480\n",
      "computing METEOR score...\n",
      "METEOR: 0.317\n",
      "computing Rouge score...\n",
      "ROUGE_L: 0.605\n",
      "computing CIDEr score...\n",
      "CIDEr: 0.686\n",
      "{'CIDEr': 0.6863426863055019, 'Bleu_4': 0.47994916643846386, 'Bleu_3': 0.4641468828298988, 'Bleu_2': 0.5203208284233435, 'Bleu_1': 0.5812843398708688, 'ROUGE_L': 0.6047780465581399, 'METEOR': 0.3165614840501212}\n"
     ]
    }
   ],
   "source": [
    "res = {x['image']:[{'image_id':x['image'], 'caption':x['answer']}] for x in results}\n",
    "gts = {}\n",
    "for img, ans_list in img2gt.items():\n",
    "    ans_list = [x['answer'] for x in ans_list]\n",
    "    tmp = []\n",
    "    for x in ans_list:\n",
    "        try:\n",
    "            tmp.append(str(x))\n",
    "        except:\n",
    "            pass\n",
    "    ans_list = tmp\n",
    "    ans_list = [{'image_id': img, 'caption': str(x)} for x in ans_list]\n",
    "    gts[img] = ans_list\n",
    "\n",
    "for img in gts.keys():\n",
    "    if img not in res.keys():\n",
    "        res[img] = [{'image_id':img, 'caption':''}]\n",
    "        \n",
    "evalObj = COCOEvalCap(gts.keys(),gts,res)\n",
    "evalObj.evaluate()\n",
    "print evalObj.eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cPickle as pkl\n",
    "prob = pkl.load(open('saved_model/%s_prob.pkl'%split))\n",
    "answer2answer_id = json.load(open('data/create_vocab/answer2answer_id.json'))\n",
    "unanswerable_labels = [answer2answer_id['unanswerable'], answer2answer_id['unsuitable']]\n",
    "img2answerable = {x['image']:x['answerable'] for x in dataset}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AP_rel: 0.8944\n",
      "AP_irrel: 0.5905\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import recall_score, average_precision_score, precision_recall_curve\n",
    "\n",
    "y_test = []\n",
    "pred = []\n",
    "\n",
    "for res in results:\n",
    "    img = res['image']\n",
    "    gt_ans = img2answerable[img]\n",
    "    y_test.append(gt_ans)\n",
    "    one_prob = prob[img]\n",
    "    one_pred = 1 - sum([one_prob[x] for x in unanswerable_labels])\n",
    "    pred.append(one_pred)\n",
    "y_test = np.array(y_test)\n",
    "pred = np.array(pred)\n",
    "\n",
    "gt_labels = np.asarray(y_test) > 0.5\n",
    "precision, recall, thresholds = precision_recall_curve(gt_labels, pred)\n",
    "average_precision = average_precision_score(gt_labels, pred)\n",
    "print \"AP_rel: %.4f\"%average_precision\n",
    "with open('saved_model/results_rel.txt','w') as fid:\n",
    "    fid.write(str(average_precision))\n",
    "    fid.write('\\n')\n",
    "    fid.write('\\n'.join(['%.4f\\t%.4f\\t%.4f'%x for x in list(zip(recall,precision,thresholds))[::-1]]))\n",
    "\n",
    "\n",
    "gt_labels_n = np.asarray(y_test) < 0.5\n",
    "pred_n = 1.0 - pred\n",
    "precision, recall, thresholds = precision_recall_curve(gt_labels_n, pred_n)\n",
    "average_precision = average_precision_score(gt_labels_n, pred_n)\n",
    "print \"AP_irrel: %.4f\"%average_precision\n",
    "with open('saved_model/results_irrel.txt','w') as fid:\n",
    "    fid.write(str(average_precision))\n",
    "    fid.write('\\n')\n",
    "    fid.write('\\n'.join(['%.4f\\t%.4f\\t%.4f'%x for x in list(zip(recall,precision,thresholds))[::-1]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}