attn_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifying OUV using GRU sequence model + Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/opt/miniconda3/bin/python'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from scipy.special import softmax\n",
    "\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torchtext\n",
    "from torchtext.data import get_tokenizer\n",
    "tokenizer = get_tokenizer('spacy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.7.0\n",
      "GPU-enabled installation? False\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version {}\".format(torch.__version__))\n",
    "print(\"GPU-enabled installation? {}\".format(torch.cuda.is_available()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Vectorization Classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocabulary(object):\n",
    "    \"\"\"Class to process text and extract vocabulary for mapping\"\"\"\n",
    "\n",
    "    def __init__(self, token_to_idx=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            token_to_idx (dict): a pre-existing map of tokens to indices\n",
    "            add_unk (bool): a flag that indicates whether to add the UNK token\n",
    "            unk_token (str): the UNK token to add into the Vocabulary\n",
    "        \"\"\"\n",
    "\n",
    "        if token_to_idx is None:\n",
    "            token_to_idx = {}\n",
    "        self._token_to_idx = token_to_idx\n",
    "\n",
    "        self._idx_to_token = {idx: token \n",
    "                              for token, idx in self._token_to_idx.items()}\n",
    "                \n",
    "    def to_serializable(self):\n",
    "        \"\"\" returns a dictionary that can be serialized \"\"\"\n",
    "        return {'token_to_idx': self._token_to_idx}\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        \"\"\" instantiates the Vocabulary from a serialized dictionary \"\"\"\n",
    "        return cls(**contents)\n",
    "\n",
    "    def add_token(self, token):\n",
    "        \"\"\"Update mapping dicts based on the token.\n",
    "\n",
    "        Args:\n",
    "            token (str): the item to add into the Vocabulary\n",
    "        Returns:\n",
    "            index (int): the integer corresponding to the token\n",
    "        \"\"\"\n",
    "        if token in self._token_to_idx:\n",
    "            index = self._token_to_idx[token]\n",
    "        else:\n",
    "            index = len(self._token_to_idx)\n",
    "            self._token_to_idx[token] = index\n",
    "            self._idx_to_token[index] = token\n",
    "        return index\n",
    "    \n",
    "    def add_many(self, tokens):\n",
    "        \"\"\"Add a list of tokens into the Vocabulary\n",
    "        \n",
    "        Args:\n",
    "            tokens (list): a list of string tokens\n",
    "        Returns:\n",
    "            indices (list): a list of indices corresponding to the tokens\n",
    "        \"\"\"\n",
    "        return [self.add_token(token) for token in tokens]\n",
    "\n",
    "    def lookup_token(self, token):\n",
    "        \"\"\"Retrieve the index associated with the token \n",
    "          or the UNK index if token isn't present.\n",
    "        \n",
    "        Args:\n",
    "            token (str): the token to look up \n",
    "        Returns:\n",
    "            index (int): the index corresponding to the token\n",
    "        Notes:\n",
    "            `unk_index` needs to be >=0 (having been added into the Vocabulary) \n",
    "              for the UNK functionality \n",
    "        \"\"\"\n",
    "        return self._token_to_idx[token]\n",
    "\n",
    "    def lookup_index(self, index):\n",
    "        \"\"\"Return the token associated with the index\n",
    "        \n",
    "        Args: \n",
    "            index (int): the index to look up\n",
    "        Returns:\n",
    "            token (str): the token corresponding to the index\n",
    "        Raises:\n",
    "            KeyError: if the index is not in the Vocabulary\n",
    "        \"\"\"\n",
    "        if index not in self._idx_to_token:\n",
    "            raise KeyError(\"the index (%d) is not in the Vocabulary\" % index)\n",
    "        return self._idx_to_token[index]\n",
    "\n",
    "    def __str__(self):\n",
    "        return \"<Vocabulary(size=%d)>\" % len(self)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._token_to_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SequenceVocabulary(Vocabulary):\n",
    "    def __init__(self, token_to_idx=None, unk_token=\"<UNK>\",\n",
    "                 mask_token=\"<MASK>\", begin_seq_token=\"<BEGIN>\",\n",
    "                 end_seq_token=\"<END>\"):\n",
    "\n",
    "        super(SequenceVocabulary, self).__init__(token_to_idx)\n",
    "\n",
    "        self._mask_token = mask_token\n",
    "        self._unk_token = unk_token\n",
    "        self._begin_seq_token = begin_seq_token\n",
    "        self._end_seq_token = end_seq_token\n",
    "\n",
    "        self.mask_index = self.add_token(self._mask_token)\n",
    "        self.unk_index = self.add_token(self._unk_token)\n",
    "        self.begin_seq_index = self.add_token(self._begin_seq_token)\n",
    "        self.end_seq_index = self.add_token(self._end_seq_token)\n",
    "\n",
    "    def to_serializable(self):\n",
    "        contents = super(SequenceVocabulary, self).to_serializable()\n",
    "        contents.update({'unk_token': self._unk_token,\n",
    "                         'mask_token': self._mask_token,\n",
    "                         'begin_seq_token': self._begin_seq_token,\n",
    "                         'end_seq_token': self._end_seq_token})\n",
    "        return contents\n",
    "    \n",
    "    def lookup_token(self, token):\n",
    "        \"\"\"Retrieve the index associated with the token \n",
    "          or the UNK index if token isn't present.\n",
    "        \n",
    "        Args:\n",
    "            token (str): the token to look up \n",
    "        Returns:\n",
    "            index (int): the index corresponding to the token\n",
    "        Notes:\n",
    "            `unk_index` needs to be >=0 (having been added into the Vocabulary) \n",
    "              for the UNK functionality \n",
    "        \"\"\"\n",
    "        if self.unk_index >= 0:\n",
    "            return self._token_to_idx.get(token, self.unk_index)\n",
    "        else:\n",
    "            return self._token_to_idx[token]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OuvVectorizer(object):\n",
    "    \"\"\" The Vectorizer which coordinates the Vocabularies and puts them to use\"\"\"\n",
    "    def __init__(self, ouv_vocab):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            review_vocab (Vocabulary): maps words to integers\n",
    "        \"\"\"\n",
    "        self.ouv_vocab = ouv_vocab\n",
    "        \n",
    "    def vectorize(self, data, vector_length = -1):\n",
    "        \"\"\"Create a collapsed one-hit vector for the ouv data\n",
    "        \n",
    "        Args:\n",
    "            data (str): the ouv description data\n",
    "            vector_length (int): an argument for forcing the length of index vector\n",
    "        Returns:\n",
    "            the vectorized data (np.ndarray)\n",
    "        \"\"\"\n",
    "        indices = []\n",
    "        indices.extend(self.ouv_vocab.lookup_token(token) for token in data.split(' '))\n",
    "        #indices.append(self.ouv_vocab.end_seq_index)\n",
    "        \n",
    "        if vector_length < 0:\n",
    "            vector_length = len(indices)\n",
    "            \n",
    "        out_vector = np.zeros(vector_length, dtype=np.int64)\n",
    "        out_vector[:len(indices)] = indices\n",
    "        out_vector[len(indices):] = self.ouv_vocab.mask_index\n",
    "        \n",
    "        return out_vector, len(indices)\n",
    "\n",
    "    @classmethod\n",
    "    def from_dataframe(cls, ouv_df, cutoff=5):\n",
    "        \"\"\"Instantiate the vectorizer from the dataset dataframe\n",
    "        \n",
    "        Args:\n",
    "            ouv_df (pandas.DataFrame): the ouv dataset\n",
    "            cutoff (int): the parameter for frequency-based filtering\n",
    "        Returns:\n",
    "            an instance of the OuvVectorizer\n",
    "        \"\"\"\n",
    "        \n",
    "        # Add top words if count > provided count\n",
    "        word_counts = Counter()\n",
    "        for data in ouv_df.data:\n",
    "            for word in data.split(' '):\n",
    "                if word not in string.punctuation:\n",
    "                    word_counts[word] += 1\n",
    "        \n",
    "        ouv_vocab = SequenceVocabulary()\n",
    "        for word, count in word_counts.items():\n",
    "            if count > cutoff:\n",
    "                ouv_vocab.add_token(word)\n",
    "\n",
    "        return cls(ouv_vocab)\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        \"\"\"Instantiate a OuvVectorizer from a serializable dictionary\n",
    "        \n",
    "        Args:\n",
    "            contents (dict): the serializable dictionary\n",
    "        Returns:\n",
    "            an instance of the OuvVectorizer class\n",
    "        \"\"\"\n",
    "        ouv_vocab = SequenceVocabulary.from_serializable(contents['ouv_vocab'])\n",
    "        \n",
    "        return cls(ouv_vocab=ouv_vocab)\n",
    "\n",
    "    def to_serializable(self):\n",
    "        \"\"\"Create the serializable dictionary for caching\n",
    "        \n",
    "        Returns:\n",
    "            contents (dict): the serializable dictionary\n",
    "        \"\"\"\n",
    "        return {'ouv_vocab': self.ouv_vocab.to_serializable()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OuvDataset(Dataset):\n",
    "    def __init__(self, ouv_df, vectorizer):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            ouv_df (pandas.DataFrame): the dataset\n",
    "            vectorizer (ReviewVectorizer): vectorizer instantiated from dataset\n",
    "        \"\"\"\n",
    "        self.ouv_df = ouv_df\n",
    "        self._vectorizer = vectorizer\n",
    "        \n",
    "        # +0 if not using begin_seq and end seq, +1 if only using begin_seq, +2 if using both begin and end seq tokens\n",
    "        measure_len = lambda context: len(context.split(\" \"))\n",
    "        self._max_seq_length = max(map(measure_len, ouv_df.data)) + 0\n",
    "\n",
    "        self.train_df = self.ouv_df[self.ouv_df.split=='train']\n",
    "        self.train_size = len(self.train_df)\n",
    "\n",
    "        self.val_df = self.ouv_df[self.ouv_df.split=='dev']\n",
    "        self.validation_size = len(self.val_df)\n",
    "\n",
    "        self.test_df = self.ouv_df[self.ouv_df.split=='test']\n",
    "        self.test_size = len(self.test_df)\n",
    "\n",
    "        self._lookup_dict = {'train': (self.train_df, self.train_size),\n",
    "                             'val': (self.val_df, self.validation_size),\n",
    "                             'test': (self.test_df, self.test_size)}\n",
    "\n",
    "        self.set_split('train')\n",
    "\n",
    "    @classmethod\n",
    "    def load_dataset_and_make_vectorizer(cls, ouv_csv, cutoff):\n",
    "        \"\"\"Load dataset and make a new vectorizer from scratch\n",
    "        \n",
    "        Args:\n",
    "            ouv_csv (str): location of the dataset\n",
    "            cutoff (int): the boundary to set the words into unknown\n",
    "        Returns:\n",
    "            an instance of OuvDataset\n",
    "        \"\"\"\n",
    "        ouv_df = pd.read_csv(ouv_csv)\n",
    "        train_ouv_df = ouv_df[ouv_df.split=='train']\n",
    "        return cls(ouv_df, OuvVectorizer.from_dataframe(train_ouv_df, cutoff=cutoff))\n",
    "    \n",
    "    @classmethod\n",
    "    def load_dataset_and_load_vectorizer(cls, ouv_csv, vectorizer_filepath):\n",
    "        \"\"\"Load dataset and the corresponding vectorizer. \n",
    "        Used in the case in the vectorizer has been cached for re-use\n",
    "        \n",
    "        Args:\n",
    "            ouv_csv (str): location of the dataset\n",
    "            vectorizer_filepath (str): location of the saved vectorizer\n",
    "        Returns:\n",
    "            an instance of OuvDataset\n",
    "        \"\"\"\n",
    "        ouv_df = pd.read_csv(ouv_csv)\n",
    "        vectorizer = cls.load_vectorizer_only(vectorizer_filepath)\n",
    "        return cls(ouv_df, vectorizer)\n",
    "\n",
    "    @staticmethod\n",
    "    def load_vectorizer_only(vectorizer_filepath):\n",
    "        \"\"\"a static method for loading the vectorizer from file\n",
    "        \n",
    "        Args:\n",
    "            vectorizer_filepath (str): the location of the serialized vectorizer\n",
    "        Returns:\n",
    "            an instance of ReviewVectorizer\n",
    "        \"\"\"\n",
    "        with open(vectorizer_filepath) as fp:\n",
    "            return OuvVectorizer.from_serializable(json.load(fp))\n",
    "\n",
    "    def save_vectorizer(self, vectorizer_filepath):\n",
    "        \"\"\"saves the vectorizer to disk using json\n",
    "        \n",
    "        Args:\n",
    "            vectorizer_filepath (str): the location to save the vectorizer\n",
    "        \"\"\"\n",
    "        with open(vectorizer_filepath, \"w\") as fp:\n",
    "            json.dump(self._vectorizer.to_serializable(), fp)\n",
    "\n",
    "    def get_vectorizer(self):\n",
    "        \"\"\" returns the vectorizer \"\"\"\n",
    "        return self._vectorizer\n",
    "\n",
    "    def set_split(self, split=\"train\"):\n",
    "        \"\"\" selects the splits in the dataset using a column in the dataframe \n",
    "        \n",
    "        Args:\n",
    "            split (str): one of \"train\", \"val\", or \"test\"\n",
    "        \"\"\"\n",
    "        self._target_split = split\n",
    "        self._target_df, self._target_size = self._lookup_dict[split]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self._target_size\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"the primary entry point method for PyTorch datasets\n",
    "        \n",
    "        Args:\n",
    "            index (int): the index to the data point \n",
    "        Returns:\n",
    "            a dictionary holding the data point's features (x_data) and component for labels (y_target and y_fuzzy)\n",
    "        \"\"\"\n",
    "        row = self._target_df.iloc[index]\n",
    "\n",
    "        ouv_vector, vec_length = \\\n",
    "            self._vectorizer.vectorize(row.data, self._max_seq_length)\n",
    "\n",
    "        true_label = \\\n",
    "            np.fromstring(row.true[1:-1],dtype=float, sep=' ')\n",
    "        \n",
    "        if len(true_label)==10:\n",
    "            true_label = np.append(true_label,0.0)\n",
    "        \n",
    "        fuzzy_label = \\\n",
    "            np.fromstring(row.fuzzy[1:-1],dtype=float, sep=' ')\n",
    "\n",
    "        return {'x_data': ouv_vector,\n",
    "                'y_target': true_label,\n",
    "                'y_fuzzy': fuzzy_label,\n",
    "                'x_length': vec_length\n",
    "               }\n",
    "\n",
    "    def get_num_batches(self, batch_size):\n",
    "        \"\"\"Given a batch size, return the number of batches in the dataset\n",
    "        \n",
    "        Args:\n",
    "            batch_size (int)\n",
    "        Returns:\n",
    "            number of batches in the dataset\n",
    "        \"\"\"\n",
    "        return len(self) // batch_size  \n",
    "    \n",
    "def generate_batches(dataset, batch_size, shuffle=True,\n",
    "                     drop_last=True, device=\"cpu\"):\n",
    "    \"\"\"\n",
    "    A generator function which wraps the PyTorch DataLoader. It will \n",
    "      ensure each tensor is on the write device location.\n",
    "    \"\"\"\n",
    "    dataloader = DataLoader(dataset=dataset, batch_size=batch_size,\n",
    "                            shuffle=shuffle, drop_last=drop_last)\n",
    "\n",
    "    for data_dict in dataloader:\n",
    "        out_data_dict = {}\n",
    "        for name, tensor in data_dict.items():\n",
    "            out_data_dict[name] = data_dict[name].to(device)\n",
    "        yield out_data_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Model: GRU sequential Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_matmul_bias(seq, weight, bias, nonlinearity=''):\n",
    "    s = None\n",
    "    bias_dim = bias.size()\n",
    "    for i in range(seq.size(0)):\n",
    "        _s = torch.mm(seq[i], weight) \n",
    "        _s_bias = _s + bias.expand(bias_dim[0], _s.size()[0]).transpose(0,1)\n",
    "        if(nonlinearity=='tanh'):\n",
    "            _s_bias = torch.tanh(_s_bias)\n",
    "        _s_bias = _s_bias.unsqueeze(0)\n",
    "        if(s is None):\n",
    "            s = _s_bias\n",
    "        else:\n",
    "            s = torch.cat((s,_s_bias),0)\n",
    "    return s.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_matmul(seq, weight, nonlinearity=''):\n",
    "    s = None\n",
    "    for i in range(seq.size(0)):\n",
    "        _s = torch.mm(seq[i], weight)\n",
    "        if(nonlinearity=='tanh'):\n",
    "            _s = torch.tanh(_s)\n",
    "        _s = _s.unsqueeze(0)\n",
    "        if(s is None):\n",
    "            s = _s\n",
    "        else:\n",
    "            s = torch.cat((s,_s),0)\n",
    "    return s.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attention_mul(rnn_outputs, att_weights):\n",
    "    attn_vectors = None\n",
    "    for i in range(rnn_outputs.size(0)):\n",
    "        h_i = rnn_outputs[i]\n",
    "        a_i = att_weights[i].unsqueeze(1).expand_as(h_i)\n",
    "        h_i = a_i * h_i\n",
    "        h_i = h_i.unsqueeze(0)\n",
    "        if(attn_vectors is None):\n",
    "            attn_vectors = h_i\n",
    "        else:\n",
    "            attn_vectors = torch.cat((attn_vectors,h_i),0)\n",
    "    return torch.sum(attn_vectors, 0).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttnGRUClassifier(nn.Module):\n",
    "    \n",
    "    def __init__(self, batch_size, embedding_size, num_embeddings, hidden_dim, num_classes, dropout_p,\n",
    "                 batch_first=True, pretrained_embeddings=None, padding_idx=0, bi = False, freeze=True):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            embedding_size (int): size of the embedding vectors\n",
    "            num_embeddings (int): number of embedding vectors\n",
    "            hidden_dim (int): the size of the hidden dimension\n",
    "            num_classes (int): the number of classes in classification\n",
    "            dropout_p (float): a dropout parameter \n",
    "            pretrained_embeddings (numpy.array): previously trained word embeddings\n",
    "                default is None. If provided, \n",
    "            padding_idx (int): an index representing a null position\n",
    "        \"\"\"\n",
    "        super(AttnGRUClassifier, self).__init__()\n",
    "\n",
    "        if pretrained_embeddings is None:\n",
    "\n",
    "            self.emb = nn.Embedding(embedding_dim=embedding_size,\n",
    "                                    num_embeddings=num_embeddings,\n",
    "                                    padding_idx=padding_idx)        \n",
    "        else:\n",
    "            pretrained_embeddings = torch.from_numpy(pretrained_embeddings).float()\n",
    "            self.emb = nn.Embedding.from_pretrained(pretrained_embeddings,\n",
    "                                    padding_idx=padding_idx,\n",
    "                                    freeze=True)\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.batch_first = batch_first\n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    "        self.padding_idx = padding_idx\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.bi = bi\n",
    "        \n",
    "        self.word_gru = nn.GRU(input_size = embedding_size, hidden_size=hidden_dim, batch_first=False,\n",
    "                         num_layers=1, bidirectional = bi)\n",
    "        \n",
    "        self.word_bias = nn.Linear((bi+1)*hidden_dim, (bi+1)*hidden_dim)\n",
    "        self.word_n_bias = nn.Linear((bi+1)*hidden_dim, 1, bias=False)\n",
    "        \n",
    "        self.fc = nn.Linear(hidden_dim * (bi+1), num_classes)\n",
    "\n",
    "    def forward(self, x_in, state_word, apply_softmax=False):\n",
    "        \"\"\"The forward pass of the classifier\n",
    "        \n",
    "        Args:\n",
    "            x_in (torch.Tensor): an input data tensor. \n",
    "                x_in.shape should be (batch, dataset._max_seq_length)\n",
    "            apply_softmax (bool): a flag for the softmax activation\n",
    "                should be false if used with the Cross Entropy losses\n",
    "        Returns:\n",
    "            the resulting tensor. tensor.shape should be (batch, num_classes)\n",
    "        \"\"\"\n",
    "        \n",
    "        # embed and permute so features are channels\n",
    "        x_embedded = self.emb(x_in)\n",
    "        padding_mask = x_in.eq(self.padding_idx)\n",
    "        length = (~padding_mask).sum(dim=1)\n",
    "        \n",
    "        if self.batch_first:\n",
    "            x_embedded = x_embedded.transpose(1,0)\n",
    "        \n",
    "        # rnn units\n",
    "        output_word, state_word = self.word_gru(x_embedded, state_word)\n",
    "        #(T,B,C)\n",
    "        word_squish = self.word_bias(output_word.view(-1,output_word.shape[-1]))\n",
    "        #(B*T,C)\n",
    "        word_squish = torch.tanh(word_squish)\n",
    "        word_attn = self.word_n_bias(word_squish).view(output_word.shape[0],output_word.shape[1],1)\n",
    "        word_attn[padding_mask.transpose(0,1)]= -np.inf\n",
    "        #(B*T,1)->(T,B,1)\n",
    "        \n",
    "        word_attn_norm = F.softmax(word_attn, dim=0).transpose(0,1)\n",
    "        #(B,T,1)\n",
    "        \n",
    "        word_attn_vec = torch.bmm(output_word.permute(1,2,0),word_attn_norm)\n",
    "        #(B,C,T)*(B,T,1)->(B,C,1)\n",
    "        \n",
    "        y_out = self.fc(self.dropout(word_attn_vec.squeeze()))\n",
    "        \n",
    "        if apply_softmax:\n",
    "            y_out = F.softmax(y_out, dim=1)\n",
    "            \n",
    "        return y_out, state_word, word_attn_norm.squeeze()\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        return Variable(torch.zeros((self.bi+1), self.batch_size, self.hidden_dim))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_k_acc_val': 0,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_1_acc': [],\n",
    "            'train_k_acc': [],\n",
    "            'train_k_jac': [],\n",
    "            'val_loss': [],\n",
    "            'val_1_acc': [],\n",
    "            'val_k_acc': [],\n",
    "            'val_k_jac': [],\n",
    "            'test_loss': -1,\n",
    "            'test_1_acc': -1,\n",
    "            'test_k_acc':-1,\n",
    "            'test_k_jac':-1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "\n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        acc_tm1, acc_t = train_state['val_k_acc'][-2:]\n",
    "\n",
    "        # If accuracy worsened\n",
    "        if acc_t <= train_state['early_stopping_best_k_acc_val']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model from sklearn\n",
    "            if acc_t > train_state['early_stopping_best_k_acc_val']:\n",
    "                train_state['early_stopping_best_k_acc_val'] = acc_t\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                \n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cross_entropy(y_pred, y_target):\n",
    "    y_target = y_target.cpu().float()\n",
    "    y_pred = y_pred.cpu().float()\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    return criterion(y_target, y_pred)\n",
    "\n",
    "def compute_1_accuracy(y_pred, y_target):\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_accuracy(y_pred, y_target, k=3):\n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_jaccard_index(y_pred, y_target, k=3):\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/\n",
    "                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))\n",
    "                            for i in range(len(y_pred))]).sum().item()\n",
    "    return jaccard / len(y_pred_indices)\n",
    "\n",
    "def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    threshold = 1.0/(k+1)\n",
    "    threshold_2 = 0.5\n",
    "    \n",
    "    if multilabel:\n",
    "        y_pred_indices = y_pred.gt(threshold_2)\n",
    "    else:\n",
    "        y_pred_indices = y_pred.gt(threshold)\n",
    "    \n",
    "    y_target_indices = y_target.gt(threshold)\n",
    "        \n",
    "    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()\n",
    "    return jaccard / len(y_pred_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_sensitive(T):\n",
    "    T = np.exp(T) - np.exp(0) + 1e-9\n",
    "    if len(T.shape)==1:\n",
    "        return T/T.sum()\n",
    "    return  T/(T.sum(axis=1).unsqueeze(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(pred, soft_targets):\n",
    "    logsoftmax = nn.LogSoftmax(dim=1)\n",
    "    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert a df to tensor to be used in pytorch\n",
    "def df_to_tensor(df):\n",
    "    device = args.device\n",
    "    return torch.from_numpy(df.values).float().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prior():\n",
    "    prior = pd.read_csv(args.prior_csv,sep=';',names=classes[:-1], skiprows=1)\n",
    "    prior['Others'] = 1\n",
    "    prior = prior.T\n",
    "    prior['Others'] = 1\n",
    "    prior = df_to_tensor(prior)\n",
    "    return(prior)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_fuzzy_label(y_target, y_fuzzy, fuzzy=False, how='uni', lbd=0):\n",
    "    '''\n",
    "    Using two sets of prediction labels and fuzziness parameters to compute the fuzzy label in the form as \n",
    "    a distribution over classes\n",
    "    \n",
    "    Args:\n",
    "    y_target (torch.Tensor) of shape (n_batch, n_classes): the true label of the ouv description\n",
    "    y_fuzzy (torch.Tensor) of shape (n_batch, n_classes): the fuzzy label of the ouv description\n",
    "    fuzzy (bool): whether or not to turn on the fuzziness option\n",
    "    how (string): the way fuzziness weights are used, one of the options in {'uni', 'prior'}\n",
    "    lbd (float): the scaler applied to the fuzziness of the label\n",
    "    \n",
    "    Returns:\n",
    "    A pytorch Tensor of shape (n_batch, n_classes): The processed label in the form of distribution that add to 1\n",
    "    '''\n",
    "    assert y_target.shape == y_fuzzy.shape, 'target labels must have the same size'\n",
    "    assert how in {'uni', 'prior', 'origin'}, '''how must be one of the two options in {'uni', 'prior'}'''\n",
    "    \n",
    "    if not fuzzy:\n",
    "        return softmax_sensitive(y_target)\n",
    "    \n",
    "    if how == 'uni':\n",
    "        y_label = y_target + lbd * y_fuzzy\n",
    "        return softmax_sensitive(y_label)\n",
    "    \n",
    "    ### TO DO ###\n",
    "    elif how == 'prior':\n",
    "        prior = get_prior()\n",
    "        y_inter = torch.matmul(y_target.float(),prior)\n",
    "        y_inter = y_inter/(y_inter.max(dim=1, keepdim=True)[0])\n",
    "        y_label = y_target + lbd * y_fuzzy * y_inter\n",
    "        return softmax_sensitive(y_label)\n",
    "    \n",
    "    else:\n",
    "        y_label = y_target + lbd\n",
    "        return softmax_sensitive(y_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### General Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if cuda:\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)\n",
    "\n",
    "def load_glove_from_file(glove_filepath):\n",
    "    \"\"\"\n",
    "    Load the GloVe embeddings \n",
    "    \n",
    "    Args:\n",
    "        glove_filepath (str): path to the glove embeddings file \n",
    "    Returns:\n",
    "        word_to_index (dict), embeddings (numpy.ndarary)\n",
    "    \"\"\"\n",
    "\n",
    "    word_to_index = {}\n",
    "    embeddings = []\n",
    "    with open(glove_filepath, \"r\") as fp:\n",
    "        for index, line in enumerate(fp):\n",
    "            line = line.split(\" \") # each line: word num1 num2 ...\n",
    "            word_to_index[line[0]] = index # word = line[0] \n",
    "            embedding_i = np.array([float(val) for val in line[1:]])\n",
    "            embeddings.append(embedding_i)\n",
    "    return word_to_index, np.stack(embeddings)\n",
    "\n",
    "def make_embedding_matrix(glove_filepath, words):\n",
    "    \"\"\"\n",
    "    Create embedding matrix for a specific set of words.\n",
    "    \n",
    "    Args:\n",
    "        glove_filepath (str): file path to the glove embeddigns\n",
    "        words (list): list of words in the dataset\n",
    "    \"\"\"\n",
    "    word_to_idx, glove_embeddings = load_glove_from_file(glove_filepath)\n",
    "    embedding_size = glove_embeddings.shape[1]\n",
    "    \n",
    "    final_embeddings = np.zeros((len(words), embedding_size))\n",
    "\n",
    "    for i, word in enumerate(words):\n",
    "        if word in word_to_idx:\n",
    "            final_embeddings[i, :] = glove_embeddings[word_to_idx[word]]\n",
    "        else:\n",
    "            embedding_i = torch.ones(1, embedding_size)\n",
    "            torch.nn.init.xavier_uniform_(embedding_i)\n",
    "            final_embeddings[i, :] = embedding_i\n",
    "\n",
    "    return final_embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Settings and Some Prep Work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expanded filepaths: \n",
      "\tmodel_storage/attn/vectorizer.json\n",
      "\tmodel_storage/attn/model.pth\n",
      "Using CUDA: False\n"
     ]
    }
   ],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    frequency_cutoff=1,\n",
    "    model_state_file='model.pth',\n",
    "    ouv_csv='Data/ouv_with_splits_full.csv',\n",
    "    #ouv_csv='Data/all_with_splits_full.csv',\n",
    "    prior_csv = 'Data/Coappearance_matrix.csv',\n",
    "    ratings_csv = 'Data/human_rates.csv',\n",
    "    save_dir='model_storage/attn/',\n",
    "    vectorizer_file='vectorizer.json',\n",
    "    # Model hyper parameters\n",
    "    glove_filepath='Data/glove/glove.6B.300d.txt', \n",
    "    use_glove=True,\n",
    "    freeze = True,\n",
    "    embedding_size=300, \n",
    "    hidden_dim=128, \n",
    "    bi = False,\n",
    "    # Training hyper parameters\n",
    "    batch_size=256,\n",
    "    early_stopping_criteria=5,\n",
    "    learning_rate=0.001,\n",
    "    l2=1e-5,\n",
    "    dropout_p=0,\n",
    "    k = 3,\n",
    "    fuzzy = True,\n",
    "    fuzzy_how = 'uni',\n",
    "    fuzzy_lambda = 0.2,\n",
    "    num_epochs=100,\n",
    "    seed=1337,\n",
    "    # Runtime options\n",
    "    catch_keyboard_interrupt=True,\n",
    "    cuda=True,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    reload_from_files=False,\n",
    ")\n",
    "\n",
    "classes = ['Criteria i', 'Criteria ii', 'Criteria iii', 'Criteria iv', 'Criteria v', 'Criteria vi', \n",
    "              'Criteria vii', 'Criteria viii', 'Criteria ix', 'Criteria x', 'Others']\n",
    "\n",
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.vectorizer_file = os.path.join(args.save_dir,\n",
    "                                        args.vectorizer_file)\n",
    "\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "    print(\"Expanded filepaths: \")\n",
    "    print(\"\\t{}\".format(args.vectorizer_file))\n",
    "    print(\"\\t{}\".format(args.model_state_file))\n",
    "    \n",
    "# Check CUDA\n",
    "if not torch.cuda.is_available():\n",
    "    args.cuda = False\n",
    "\n",
    "print(\"Using CUDA: {}\".format(args.cuda))\n",
    "\n",
    "args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "\n",
    "# Set seed for reproducibility\n",
    "#set_seed_everywhere(args.seed, args.cuda)\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using pre-trained embeddings\n"
     ]
    }
   ],
   "source": [
    "if args.reload_from_files:\n",
    "    # training from a checkpoint\n",
    "    dataset = OuvDataset.load_dataset_and_load_vectorizer(args.ouv_csv, args.vectorizer_file)\n",
    "\n",
    "else:\n",
    "    # create dataset and vectorizer\n",
    "    dataset = OuvDataset.load_dataset_and_make_vectorizer(args.ouv_csv, cutoff=args.frequency_cutoff)\n",
    "    dataset.save_vectorizer(args.vectorizer_file)    \n",
    "\n",
    "vectorizer = dataset.get_vectorizer()\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "# Use GloVe or randomly initialized embeddings\n",
    "if args.use_glove:\n",
    "    words = vectorizer.ouv_vocab._token_to_idx.keys()\n",
    "    embeddings = make_embedding_matrix(glove_filepath=args.glove_filepath, \n",
    "                                       words=words)\n",
    "    print(\"Using pre-trained embeddings\")\n",
    "else:\n",
    "    print(\"Not using pre-trained embeddings\")\n",
    "    embeddings = None\n",
    "classifier = AttnGRUClassifier(batch_size=args.batch_size,\n",
    "                               embedding_size=args.embedding_size,\n",
    "                               num_embeddings=len(vectorizer.ouv_vocab),\n",
    "                            hidden_dim=args.hidden_dim, \n",
    "                            num_classes=len(classes), \n",
    "                            dropout_p=args.dropout_p,\n",
    "                            pretrained_embeddings=embeddings,\n",
    "                            padding_idx=0,\n",
    "                            bi = args.bi,\n",
    "                            freeze = args.freeze)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Trained Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 1 LS Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:\n",
    "    hyperdict_fuzzy = pickle.load(fp)\n",
    "    train_state = hyperdict_fuzzy[('uni',0.2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AttnGRUClassifier(\n",
       "  (emb): Embedding(6047, 300, padding_idx=0)\n",
       "  (dropout): Dropout(p=0, inplace=False)\n",
       "  (word_gru): GRU(300, 128)\n",
       "  (word_bias): Linear(in_features=128, out_features=128, bias=True)\n",
       "  (word_n_bias): Linear(in_features=128, out_features=1, bias=False)\n",
       "  (fc): Linear(in_features=128, out_features=11, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.load_state_dict(torch.load(args.save_dir+'1337/model.pth',map_location=torch.device('cpu')))\n",
    "classifier.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 2 Baseline w/o LS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:\n",
    "    hyperdict_fuzzy = pickle.load(fp)\n",
    "    train_state = hyperdict_fuzzy[('uni',0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AttnGRUClassifier(\n",
       "  (emb): Embedding(6047, 300, padding_idx=0)\n",
       "  (dropout): Dropout(p=0, inplace=False)\n",
       "  (word_gru): GRU(300, 128)\n",
       "  (word_bias): Linear(in_features=128, out_features=128, bias=True)\n",
       "  (word_n_bias): Linear(in_features=128, out_features=1, bias=False)\n",
       "  (fc): Linear(in_features=128, out_features=11, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.load_state_dict(torch.load(args.save_dir+'baseline/model.pth',map_location=torch.device('cpu')))\n",
    "classifier.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "183179"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "count_parameters(classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "# compute the loss & accuracy on the test set using the best available model\n",
    "loss_func = cross_entropy\n",
    "#train_state = hyperdict[best_config]\n",
    "#classifier.load_state_dict(torch.load(train_state['model_filename']))\n",
    "#classifier = classifier.to(args.device)\n",
    "\n",
    "dataset.set_split('test')\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "running_loss = 0.\n",
    "running_1_acc = 0.\n",
    "running_k_acc = 0.\n",
    "running_k_jac = 0.\n",
    "classifier.eval()\n",
    "\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    state_word = classifier.init_hidden().to(args.device)\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # step 3. compute the output\n",
    "    y_pred,state_word,_ = classifier(X, state_word)\n",
    "\n",
    "\n",
    "    # compute the loss\n",
    "    loss = loss_func(y_pred, Y)\n",
    "    loss_t = loss.item()\n",
    "    running_loss += (loss_t - running_loss) / (batch_index + 1)\n",
    "\n",
    "    # compute the accuracy\n",
    "    acc_1_t = compute_1_accuracy(y_pred, Y)\n",
    "    acc_k_t = compute_k_accuracy(y_pred, Y, args.k)\n",
    "    jac_k_t = compute_jaccard_index(y_pred, Y, args.k)\n",
    "\n",
    "    running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)\n",
    "    running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)\n",
    "    running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)\n",
    "\n",
    "train_state['test_loss'] = running_loss\n",
    "train_state['test_1_acc'] = running_1_acc\n",
    "train_state['test_k_acc'] = running_k_acc\n",
    "train_state['test_k_jac'] = running_k_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 5,\n",
       " 'early_stopping_best_k_acc_val': 91.796875,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 15,\n",
       " 'train_loss': [2.198943771596139,\n",
       "  1.7894909588827341,\n",
       "  1.6077453631286176,\n",
       "  1.51393375583656,\n",
       "  1.4374524229319623,\n",
       "  1.3765786128704782,\n",
       "  1.3162432918650375,\n",
       "  1.2749618684771529,\n",
       "  1.2391568531602377,\n",
       "  1.2141996012523106,\n",
       "  1.1801471967092538,\n",
       "  1.164778186948496,\n",
       "  1.1533445934247148,\n",
       "  1.1398225917144127,\n",
       "  1.1235810948209222,\n",
       "  1.1185056999615688],\n",
       " 'train_1_acc': [23.322610294117645,\n",
       "  35.52389705882353,\n",
       "  43.635110294117645,\n",
       "  48.25367647058823,\n",
       "  53.216911764705884,\n",
       "  58.18014705882353,\n",
       "  61.465992647058826,\n",
       "  64.15441176470588,\n",
       "  65.6939338235294,\n",
       "  67.6470588235294,\n",
       "  68.63511029411765,\n",
       "  70.1516544117647,\n",
       "  70.31250000000001,\n",
       "  71.30055147058822,\n",
       "  72.74816176470588,\n",
       "  72.47242647058823],\n",
       " 'train_k_acc': [54.940257352941174,\n",
       "  71.92095588235296,\n",
       "  80.67555147058825,\n",
       "  83.61672794117648,\n",
       "  87.61488970588235,\n",
       "  89.65992647058822,\n",
       "  91.08455882352942,\n",
       "  92.41727941176471,\n",
       "  93.12959558823528,\n",
       "  93.74999999999999,\n",
       "  94.76102941176471,\n",
       "  94.82996323529412,\n",
       "  95.08272058823529,\n",
       "  95.33547794117646,\n",
       "  95.65716911764703,\n",
       "  95.77205882352942],\n",
       " 'train_k_jac': [0.21376051008701322,\n",
       "  0.2693104989388409,\n",
       "  0.26399731460739584,\n",
       "  0.26328617772635293,\n",
       "  0.26682122314677514,\n",
       "  0.26655725170584293,\n",
       "  0.27056663176592666,\n",
       "  0.271926157614764,\n",
       "  0.2729790386031656,\n",
       "  0.27450214414035573,\n",
       "  0.2743612720685847,\n",
       "  0.27423215964261227,\n",
       "  0.27417452545727,\n",
       "  0.2731095219359678,\n",
       "  0.27334286360179677,\n",
       "  0.2718733601710376],\n",
       " 'val_loss': [1.9995111342616438,\n",
       "  1.6775607431369781,\n",
       "  1.5558905001692154,\n",
       "  1.4742230748464182,\n",
       "  1.4265134928932655,\n",
       "  1.3781640235940296,\n",
       "  1.338012610746126,\n",
       "  1.3182769913359036,\n",
       "  1.3259802173990183,\n",
       "  1.3191348379823213,\n",
       "  1.2710393319155764,\n",
       "  1.2883886707227603,\n",
       "  1.3155775213060168,\n",
       "  1.3060447495332408,\n",
       "  1.314217976469557,\n",
       "  1.2952697589811657],\n",
       " 'val_1_acc': [27.34375,\n",
       "  39.2578125,\n",
       "  49.8046875,\n",
       "  52.1484375,\n",
       "  57.2265625,\n",
       "  58.203125,\n",
       "  59.5703125,\n",
       "  60.546875,\n",
       "  61.9140625,\n",
       "  62.890625,\n",
       "  64.2578125,\n",
       "  63.8671875,\n",
       "  63.0859375,\n",
       "  63.28125,\n",
       "  62.6953125,\n",
       "  62.6953125],\n",
       " 'val_k_acc': [65.625,\n",
       "  74.21875,\n",
       "  77.9296875,\n",
       "  85.3515625,\n",
       "  86.71875,\n",
       "  87.890625,\n",
       "  88.8671875,\n",
       "  89.6484375,\n",
       "  88.8671875,\n",
       "  89.0625,\n",
       "  91.796875,\n",
       "  91.015625,\n",
       "  89.2578125,\n",
       "  90.0390625,\n",
       "  89.84375,\n",
       "  90.625],\n",
       " 'val_k_jac': [0.25359003245830536,\n",
       "  0.2598167955875397,\n",
       "  0.2694893926382065,\n",
       "  0.2690941244363785,\n",
       "  0.26591797173023224,\n",
       "  0.2720889151096344,\n",
       "  0.27075427770614624,\n",
       "  0.26803384721279144,\n",
       "  0.27114491164684296,\n",
       "  0.2703962028026581,\n",
       "  0.27419085800647736,\n",
       "  0.2726237177848816,\n",
       "  0.2627464830875397,\n",
       "  0.26754558086395264,\n",
       "  0.267578125,\n",
       "  0.2671316862106323],\n",
       " 'test_loss': 1.3303618426681183,\n",
       " 'test_1_acc': 61.5234375,\n",
       " 'test_k_acc': 90.234375,\n",
       " 'test_k_jac': 0.20810548216104507,\n",
       " 'model_filename': 'model_storage/rnn/attn/multigroup/model.pth'}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#LS Model\n",
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 5,\n",
       " 'early_stopping_best_k_acc_val': 91.6015625,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 15,\n",
       " 'train_loss': [2.1776003070893335,\n",
       "  1.7192871886354384,\n",
       "  1.478551664518919,\n",
       "  1.3462597753399255,\n",
       "  1.232127564134482,\n",
       "  1.1414252035724703,\n",
       "  1.0566094870754879,\n",
       "  0.9943482124976606,\n",
       "  0.9379362961109343,\n",
       "  0.8935166259121269,\n",
       "  0.8489513760790851,\n",
       "  0.8208194994694975,\n",
       "  0.7761225418543545,\n",
       "  0.7242527879920942,\n",
       "  0.6912963192212869,\n",
       "  0.6689500920684159],\n",
       " 'train_1_acc': [23.253676470588236,\n",
       "  35.66176470588236,\n",
       "  43.79595588235294,\n",
       "  49.333639705882355,\n",
       "  54.18198529411764,\n",
       "  58.478860294117645,\n",
       "  61.4889705882353,\n",
       "  64.49908088235294,\n",
       "  66.75091911764706,\n",
       "  67.578125,\n",
       "  69.04871323529412,\n",
       "  70.22058823529413,\n",
       "  72.44944852941178,\n",
       "  73.87408088235296,\n",
       "  75.34466911764706,\n",
       "  76.63143382352942],\n",
       " 'train_k_acc': [54.526654411764696,\n",
       "  71.36948529411765,\n",
       "  80.53768382352939,\n",
       "  83.73161764705883,\n",
       "  87.68382352941177,\n",
       "  89.45312499999999,\n",
       "  90.60202205882354,\n",
       "  92.1875,\n",
       "  93.19852941176471,\n",
       "  93.58915441176471,\n",
       "  94.16360294117648,\n",
       "  94.66911764705881,\n",
       "  95.05974264705881,\n",
       "  95.7720588235294,\n",
       "  96.36948529411765,\n",
       "  96.50735294117646],\n",
       " 'train_k_jac': [0.1596206479212817,\n",
       "  0.19430530334220214,\n",
       "  0.18900533458765814,\n",
       "  0.18747757638201995,\n",
       "  0.18920310805825627,\n",
       "  0.18982214436811562,\n",
       "  0.19471316127216112,\n",
       "  0.19746394718394567,\n",
       "  0.19994638421956232,\n",
       "  0.20318736837190737,\n",
       "  0.20523049375590158,\n",
       "  0.20790577723699458,\n",
       "  0.20907847758601691,\n",
       "  0.21105294017230763,\n",
       "  0.21127888735602887,\n",
       "  0.21209979933850906],\n",
       " 'val_loss': [1.9706743463313865,\n",
       "  1.5732477147546224,\n",
       "  1.40990879140604,\n",
       "  1.2893677334690272,\n",
       "  1.2187500683532062,\n",
       "  1.156384475023406,\n",
       "  1.104636655511745,\n",
       "  1.07505843646112,\n",
       "  1.088307064412295,\n",
       "  1.074497368500911,\n",
       "  1.0062854060974278,\n",
       "  1.0433536913231696,\n",
       "  1.0958730789968707,\n",
       "  1.0597411376987298,\n",
       "  1.0943367823853491,\n",
       "  1.0767749237064974],\n",
       " 'val_1_acc': [27.34375,\n",
       "  39.2578125,\n",
       "  49.8046875,\n",
       "  53.7109375,\n",
       "  57.6171875,\n",
       "  58.984375,\n",
       "  60.15625,\n",
       "  61.9140625,\n",
       "  62.109375,\n",
       "  61.328125,\n",
       "  64.2578125,\n",
       "  64.6484375,\n",
       "  62.890625,\n",
       "  62.109375,\n",
       "  61.5234375,\n",
       "  61.328125],\n",
       " 'val_k_acc': [62.109375,\n",
       "  74.21875,\n",
       "  78.7109375,\n",
       "  86.328125,\n",
       "  85.9375,\n",
       "  88.28125,\n",
       "  87.5,\n",
       "  88.671875,\n",
       "  88.4765625,\n",
       "  88.0859375,\n",
       "  91.6015625,\n",
       "  91.6015625,\n",
       "  89.84375,\n",
       "  89.6484375,\n",
       "  89.453125,\n",
       "  89.84375],\n",
       " 'val_k_jac': [0.17997116595506668,\n",
       "  0.1874535009264946,\n",
       "  0.18973447382450104,\n",
       "  0.18885324150323868,\n",
       "  0.18633278459310532,\n",
       "  0.1902436837553978,\n",
       "  0.19175735861063004,\n",
       "  0.195382259786129,\n",
       "  0.19956287741661072,\n",
       "  0.20066963881254196,\n",
       "  0.20581287145614624,\n",
       "  0.20630580186843872,\n",
       "  0.20412945747375488,\n",
       "  0.20614304393529892,\n",
       "  0.20483630895614624,\n",
       "  0.20654761791229248],\n",
       " 'test_loss': 1.4468541991034296,\n",
       " 'test_1_acc': 60.546875,\n",
       " 'test_k_acc': 91.40625,\n",
       " 'test_k_jac': 0.20890532433986664,\n",
       " 'model_filename': 'model_storage/rnn/attn/multigroup/model.pth'}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Baseline\n",
    "train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"([.,!?])\", r\" \\1 \", text)\n",
    "    text = re.sub(r\"[^a-zA-Z.,!?]+\", r\" \", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_rating(text, classifier, vectorizer, classes, max_length=dataset._max_seq_length, k=1):\n",
    "    \"\"\"Predict the rating of a review\n",
    "    \n",
    "    Args:\n",
    "        text (str): the text of the description\n",
    "        classifier (ReviewClassifier): the trained model\n",
    "        vectorizer (ReviewVectorizer): the corresponding vectorizer\n",
    "        classes (list of str): The name of the ouv classes\n",
    "        k (int): show the largest k prediction, default to 1\n",
    "    \"\"\"\n",
    "    ouv = preprocess_text(text)\n",
    "    \n",
    "    classifier.eval()\n",
    "    state_word = classifier.init_hidden().to(args.device)\n",
    "    vectorized_ouv = torch.tensor(vectorizer.vectorize(ouv, vector_length = max_length)[0])\n",
    "    X = vectorized_ouv.view(1,-1).repeat(args.batch_size,1,1)\n",
    "    result = classifier(vectorized_ouv.unsqueeze(0).repeat(args.batch_size,1),state_word, apply_softmax=True)[0][0]\n",
    "    \n",
    "    if k==1:\n",
    "        pred_id = result.argmax().item()\n",
    "        return (classes[pred_id], result[pred_id])\n",
    "    else:\n",
    "        pred_indices = [i.item() for i in result.topk(k)[1]]\n",
    "        output = []\n",
    "        for pred_id in pred_indices:\n",
    "            output.append((classes[pred_id],result[pred_id].item()))\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is a very old building dating back to 13th century -> Criteria iv with a probability of 0.46\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'this is a very old building dating back to 13th century'\n",
    "\n",
    "prediction = predict_rating(test_ouv,classifier,vectorizer,classes)\n",
    "print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "this is a very old building dating back to 13th century -> Criteria iv with a probability of 0.46\n",
      "this is a very old building dating back to 13th century -> Criteria iii with a probability of 0.41\n",
      "this is a very old building dating back to 13th century -> Criteria ii with a probability of 0.04\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'this is a very old building dating back to 13th century'\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "The particular layout of the complex is unique to this site -> Criteria iv with a probability of 0.59\n",
      "The particular layout of the complex is unique to this site -> Criteria ii with a probability of 0.10\n",
      "The particular layout of the complex is unique to this site -> Criteria iii with a probability of 0.10\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'The particular layout of the complex is unique to this site'\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty san marco , palazzo ducale , san zanipolo , scuola di san \n",
      "marco , frari and scuola di san rocco , san giorgio maggiore , etc .  -> Criteria i with a probability of 0.73\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty san marco , palazzo ducale , san zanipolo , scuola di san \n",
      "marco , frari and scuola di san rocco , san giorgio maggiore , etc .  -> Criteria vii with a probability of 0.06\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty san marco , palazzo ducale , san zanipolo , scuola di san \n",
      "marco , frari and scuola di san rocco , san giorgio maggiore , etc .  -> Criteria iv with a probability of 0.06\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
    "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
    "age are represented by monuments of incomparable beauty san marco , palazzo ducale , san zanipolo , scuola di san \n",
    "marco , frari and scuola di san rocco , san giorgio maggiore , etc . '''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed: 0.22770094871520996\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "class Timer(object):\n",
    "    def __init__(self, name=None):\n",
    "        self.name = name\n",
    "\n",
    "    def __enter__(self):\n",
    "        self.tstart = time.time()\n",
    "\n",
    "    def __exit__(self, type, value, traceback):\n",
    "        if self.name:\n",
    "            print('[%s]' % self.name,)\n",
    "        print('Elapsed: %s' % (time.time() - self.tstart))\n",
    "        \n",
    "set_seed_everywhere(args.seed, args.cuda)        \n",
    "test_ouv = 'The particular layout of the complex is unique to this site'\n",
    "k=3\n",
    "with Timer():\n",
    "    predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference with Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "def show_attention(text, classifier, vectorizer, classes, max_length=dataset._max_seq_length, k=1):\n",
    "    \"\"\"Predict the rating of a review\n",
    "    \n",
    "    Args:\n",
    "        text (str): the text of the description\n",
    "        classifier (ReviewClassifier): the trained model\n",
    "        vectorizer (ReviewVectorizer): the corresponding vectorizer\n",
    "        classes (list of str): The name of the ouv classes\n",
    "        k (int): show the largest k prediction, default to 1\n",
    "    \"\"\"\n",
    "    ouv = preprocess_text(text)\n",
    "    ouv_len = len(ouv.split())\n",
    "    \n",
    "    classifier.eval()\n",
    "    state_word = classifier.init_hidden().to(args.device)\n",
    "    vectorized_ouv = torch.tensor(vectorizer.vectorize(ouv, vector_length = max_length)[0])\n",
    "    X = vectorized_ouv.view(1,-1).repeat(args.batch_size,1,1)\n",
    "    result = classifier(vectorized_ouv.unsqueeze(0).repeat(args.batch_size,1),state_word, apply_softmax=True)[-1][0]\n",
    "    attn = result.detach().numpy()\n",
    "    \n",
    "    tokens = ouv.split(' ')\n",
    "    print_token = ['\\033[0;39;44m'+tokens[i] if attn[i]>1.0/len(tokens) \n",
    "                   else '\\033[0;39;107m'+tokens[i] if attn[i]>0.5/len(tokens)\n",
    "                   else '\\033[0;39;47m'+tokens[i] if attn[i]>0.1/len(tokens)\n",
    "                   else '\\033[0;39;49m'+tokens[i]\n",
    "                   for i in range(len(tokens))]\n",
    "    print(' '.join(print_token))\n",
    "    \n",
    "    return attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Namespace(batch_size=256, bi=False, catch_keyboard_interrupt=True, cuda=False, device=device(type='cpu'), dropout_p=0, early_stopping_criteria=5, embedding_size=300, expand_filepaths_to_save_dir=True, freeze=True, frequency_cutoff=1, fuzzy=True, fuzzy_how='uni', fuzzy_lambda=0.2, glove_filepath='Data/glove/glove.6B.300d.txt', hidden_dim=128, k=3, l2=1e-05, learning_rate=0.001, model_state_file='model_storage/attn/model.pth', num_epochs=100, ouv_csv='Data/ouv_with_splits_full.csv', prior_csv='Data/Coappearance_matrix.csv', ratings_csv='Data/human_rates.csv', reload_from_files=False, save_dir='model_storage/attn/', seed=1337, use_glove=True, vectorizer_file='model_storage/attn/vectorizer.json')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;49mthis \u001b[0;39;49mis \u001b[0;39;49ma \u001b[0;39;49mvery \u001b[0;39;49mold \u001b[0;39;44mbuilding \u001b[0;39;44mdating \u001b[0;39;107mback \u001b[0;39;107mto \u001b[0;39;47mth \u001b[0;39;44mcentury\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1.2716800e-03, 5.7590933e-04, 3.2928781e-04, 1.0889025e-03,\n",
       "       3.8831106e-03, 9.5687523e-02, 6.2945044e-01, 5.8780748e-02,\n",
       "       5.1390026e-02, 1.5178600e-02, 1.4236379e-01, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00], dtype=float32)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k=3\n",
    "show_attention('This is a very old building dating back to 13th century',classifier,vectorizer,classes,k=k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;107msaint \u001b[0;39;107mchristopher \u001b[0;39;47m, \u001b[0;39;44msaint \u001b[0;39;47mvincent \u001b[0;39;47mferreri \u001b[0;39;47m, \u001b[0;39;44msaint \u001b[0;39;107msebastian \u001b[0;39;47mare \u001b[0;39;47mdepicted \u001b[0;39;49min \u001b[0;39;49mthis \u001b[0;39;44maltar \u001b[0;39;44m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.04375633, 0.05424835, 0.01664908, 0.07758369, 0.02839746,\n",
       "       0.02402568, 0.02167705, 0.08286478, 0.03810453, 0.01085976,\n",
       "       0.01902153, 0.00453314, 0.00402649, 0.49807873, 0.07617339,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        ], dtype=float32)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k=3\n",
    "show_attention('Saint Christopher, Saint Vincent Ferreri, Saint Sebastian (are depicted in this Altar)',classifier,vectorizer,classes,k=k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;49mthe \u001b[0;39;107marchitecture \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mimperial \u001b[0;39;47mpalace \u001b[0;39;47mcomplexes \u001b[0;39;47m, \u001b[0;39;47mparticularly \u001b[0;39;49min \u001b[0;39;47mshenyang \u001b[0;39;49m, \u001b[0;39;49mexhibits \u001b[0;39;49man \u001b[0;39;47mimportant \u001b[0;39;47minterchange \u001b[0;39;49mof \u001b[0;39;44minfluences \u001b[0;39;47mof \u001b[0;39;44mtraditional \u001b[0;39;44marchitecture \u001b[0;39;47mand \u001b[0;39;44mchinese \u001b[0;39;47mpalace \u001b[0;39;44marchitecture \u001b[0;39;107mparticularly \u001b[0;39;47min \u001b[0;39;49mthe \u001b[0;39;49mth \u001b[0;39;49mand \u001b[0;39;49mth \u001b[0;39;47mcenturies \u001b[0;39;49m. \u001b[0;39;49m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2.1641505e-05, 1.5225343e-02, 3.6769002e-04, 1.2306761e-04,\n",
       "       3.3024438e-02, 9.5179090e-03, 4.4805603e-03, 5.5463151e-03,\n",
       "       3.5013244e-03, 1.3319124e-03, 3.4408979e-03, 2.7808642e-03,\n",
       "       2.4274683e-03, 1.1310581e-03, 3.7345218e-03, 5.6605279e-03,\n",
       "       1.3185071e-03, 4.6892059e-01, 4.1425726e-03, 3.5524447e-02,\n",
       "       1.6723141e-01, 8.8114878e-03, 3.0805282e-02, 1.2297292e-02,\n",
       "       1.3630727e-01, 1.7332496e-02, 4.0842174e-03, 1.3076827e-03,\n",
       "       1.5618989e-03, 1.9770386e-03, 1.2900901e-03, 9.7581586e-03,\n",
       "       2.4993871e-03, 2.5146264e-03, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00], dtype=float32)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_attention('''The architecture of the Imperial Palace complexes, particularly in Shenyang, \n",
    "exhibits an important interchange of influences of traditional architecture and Chinese palace \n",
    "architecture particularly in the 17th and 18th centuries.''',classifier,vectorizer,classes,k=k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = '''With the unusualness of an archaeological site which still breathes life, Venice bears testimony unto \n",
    "itself. This mistress of the seas is a link between the East and the West, between Islam and Christianity and \n",
    "lives on through thousands of monuments and vestiges of a time gone by.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;49mwith \u001b[0;39;49mthe \u001b[0;39;49munusualness \u001b[0;39;49mof \u001b[0;39;49man \u001b[0;39;44marchaeological \u001b[0;39;47msite \u001b[0;39;49mwhich \u001b[0;39;47mstill \u001b[0;39;47mbreathes \u001b[0;39;107mlife \u001b[0;39;47m, \u001b[0;39;44mvenice \u001b[0;39;107mbears \u001b[0;39;44mtestimony \u001b[0;39;44munto \u001b[0;39;107mitself \u001b[0;39;107m. \u001b[0;39;49mthis \u001b[0;39;107mmistress \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mseas \u001b[0;39;49mis \u001b[0;39;49ma \u001b[0;39;49mlink \u001b[0;39;49mbetween \u001b[0;39;49mthe \u001b[0;39;47meast \u001b[0;39;47mand \u001b[0;39;49mthe \u001b[0;39;47mwest \u001b[0;39;47m, \u001b[0;39;47mbetween \u001b[0;39;47mislam \u001b[0;39;47mand \u001b[0;39;44mchristianity \u001b[0;39;47mand \u001b[0;39;107mlives \u001b[0;39;49mon \u001b[0;39;47mthrough \u001b[0;39;47mthousands \u001b[0;39;49mof \u001b[0;39;47mmonuments \u001b[0;39;47mand \u001b[0;39;44mvestiges \u001b[0;39;49mof \u001b[0;39;49ma \u001b[0;39;47mtime \u001b[0;39;47mgone \u001b[0;39;107mby \u001b[0;39;47m. \u001b[0;39;47m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([3.5281535e-04, 5.1882460e-05, 4.6675003e-04, 9.4774259e-05,\n",
       "       1.5358051e-04, 2.2895613e-01, 3.5995883e-03, 1.5663307e-03,\n",
       "       4.2108442e-03, 4.1944478e-03, 1.3772404e-02, 5.7633081e-03,\n",
       "       2.6178673e-02, 1.3602166e-02, 3.3615637e-01, 4.8800863e-02,\n",
       "       1.3284966e-02, 1.5416360e-02, 1.8123721e-03, 1.0915331e-02,\n",
       "       1.3309506e-03, 9.4688218e-04, 4.9420852e-02, 1.7434456e-03,\n",
       "       2.9540571e-04, 1.0651515e-03, 4.0065896e-04, 2.0084185e-04,\n",
       "       8.2686748e-03, 2.1177977e-03, 6.4169656e-04, 4.6805656e-03,\n",
       "       7.7787614e-03, 2.2067758e-03, 8.7849200e-03, 3.3839729e-03,\n",
       "       4.7267392e-02, 4.2197164e-03, 1.4599891e-02, 1.8816863e-03,\n",
       "       7.7002859e-03, 6.1622863e-03, 1.0209952e-03, 8.3722593e-03,\n",
       "       2.7325631e-03, 3.6731102e-02, 1.2392439e-03, 6.2849157e-04,\n",
       "       4.4447095e-03, 8.7236138e-03, 1.4838366e-02, 9.1225700e-03,\n",
       "       7.6975045e-03, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00], dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_attention(test,classifier,vectorizer,classes,k=k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Criteria iii', 0.6642264127731323),\n",
       " ('Criteria vi', 0.19791539013385773),\n",
       " ('Criteria iv', 0.04785666987299919)]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_rating(test,classifier,vectorizer,classes,k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Criteria ii', 0.8414957523345947),\n",
       " ('Criteria iv', 0.10986685007810593),\n",
       " ('Criteria iii', 0.014939364045858383)]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_rating('''The architecture of the Imperial Palace complexes, particularly in Shenyang, \n",
    "exhibits an important interchange of influences of traditional architecture and Chinese palace \n",
    "architecture particularly in the 17th and 18th centuries.''',classifier,vectorizer,classes,k=k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_f = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world'''\n",
    "sentence_f1 = '''from torcellos cathedral to the church of santa maria della salute'''\n",
    "sentence_g = '''the years of the republics extraordinary golden age are represented by monuments of incomparable beauty'''\n",
    "sentence_h = '''san marco , palazzo ducale , san zanipolo , scuola di san marco , frari and scuola di san rocco , san giorgio maggiore , etc'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;49mthe \u001b[0;39;47mlagoon \u001b[0;39;49mof \u001b[0;39;47mvenice \u001b[0;39;49malso \u001b[0;39;49mhas \u001b[0;39;49mone \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;49mhighest \u001b[0;39;49mconcentrations \u001b[0;39;49mof \u001b[0;39;44mmasterpieces \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;49mworld\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([8.9620677e-05, 1.0345962e-02, 2.8579152e-04, 7.6073254e-03,\n",
       "       6.3028967e-04, 5.3201697e-04, 9.5164462e-05, 1.2912272e-04,\n",
       "       9.8319950e-05, 9.8977226e-04, 1.9344218e-03, 1.4400504e-04,\n",
       "       9.6901965e-01, 4.6469267e-03, 1.9053022e-04, 3.2611040e-03,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 0.0000000e+00,\n",
       "       0.0000000e+00, 0.0000000e+00, 0.0000000e+00], dtype=float32)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_attention(sentence_f,classifier,vectorizer,classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;47mfrom \u001b[0;39;47mtorcellos \u001b[0;39;47mcathedral \u001b[0;39;47mto \u001b[0;39;49mthe \u001b[0;39;44mchurch \u001b[0;39;47mof \u001b[0;39;47msanta \u001b[0;39;44mmaria \u001b[0;39;44mdella \u001b[0;39;107msalute\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.01298109, 0.01378309, 0.0169484 , 0.01023767, 0.00248664,\n",
       "       0.25505283, 0.01498654, 0.02350277, 0.38028172, 0.19383384,\n",
       "       0.07590542, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        ], dtype=float32)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_attention(sentence_f1,classifier,vectorizer,classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;49mthe \u001b[0;39;49myears \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;49mrepublics \u001b[0;39;47mextraordinary \u001b[0;39;44mgolden \u001b[0;39;44mage \u001b[0;39;47mare \u001b[0;39;47mrepresented \u001b[0;39;47mby \u001b[0;39;107mmonuments \u001b[0;39;49mof \u001b[0;39;44mincomparable \u001b[0;39;44mbeauty\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2.51027581e-04, 5.72370132e-04, 1.19964141e-04, 6.98789736e-05,\n",
       "       9.44014173e-04, 7.49043375e-03, 1.66301891e-01, 2.30476528e-01,\n",
       "       8.18199571e-03, 1.64250173e-02, 3.22422758e-02, 3.69247682e-02,\n",
       "       2.32940912e-03, 1.03519686e-01, 3.94150734e-01, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00], dtype=float32)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_attention(sentence_g,classifier,vectorizer,classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = '''The designated property area includes all elements embodying the values in the creativity, influence, \n",
    "historic evidence, and architectural exemplar, with the historical scale, architectural types, and other \n",
    "components, as well as the techniques and artistic achievements of Chinese palace buildings after the 15th \n",
    "century, particularly in the 17th to 18th century, well preserved'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Criteria i', 0.733273983001709),\n",
       " ('Criteria vii', 0.10703416168689728),\n",
       " ('Criteria viii', 0.03875169903039932)]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_rating(sentence_f,classifier,vectorizer,classes,k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Criteria vi', 0.3464215397834778),\n",
       " ('Criteria iv', 0.2133128046989441),\n",
       " ('Criteria ii', 0.1942373812198639)]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_rating(sentence_f1,classifier,vectorizer,classes,k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Criteria i', 0.41988039016723633),\n",
       " ('Criteria iii', 0.1634589433670044),\n",
       " ('Criteria vi', 0.15812163054943085)]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_rating(sentence_g,classifier,vectorizer,classes,k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Criteria i', 0.7399744987487793),\n",
       " ('Criteria vii', 0.07228098809719086),\n",
       " ('Criteria iv', 0.04865434020757675)]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_rating(sentence_f + sentence_f1 + sentence_g,classifier,vectorizer,classes,k=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Producing Highlighted Ratings for Human Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "rates = pd.read_csv(args.ratings_csv, sep ='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;49mwith \u001b[0;39;49mthe \u001b[0;39;49munusualness \u001b[0;39;49mof \u001b[0;39;49man \u001b[0;39;44marchaeological \u001b[0;39;49msite \u001b[0;39;49mwhich \u001b[0;39;47mstill \u001b[0;39;47mbreathes \u001b[0;39;47mlife \u001b[0;39;47m, \u001b[0;39;107mvenice \u001b[0;39;47mbears \u001b[0;39;44mtestimony \u001b[0;39;44munto \u001b[0;39;47mitself\n",
      "\n",
      "\u001b[0;39;107mvenice \u001b[0;39;107msymbolizes \u001b[0;39;49mthe \u001b[0;39;44mpeoples \u001b[0;39;47mvictorious \u001b[0;39;47mstruggle \u001b[0;39;49magainst \u001b[0;39;49mthe \u001b[0;39;47melements \u001b[0;39;47mas \u001b[0;39;47mthey \u001b[0;39;47mmanaged \u001b[0;39;47mto \u001b[0;39;44mmaster \u001b[0;39;47ma \u001b[0;39;44mhostile \u001b[0;39;44mnature\n",
      "\n",
      "\u001b[0;39;49mit \u001b[0;39;49mwas \u001b[0;39;49mfrom \u001b[0;39;44mvenice \u001b[0;39;107mthat \u001b[0;39;107mmarco \u001b[0;39;44mpolo \u001b[0;39;44mnum \u001b[0;39;107mnum \u001b[0;39;107mset \u001b[0;39;107mout \u001b[0;39;47min \u001b[0;39;107msearch \u001b[0;39;47mof \u001b[0;39;44mchina \u001b[0;39;44m, \u001b[0;39;44mannam \u001b[0;39;107m, \u001b[0;39;107mtonkin \u001b[0;39;107m, \u001b[0;39;47msumatra \u001b[0;39;47m, \u001b[0;39;47mindia \u001b[0;39;49mand \u001b[0;39;44mpersia\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;44mqueen \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mseas \u001b[0;39;107m, \u001b[0;39;47mheroically \u001b[0;39;44mperched \u001b[0;39;49mon \u001b[0;39;47mher \u001b[0;39;47mtiny \u001b[0;39;47mislands \u001b[0;39;47m, \u001b[0;39;47mextended \u001b[0;39;47mher \u001b[0;39;44mhorizon \u001b[0;39;49mwell \u001b[0;39;44mbeyond \u001b[0;39;49mthe \u001b[0;39;44mlagoon \u001b[0;39;44m, \u001b[0;39;49mthe \u001b[0;39;44madriatic \u001b[0;39;47mand \u001b[0;39;49mthe \u001b[0;39;44mmediterranean\n",
      "\n",
      "\u001b[0;39;49mthis \u001b[0;39;47mmistress \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mseas \u001b[0;39;47mis \u001b[0;39;49ma \u001b[0;39;47mlink \u001b[0;39;49mbetween \u001b[0;39;49mthe \u001b[0;39;107meast \u001b[0;39;49mand \u001b[0;39;49mthe \u001b[0;39;47mwest \u001b[0;39;47m, \u001b[0;39;49mbetween \u001b[0;39;47mislam \u001b[0;39;47mand \u001b[0;39;44mchristianity \u001b[0;39;107mand \u001b[0;39;44mlives \u001b[0;39;47mon \u001b[0;39;44mthrough \u001b[0;39;107mthousands \u001b[0;39;47mof \u001b[0;39;44mmonuments \u001b[0;39;47mand \u001b[0;39;44mvestiges \u001b[0;39;47mof \u001b[0;39;49ma \u001b[0;39;107mtime \u001b[0;39;44mgone \u001b[0;39;44mby \u001b[0;39;44m. \u001b[0;39;44m\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49mcity \u001b[0;39;49mis \u001b[0;39;49malso \u001b[0;39;49mdirectly \u001b[0;39;49mand \u001b[0;39;107mtangibly \u001b[0;39;47massociated \u001b[0;39;49mwith \u001b[0;39;49mthe \u001b[0;39;47mhistory \u001b[0;39;49mof \u001b[0;39;44mhumankind\n",
      "\n",
      "\u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;49mmediterranean \u001b[0;39;49marea \u001b[0;39;49m, \u001b[0;39;49mthe \u001b[0;39;47mlagoon \u001b[0;39;49mof \u001b[0;39;47mvenice \u001b[0;39;49mrepresents \u001b[0;39;49man \u001b[0;39;47moutstanding \u001b[0;39;47mexample \u001b[0;39;49mof \u001b[0;39;49ma \u001b[0;39;44msemi \u001b[0;39;107mlacustral \u001b[0;39;44mhabitat \u001b[0;39;47mwhich \u001b[0;39;49mhas \u001b[0;39;49mbecome \u001b[0;39;44mvulnerable \u001b[0;39;49mas \u001b[0;39;49ma \u001b[0;39;49mresult \u001b[0;39;49mof \u001b[0;39;44mirreversible \u001b[0;39;44mnatural \u001b[0;39;49mand \u001b[0;39;47mclimate \u001b[0;39;49mchanges\n",
      "\n",
      "\u001b[0;39;47mvenice \u001b[0;39;47mpossesses \u001b[0;39;49man \u001b[0;39;44mincomparable \u001b[0;39;47mseries \u001b[0;39;49mof \u001b[0;39;44marchitectural \u001b[0;39;44mensembles \u001b[0;39;44millustrating \u001b[0;39;49mthe \u001b[0;39;47mhight \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;47mrepublics \u001b[0;39;44msplendour\n",
      "\n",
      "\u001b[0;39;49mbut \u001b[0;39;49mwhen \u001b[0;39;49mit \u001b[0;39;49mbegan \u001b[0;39;49mto \u001b[0;39;49mlose \u001b[0;39;49mits \u001b[0;39;49mpower \u001b[0;39;49mover \u001b[0;39;49mthe \u001b[0;39;47mseas \u001b[0;39;49m, \u001b[0;39;49mvenice \u001b[0;39;44mexerted \u001b[0;39;49mits \u001b[0;39;44minfluence \u001b[0;39;47min \u001b[0;39;49ma \u001b[0;39;47mvery \u001b[0;39;49mdifferent \u001b[0;39;44mmanner \u001b[0;39;107m, \u001b[0;39;47mthanks \u001b[0;39;47mto \u001b[0;39;49mits \u001b[0;39;47mgreat \u001b[0;39;44mpainters\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;44minfluence \u001b[0;39;49mof \u001b[0;39;107mvenice \u001b[0;39;49mon \u001b[0;39;49mthe \u001b[0;39;44mdevelopment \u001b[0;39;47mof \u001b[0;39;44marchitecture \u001b[0;39;107mand \u001b[0;39;44mmonumental \u001b[0;39;44marts \u001b[0;39;47mis \u001b[0;39;44mconsiderable \u001b[0;39;47mfirst \u001b[0;39;107mthrough \u001b[0;39;47mthe \u001b[0;39;47mserenissimas \u001b[0;39;47mfondachi \u001b[0;39;47mor \u001b[0;39;47mtrading \u001b[0;39;47mstations \u001b[0;39;47m, \u001b[0;39;49malong \u001b[0;39;49mthe \u001b[0;39;47mdalmatian \u001b[0;39;47mcoast \u001b[0;39;49m, \u001b[0;39;49min \u001b[0;39;44masia \u001b[0;39;107mminor \u001b[0;39;47mand \u001b[0;39;47min \u001b[0;39;47megypt \u001b[0;39;47m, \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;49mislands \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;49mionian \u001b[0;39;49msea \u001b[0;39;49m, \u001b[0;39;49mthe \u001b[0;39;49mpeloponnesus \u001b[0;39;49m, \u001b[0;39;49mcrete \u001b[0;39;49m, \u001b[0;39;49mand \u001b[0;39;49mcyprus \u001b[0;39;49m, \u001b[0;39;49mwhere \u001b[0;39;49mthe \u001b[0;39;47mmonuments \u001b[0;39;49mwere \u001b[0;39;47mclearly \u001b[0;39;47mbuilt \u001b[0;39;49mfollowing \u001b[0;39;107mvenetian \u001b[0;39;47mmodels\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49mcity \u001b[0;39;49mis \u001b[0;39;47mbuilt \u001b[0;39;49mon \u001b[0;39;47mnum \u001b[0;39;47msmall \u001b[0;39;49mislands \u001b[0;39;49mand \u001b[0;39;47mseems \u001b[0;39;47mto \u001b[0;39;47mfloat \u001b[0;39;49mon \u001b[0;39;49mthe \u001b[0;39;107mwaters \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mlagoon \u001b[0;39;47m, \u001b[0;39;49mcomposing \u001b[0;39;49man \u001b[0;39;47munforgettable \u001b[0;39;44mlandscape \u001b[0;39;107mwhose \u001b[0;39;47mimponderable \u001b[0;39;44mbeauty \u001b[0;39;44minspired \u001b[0;39;107mcanaletto \u001b[0;39;47m, \u001b[0;39;47mguardi \u001b[0;39;47m, \u001b[0;39;47mturner \u001b[0;39;49mand \u001b[0;39;49mmany \u001b[0;39;49mother \u001b[0;39;44mpainters\n",
      "\n",
      "\u001b[0;39;49mhis \u001b[0;39;47mtomb \u001b[0;39;49mat \u001b[0;39;47msan \u001b[0;39;44mlorenzo \u001b[0;39;47mrecalls \u001b[0;39;49mthe \u001b[0;39;47mrole \u001b[0;39;47mof \u001b[0;39;44mvenetian \u001b[0;39;44mmerchants \u001b[0;39;47min \u001b[0;39;47mthe \u001b[0;39;44mdiscovery \u001b[0;39;47mof \u001b[0;39;47mthe \u001b[0;39;44mworld \u001b[0;39;107mafter \u001b[0;39;47mthe \u001b[0;39;47marabs \u001b[0;39;47m, \u001b[0;39;47mbut \u001b[0;39;47mwell \u001b[0;39;47mbefore \u001b[0;39;49mthe \u001b[0;39;47mportuguese \u001b[0;39;47m. \u001b[0;39;107m\n",
      "\n",
      "\u001b[0;39;49min \u001b[0;39;49mthis \u001b[0;39;49mcoherent \u001b[0;39;44mecosystem \u001b[0;39;47mwhere \u001b[0;39;49mthe \u001b[0;39;47mmuddy \u001b[0;39;49mshelves \u001b[0;39;49malternately \u001b[0;39;49mabove \u001b[0;39;49mand \u001b[0;39;49mbelow \u001b[0;39;47mwater \u001b[0;39;47mlevel \u001b[0;39;49mare \u001b[0;39;49mas \u001b[0;39;49mimportant \u001b[0;39;49mas \u001b[0;39;49mthe \u001b[0;39;49mislands \u001b[0;39;49m, \u001b[0;39;49mpile \u001b[0;39;44mdwellings \u001b[0;39;47m, \u001b[0;39;44mfishing \u001b[0;39;44mvillages \u001b[0;39;47mand \u001b[0;39;47mrice \u001b[0;39;47mfields \u001b[0;39;47mneed \u001b[0;39;49mto \u001b[0;39;49mbe \u001b[0;39;44mprotected \u001b[0;39;44mno \u001b[0;39;47mless \u001b[0;39;49mthan \u001b[0;39;49mthe \u001b[0;39;49mpalazzi \u001b[0;39;49mand \u001b[0;39;47mchurches \u001b[0;39;47m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;47mlagoon \u001b[0;39;49mof \u001b[0;39;47mvenice \u001b[0;39;49malso \u001b[0;39;49mhas \u001b[0;39;49mone \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;49mhighest \u001b[0;39;49mconcentrations \u001b[0;39;49mof \u001b[0;39;44mmasterpieces \u001b[0;39;47min \u001b[0;39;49mthe \u001b[0;39;47mworld \u001b[0;39;49mfrom \u001b[0;39;49mtorcellos \u001b[0;39;47mcathedral \u001b[0;39;49mto \u001b[0;39;49mthe \u001b[0;39;47mchurch \u001b[0;39;49mof \u001b[0;39;49msanta \u001b[0;39;107mmaria \u001b[0;39;47mdella \u001b[0;39;47msalute \u001b[0;39;47m. \u001b[0;39;49mthe \u001b[0;39;49myears \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;49mrepublics \u001b[0;39;47mextraordinary \u001b[0;39;44mgolden \u001b[0;39;44mage \u001b[0;39;47mare \u001b[0;39;47mrepresented \u001b[0;39;107mby \u001b[0;39;107mmonuments \u001b[0;39;49mof \u001b[0;39;44mincomparable \u001b[0;39;44mbeauty \u001b[0;39;107msan \u001b[0;39;47mmarco \u001b[0;39;47m, \u001b[0;39;44mpalazzo \u001b[0;39;47mducale \u001b[0;39;47m, \u001b[0;39;49msan \u001b[0;39;49mzanipolo \u001b[0;39;47m, \u001b[0;39;47mscuola \u001b[0;39;47mdi \u001b[0;39;49msan \u001b[0;39;49mmarco \u001b[0;39;47m, \u001b[0;39;47mfrari \u001b[0;39;49mand \u001b[0;39;49mscuola \u001b[0;39;47mdi \u001b[0;39;49msan \u001b[0;39;49mrocco \u001b[0;39;47m, \u001b[0;39;49msan \u001b[0;39;107mgiorgio \u001b[0;39;47mmaggiore \u001b[0;39;47m, \u001b[0;39;107metc \u001b[0;39;47m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;49mbellini \u001b[0;39;49mand \u001b[0;39;49mgiorgione \u001b[0;39;49m, \u001b[0;39;49mthen \u001b[0;39;49mtiziano \u001b[0;39;49m, \u001b[0;39;49mtintoretto \u001b[0;39;49m, \u001b[0;39;49mveronese \u001b[0;39;49mand \u001b[0;39;49mtiepolo \u001b[0;39;49mcompletely \u001b[0;39;49mchanged \u001b[0;39;49mthe \u001b[0;39;47mperception \u001b[0;39;49mof \u001b[0;39;47mspace \u001b[0;39;47m, \u001b[0;39;49mlight \u001b[0;39;49mand \u001b[0;39;49mcolour \u001b[0;39;47mthus \u001b[0;39;47mleaving \u001b[0;39;49ma \u001b[0;39;47mdecisive \u001b[0;39;44mmark \u001b[0;39;49mon \u001b[0;39;49mthe \u001b[0;39;44mdevelopment \u001b[0;39;47mof \u001b[0;39;44mpainting \u001b[0;39;107mand \u001b[0;39;44mdecorative \u001b[0;39;44marts \u001b[0;39;107min \u001b[0;39;47mthe \u001b[0;39;47mwhole \u001b[0;39;49mof \u001b[0;39;107meurope \u001b[0;39;47m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;49mthese \u001b[0;39;49mtemporary \u001b[0;39;44msettlements \u001b[0;39;44mgradually \u001b[0;39;47mbecame \u001b[0;39;107mpermanent \u001b[0;39;47mand \u001b[0;39;49mthe \u001b[0;39;47minitial \u001b[0;39;44mrefuge \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;47mland \u001b[0;39;44mdwelling \u001b[0;39;107mpeasants \u001b[0;39;49mand \u001b[0;39;47mfishermen \u001b[0;39;107mbecame \u001b[0;39;49ma \u001b[0;39;44mmaritime \u001b[0;39;44mpower\n",
      "\n",
      "\u001b[0;39;47mhowever \u001b[0;39;47m, \u001b[0;39;49mat \u001b[0;39;49mthe \u001b[0;39;49mheart \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;107mlagoon \u001b[0;39;47m, \u001b[0;39;44mvenice \u001b[0;39;47mitself \u001b[0;39;47mstood \u001b[0;39;47mas \u001b[0;39;49mone \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mgreatest \u001b[0;39;44mcapitals \u001b[0;39;107min \u001b[0;39;47mthe \u001b[0;39;44mmedieval \u001b[0;39;44mworld\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[0;39;49mvenice \u001b[0;39;49mand \u001b[0;39;49mits \u001b[0;39;49mlagoon \u001b[0;39;49mform \u001b[0;39;49man \u001b[0;39;49minseparable \u001b[0;39;49mwhole \u001b[0;39;49mof \u001b[0;39;49mwhich \u001b[0;39;49mthe \u001b[0;39;49mcity \u001b[0;39;49mof \u001b[0;39;47mvenice \u001b[0;39;49mis \u001b[0;39;49mthe \u001b[0;39;49mpulsating \u001b[0;39;49mhistoric \u001b[0;39;49mheart \u001b[0;39;49mand \u001b[0;39;49ma \u001b[0;39;49munique \u001b[0;39;44martistic \u001b[0;39;44machievement\n",
      "\n",
      "\u001b[0;39;47mhuman \u001b[0;39;44minterventions \u001b[0;39;47mshow \u001b[0;39;47mhigh \u001b[0;39;47mtechnical \u001b[0;39;49mand \u001b[0;39;44mcreative \u001b[0;39;44mskills \u001b[0;39;47min \u001b[0;39;49mthe \u001b[0;39;47mrealization \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;107mhydraulic \u001b[0;39;49mand \u001b[0;39;44marchitectural \u001b[0;39;44mworks \u001b[0;39;47min \u001b[0;39;49mthe \u001b[0;39;47mlagoon \u001b[0;39;49marea\n",
      "\n",
      "\u001b[0;39;49mover \u001b[0;39;49mthe \u001b[0;39;47mcenturies \u001b[0;39;47m, \u001b[0;39;47mduring \u001b[0;39;49mthe \u001b[0;39;47mentire \u001b[0;39;44mperiod \u001b[0;39;47mof \u001b[0;39;49mthe \u001b[0;39;44mexpansion \u001b[0;39;47mof \u001b[0;39;44mvenice \u001b[0;39;44m, \u001b[0;39;44mwhen \u001b[0;39;47mit \u001b[0;39;44mwas \u001b[0;39;44mobliged \u001b[0;39;107mto \u001b[0;39;44mdefend \u001b[0;39;49mits \u001b[0;39;47mtrading \u001b[0;39;107mmarkets \u001b[0;39;47magainst \u001b[0;39;49mthe \u001b[0;39;44mcommercial \u001b[0;39;44mundertakings \u001b[0;39;47mof \u001b[0;39;47mthe \u001b[0;39;47marabs \u001b[0;39;107m, \u001b[0;39;49mthe \u001b[0;39;107mgenoese \u001b[0;39;47mand \u001b[0;39;49mthe \u001b[0;39;44mottoman \u001b[0;39;107mturks \u001b[0;39;107m, \u001b[0;39;44mvenice \u001b[0;39;44mnever \u001b[0;39;44mceased \u001b[0;39;47mto \u001b[0;39;107mconsolidate \u001b[0;39;49mits \u001b[0;39;49mposition \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;107mlagoon \u001b[0;39;47m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49munique \u001b[0;39;44mcultural \u001b[0;39;44mheritage \u001b[0;39;107maccumulated \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;107mlagoon \u001b[0;39;49mover \u001b[0;39;49mthe \u001b[0;39;47mcenturies \u001b[0;39;49mis \u001b[0;39;49mattested \u001b[0;39;47mby \u001b[0;39;49mthe \u001b[0;39;44mdiscovery \u001b[0;39;49mof \u001b[0;39;49mimportant \u001b[0;39;44marchaeological \u001b[0;39;44msettlements \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;47maltino \u001b[0;39;47marea \u001b[0;39;49mand \u001b[0;39;49mother \u001b[0;39;49msites \u001b[0;39;49mon \u001b[0;39;49mthe \u001b[0;39;47mmainland \u001b[0;39;47m, \u001b[0;39;49mwhich \u001b[0;39;49mwere \u001b[0;39;49mimportant \u001b[0;39;44mcommunication \u001b[0;39;47mand \u001b[0;39;44mtrade \u001b[0;39;107mhubs \u001b[0;39;107m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;47mvenice \u001b[0;39;49mand \u001b[0;39;49mits \u001b[0;39;49mlagoon \u001b[0;39;44mlandscape \u001b[0;39;49mis \u001b[0;39;49mthe \u001b[0;39;49mresult \u001b[0;39;49mof \u001b[0;39;49ma \u001b[0;39;47mdynamic \u001b[0;39;44mprocess \u001b[0;39;49mwhich \u001b[0;39;47millustrates \u001b[0;39;49mthe \u001b[0;39;47minteraction \u001b[0;39;49mbetween \u001b[0;39;47mpeople \u001b[0;39;47mand \u001b[0;39;49mthe \u001b[0;39;44mecosystem \u001b[0;39;47mof \u001b[0;39;49mtheir \u001b[0;39;44mnatural \u001b[0;39;44menvironment \u001b[0;39;49mover \u001b[0;39;49mtime\n",
      "\n",
      "\u001b[0;39;49mwhen \u001b[0;39;49ma \u001b[0;39;49mgroup \u001b[0;39;49mof \u001b[0;39;49mtiny \u001b[0;39;49mislands \u001b[0;39;49mwere \u001b[0;39;49mconsolidated \u001b[0;39;49mand \u001b[0;39;49morganized \u001b[0;39;49min \u001b[0;39;49ma \u001b[0;39;49munique \u001b[0;39;44murban \u001b[0;39;44msystem \u001b[0;39;44m, \u001b[0;39;47mnothing \u001b[0;39;49mremained \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mprimitive \u001b[0;39;107mtopography \u001b[0;39;49mbut \u001b[0;39;49mwhat \u001b[0;39;49mbecame \u001b[0;39;107mcanals \u001b[0;39;47m, \u001b[0;39;47msuch \u001b[0;39;47mas \u001b[0;39;49mthe \u001b[0;39;47mgiudecca \u001b[0;39;107mcanal \u001b[0;39;47m, \u001b[0;39;107mst \u001b[0;39;44mmark \u001b[0;39;47ms \u001b[0;39;107mcanal \u001b[0;39;47mand \u001b[0;39;49mthe \u001b[0;39;107mgreat \u001b[0;39;44mcanal \u001b[0;39;107m, \u001b[0;39;47mand \u001b[0;39;49ma \u001b[0;39;44mnetwork \u001b[0;39;47mof \u001b[0;39;107msmall \u001b[0;39;107mrii \u001b[0;39;47mthat \u001b[0;39;47mare \u001b[0;39;49mthe \u001b[0;39;44mveritable \u001b[0;39;44marteries \u001b[0;39;49mof \u001b[0;39;49ma \u001b[0;39;47mcity \u001b[0;39;47mon \u001b[0;39;107mwater \u001b[0;39;107m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;47mfrom \u001b[0;39;47mtorcello \u001b[0;39;49mto \u001b[0;39;49mthe \u001b[0;39;49mnorth \u001b[0;39;49mto \u001b[0;39;47mchioggia \u001b[0;39;47mto \u001b[0;39;49mthe \u001b[0;39;49msouth \u001b[0;39;47m, \u001b[0;39;47malmost \u001b[0;39;49mevery \u001b[0;39;49msmall \u001b[0;39;47misland \u001b[0;39;47mhad \u001b[0;39;49mits \u001b[0;39;47mown \u001b[0;39;44msettlement \u001b[0;39;44m, \u001b[0;39;107mtown \u001b[0;39;44m, \u001b[0;39;44mfishing \u001b[0;39;44mvillage \u001b[0;39;44mand \u001b[0;39;44martisan \u001b[0;39;44mvillage \u001b[0;39;44mmurano \u001b[0;39;44m\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;44minfluence \u001b[0;39;49mof \u001b[0;39;47mvenice \u001b[0;39;49mon \u001b[0;39;49mthe \u001b[0;39;44mdevelopment \u001b[0;39;49mof \u001b[0;39;44marchitecture \u001b[0;39;47mand \u001b[0;39;107mmonumental \u001b[0;39;44marts \u001b[0;39;47mhas \u001b[0;39;47mbeen \u001b[0;39;44mconsiderable \u001b[0;39;47m. \u001b[0;39;47m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;47mfounded \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;49mnum \u001b[0;39;49mth \u001b[0;39;47mcentury \u001b[0;39;47mad \u001b[0;39;47mand \u001b[0;39;44mspread \u001b[0;39;47mover \u001b[0;39;47mnum \u001b[0;39;47msmall \u001b[0;39;49mislands \u001b[0;39;49m, \u001b[0;39;107mvenice \u001b[0;39;107mbecame \u001b[0;39;49ma \u001b[0;39;47mmajor \u001b[0;39;44mmaritime \u001b[0;39;44mpower \u001b[0;39;47min \u001b[0;39;47mthe \u001b[0;39;107mnum \u001b[0;39;47mth \u001b[0;39;44mcentury\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49mwhole \u001b[0;39;49mcity \u001b[0;39;49mis \u001b[0;39;49man \u001b[0;39;47mextraordinary \u001b[0;39;44marchitectural \u001b[0;39;44mmasterpiece \u001b[0;39;47min \u001b[0;39;49mwhich \u001b[0;39;47meven \u001b[0;39;49mthe \u001b[0;39;49msmallest \u001b[0;39;47mbuilding \u001b[0;39;49mcontains \u001b[0;39;47mworks \u001b[0;39;47mby \u001b[0;39;49msome \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;47mworld \u001b[0;39;49ms \u001b[0;39;47mgreatest \u001b[0;39;44martists \u001b[0;39;47msuch \u001b[0;39;47mas \u001b[0;39;49mgiorgione \u001b[0;39;49m, \u001b[0;39;49mtitian \u001b[0;39;49m, \u001b[0;39;49mtintoretto \u001b[0;39;49m, \u001b[0;39;49mveronese \u001b[0;39;49mand \u001b[0;39;49mothers \u001b[0;39;49m. \u001b[0;39;49m\n",
      "\n",
      "\u001b[0;39;47mfounded \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;49mnum \u001b[0;39;49mth \u001b[0;39;47mcentury \u001b[0;39;47mand \u001b[0;39;44mspread \u001b[0;39;47mover \u001b[0;39;47mnum \u001b[0;39;47msmall \u001b[0;39;49mislands \u001b[0;39;49m, \u001b[0;39;107mvenice \u001b[0;39;107mbecame \u001b[0;39;49ma \u001b[0;39;47mmajor \u001b[0;39;44mmaritime \u001b[0;39;44mpower \u001b[0;39;47min \u001b[0;39;47mthe \u001b[0;39;107mnum \u001b[0;39;47mth \u001b[0;39;44mcentury\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49mwhole \u001b[0;39;49mcity \u001b[0;39;49mis \u001b[0;39;49man \u001b[0;39;47mextraordinary \u001b[0;39;44marchitectural \u001b[0;39;44mmasterpiece \u001b[0;39;47min \u001b[0;39;49mwhich \u001b[0;39;47meven \u001b[0;39;49mthe \u001b[0;39;49msmallest \u001b[0;39;47mbuilding \u001b[0;39;49mcontains \u001b[0;39;47mworks \u001b[0;39;47mby \u001b[0;39;49msome \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;47mworld \u001b[0;39;49ms \u001b[0;39;47mgreatest \u001b[0;39;44martists \u001b[0;39;47msuch \u001b[0;39;47mas \u001b[0;39;49mgiorgione \u001b[0;39;49m, \u001b[0;39;49mtitian \u001b[0;39;49m, \u001b[0;39;49mtintoretto \u001b[0;39;49m, \u001b[0;39;49mveronese \u001b[0;39;49mand \u001b[0;39;49mothers \u001b[0;39;49m. \u001b[0;39;49m\n",
      "\n",
      "\u001b[0;39;47mif \u001b[0;39;49mthe \u001b[0;39;47mbuildings \u001b[0;39;107mwould \u001b[0;39;47mbe \u001b[0;39;107mstraight \u001b[0;39;107m, \u001b[0;39;44mpainted \u001b[0;39;47mwell \u001b[0;39;44mmaintained \u001b[0;39;107m, \u001b[0;39;47mi \u001b[0;39;107mthink \u001b[0;39;44mvenice \u001b[0;39;44mwithout \u001b[0;39;47mthis \u001b[0;39;44mpatina \u001b[0;39;44m, \u001b[0;39;49mthe \u001b[0;39;107mgondolas \u001b[0;39;107mgondolieri \u001b[0;39;107m, \u001b[0;39;47mchannels \u001b[0;39;107m, \u001b[0;39;107mlittle \u001b[0;39;107mbridges \u001b[0;39;49mthe \u001b[0;39;47mat \u001b[0;39;47mall \u001b[0;39;47mtime \u001b[0;39;107mbusy \u001b[0;39;44mcanal \u001b[0;39;44mgrande \u001b[0;39;107mall \u001b[0;39;107mthese \u001b[0;39;47mmain \u001b[0;39;44mingredients \u001b[0;39;44maltogether \u001b[0;39;107mexpress \u001b[0;39;47mthe \u001b[0;39;44mromantic \u001b[0;39;44mcharm \u001b[0;39;47mof \u001b[0;39;44mvenice\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;47mlower \u001b[0;39;47mones \u001b[0;39;49mare \u001b[0;39;47minlaid \u001b[0;39;47mworked \u001b[0;39;49mwith \u001b[0;39;49mfigures \u001b[0;39;49mof \u001b[0;39;44mbuildings \u001b[0;39;44m, \u001b[0;39;107mstreets \u001b[0;39;44m, \u001b[0;39;44mfields \u001b[0;39;107mand \u001b[0;39;44mwells \u001b[0;39;47min \u001b[0;39;44mforeshortening \u001b[0;39;44mand \u001b[0;39;44mperspective\n",
      "\n",
      "\u001b[0;39;44mpeople \u001b[0;39;44mspoke \u001b[0;39;47mtogether \u001b[0;39;107m, \u001b[0;39;47mthey \u001b[0;39;107mlooked \u001b[0;39;47mat \u001b[0;39;107moneself \u001b[0;39;44m, \u001b[0;39;107mthey \u001b[0;39;44mlistened \u001b[0;39;107mand \u001b[0;39;107mthey \u001b[0;39;44mhad \u001b[0;39;44mtime\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49mlarge \u001b[0;39;47mchoir \u001b[0;39;49m, \u001b[0;39;49mlocated \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;49mcentral \u001b[0;39;49mnave \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mbasilica \u001b[0;39;107m, \u001b[0;39;49mis \u001b[0;39;49mthe \u001b[0;39;49monly \u001b[0;39;49mone \u001b[0;39;49min \u001b[0;39;47mitaly \u001b[0;39;47mto \u001b[0;39;47mhave \u001b[0;39;47mmaintained \u001b[0;39;49mthe \u001b[0;39;47moriginal \u001b[0;39;49mposition \u001b[0;39;47min \u001b[0;39;49mfront \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;49mmain \u001b[0;39;44maltar \u001b[0;39;44m, \u001b[0;39;107maccording \u001b[0;39;47mto \u001b[0;39;44mmedieval \u001b[0;39;107muse\n",
      "\n",
      "\u001b[0;39;49mit \u001b[0;39;49mwas \u001b[0;39;47moriginally \u001b[0;39;49mthe \u001b[0;39;47mhome \u001b[0;39;47mto \u001b[0;39;47mone \u001b[0;39;47mof \u001b[0;39;47mthe \u001b[0;39;44mscuole \u001b[0;39;44mgrandi \u001b[0;39;47mof \u001b[0;39;44mvenice \u001b[0;39;44m, \u001b[0;39;44mor \u001b[0;39;107msix \u001b[0;39;44mmajor \u001b[0;39;44mconfraternities \u001b[0;39;44m, \u001b[0;39;107mbut \u001b[0;39;47mis \u001b[0;39;47mnow \u001b[0;39;49mthe \u001b[0;39;49mcity \u001b[0;39;47ms \u001b[0;39;107mhospital\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[0;39;44msaint \u001b[0;39;44mchristopher \u001b[0;39;47m, \u001b[0;39;44msaint \u001b[0;39;107mvincent \u001b[0;39;107mferreri \u001b[0;39;107m, \u001b[0;39;44msaint \u001b[0;39;107msebastian \u001b[0;39;107mss\n",
      "\n",
      "\u001b[0;39;47mstructural \u001b[0;39;47mmodel \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mbasilica \u001b[0;39;47mshowing \u001b[0;39;49ma \u001b[0;39;44mcross \u001b[0;39;107msection \u001b[0;39;47mof \u001b[0;39;47mthe \u001b[0;39;107mcentral \u001b[0;39;107mpart \u001b[0;39;47mof \u001b[0;39;47mthe \u001b[0;39;44mchurch \u001b[0;39;107mwith \u001b[0;39;47mthe \u001b[0;39;44mdomes \u001b[0;39;47mof \u001b[0;39;49mthe \u001b[0;39;107mascension \u001b[0;39;107m, \u001b[0;39;49mthe \u001b[0;39;47mchancel \u001b[0;39;47mand \u001b[0;39;49mthe \u001b[0;39;49msouth \u001b[0;39;47mtransept\n",
      "\n",
      "\u001b[0;39;44msculpted \u001b[0;39;44mby \u001b[0;39;47mpietro \u001b[0;39;107mda \u001b[0;39;47msal \u001b[0;39;49min \u001b[0;39;47m, \u001b[0;39;49mthe \u001b[0;39;107mstatue \u001b[0;39;44mpresents \u001b[0;39;47ma \u001b[0;39;44mcrouching \u001b[0;39;44m, \u001b[0;39;44mnaked \u001b[0;39;44mhunchback \u001b[0;39;47msupporting \u001b[0;39;49ma \u001b[0;39;47msmall \u001b[0;39;44mflight \u001b[0;39;47mof \u001b[0;39;44msteps\n",
      "\n",
      "\u001b[0;39;47mif \u001b[0;39;49mthe \u001b[0;39;47mbuildings \u001b[0;39;107mwould \u001b[0;39;47mbe \u001b[0;39;107mstraight \u001b[0;39;107m, \u001b[0;39;44mpainted \u001b[0;39;47mwell \u001b[0;39;44mmaintained \u001b[0;39;107m, \u001b[0;39;47mi \u001b[0;39;107mthink \u001b[0;39;44mvenice \u001b[0;39;44mwithout \u001b[0;39;47mthis \u001b[0;39;44mpatina \u001b[0;39;44m, \u001b[0;39;49mthe \u001b[0;39;107mgondolas \u001b[0;39;107mgondolieri \u001b[0;39;107m, \u001b[0;39;47mchannels \u001b[0;39;107m, \u001b[0;39;107mlittle \u001b[0;39;107mbridges \u001b[0;39;49mthe \u001b[0;39;47mat \u001b[0;39;47mall \u001b[0;39;47mtime \u001b[0;39;107mbusy \u001b[0;39;44mcanal \u001b[0;39;44mgrande \u001b[0;39;107mall \u001b[0;39;107mthese \u001b[0;39;47mmain \u001b[0;39;44mingredients \u001b[0;39;44maltogether \u001b[0;39;107mexpress \u001b[0;39;47mthe \u001b[0;39;44mromantic \u001b[0;39;44mcharm \u001b[0;39;47mof \u001b[0;39;44mvenice\n",
      "\n",
      "\u001b[0;39;44mmark \u001b[0;39;47ms \u001b[0;39;47mbasilic \u001b[0;39;49muntil \u001b[0;39;49m, \u001b[0;39;49mafter \u001b[0;39;47mrestoration \u001b[0;39;49mthey \u001b[0;39;47mwere \u001b[0;39;47mthen \u001b[0;39;107mmoved \u001b[0;39;47mto \u001b[0;39;49mthe \u001b[0;39;44mmuseum \u001b[0;39;107min \u001b[0;39;47mfor \u001b[0;39;47mreasons \u001b[0;39;47mof \u001b[0;39;44mconservation \u001b[0;39;47mand \u001b[0;39;47mreplaced \u001b[0;39;47mby \u001b[0;39;47mreproductions \u001b[0;39;49mit \u001b[0;39;47mwas \u001b[0;39;47mmade \u001b[0;39;44mby \u001b[0;39;107mlost \u001b[0;39;107mwax \u001b[0;39;107mcasting \u001b[0;39;107musing \u001b[0;39;49mthe \u001b[0;39;107mso \u001b[0;39;47mcalled \u001b[0;39;107mindirect \u001b[0;39;44mmethod \u001b[0;39;44m, \u001b[0;39;47min \u001b[0;39;47man \u001b[0;39;44malloy \u001b[0;39;47mwith \u001b[0;39;47ma \u001b[0;39;44mvery \u001b[0;39;44mhigh \u001b[0;39;44mpercentage \u001b[0;39;47mof \u001b[0;39;44mcopper \u001b[0;39;107m, \u001b[0;39;47ma \u001b[0;39;44mrequirement \u001b[0;39;47mfor \u001b[0;39;49mthe \u001b[0;39;107mprocedure \u001b[0;39;47mof \u001b[0;39;44mmercury \u001b[0;39;107mgilding\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;44mvenetian \u001b[0;39;107marsenal \u001b[0;39;107mitalian \u001b[0;39;107marsenale \u001b[0;39;44mdi \u001b[0;39;107mvenezia \u001b[0;39;47mis \u001b[0;39;49ma \u001b[0;39;47mcomplex \u001b[0;39;47mof \u001b[0;39;44mformer \u001b[0;39;44mshipyards \u001b[0;39;107mand \u001b[0;39;44marmories \u001b[0;39;44mclustered \u001b[0;39;47mtogether \u001b[0;39;47min \u001b[0;39;47mthe \u001b[0;39;47mcity \u001b[0;39;47mof \u001b[0;39;44mvenice \u001b[0;39;107min \u001b[0;39;44mnorthern \u001b[0;39;44mitaly\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;47mfeast \u001b[0;39;49mof \u001b[0;39;49mour \u001b[0;39;47mlady \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;47mrosary \u001b[0;39;44mhad \u001b[0;39;44mbeen \u001b[0;39;44mmade \u001b[0;39;44mpart \u001b[0;39;107mof \u001b[0;39;107mthe \u001b[0;39;44mcalendar \u001b[0;39;47mof \u001b[0;39;44msaints \u001b[0;39;107min \u001b[0;39;44mafter \u001b[0;39;47ma \u001b[0;39;44mvictory \u001b[0;39;107mover \u001b[0;39;47mthe \u001b[0;39;44mturks\n",
      "\n",
      "\u001b[0;39;47muntil \u001b[0;39;44mvenice \u001b[0;39;107ms \u001b[0;39;47mislands \u001b[0;39;44mcould \u001b[0;39;47monly \u001b[0;39;47mbe \u001b[0;39;107maccessed \u001b[0;39;107mby \u001b[0;39;44mriver \u001b[0;39;107mboats \u001b[0;39;107mor \u001b[0;39;107mrailway\n",
      "\n",
      "\u001b[0;39;49mall \u001b[0;39;44mdecorative \u001b[0;39;44mlevels \u001b[0;39;47mare \u001b[0;39;47mpunctuated \u001b[0;39;47mby \u001b[0;39;47melegant \u001b[0;39;44mbelt \u001b[0;39;47mcourses\n",
      "\n",
      "\u001b[0;39;49mit \u001b[0;39;49mserved \u001b[0;39;49mat \u001b[0;39;49mvarious \u001b[0;39;47mtimes \u001b[0;39;47mas \u001b[0;39;49ma \u001b[0;39;44mgathering \u001b[0;39;47mplace \u001b[0;39;47mfor \u001b[0;39;107mnobles \u001b[0;39;47mand \u001b[0;39;47mfor \u001b[0;39;107mmeetings \u001b[0;39;47mof \u001b[0;39;49mthe \u001b[0;39;107mprocurators \u001b[0;39;49mof \u001b[0;39;44msaint \u001b[0;39;44mmark \u001b[0;39;107m, \u001b[0;39;47mthe \u001b[0;39;44mvenetian \u001b[0;39;107mofficials \u001b[0;39;107mresponsible \u001b[0;39;107mprincipally \u001b[0;39;47mfor \u001b[0;39;49mthe \u001b[0;39;107madministration \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;44mtreasury \u001b[0;39;47mof \u001b[0;39;44msaint \u001b[0;39;44mmark \u001b[0;39;107ms \u001b[0;39;44mbasilica \u001b[0;39;107mand \u001b[0;39;47mfor \u001b[0;39;47mthe \u001b[0;39;44mpublic \u001b[0;39;107mbuildings \u001b[0;39;47maround \u001b[0;39;44msaint \u001b[0;39;44mmark \u001b[0;39;44ms \u001b[0;39;47msquare\n",
      "\n",
      "\u001b[0;39;49min \u001b[0;39;49m, \u001b[0;39;49mthe \u001b[0;39;49mstation \u001b[0;39;49mwas \u001b[0;39;49mfinalized \u001b[0;39;49mon \u001b[0;39;49ma \u001b[0;39;44mdesign \u001b[0;39;44mby \u001b[0;39;49mthe \u001b[0;39;44marchitect \u001b[0;39;107mpaul \u001b[0;39;47mperilli\n",
      "\n",
      "\u001b[0;39;44mcatherine \u001b[0;39;107m, \u001b[0;39;49mthe \u001b[0;39;107mqueen \u001b[0;39;49mof \u001b[0;39;47malexandria \u001b[0;39;107mwho \u001b[0;39;47m, \u001b[0;39;47maccording \u001b[0;39;47mto \u001b[0;39;47mlegend \u001b[0;39;47m, \u001b[0;39;44mlived \u001b[0;39;47min \u001b[0;39;49mthe \u001b[0;39;49mthird \u001b[0;39;107mcentury \u001b[0;39;107m, \u001b[0;39;47mis \u001b[0;39;107mdepicted \u001b[0;39;47mhere \u001b[0;39;47min \u001b[0;39;49mthe \u001b[0;39;47mmost \u001b[0;39;47mfamous \u001b[0;39;47mepisode \u001b[0;39;47mof \u001b[0;39;44mher \u001b[0;39;44mlife \u001b[0;39;44mthough \u001b[0;39;47mit \u001b[0;39;47mhas \u001b[0;39;107mnot \u001b[0;39;107mbeen \u001b[0;39;44mconfirmed \u001b[0;39;44mby \u001b[0;39;107mcanonical \u001b[0;39;107mlaw \u001b[0;39;44mher \u001b[0;39;107mmystic \u001b[0;39;107mmarriage \u001b[0;39;47mwith \u001b[0;39;44mchrist \u001b[0;39;44m, \u001b[0;39;44msymbolised \u001b[0;39;44mby \u001b[0;39;47mthe \u001b[0;39;107mlamb \u001b[0;39;107mplaced \u001b[0;39;44mby \u001b[0;39;47mthe \u001b[0;39;107mchild \u001b[0;39;47min \u001b[0;39;44mher \u001b[0;39;47mright \u001b[0;39;107mhand\n",
      "\n",
      "\u001b[0;39;107mmary \u001b[0;39;49mafter \u001b[0;39;49mvenetians \u001b[0;39;47mbelieve \u001b[0;39;47mprayers \u001b[0;39;49mto \u001b[0;39;47mher \u001b[0;39;47mwere \u001b[0;39;47manswered \u001b[0;39;47mwhen \u001b[0;39;47mthey \u001b[0;39;47mprayed \u001b[0;39;49mfor \u001b[0;39;44mprotection \u001b[0;39;47magainst \u001b[0;39;47msickness \u001b[0;39;47m. \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;107mpeter \u001b[0;39;47m, \u001b[0;39;47mwearing \u001b[0;39;49ma \u001b[0;39;47mdelightful \u001b[0;39;47mblue \u001b[0;39;47mvestment \u001b[0;39;49mwith \u001b[0;39;49ma \u001b[0;39;47mglorious \u001b[0;39;47myellow \u001b[0;39;44mmantle \u001b[0;39;44mdraped \u001b[0;39;49mover \u001b[0;39;49mhis \u001b[0;39;49mknees \u001b[0;39;47m, \u001b[0;39;47minterrupts \u001b[0;39;49mhis \u001b[0;39;49mreading \u001b[0;39;49mand \u001b[0;39;47mpresents \u001b[0;39;47mto \u001b[0;39;44mher \u001b[0;39;107mjacopo \u001b[0;39;107mpesaro \u001b[0;39;107m, \u001b[0;39;49mthe \u001b[0;39;107mbishop \u001b[0;39;47mand \u001b[0;39;47mcommander \u001b[0;39;47mof \u001b[0;39;107mtwenty \u001b[0;39;107mpapal \u001b[0;39;107mgalleys \u001b[0;39;47min \u001b[0;39;49mthe \u001b[0;39;44mtriumphant \u001b[0;39;44mbattle \u001b[0;39;47magainst \u001b[0;39;49mthe \u001b[0;39;107mturks \u001b[0;39;47mat \u001b[0;39;47msanta \u001b[0;39;47mmaura \u001b[0;39;107m\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49mbanks \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;49mgrand \u001b[0;39;47mcanal \u001b[0;39;49mare \u001b[0;39;49mlined \u001b[0;39;49mwith \u001b[0;39;49mmore \u001b[0;39;49mthan \u001b[0;39;47mbuildings \u001b[0;39;47m, \u001b[0;39;47mmost \u001b[0;39;49mof \u001b[0;39;49mwhich \u001b[0;39;47mdate \u001b[0;39;47mfrom \u001b[0;39;49mthe \u001b[0;39;49mth \u001b[0;39;47mto \u001b[0;39;49mthe \u001b[0;39;49mth \u001b[0;39;107mcentury \u001b[0;39;47m, \u001b[0;39;47mand \u001b[0;39;107mdemonstrate \u001b[0;39;49mthe \u001b[0;39;44mwelfare \u001b[0;39;47mand \u001b[0;39;44mart \u001b[0;39;44mcreated \u001b[0;39;44mby \u001b[0;39;47mthe \u001b[0;39;47mrepublic \u001b[0;39;47mof \u001b[0;39;44mvenice\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49msquare \u001b[0;39;47mbecame \u001b[0;39;47mpaved \u001b[0;39;47mwith \u001b[0;39;47mbricks \u001b[0;39;47min \u001b[0;39;47min \u001b[0;39;47ma \u001b[0;39;44mherringbone \u001b[0;39;44mpattern\n",
      "\n",
      "\u001b[0;39;107mmark \u001b[0;39;47m, \u001b[0;39;49malong \u001b[0;39;49mwith \u001b[0;39;49msome \u001b[0;39;49mepisodes \u001b[0;39;49mfrom \u001b[0;39;49mthe \u001b[0;39;44mvenetian \u001b[0;39;47mlegend \u001b[0;39;47m, \u001b[0;39;47mwhich \u001b[0;39;47mcelebrates \u001b[0;39;49mthe \u001b[0;39;44msaint \u001b[0;39;107ms \u001b[0;39;44mprotection \u001b[0;39;47mof \u001b[0;39;49mthe \u001b[0;39;47mcity\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;49mfacade \u001b[0;39;49mis \u001b[0;39;49man \u001b[0;39;49mexample \u001b[0;39;49mof \u001b[0;39;49mthe \u001b[0;39;107mvenetian \u001b[0;39;44mgothic \u001b[0;39;44mbyzantine \u001b[0;39;44mstyle\n",
      "\n",
      "\u001b[0;39;44mfresco \u001b[0;39;47mfrom \u001b[0;39;49mthe \u001b[0;39;49msala \u001b[0;39;49mdel \u001b[0;39;49mmaggior \u001b[0;39;49mconsiglio \u001b[0;39;47m, \u001b[0;39;44mdestroyed \u001b[0;39;47m, \u001b[0;39;47mdetached \u001b[0;39;47mfrom \u001b[0;39;47mbehind \u001b[0;39;47mtintorettos \u001b[0;39;47mparadise\n",
      "\n",
      "\u001b[0;39;49mtheodore \u001b[0;39;49mwas \u001b[0;39;49ma \u001b[0;39;47mrd \u001b[0;39;107mcentury \u001b[0;39;44mroman \u001b[0;39;47msoldier \u001b[0;39;107mwho \u001b[0;39;107mwas \u001b[0;39;107mnot \u001b[0;39;47mwilling \u001b[0;39;47mto \u001b[0;39;44mworship \u001b[0;39;44mpagan \u001b[0;39;47mdeities \u001b[0;39;49mand \u001b[0;39;47mwas \u001b[0;39;44mtherefore \u001b[0;39;107mcondemned \u001b[0;39;47mto \u001b[0;39;47mdeath\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;39;49mon \u001b[0;39;49mthe \u001b[0;39;44maltar \u001b[0;39;107m, \u001b[0;39;47mbetween \u001b[0;39;49ma \u001b[0;39;107mrich \u001b[0;39;47mand \u001b[0;39;47melegant \u001b[0;39;107mcornice \u001b[0;39;107m, \u001b[0;39;44mprobably \u001b[0;39;44msculpted \u001b[0;39;44mby \u001b[0;39;44mjacopo \u001b[0;39;107mfrom \u001b[0;39;107mfaenza \u001b[0;39;107m, \u001b[0;39;47mthere \u001b[0;39;47mis \u001b[0;39;49ma \u001b[0;39;44mbeautiful \u001b[0;39;44mpolyptych \u001b[0;39;107msigned \u001b[0;39;44mby \u001b[0;39;107mbartolom \u001b[0;39;107meo \u001b[0;39;107mvivarini \u001b[0;39;47m\n",
      "\n",
      "\u001b[0;39;107m \u001b[0;39;49mas \u001b[0;39;47msatire \u001b[0;39;47mwas \u001b[0;39;47mforbidden \u001b[0;39;44mby \u001b[0;39;107mlaw \u001b[0;39;47min \u001b[0;39;107mveneto \u001b[0;39;44m, \u001b[0;39;44mpeople \u001b[0;39;107mwrote \u001b[0;39;107mpoems \u001b[0;39;47mand \u001b[0;39;107mstories \u001b[0;39;47magainst \u001b[0;39;47mthe \u001b[0;39;107mnobility \u001b[0;39;47mwhich \u001b[0;39;44mwas \u001b[0;39;47mthe \u001b[0;39;107monly \u001b[0;39;47mone \u001b[0;39;44mto \u001b[0;39;44mhold \u001b[0;39;44mpower \u001b[0;39;107mand \u001b[0;39;44mput \u001b[0;39;44mthem \u001b[0;39;107mup \u001b[0;39;47mon \u001b[0;39;47mthe \u001b[0;39;44mstatue \u001b[0;39;44msigning \u001b[0;39;44mthem \u001b[0;39;44mwith \u001b[0;39;47mits \u001b[0;39;47mname\n",
      "\n",
      "\u001b[0;39;47m \u001b[0;39;49mit \u001b[0;39;49mis \u001b[0;39;49mthe \u001b[0;39;47mtallest \u001b[0;39;47mstructure \u001b[0;39;49min \u001b[0;39;44mvenice \u001b[0;39;47mand \u001b[0;39;49mis \u001b[0;39;47mcolloquially \u001b[0;39;47mtermed \u001b[0;39;49mel \u001b[0;39;47mpar \u001b[0;39;49mn \u001b[0;39;49mde \u001b[0;39;107mcasa \u001b[0;39;49mthe \u001b[0;39;44mmaster \u001b[0;39;47mof \u001b[0;39;47mthe \u001b[0;39;44mhouse \u001b[0;39;107m\n",
      "\n",
      "\u001b[0;39;47mbeeing \u001b[0;39;49mon \u001b[0;39;49mmy \u001b[0;39;49mway \u001b[0;39;49mto \u001b[0;39;49mthe \u001b[0;39;49mhotel \u001b[0;39;49min \u001b[0;39;49mthe \u001b[0;39;47mearly \u001b[0;39;47mmorning \u001b[0;39;47m, \u001b[0;39;47mi \u001b[0;39;47mtook \u001b[0;39;49ma \u001b[0;39;47mpicture \u001b[0;39;47mfrom \u001b[0;39;49mthe \u001b[0;39;44mbasilica \u001b[0;39;44mdi \u001b[0;39;47msanta \u001b[0;39;44mmaria \u001b[0;39;107mdella \u001b[0;39;47msalute \u001b[0;39;47mimmediately \u001b[0;39;47mafter \u001b[0;39;47marriving \u001b[0;39;49min \u001b[0;39;107mvenice\n",
      "\n",
      "\u001b[0;39;49mthe \u001b[0;39;47menclosed \u001b[0;39;47mbridge \u001b[0;39;49mis \u001b[0;39;49mmade \u001b[0;39;49mof \u001b[0;39;47mwhite \u001b[0;39;44mlimestone \u001b[0;39;107m, \u001b[0;39;47mhas \u001b[0;39;47mwindows \u001b[0;39;49mwith \u001b[0;39;107mstone \u001b[0;39;47mbars \u001b[0;39;107m, \u001b[0;39;107mpasses \u001b[0;39;49mover \u001b[0;39;49mthe \u001b[0;39;107mrio \u001b[0;39;44mdi \u001b[0;39;44mpalazzo \u001b[0;39;44m, \u001b[0;39;47mand \u001b[0;39;107mconnects \u001b[0;39;47mthe \u001b[0;39;47mnew \u001b[0;39;44mprison \u001b[0;39;44mprigioni \u001b[0;39;44mnuove \u001b[0;39;107mto \u001b[0;39;47mthe \u001b[0;39;107minterrogation \u001b[0;39;44mrooms \u001b[0;39;47min \u001b[0;39;47mthe \u001b[0;39;44mdoge \u001b[0;39;107ms \u001b[0;39;44mpalace\n",
      "\n"
     ]
    }
   ],
   "source": [
    "k=3\n",
    "for i in range(len(rates)):\n",
    "    show_attention(rates['data'].iloc[i],classifier,vectorizer,classes,k=k)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpretability"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experimenting with the effect of Embedding fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "ouv_df = pd.read_csv(args.ouv_csv)\n",
    "word_counts = Counter()\n",
    "for data in ouv_df.data:\n",
    "    for word in data.split(' '):\n",
    "        if word not in string.punctuation:\n",
    "            word_counts[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngrams_iterator(token_list, ngrams):\n",
    "    def _get_ngrams(n):\n",
    "        return zip(*[token_list[i:] for i in range(n)])\n",
    "    for x in token_list:\n",
    "        yield x\n",
    "    for n in range(2, ngrams+1):\n",
    "        for x in _get_ngrams(n):\n",
    "            yield ' '.join(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "ouv_df = pd.read_csv(args.ouv_csv)\n",
    "word_counts = Counter()\n",
    "for data in ouv_df.data:\n",
    "    token_list = data.split(' ')\n",
    "    for word in ngrams_iterator(token_list, 5):\n",
    "        temp = 0\n",
    "        for element in word:\n",
    "            if element in string.punctuation:\n",
    "                temp = 1\n",
    "                break\n",
    "        if temp==0:\n",
    "            word_counts[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2353"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = [word for word, count in word_counts.items() if count>15 and count<600]\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer_tokens_importance(vocab, classifier, vectorizer, classes, k=50):\n",
    "    \"\"\"Predict the rating of a review\n",
    "    \n",
    "    Args:\n",
    "        vocab (list of str): the whole vocabulary\n",
    "        classifier (ReviewClassifier): the trained model\n",
    "        vectorizer (ReviewVectorizer): the corresponding vectorizer\n",
    "        classes (list of str): The name of the ouv classes\n",
    "        k (int): show the largest k prediction, default to 1\n",
    "    \"\"\"\n",
    "    vectorized_token = []    \n",
    "    for token in vocab:\n",
    "        vectorized_token.append(torch.tensor(vectorizer.vectorize(token, vector_length=dataset._max_seq_length)[0]))\n",
    "    \n",
    "    result=torch.zeros((int(len(vocab)/args.batch_size)+1)*args.batch_size,len(classes))\n",
    "    #print(result.shape)\n",
    "    for i in range(int(len(vocab)/args.batch_size)+1):\n",
    "        state_word = classifier.init_hidden()\n",
    "        X = torch.stack(vectorized_token[i*args.batch_size:(i+1)*args.batch_size])\n",
    "        X = torch.cat([X,torch.zeros(args.batch_size-X.shape[0], dataset._max_seq_length).long()])\n",
    "        #print(X.shape)\n",
    "        classifier.eval()\n",
    "        res,state_word,_ = classifier(X,state_word, apply_softmax=True)\n",
    "        #print(res.shape)\n",
    "        result[i*args.batch_size:(i+1)*args.batch_size]=res\n",
    "        #print(result.shape)\n",
    "    result = result[:len(vocab)]\n",
    "    \n",
    "    vocab_id = result[1:].topk(k, dim=0)[1]\n",
    "    vocab_weight = result[1:].topk(k, dim=0)[0]\n",
    "    return vocab_id, vocab_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_k = infer_tokens_importance(vocab, classifier, vectorizer, classes, k=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=10):\n",
    "    \n",
    "    vocab_id = infer_tokens_importance(vocab, classifier, vectorizer, classes, k)[0]\n",
    "    df = pd.DataFrame(columns = classes)\n",
    "    for i in range(len(classes)):\n",
    "        \n",
    "        indices = vocab_id[:,i].tolist()\n",
    "        words = pd.Series([vocab[j] for j in indices])\n",
    "        df[classes[i]] = words\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Criteria i</th>\n",
       "      <th>Criteria ii</th>\n",
       "      <th>Criteria iii</th>\n",
       "      <th>Criteria iv</th>\n",
       "      <th>Criteria v</th>\n",
       "      <th>Criteria vi</th>\n",
       "      <th>Criteria vii</th>\n",
       "      <th>Criteria viii</th>\n",
       "      <th>Criteria ix</th>\n",
       "      <th>Criteria x</th>\n",
       "      <th>Others</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>of human creative</td>\n",
       "      <td>more than</td>\n",
       "      <td>of early</td>\n",
       "      <td>example of a type</td>\n",
       "      <td>a period</td>\n",
       "      <td>political</td>\n",
       "      <td>the great</td>\n",
       "      <td>impressive</td>\n",
       "      <td>is one</td>\n",
       "      <td>include the</td>\n",
       "      <td>royal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a unique and</td>\n",
       "      <td>manner</td>\n",
       "      <td>the archaeological</td>\n",
       "      <td>dominant</td>\n",
       "      <td>a cultural</td>\n",
       "      <td>indigenous</td>\n",
       "      <td>exceptional natural</td>\n",
       "      <td>this region</td>\n",
       "      <td>and biological</td>\n",
       "      <td>bird species</td>\n",
       "      <td>in the area</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the exceptional</td>\n",
       "      <td>an exceptional</td>\n",
       "      <td>archaeological site of</td>\n",
       "      <td>styles</td>\n",
       "      <td>the cultural landscape</td>\n",
       "      <td>is still</td>\n",
       "      <td>of exceptional</td>\n",
       "      <td>the architecture of</td>\n",
       "      <td>zone</td>\n",
       "      <td>exhibit</td>\n",
       "      <td>erosion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a masterpiece of</td>\n",
       "      <td>interchange of</td>\n",
       "      <td>culture of</td>\n",
       "      <td>colonization</td>\n",
       "      <td>design of</td>\n",
       "      <td>holy</td>\n",
       "      <td>geological</td>\n",
       "      <td>of monuments</td>\n",
       "      <td>geological processes</td>\n",
       "      <td>bird</td>\n",
       "      <td>include</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unique artistic achievement</td>\n",
       "      <td>on the development of</td>\n",
       "      <td>culture of the</td>\n",
       "      <td>are outstanding examples of</td>\n",
       "      <td>bronze age</td>\n",
       "      <td>testify to</td>\n",
       "      <td>man</td>\n",
       "      <td>succession</td>\n",
       "      <td>processes</td>\n",
       "      <td>critically endangered</td>\n",
       "      <td>immense</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>masterpiece of</td>\n",
       "      <td>architecture in the</td>\n",
       "      <td>exceptional testimony to the</td>\n",
       "      <td>example of an architectural</td>\n",
       "      <td>ad</td>\n",
       "      <td>impact</td>\n",
       "      <td>trees</td>\n",
       "      <td>features</td>\n",
       "      <td>back</td>\n",
       "      <td>greatest</td>\n",
       "      <td>rome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>a unique artistic</td>\n",
       "      <td>development of architecture</td>\n",
       "      <td>over a period</td>\n",
       "      <td>outstanding example of a</td>\n",
       "      <td>is an</td>\n",
       "      <td>islam</td>\n",
       "      <td>glaciers</td>\n",
       "      <td>changes</td>\n",
       "      <td>surrounded</td>\n",
       "      <td>plant and</td>\n",
       "      <td>wild</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>decoration</td>\n",
       "      <td>profound influence</td>\n",
       "      <td>the modern</td>\n",
       "      <td>outstanding example of the</td>\n",
       "      <td>of a type of</td>\n",
       "      <td>the most complete</td>\n",
       "      <td>deep</td>\n",
       "      <td>basin</td>\n",
       "      <td>the development of the</td>\n",
       "      <td>also contains</td>\n",
       "      <td>the eastern</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>is a masterpiece</td>\n",
       "      <td>profound influence on</td>\n",
       "      <td>bears exceptional testimony</td>\n",
       "      <td>outstanding examples of</td>\n",
       "      <td>deciduous</td>\n",
       "      <td>example of a traditional</td>\n",
       "      <td>altitude</td>\n",
       "      <td>record</td>\n",
       "      <td>worldwide</td>\n",
       "      <td>critically</td>\n",
       "      <td>north america</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>artistic achievement</td>\n",
       "      <td>exerted</td>\n",
       "      <td>bears exceptional testimony to</td>\n",
       "      <td>outstanding example of an</td>\n",
       "      <td>the landscape</td>\n",
       "      <td>documented</td>\n",
       "      <td>metres</td>\n",
       "      <td>i</td>\n",
       "      <td>series</td>\n",
       "      <td>the islands</td>\n",
       "      <td>time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>unique artistic</td>\n",
       "      <td>and landscape</td>\n",
       "      <td>bears exceptional</td>\n",
       "      <td>are an outstanding example</td>\n",
       "      <td>century ad</td>\n",
       "      <td>which has</td>\n",
       "      <td>resources</td>\n",
       "      <td>elsewhere</td>\n",
       "      <td>changes in</td>\n",
       "      <td>mammals</td>\n",
       "      <td>an extraordinary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>interior</td>\n",
       "      <td>architecture in</td>\n",
       "      <td>economic and</td>\n",
       "      <td>is an outstanding example</td>\n",
       "      <td>create</td>\n",
       "      <td>species such as</td>\n",
       "      <td>abundant</td>\n",
       "      <td>array</td>\n",
       "      <td>remaining</td>\n",
       "      <td>are</td>\n",
       "      <td>socio economic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>a masterpiece</td>\n",
       "      <td>the influence</td>\n",
       "      <td>the site</td>\n",
       "      <td>the new</td>\n",
       "      <td>cultural landscape</td>\n",
       "      <td>represented by</td>\n",
       "      <td>it represents</td>\n",
       "      <td>dominated</td>\n",
       "      <td>level</td>\n",
       "      <td>of this</td>\n",
       "      <td>east asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>represents a</td>\n",
       "      <td>of architecture</td>\n",
       "      <td>itself</td>\n",
       "      <td>imperial</td>\n",
       "      <td>the sacred</td>\n",
       "      <td>of the site</td>\n",
       "      <td>rugged</td>\n",
       "      <td>example of a type of</td>\n",
       "      <td>earth</td>\n",
       "      <td>species of</td>\n",
       "      <td>fields</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>overall</td>\n",
       "      <td>a profound influence</td>\n",
       "      <td>the archaeological site</td>\n",
       "      <td>the establishment of</td>\n",
       "      <td>the urban</td>\n",
       "      <td>had a</td>\n",
       "      <td>a large</td>\n",
       "      <td>mountain</td>\n",
       "      <td>the church of</td>\n",
       "      <td>diversity of</td>\n",
       "      <td>the sea</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>contains</td>\n",
       "      <td>a profound</td>\n",
       "      <td>monument</td>\n",
       "      <td>administrative</td>\n",
       "      <td>a profound influence on</td>\n",
       "      <td>field</td>\n",
       "      <td>group of</td>\n",
       "      <td>scenery</td>\n",
       "      <td>wetlands</td>\n",
       "      <td>are still</td>\n",
       "      <td>surrounding</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>sculpture</td>\n",
       "      <td>asia</td>\n",
       "      <td>many of</td>\n",
       "      <td>of northern</td>\n",
       "      <td>bridge</td>\n",
       "      <td>directly and tangibly associated</td>\n",
       "      <td>on the</td>\n",
       "      <td>illustrating</td>\n",
       "      <td>found in</td>\n",
       "      <td>over a</td>\n",
       "      <td>an important interchange of</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>creative</td>\n",
       "      <td>the first</td>\n",
       "      <td>of one</td>\n",
       "      <td>organization</td>\n",
       "      <td>a series of</td>\n",
       "      <td>outstanding universal</td>\n",
       "      <td>result of</td>\n",
       "      <td>evolution</td>\n",
       "      <td>endemic to the</td>\n",
       "      <td>species</td>\n",
       "      <td>biological</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>can be</td>\n",
       "      <td>ritual</td>\n",
       "      <td>a unique testimony</td>\n",
       "      <td>vestiges</td>\n",
       "      <td>desert</td>\n",
       "      <td>painted</td>\n",
       "      <td>waterfalls</td>\n",
       "      <td>gulf</td>\n",
       "      <td>sand</td>\n",
       "      <td>animal species</td>\n",
       "      <td>production</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>and artistic</td>\n",
       "      <td>exhibits</td>\n",
       "      <td>an exceptional testimony to</td>\n",
       "      <td>a model</td>\n",
       "      <td>witness to</td>\n",
       "      <td>became the</td>\n",
       "      <td>of major</td>\n",
       "      <td>adaptation of</td>\n",
       "      <td>evergreen</td>\n",
       "      <td>tree</td>\n",
       "      <td>thus</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Criteria i                  Criteria ii  \\\n",
       "0             of human creative                    more than   \n",
       "1                  a unique and                       manner   \n",
       "2               the exceptional               an exceptional   \n",
       "3              a masterpiece of               interchange of   \n",
       "4   unique artistic achievement        on the development of   \n",
       "5                masterpiece of          architecture in the   \n",
       "6             a unique artistic  development of architecture   \n",
       "7                    decoration           profound influence   \n",
       "8              is a masterpiece        profound influence on   \n",
       "9          artistic achievement                      exerted   \n",
       "10              unique artistic                and landscape   \n",
       "11                     interior              architecture in   \n",
       "12                a masterpiece                the influence   \n",
       "13                 represents a              of architecture   \n",
       "14                      overall         a profound influence   \n",
       "15                     contains                   a profound   \n",
       "16                    sculpture                         asia   \n",
       "17                     creative                    the first   \n",
       "18                       can be                       ritual   \n",
       "19                 and artistic                     exhibits   \n",
       "\n",
       "                      Criteria iii                  Criteria iv  \\\n",
       "0                         of early            example of a type   \n",
       "1               the archaeological                     dominant   \n",
       "2           archaeological site of                       styles   \n",
       "3                       culture of                 colonization   \n",
       "4                   culture of the  are outstanding examples of   \n",
       "5     exceptional testimony to the  example of an architectural   \n",
       "6                    over a period     outstanding example of a   \n",
       "7                       the modern   outstanding example of the   \n",
       "8      bears exceptional testimony      outstanding examples of   \n",
       "9   bears exceptional testimony to    outstanding example of an   \n",
       "10               bears exceptional   are an outstanding example   \n",
       "11                    economic and    is an outstanding example   \n",
       "12                        the site                      the new   \n",
       "13                          itself                     imperial   \n",
       "14         the archaeological site         the establishment of   \n",
       "15                        monument               administrative   \n",
       "16                         many of                  of northern   \n",
       "17                          of one                 organization   \n",
       "18              a unique testimony                     vestiges   \n",
       "19     an exceptional testimony to                      a model   \n",
       "\n",
       "                 Criteria v                       Criteria vi  \\\n",
       "0                  a period                         political   \n",
       "1                a cultural                        indigenous   \n",
       "2    the cultural landscape                          is still   \n",
       "3                 design of                              holy   \n",
       "4                bronze age                        testify to   \n",
       "5                        ad                            impact   \n",
       "6                     is an                             islam   \n",
       "7              of a type of                 the most complete   \n",
       "8                 deciduous          example of a traditional   \n",
       "9             the landscape                        documented   \n",
       "10               century ad                         which has   \n",
       "11                   create                   species such as   \n",
       "12       cultural landscape                    represented by   \n",
       "13               the sacred                       of the site   \n",
       "14                the urban                             had a   \n",
       "15  a profound influence on                             field   \n",
       "16                   bridge  directly and tangibly associated   \n",
       "17              a series of             outstanding universal   \n",
       "18                   desert                           painted   \n",
       "19               witness to                        became the   \n",
       "\n",
       "           Criteria vii         Criteria viii             Criteria ix  \\\n",
       "0             the great            impressive                  is one   \n",
       "1   exceptional natural           this region          and biological   \n",
       "2        of exceptional   the architecture of                    zone   \n",
       "3            geological          of monuments    geological processes   \n",
       "4                   man            succession               processes   \n",
       "5                 trees              features                    back   \n",
       "6              glaciers               changes              surrounded   \n",
       "7                  deep                 basin  the development of the   \n",
       "8              altitude                record               worldwide   \n",
       "9                metres                     i                  series   \n",
       "10            resources             elsewhere              changes in   \n",
       "11             abundant                 array               remaining   \n",
       "12        it represents             dominated                   level   \n",
       "13               rugged  example of a type of                   earth   \n",
       "14              a large              mountain           the church of   \n",
       "15             group of               scenery                wetlands   \n",
       "16               on the          illustrating                found in   \n",
       "17            result of             evolution          endemic to the   \n",
       "18           waterfalls                  gulf                    sand   \n",
       "19             of major         adaptation of               evergreen   \n",
       "\n",
       "               Criteria x                       Others  \n",
       "0             include the                        royal  \n",
       "1            bird species                  in the area  \n",
       "2                 exhibit                      erosion  \n",
       "3                    bird                      include  \n",
       "4   critically endangered                      immense  \n",
       "5                greatest                         rome  \n",
       "6               plant and                         wild  \n",
       "7           also contains                  the eastern  \n",
       "8              critically                north america  \n",
       "9             the islands                         time  \n",
       "10                mammals             an extraordinary  \n",
       "11                    are               socio economic  \n",
       "12                of this                    east asia  \n",
       "13             species of                       fields  \n",
       "14           diversity of                      the sea  \n",
       "15              are still                  surrounding  \n",
       "16                 over a  an important interchange of  \n",
       "17                species                   biological  \n",
       "18         animal species                   production  \n",
       "19                   tree                         thus  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=50).to_csv(args.save_dir+'top_words.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('test')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_test = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    classifier.eval()\n",
    "    state_word = classifier.init_hidden()\n",
    "    y_pred,state_word,_ = classifier(X,state_word)\n",
    "    \n",
    "    conf_mat_test = np.add(conf_mat_test,confusion_matrix(Y.argmax(axis=1), y_pred.argmax(axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[20.,  3.,  2.,  6.,  1.,  3.,  1.,  0.,  0.,  0.],\n",
       "       [ 1., 46.,  4., 13.,  4.,  4.,  1.,  1.,  0.,  0.],\n",
       "       [ 1.,  5., 40., 11.,  3.,  2.,  1.,  0.,  0.,  0.],\n",
       "       [ 3., 13., 11., 45.,  6.,  0.,  2.,  1.,  0.,  0.],\n",
       "       [ 0.,  0.,  3., 13., 12.,  0.,  2.,  0.,  2.,  0.],\n",
       "       [ 0.,  4., 11.,  4.,  1., 22.,  0.,  1.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  2.,  1., 30.,  4.,  1.,  4.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  5., 18.,  5.,  2.],\n",
       "       [ 0.,  0.,  2.,  0.,  1.,  0.,  3.,  1., 28., 12.],\n",
       "       [ 0.,  0.,  1.,  1.,  0.,  0.,  1.,  0.,  7., 54.]])"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('val')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_val = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    state_word = classifier.init_hidden()\n",
    "    y_pred,state_word,_ = classifier(X,state_word)\n",
    "    \n",
    "    conf_mat_val = np.add(conf_mat_val,confusion_matrix(Y.argmax(axis=1), y_pred.argmax(axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[14.,  5.,  3.,  9.,  0.,  4.,  0.,  0.,  0.,  0.],\n",
       "       [ 3., 39.,  6., 12.,  1.,  5.,  0.,  0.,  0.,  0.],\n",
       "       [ 3.,  5., 51., 10.,  3.,  7.,  0.,  0.,  0.,  0.],\n",
       "       [ 1., 14.,  9., 45.,  7.,  5.,  0.,  1.,  0.,  0.],\n",
       "       [ 1.,  1.,  2.,  7.,  9.,  1.,  1.,  1.,  2.,  0.],\n",
       "       [ 0.,  6.,  5.,  1.,  0., 28.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  1.,  0., 31.,  1.,  2.,  4.],\n",
       "       [ 0.,  0.,  1.,  1.,  0.,  1.,  6., 26.,  3.,  2.],\n",
       "       [ 0.,  0.,  1.,  0.,  1.,  0.,  2.,  2., 22., 12.],\n",
       "       [ 0.,  0.,  0.,  0.,  1.,  0.,  3.,  0.,  5., 57.]])"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('train')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_train = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    state_word = classifier.init_hidden()\n",
    "    y_pred,state_word,_ = classifier(X,state_word)\n",
    "    \n",
    "    conf_mat_train = np.add(conf_mat_train,confusion_matrix(Y.argmax(axis=1), y_pred.argmax(axis=1),labels=range(len(classes)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[196.,  26.,  10.,  76.,   2.,   8.,   2.,   0.,   0.,   0.],\n",
       "       [ 20., 402.,  35., 110.,   9.,  28.,   0.,   0.,   0.,   0.],\n",
       "       [ 16.,  51., 388.,  95.,  24.,  49.,   2.,   3.,   2.,   3.],\n",
       "       [ 25.,  94.,  53., 529.,  19.,  26.,   1.,   2.,   1.,   0.],\n",
       "       [  2.,   8.,  24.,  41., 117.,   1.,   3.,   2.,   4.,   1.],\n",
       "       [  6.,  32.,  49.,  12.,   4., 209.,   2.,   0.,   1.,   0.],\n",
       "       [  1.,   0.,   3.,   0.,   4.,   1., 277.,  25.,  23.,  34.],\n",
       "       [  0.,   0.,   0.,   0.,   1.,   1.,  30., 203.,  15.,   3.],\n",
       "       [  0.,   0.,   1.,   0.,   0.,   0.,  19.,   9., 242.,  84.],\n",
       "       [  0.,   0.,   1.,   0.,   0.,   1.,  13.,   2.,  33., 501.]])"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'confusion_matrix.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'baseline_confusion_matrix.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/row.sum()\n",
    "        recall = val/col.sum()\n",
    "        F1 = 2*(precision*recall)/(precision+recall)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test'] = per_class_metrics(conf_mat_test, classes[:-1])\n",
    "metrics_dict['val'] = per_class_metrics(conf_mat_val, classes[:-1])\n",
    "metrics_dict['train'] = per_class_metrics(conf_mat_train, classes[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'baseline_per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try on totally Unseen Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "#args.ouv_csv='Data/ouv_with_splits_full.csv',\n",
    "new_ouv_csv='Data/sd_full.csv'\n",
    "#args.reload_from_files=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_jac_k_accuracy(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "        \n",
    "    n_correct = torch.tensor([torch.tensor([y_pred_indices[j][i] in y_target_indices[j] for i in range(k)]).sum()>0 \n",
    "                              for j in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_jac_1_accuracy(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    y_pred_indices = y_pred.topk(1, dim=1)[1]\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "        \n",
    "    n_correct = torch.tensor([torch.tensor([y_pred_indices[j] in y_target_indices[j] for i in range(k)]).sum()>0 \n",
    "                              for j in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed: 5.034240961074829\n"
     ]
    }
   ],
   "source": [
    "with Timer():\n",
    "    loss_func = cross_entropy\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    train_state = make_train_state(args)\n",
    "    dataset = OuvDataset.load_dataset_and_load_vectorizer(new_ouv_csv, args.vectorizer_file)\n",
    "\n",
    "    dataset.set_split('val')\n",
    "    verbose=False\n",
    "    try:\n",
    "        # Iterate over training dataset\n",
    "\n",
    "        # setup: batch generator, set loss and acc to 0, set train mode on\n",
    "        dataset.set_split('val')\n",
    "        batch_generator = generate_batches(dataset, \n",
    "                                           batch_size=args.batch_size, \n",
    "                                           device=args.device)\n",
    "        running_loss = 0.0\n",
    "        running_1_acc = 0.0\n",
    "        running_k_acc = 0.0\n",
    "        running_k_jac = 0.0\n",
    "        classifier.eval()\n",
    "\n",
    "        for batch_index, batch_dict in enumerate(batch_generator):\n",
    "\n",
    "            # step 2. get the data compute fuzzy labels\n",
    "            state_word = classifier.init_hidden().to(args.device)\n",
    "            X = batch_dict['x_data']\n",
    "\n",
    "            y_target = batch_dict['y_target']\n",
    "            y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "            Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                                    how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "            # step 3. compute the output\n",
    "            with torch.no_grad():\n",
    "                y_pred,state_word,_ = classifier(X, state_word)\n",
    "\n",
    "            # step 4. compute the loss\n",
    "            loss = loss_func(y_pred, Y)\n",
    "            loss_t = loss.item()\n",
    "            running_loss += (loss_t - running_loss) / (batch_index + 1)\n",
    "\n",
    "            # -----------------------------------------\n",
    "            # compute the accuracies\n",
    "            acc_1_t = compute_jac_1_accuracy(y_pred, y_target)\n",
    "            acc_k_t = compute_jac_k_accuracy(y_pred, y_target, args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_pred, y_target, len(classes))\n",
    "\n",
    "            running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)\n",
    "            running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)\n",
    "            running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)\n",
    "\n",
    "            # update bar\n",
    "            if verbose:\n",
    "                val_bar.set_postfix(loss=running_loss, \n",
    "                                acc_1=running_1_acc,\n",
    "                                acc_k=running_k_acc,\n",
    "                                jac_k=running_k_jac,\n",
    "                                epoch=epoch_index)\n",
    "                val_bar.update()\n",
    "\n",
    "        train_state['val_loss'].append(running_loss)\n",
    "        train_state['val_1_acc'].append(running_1_acc)\n",
    "        train_state['val_k_acc'].append(running_k_acc)\n",
    "        train_state['val_k_jac'].append(running_k_jac)\n",
    "\n",
    "        if verbose:\n",
    "            train_bar.n = 0\n",
    "            val_bar.n = 0\n",
    "            epoch_bar.update()\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 0,\n",
       " 'early_stopping_best_k_acc_val': 0,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 0,\n",
       " 'train_loss': [],\n",
       " 'train_1_acc': [],\n",
       " 'train_k_acc': [],\n",
       " 'train_k_jac': [],\n",
       " 'val_loss': [2.016925685455641],\n",
       " 'val_1_acc': [66.35416666666669],\n",
       " 'val_k_acc': [94.0625],\n",
       " 'val_k_jac': [0.4140845318635305],\n",
       " 'test_loss': -1,\n",
       " 'test_1_acc': -1,\n",
       " 'test_k_acc': -1,\n",
       " 'test_k_jac': -1,\n",
       " 'model_filename': 'model_storage/attn/model.pth'}"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 0,\n",
       " 'early_stopping_best_k_acc_val': 0,\n",
       " 'learning_rate': 0.001,\n",
       " 'epoch_index': 0,\n",
       " 'train_loss': [],\n",
       " 'train_1_acc': [],\n",
       " 'train_k_acc': [],\n",
       " 'train_k_jac': [],\n",
       " 'val_loss': [2.4428400123202985],\n",
       " 'val_1_acc': [64.27083333333333],\n",
       " 'val_k_acc': [92.70833333333334],\n",
       " 'val_k_jac': [0.3412336905797323],\n",
       " 'test_loss': -1,\n",
       " 'test_1_acc': -1,\n",
       " 'test_k_acc': -1,\n",
       " 'test_k_jac': -1,\n",
       " 'model_filename': 'model_storage/attn/model.pth'}"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## END"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}