Notebooks/ImageColorizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NdHkV9wFxJeN"
   },
   "source": [
    "## All Imports Merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 129
    },
    "colab_type": "code",
    "id": "buVILqYyxJeU",
    "outputId": "0bb56804-bafd-4e44-e3b2-fd4a1603ff4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pillow\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/62/94/5430ebaa83f91cc7a9f687ff5238e26164a779cca2ef9903232268b0a318/Pillow-5.3.0-cp36-cp36m-manylinux1_x86_64.whl (2.0MB)\n",
      "\u001b[K    100% |████████████████████████████████| 2.0MB 32.1MB/s \n",
      "\u001b[?25hInstalling collected packages: pillow\n",
      "Successfully installed pillow-5.3.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<script src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.3/latest.js?config=default'></script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from skimage.color import rgb2lab, lab2rgb\n",
    "from skimage.io import imread\n",
    "from skimage.transform import resize\n",
    "import sklearn.neighbors as ne\n",
    "from sklearn.model_selection import train_test_split\n",
    "import scipy.misc\n",
    "\n",
    "from math import sqrt, pi\n",
    "import time\n",
    "import os\n",
    "from os import listdir, walk\n",
    "from os.path import join, isfile, isdir\n",
    "import pdb\n",
    "import random\n",
    "import sys\n",
    "import getopt\n",
    "\n",
    "# http://pytorch.org/\n",
    "from os.path import exists\n",
    "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
    "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
    "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n",
    "accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'\n",
    "\n",
    "!pip install -q http://download.pytorch.org/whl/{accelerator}/torch-0.4.1-{platform}-linux_x86_64.whl torchvision\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "import torchvision.datasets as dsets\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from google.colab import drive\n",
    "\n",
    "\n",
    "!pip install --no-cache-dir -I pillow\n",
    "\n",
    "from IPython.display import Math, HTML\n",
    "display(HTML(\"<script src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.3/\"\n",
    "               \"latest.js?config=default'></script>\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 129
    },
    "colab_type": "code",
    "id": "T0LeRr1Zxz0T",
    "outputId": "1277743f-35c8-484c-d4e1-78d86129f8b2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "cuda = True if torch.cuda.is_available() else False\n",
    "drive.mount('/content/gdrive')\n",
    "#defining the main path of the drive where all contents are saved.\n",
    "StatePath = \"gdrive/My Drive/AIProject/PytorchVersion\"\n",
    "DatasetPath = StatePath+\"/flowers\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xcG9BCgDyl2P"
   },
   "outputs": [],
   "source": [
    "os.makedirs(StatePath, exist_ok=True)\n",
    "os.makedirs(StatePath+\"/states\", exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CmLPhAu4Fx4R"
   },
   "source": [
    "## Hyper Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mIIEfq_WA0cv"
   },
   "outputs": [],
   "source": [
    "epochs = 1000\n",
    "batch_size = 10\n",
    "imageSize = 128\n",
    "learningRate = 0.001\n",
    "print_freq = 10\n",
    "save_freq = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ue7PhfaExJel"
   },
   "source": [
    "## Color Utilities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AHYQ3hXZg53j"
   },
   "source": [
    "The ab colorspace was quantized into bins with grid size 10. The number of quantized ab values $Q = 313$. These qauntized values are kept in $\\texttt{pts_in_hull.npy}$. The following class $\\texttt{NNEncode}$ implements important functions as discussed in research paper.\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "The function $\\texttt{imgEncodeTorch}$ implements the $H_{gt}^{-1}$ function which converts ground truth colors to a vector $Z$ using a soft encoding scheme. Here the $ab$ colorspace (ground truth) is encoded into quantized $ab$ space according to the file $\\texttt{pts_in_hull.npy}$.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "i99iXmotxJep"
   },
   "outputs": [],
   "source": [
    "class NNEncode():\n",
    "    def __init__(self, NN=5, sigma=5, km_filepath=join(StatePath, 'static', 'pts_in_hull.npy'), train=True, location='cuda'):\n",
    "        self.cc = np.load(km_filepath)\n",
    "        self.NN = int(NN)\n",
    "        self.sigma = sigma\n",
    "        self.nbrs = ne.NearestNeighbors(\n",
    "            n_neighbors=NN, algorithm='ball_tree').fit(self.cc)\n",
    "        if train:\n",
    "            self.weights = torch.load(StatePath+'/static/weights_test')\n",
    "            if ('cuda' in location):\n",
    "                self.weights = self.weights.cuda()\n",
    "\n",
    "\n",
    "    # computes soft encoding of ground truth ab image, multiplied by weight (for class rebalancing)\n",
    "    #for training\n",
    "    def imgEncodeTorch(self, abimg):\n",
    "        abimg = abimg.cuda()\n",
    "        w, h = abimg.shape[1], abimg.shape[2]\n",
    "        label = torch.zeros((w*h, 313))\n",
    "        label = label.cuda()\n",
    "\n",
    "        (dists, indexes) = self.nbrs.kneighbors(\n",
    "            abimg.view(abimg.shape[0], -1).t(), self.NN)\n",
    "        dists = torch.from_numpy(dists).float().cuda()\n",
    "        indexes = torch.from_numpy(indexes).cuda()\n",
    "\n",
    "        weights = torch.exp(-dists**2/(2*self.sigma**2)).cuda()\n",
    "        weights = weights/torch.sum(weights, dim=1).view(-1, 1)\n",
    "\n",
    "        pixel_indexes = torch.Tensor.long(torch.arange(\n",
    "            start=0, end=abimg.shape[1]*abimg.shape[2])[:, np.newaxis])\n",
    "        pixel_indexes = pixel_indexes.cuda()\n",
    "        label[pixel_indexes, indexes] = weights\n",
    "        label = label.t().contiguous().view(313, w, h)\n",
    "\n",
    "        rebal_indexes = indexes[:, 0]\n",
    "        rebal_weights = self.weights[rebal_indexes]\n",
    "        rebal_weights = rebal_weights.view(w, h)\n",
    "        rebal_label = rebal_weights * label\n",
    "\n",
    "        return rebal_label\n",
    "    def bin2color(self, idx):\n",
    "        return self.cc[idx]\n",
    "    def uint_color2tanh_range(img):\n",
    "        return img / 128.0 - 1.0\n",
    "    def tanh_range2uint_color(img):\n",
    "        return (img * 128.0 + 128.0).astype(np.uint8)\n",
    "    def modelimg2cvimg(img):\n",
    "        cvimg = np.array(img[0, :, :, :]).transpose(1, 2, 0)\n",
    "        return tanh_range2uint_color(cvimg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Xd38ymvfoJjM"
   },
   "source": [
    "This function is implemented to save the results of every $10^{th}$ epoch and show us how the model is learning an image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "49NhFWpnHCk2"
   },
   "outputs": [],
   "source": [
    "def sample_image(grayImage, predImage, actualImage, batch, index):\n",
    "    gen_imgs = np.concatenate((predImage, actualImage), axis=1)\n",
    "    os.makedirs(StatePath+\"/images/\"+str(batch), exist_ok=True)\n",
    "    scipy.misc.imsave(StatePath+\"/images/\"+str(batch)+\"/\"+str(index)+'.jpg', gen_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "V-vglyEPxJfd"
   },
   "source": [
    "## Making Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "whDozcseogzQ"
   },
   "source": [
    "This function is used to make train, validate and tests datasets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NqTl9kj_xJfi"
   },
   "outputs": [],
   "source": [
    "class CustomImages(Dataset):\n",
    "    def __init__(self, root, train=True, val=False, color_space='lab', transform=None, test_size=0.1, val_size=0.125, location='cuda'):\n",
    "\n",
    "        self.root_dir = root\n",
    "        all_files = []\n",
    "        for r, _, files in walk(self.root_dir):\n",
    "          for f in files:\n",
    "            if f.endswith('.jpg'):\n",
    "              all_files.append(join(r, f))\n",
    "        \n",
    "        train_val_files, test_files = train_test_split(\n",
    "            all_files, test_size=test_size, random_state=69)\n",
    "        \n",
    "        \n",
    "        train_files, val_files = train_test_split(train_val_files,\n",
    "                                                  test_size=val_size, random_state=69)\n",
    "        \n",
    "        if (train and val):\n",
    "            self.filenames = val_files\n",
    "        elif train:\n",
    "            self.filenames = train_files\n",
    "        else:\n",
    "            self.filenames = test_files\n",
    "\n",
    "        self.color_space = color_space\n",
    "        if (self.color_space not in ['rgb', 'lab']):\n",
    "            raise(NotImplementedError)\n",
    "        self.transform = transform\n",
    "        self.location = location\n",
    "        self.nnenc = NNEncode(location=self.location)\n",
    "        self.train = train\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.filenames)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img = imread(self.filenames[idx])\n",
    "        if self.color_space == 'lab':\n",
    "            img = rgb2lab(img)\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        bwimg = img[:, :, 0:1].transpose(2, 0, 1)\n",
    "        bwimg = torch.from_numpy(bwimg).float()\n",
    "        abimg = img[:, :, 1:].transpose(2, 0, 1)    # abimg dim: 2, h, w\n",
    "        abimg = torch.from_numpy(abimg).float()\n",
    "        label = -1\n",
    "        if (self.train):\n",
    "            if ('cuda' in self.location):\n",
    "                label = self.nnenc.imgEncodeTorch(abimg)\n",
    "            #else:\n",
    "             #   label = self.nnenc.imgEncode(abimg)\n",
    "        return (bwimg, label, abimg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GKk6rgWro374"
   },
   "source": [
    "If the image is of size greater than 128 by 128, we will rescale it using the following function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Xo74vlTVxJfw"
   },
   "outputs": [],
   "source": [
    "class Rescale(object):\n",
    "\n",
    "    def __init__(self, output_size):\n",
    "        assert isinstance(output_size, (int, tuple))\n",
    "        self.output_size = output_size\n",
    "\n",
    "    def __call__(self, sample):\n",
    "        image = sample\n",
    "        h, w = image.shape[:2]\n",
    "        if isinstance(self.output_size, int):\n",
    "            if h > w:\n",
    "                new_h, new_w = self.output_size * h / w, self.output_size\n",
    "            else:\n",
    "                new_h, new_w = self.output_size, self.output_size * w / h\n",
    "        else:\n",
    "            new_h, new_w = self.output_size\n",
    "        new_h, new_w = int(new_h), int(new_w)\n",
    "        img = resize(image, (new_h, new_w))[:self.output_size, :self.output_size, :]\n",
    "        return img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4aEq6718xJgA"
   },
   "source": [
    "## Class Rebalancing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "w4Aav9-bpY-9"
   },
   "source": [
    "The loss function is dominated by desaturated $ab$ values if the distribution of $ab$ values is strongly biased towards low ab values. \n",
    "\n",
    "This biasness is removed by reweighting the loss of each pixel at train time based on the pixel color rarity. Each pixel is weighed by factor $w \\in R^Q$, based on its closest $ab$ bin.\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "t0GoJmLpxJgJ"
   },
   "outputs": [],
   "source": [
    "# calculate the weight for each bin based on empirical probability, for class rebalancing\n",
    "# only needs to be run once\n",
    "def cal_emp_weights(dset, bins_num=313, sigma=5, lamda=0.5):\n",
    "    cc = np.load(os.path.join(StatePath, 'static', 'pts_in_hull.npy'))\n",
    "    nbrs = ne.NearestNeighbors(n_neighbors=1, algorithm='ball_tree').fit(cc)\n",
    "\n",
    "    bins_prob = torch.zeros(bins_num)\n",
    "\n",
    "    print('Dataset length:', len(dset))\n",
    "    for i in range(len(dset)):\n",
    "        if (i%100==0):\n",
    "            print('Reading Image:', i)\n",
    "        _, _, abimg = dset[i]\n",
    "        _, indexes = nbrs.kneighbors(abimg.view(abimg.shape[0],-1).t(), 1)\n",
    "        bins_prob[torch.from_numpy(indexes).view(-1)] += 1\n",
    "    bins_sum = bins_prob.sum()\n",
    "    bins_prob /= bins_sum\n",
    "\n",
    "    w = 1/((1 - lamda) * bins_prob + lamda / bins_num)\n",
    "    w /= ((bins_prob * w).sum())\n",
    "    torch.save(w, StatePath+'/static/weights_test')\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 329
    },
    "colab_type": "code",
    "id": "OqaswMd-xJgR",
    "outputId": "0666a0e1-cbac-49e7-d848-dd46c48debad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final lenght 1440\n",
      "Dataset length: 1440\n",
      "Reading Image: 0\n",
      "Reading Image: 100\n",
      "Reading Image: 200\n",
      "Reading Image: 300\n",
      "Reading Image: 400\n",
      "Reading Image: 500\n",
      "Reading Image: 600\n",
      "Reading Image: 700\n",
      "Reading Image: 800\n",
      "Reading Image: 900\n",
      "Reading Image: 1000\n",
      "Reading Image: 1100\n",
      "Reading Image: 1200\n",
      "Reading Image: 1300\n",
      "Reading Image: 1400\n"
     ]
    }
   ],
   "source": [
    "entire_dataset = CustomImages(DatasetPath, train=True, test_size=0.1, val_size=0) #40 images for test\n",
    "print(\"final lenght\",len(entire_dataset))\n",
    "a = cal_emp_weights(entire_dataset, 313)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Xsf0oOsPxJg0"
   },
   "source": [
    "## Loss Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rV4ulSfit8WW"
   },
   "source": [
    "Euclidean loss is not robust to the inherent ambiguity and multimodal\n",
    "nature of the colorization problem. If an image can contain a set of distinct $ab$ values, the optimal solution to the Euclidean loss will be the mean of the set. In color prediction, this averaging effect favors grayish, desaturated results. Thus, the research paper uses multinomial cross entropy loss to element desaturation of images. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E9YQAuPMxJg4"
   },
   "outputs": [],
   "source": [
    "print(1)\n",
    "class MultinomialCELoss(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MultinomialCELoss, self).__init__()\n",
    "\n",
    "    # x dim: n, q, h, w\n",
    "    # y dim: n, q, h, w\n",
    "    # n number of cases\n",
    "    # h, w height width\n",
    "    # q number of bins\n",
    "    # output: loss, as a float\n",
    "    def forward(self, x, y):\n",
    "        # softmax \n",
    "        x = x + 1e-8 #add a small number in x to avoid number 0.\n",
    "        x = torch.log(x)\n",
    "        zlogz = y*x\n",
    "        loss = - zlogz.sum()\n",
    "        loss /= (x.shape[0] * x.shape[2] * x.shape[3])\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sXscp1dIxaQv"
   },
   "source": [
    "## CNN architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f5p2bwpayuNI"
   },
   "source": [
    "This architecture uses multiple layers of CNN and maps the image pixels to a probability distribution of depth $313$. This result is described as $\\hat Z$ in the research paper. The probability distribution that the model learns is then evaluated with the multinomial loss function described above.\n",
    "\n",
    "$L_{cl}(\\hat Z, Z) = -\\sum{v(Z_{h,w})} \\sum Z_{h,w,q} log (\\hat Z_{h,w,q}) $\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yo_mVmzWxJg9"
   },
   "outputs": [],
   "source": [
    "class ColorfulColorizer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ColorfulColorizer, self).__init__()\n",
    "\n",
    "        self.op_1 = nn.Sequential(\n",
    "            nn.Conv2d(1, 64, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(64),\n",
    "        )\n",
    "        self.op_2 = nn.Sequential(\n",
    "            nn.Conv2d(64, 128, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(128, 128, kernel_size=3, stride=2, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(128)\n",
    "        )\n",
    "        self.op_3 = nn.Sequential(\n",
    "            nn.Conv2d(128, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, stride=2, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(256)\n",
    "        )\n",
    "        self.op_4 = nn.Sequential(\n",
    "            nn.Conv2d(256, 512, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(512)\n",
    "        )\n",
    "        self.op_5 = nn.Sequential(\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=2, dilation=2),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=2, dilation=2),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=2, dilation=2),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(512)\n",
    "        )\n",
    "        self.op_6 = nn.Sequential(\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=2, dilation=2),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=2, dilation=2),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=2, dilation=2),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(512)\n",
    "        )\n",
    "        self.op_7 = nn.Sequential(\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(512, 512, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(512)\n",
    "        )\n",
    "        self.op_8 = nn.Sequential(\n",
    "            nn.UpsamplingBilinear2d(scale_factor=2),\n",
    "            nn.Conv2d(512, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(256, 313, kernel_size=1),\n",
    "            nn.UpsamplingBilinear2d(scale_factor=4)\n",
    "        )\n",
    "\n",
    "        self.op_9 = nn.Sequential(\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "        self.op_1.apply(self.init_weights)\n",
    "        self.op_2.apply(self.init_weights)\n",
    "        self.op_3.apply(self.init_weights)\n",
    "        self.op_4.apply(self.init_weights)\n",
    "        self.op_5.apply(self.init_weights)\n",
    "        self.op_6.apply(self.init_weights)\n",
    "        self.op_7.apply(self.init_weights)\n",
    "        self.op_8.apply(self.init_weights)\n",
    "\n",
    "    def init_weights(self, m):\n",
    "        if type(m) == nn.Conv2d:\n",
    "            torch.nn.init.xavier_uniform_(m.weight)\n",
    "            m.bias.data.fill_(0.01)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.op_1(x)\n",
    "        out = self.op_2(out)\n",
    "        out = self.op_3(out)\n",
    "        out = self.op_4(out)\n",
    "        out = self.op_5(out)\n",
    "        out = self.op_6(out)\n",
    "        out = self.op_7(out)\n",
    "        out = self.op_8(out)\n",
    "        out = self.op_9(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nSXJy1CnxJhE"
   },
   "source": [
    "## Main - Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ytkib8IFxJhG"
   },
   "outputs": [],
   "source": [
    "def main(dset_root, batch_size, num_epochs, print_freq, encoder, criterion,\n",
    "         optimizer, step_every_iteration=False):\n",
    "    continue_training = True\n",
    "    location = 'cuda'\n",
    "    rescale = Rescale(imageSize)\n",
    "    train_dataset = CustomImages(\n",
    "        root=dset_root, train=True, location=location, transform=rescale, test_size=0)\n",
    "    \n",
    "    val_dataset = CustomImages(\n",
    "        root=dset_root, train=True, val=True, location=location, transform=rescale) #val files\n",
    "    \n",
    "    \n",
    "    train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                               batch_size=batch_size,\n",
    "                                               shuffle=True)\n",
    "\n",
    "    val_loader = torch.utils.data.DataLoader(dataset=val_dataset,\n",
    "                                               batch_size=batch_size,\n",
    "                                               shuffle=True)\n",
    "\n",
    "    if continue_training and os.path.isfile('best_model.pkl'):\n",
    "        encoder.load_state_dict(torch.load(\n",
    "            'best_model.pkl', map_location=location))\n",
    "        print('Model loaded!')\n",
    "\n",
    "\n",
    "    if 'cuda' in location:\n",
    "        print('Using:', torch.cuda.get_device_name(torch.cuda.current_device()))\n",
    "        encoder.cuda()\n",
    "        criterion.cuda()\n",
    "\n",
    "    best_loss = 100\n",
    "    losses = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        # train for one epoch\n",
    "        epoch_losses = train(train_loader, encoder, criterion, optimizer, epoch, location, step_every_iteration, num_epochs, print_freq)\n",
    "        losses.append(epoch_losses)\n",
    "\n",
    "        if epoch % save_freq == 0:\n",
    "          save_checkpoint(encoder.state_dict(), str(epoch)+\".pkl\")\n",
    "          save_model_results(train_dataset, encoder, epoch)\n",
    "          # coloring 5 random images and saving the output\n",
    "          \n",
    "          \n",
    "          \n",
    "\n",
    "        # evaluate on validation set\n",
    "        val_loss = validate(val_loader, encoder, criterion, location, num_epochs, print_freq)\n",
    "#         if (not step_every_iteration):\n",
    "#             scheduler.step(val_loss.data.item())\n",
    "        is_best = val_loss.data.item() < best_loss\n",
    "\n",
    "        if is_best:\n",
    "            print('New best score! Model saved as best_model.pkl')\n",
    "            best_loss = val_loss.data.item()\n",
    "            save_checkpoint(encoder.state_dict(), is_best)\n",
    "    return losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "01Y3a_TlxJhZ"
   },
   "outputs": [],
   "source": [
    "def save_checkpoint(state, is_best=False, filename='colorizer2.pkl'):\n",
    "    torch.save(state, StatePath+\"/states/\"+filename)\n",
    "    if is_best:\n",
    "        torch.save(state, 'best_model.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_j_vGFvz1pCb"
   },
   "source": [
    "After calculating the loss of each image between the ground truth encoded/ quantized ab space and the learned probability distribution $\\hat Z$, the prediction of ab colorspace of images is done via taking annealed mean of the learned probability distribution. This is because taking mean of this distribution poses the same problems as they were with computing Euclidean Loss, desaturated images. Hence a function $H(\\hat Z_{h,w})$ which takes the learned probability distribution as an input is implemented as described in research paper, and it outputs the annealed mean of the distribution for every pixel. This gives us the predicted ab colorspace for that image which is then converted to rgb colorspace to give results. \n",
    "\n",
    "According to the research paper, a temperature value $T = 0.38$ captures the vibrancy of the mode while maintaining the spatial coherence of the mean. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1o14wpsGKTSy"
   },
   "outputs": [],
   "source": [
    "def save_model_results(dset, model, batchesDone, location='cuda'):\n",
    "  test_cases = np.floor(np.random.rand(5) * len(dset)).astype(int)\n",
    "  test_cases = np.append(test_cases, [0], 0)\n",
    "  outputs = []\n",
    "  images = []\n",
    "  labels = []\n",
    "  for c in test_cases:\n",
    "      image,_, label = dset[c]\n",
    "      image = image.unsqueeze(0)\n",
    "      with torch.no_grad():\n",
    "          if 'cuda' in location:\n",
    "            image = image.cuda()\n",
    "            label = label.cuda()\n",
    "          images.append(image)\n",
    "          labels.append(label)\n",
    "          output = model(image)\n",
    "          outputs.append(output)\n",
    "          \n",
    "  T = 0.38\n",
    "  q = 313  # number of colours\n",
    "  nnenc = NNEncode()\n",
    "  bin_index = np.arange(q)\n",
    "  ab_list = nnenc.bin2color(bin_index)\n",
    "  for i in range(len(test_cases)):\n",
    "    l_layer = images[i].data[0].cpu().numpy()\n",
    "    bin_probabilities = outputs[i].data[0].cpu().numpy()  # bin_probabilities dim: q, h, w\n",
    "    ab_label = labels[i].data.cpu().numpy().astype('float64')\n",
    "\n",
    "    # convert bin_probab -> ab_pred\n",
    "    bin_probabilities = np.exp(np.log(bin_probabilities)/T)\n",
    "    bin_sum = bin_probabilities.sum(0)\n",
    "    bin_sum = bin_sum.reshape((1, bin_sum.shape[0], bin_sum.shape[1]))\n",
    "    bin_probabilities /= bin_sum\n",
    "\n",
    "    # ab_pred dim: 2, h, w\n",
    "    ab_pred = (bin_probabilities[:, np.newaxis, :, :] * ab_list[:, :, np.newaxis, np.newaxis]).sum(0)\n",
    "\n",
    "    img_input = l_layer[0]\n",
    "#     img_input = np.concatenate((l_layer, torch.zeros([2,128,128])), axis=0)\n",
    "    img_pred = np.concatenate((l_layer, ab_pred), axis=0)\n",
    "    img_actual = np.concatenate((l_layer, ab_label), axis=0)\n",
    "    \n",
    "#     img_input = lab2rgb(img_input.transpose(1, 2, 0))\n",
    "    img_pred = lab2rgb(img_pred.transpose(1, 2, 0))\n",
    "    img_actual = lab2rgb(img_actual.transpose(1, 2, 0))\n",
    "    \n",
    "    sample_image(img_input, img_pred, img_actual, batchesDone, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jZWQlvaxxJhj"
   },
   "outputs": [],
   "source": [
    "def train(train_loader, model, criterion, optimizer, epoch,\n",
    "          location, step_every_iteration,num_epochs, print_freq):\n",
    "    batch_time = AverageMeter()\n",
    "    data_time = AverageMeter()\n",
    "    losses = AverageMeter()\n",
    "    epoch_losses = []\n",
    "\n",
    "    # switch to train mode\n",
    "    model.train()\n",
    "\n",
    "    end = time.time()\n",
    "    for i, (image, target, _) in enumerate(train_loader):\n",
    "        # measure data loading time\n",
    "        data_time.update(time.time() - end)\n",
    "\n",
    "        image_var = Variable(image)\n",
    "        target_var = Variable(target)\n",
    "\n",
    "        if 'cuda' in location:\n",
    "            image_var = image_var.cuda()\n",
    "            target_var = target_var.cuda()\n",
    "\n",
    "        # compute output\n",
    "        output = model(image_var)\n",
    "        \n",
    "        loss = criterion(output, target_var)\n",
    "        \n",
    "        losses.update(loss.data.item(), image.size(0))\n",
    "        epoch_losses.append(loss.data.item())\n",
    "        \n",
    "        # compute gradient and do SGD step\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # measure elapsed time\n",
    "        batch_time.update(time.time() - end)\n",
    "        end = time.time()\n",
    "        batchDone = epoch * len(train_loader) + i\n",
    "        \n",
    "        if batchDone % print_freq == 0:\n",
    "          print('Epoch: [{0}/{1}][{2}/{3}]\\t'\n",
    "            'BatchTime(Average) {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n",
    "            'DataTime(Average) {data_time.val:.3f} ({data_time.avg:.3f})\\t'\n",
    "            'Loss(Average) {loss.val:.4f} ({loss.avg:.4f})\\t'\n",
    "            .format(\n",
    "             epoch, num_epochs, i, len(train_loader), batch_time=batch_time,\n",
    "              data_time=data_time, loss=losses))\n",
    "    return epoch_losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PDtVJatDxJhu"
   },
   "outputs": [],
   "source": [
    "def validate(val_loader, model, criterion, location,num_epochs, print_freq):\n",
    "    batch_time = AverageMeter()\n",
    "    losses = AverageMeter()\n",
    "    loss = 0\n",
    "    # switch to evaluate mode\n",
    "    model.eval()\n",
    "\n",
    "    end = time.time()\n",
    "    for i, (image, target, _) in enumerate(val_loader):\n",
    "        with torch.no_grad():\n",
    "          image_var = Variable(image)\n",
    "          target_var = Variable(target)\n",
    "\n",
    "        if 'cuda' in location:\n",
    "            image_var = image_var.cuda()\n",
    "            target_var = target_var.cuda()\n",
    "\n",
    "        # compute output\n",
    "        output = model(image_var)\n",
    "        loss = criterion(output, target_var)\n",
    "        losses.update(loss.data.item(), image.size(0))\n",
    "\n",
    "        # measure elapsed time\n",
    "        batch_time.update(time.time() - end)\n",
    "        end = time.time()\n",
    "\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lvqK8IDNxJh1"
   },
   "outputs": [],
   "source": [
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 9477
    },
    "colab_type": "code",
    "id": "-Jj417HKxJiG",
    "outputId": "cac1f158-a085-446d-c593-a9b3570d23cb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using: Tesla K80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/skimage/transform/_warps.py:84: UserWarning: The default mode, 'constant', will be changed to 'reflect' in skimage 0.15.\n",
      "  warn(\"The default mode, 'constant', will be changed to 'reflect' in \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/upsampling.py:225: UserWarning: nn.UpsamplingBilinear2d is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.UpsamplingBilinear2d is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/upsampling.py:122: UserWarning: nn.Upsampling is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.Upsampling is deprecated. Use nn.functional.interpolate instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [0/1000][0/140]\tBatchTime(Average) 0.885 (0.885)\tDataTime(Average) 0.626 (0.626)\tLoss(Average) 4.8431 (4.8431)\t\n",
      "Epoch: [0/1000][10/140]\tBatchTime(Average) 2.037 (1.664)\tDataTime(Average) 1.843 (1.464)\tLoss(Average) 4.9223 (4.8267)\t\n",
      "Epoch: [0/1000][20/140]\tBatchTime(Average) 1.266 (1.598)\tDataTime(Average) 1.069 (1.402)\tLoss(Average) 5.8862 (4.9357)\t\n",
      "Epoch: [0/1000][30/140]\tBatchTime(Average) 2.144 (1.618)\tDataTime(Average) 1.951 (1.423)\tLoss(Average) 4.9003 (4.8795)\t\n",
      "Epoch: [0/1000][40/140]\tBatchTime(Average) 1.503 (1.599)\tDataTime(Average) 1.311 (1.405)\tLoss(Average) 4.2474 (4.8750)\t\n",
      "Epoch: [0/1000][50/140]\tBatchTime(Average) 2.494 (1.616)\tDataTime(Average) 2.309 (1.423)\tLoss(Average) 5.0968 (4.8343)\t\n",
      "Epoch: [0/1000][60/140]\tBatchTime(Average) 1.988 (1.647)\tDataTime(Average) 1.794 (1.452)\tLoss(Average) 4.7179 (4.7546)\t\n",
      "Epoch: [0/1000][70/140]\tBatchTime(Average) 1.354 (1.630)\tDataTime(Average) 1.161 (1.436)\tLoss(Average) 5.6158 (4.7691)\t\n",
      "Epoch: [0/1000][80/140]\tBatchTime(Average) 1.652 (1.615)\tDataTime(Average) 1.457 (1.421)\tLoss(Average) 5.4723 (4.8030)\t\n",
      "Epoch: [0/1000][90/140]\tBatchTime(Average) 1.264 (1.593)\tDataTime(Average) 1.072 (1.400)\tLoss(Average) 4.8581 (4.7881)\t\n",
      "Epoch: [0/1000][100/140]\tBatchTime(Average) 1.365 (1.618)\tDataTime(Average) 1.173 (1.424)\tLoss(Average) 4.6080 (4.7927)\t\n",
      "Epoch: [0/1000][110/140]\tBatchTime(Average) 1.614 (1.615)\tDataTime(Average) 1.416 (1.422)\tLoss(Average) 4.5470 (4.7852)\t\n",
      "Epoch: [0/1000][120/140]\tBatchTime(Average) 2.421 (1.616)\tDataTime(Average) 2.226 (1.422)\tLoss(Average) 5.0468 (4.8051)\t\n",
      "Epoch: [0/1000][130/140]\tBatchTime(Average) 2.012 (1.609)\tDataTime(Average) 1.817 (1.415)\tLoss(Average) 4.2725 (4.8174)\t\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: DeprecationWarning: `imsave` is deprecated!\n",
      "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``imageio.imwrite`` instead.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best score! Model saved as best_model.pkl\n",
      "Epoch: [1/1000][0/140]\tBatchTime(Average) 1.055 (1.055)\tDataTime(Average) 0.876 (0.876)\tLoss(Average) 5.8726 (5.8726)\t\n",
      "Epoch: [1/1000][10/140]\tBatchTime(Average) 1.292 (1.304)\tDataTime(Average) 1.097 (1.110)\tLoss(Average) 5.9741 (4.9421)\t\n",
      "Epoch: [1/1000][20/140]\tBatchTime(Average) 1.266 (1.295)\tDataTime(Average) 1.069 (1.100)\tLoss(Average) 4.2507 (4.8344)\t\n",
      "Epoch: [1/1000][30/140]\tBatchTime(Average) 1.333 (1.287)\tDataTime(Average) 1.127 (1.092)\tLoss(Average) 4.9865 (4.9288)\t\n",
      "Epoch: [1/1000][40/140]\tBatchTime(Average) 1.316 (1.288)\tDataTime(Average) 1.119 (1.093)\tLoss(Average) 5.0666 (4.8857)\t\n",
      "Epoch: [1/1000][50/140]\tBatchTime(Average) 1.265 (1.284)\tDataTime(Average) 1.070 (1.089)\tLoss(Average) 4.8100 (4.8825)\t\n",
      "Epoch: [1/1000][60/140]\tBatchTime(Average) 1.309 (1.284)\tDataTime(Average) 1.112 (1.089)\tLoss(Average) 4.9589 (4.8713)\t\n",
      "Epoch: [1/1000][70/140]\tBatchTime(Average) 1.333 (1.287)\tDataTime(Average) 1.140 (1.091)\tLoss(Average) 4.0681 (4.8384)\t\n",
      "Epoch: [1/1000][80/140]\tBatchTime(Average) 1.229 (1.288)\tDataTime(Average) 1.031 (1.092)\tLoss(Average) 4.4560 (4.8448)\t\n",
      "Epoch: [1/1000][90/140]\tBatchTime(Average) 1.307 (1.288)\tDataTime(Average) 1.096 (1.092)\tLoss(Average) 4.1727 (4.8512)\t\n",
      "Epoch: [1/1000][100/140]\tBatchTime(Average) 1.262 (1.286)\tDataTime(Average) 1.067 (1.091)\tLoss(Average) 5.3607 (4.8574)\t\n",
      "Epoch: [1/1000][110/140]\tBatchTime(Average) 1.347 (1.287)\tDataTime(Average) 1.149 (1.091)\tLoss(Average) 4.5793 (4.8151)\t\n",
      "Epoch: [1/1000][120/140]\tBatchTime(Average) 1.244 (1.284)\tDataTime(Average) 1.053 (1.088)\tLoss(Average) 4.2361 (4.8178)\t\n",
      "Epoch: [1/1000][130/140]\tBatchTime(Average) 1.254 (1.282)\tDataTime(Average) 1.061 (1.087)\tLoss(Average) 5.4965 (4.7939)\t\n",
      "New best score! Model saved as best_model.pkl\n",
      "Epoch: [2/1000][0/140]\tBatchTime(Average) 1.014 (1.014)\tDataTime(Average) 0.834 (0.834)\tLoss(Average) 4.8313 (4.8313)\t\n",
      "Epoch: [2/1000][10/140]\tBatchTime(Average) 1.275 (1.260)\tDataTime(Average) 1.084 (1.065)\tLoss(Average) 5.1386 (5.0315)\t\n",
      "Epoch: [2/1000][20/140]\tBatchTime(Average) 1.197 (1.246)\tDataTime(Average) 1.005 (1.052)\tLoss(Average) 4.8792 (4.9227)\t\n",
      "Epoch: [2/1000][30/140]\tBatchTime(Average) 1.217 (1.243)\tDataTime(Average) 1.026 (1.049)\tLoss(Average) 4.4850 (4.8329)\t\n",
      "Epoch: [2/1000][40/140]\tBatchTime(Average) 1.236 (1.240)\tDataTime(Average) 1.034 (1.046)\tLoss(Average) 4.1059 (4.7817)\t\n",
      "Epoch: [2/1000][50/140]\tBatchTime(Average) 1.217 (1.238)\tDataTime(Average) 1.033 (1.045)\tLoss(Average) 4.5007 (4.7532)\t\n",
      "Epoch: [2/1000][60/140]\tBatchTime(Average) 1.235 (1.238)\tDataTime(Average) 1.042 (1.045)\tLoss(Average) 4.4183 (4.7421)\t\n",
      "Epoch: [2/1000][70/140]\tBatchTime(Average) 1.248 (1.239)\tDataTime(Average) 1.056 (1.046)\tLoss(Average) 5.8222 (4.7429)\t\n",
      "Epoch: [2/1000][80/140]\tBatchTime(Average) 1.229 (1.239)\tDataTime(Average) 1.031 (1.046)\tLoss(Average) 4.8962 (4.7500)\t\n",
      "Epoch: [2/1000][90/140]\tBatchTime(Average) 1.244 (1.240)\tDataTime(Average) 1.055 (1.046)\tLoss(Average) 5.0103 (4.7548)\t\n",
      "Epoch: [2/1000][100/140]\tBatchTime(Average) 1.275 (1.241)\tDataTime(Average) 1.076 (1.047)\tLoss(Average) 5.0816 (4.7727)\t\n",
      "Epoch: [2/1000][110/140]\tBatchTime(Average) 1.221 (1.240)\tDataTime(Average) 1.028 (1.047)\tLoss(Average) 5.2300 (4.7734)\t\n",
      "Epoch: [2/1000][120/140]\tBatchTime(Average) 1.241 (1.241)\tDataTime(Average) 1.053 (1.047)\tLoss(Average) 4.5550 (4.7817)\t\n",
      "Epoch: [2/1000][130/140]\tBatchTime(Average) 1.211 (1.241)\tDataTime(Average) 1.016 (1.047)\tLoss(Average) 5.3374 (4.7712)\t\n",
      "Epoch: [3/1000][0/140]\tBatchTime(Average) 1.007 (1.007)\tDataTime(Average) 0.826 (0.826)\tLoss(Average) 4.7159 (4.7159)\t\n",
      "Epoch: [3/1000][10/140]\tBatchTime(Average) 1.258 (1.236)\tDataTime(Average) 1.072 (1.042)\tLoss(Average) 4.5899 (4.8332)\t\n",
      "Epoch: [3/1000][20/140]\tBatchTime(Average) 1.240 (1.244)\tDataTime(Average) 1.054 (1.050)\tLoss(Average) 4.9775 (4.7025)\t\n",
      "Epoch: [3/1000][30/140]\tBatchTime(Average) 1.265 (1.247)\tDataTime(Average) 1.067 (1.054)\tLoss(Average) 4.5300 (4.7346)\t\n",
      "Epoch: [3/1000][40/140]\tBatchTime(Average) 1.259 (1.250)\tDataTime(Average) 1.063 (1.056)\tLoss(Average) 6.1719 (4.8006)\t\n",
      "Epoch: [3/1000][50/140]\tBatchTime(Average) 1.252 (1.250)\tDataTime(Average) 1.055 (1.056)\tLoss(Average) 3.8219 (4.7471)\t\n",
      "Epoch: [3/1000][60/140]\tBatchTime(Average) 1.287 (1.253)\tDataTime(Average) 1.095 (1.059)\tLoss(Average) 4.3568 (4.6877)\t\n",
      "Epoch: [3/1000][70/140]\tBatchTime(Average) 1.276 (1.255)\tDataTime(Average) 1.086 (1.061)\tLoss(Average) 6.0002 (4.7385)\t\n",
      "Epoch: [3/1000][80/140]\tBatchTime(Average) 1.284 (1.256)\tDataTime(Average) 1.091 (1.062)\tLoss(Average) 5.1611 (4.7307)\t\n",
      "Epoch: [3/1000][90/140]\tBatchTime(Average) 1.289 (1.257)\tDataTime(Average) 1.093 (1.063)\tLoss(Average) 4.9934 (4.6822)\t\n",
      "Epoch: [3/1000][100/140]\tBatchTime(Average) 1.268 (1.258)\tDataTime(Average) 1.076 (1.064)\tLoss(Average) 5.0358 (4.6687)\t\n",
      "Epoch: [3/1000][110/140]\tBatchTime(Average) 1.253 (1.258)\tDataTime(Average) 1.052 (1.064)\tLoss(Average) 5.5574 (4.6842)\t\n",
      "Epoch: [3/1000][120/140]\tBatchTime(Average) 1.239 (1.258)\tDataTime(Average) 1.045 (1.064)\tLoss(Average) 4.4936 (4.6943)\t\n",
      "Epoch: [3/1000][130/140]\tBatchTime(Average) 1.275 (1.259)\tDataTime(Average) 1.076 (1.064)\tLoss(Average) 5.1449 (4.6923)\t\n",
      "Epoch: [4/1000][0/140]\tBatchTime(Average) 1.038 (1.038)\tDataTime(Average) 0.856 (0.856)\tLoss(Average) 4.7895 (4.7895)\t\n",
      "Epoch: [4/1000][10/140]\tBatchTime(Average) 1.242 (1.244)\tDataTime(Average) 1.052 (1.051)\tLoss(Average) 4.9062 (4.6931)\t\n",
      "Epoch: [4/1000][20/140]\tBatchTime(Average) 1.261 (1.243)\tDataTime(Average) 1.069 (1.050)\tLoss(Average) 4.8485 (4.6762)\t\n",
      "Epoch: [4/1000][30/140]\tBatchTime(Average) 1.266 (1.249)\tDataTime(Average) 1.077 (1.055)\tLoss(Average) 5.9253 (4.7750)\t\n",
      "Epoch: [4/1000][40/140]\tBatchTime(Average) 1.251 (1.253)\tDataTime(Average) 1.060 (1.059)\tLoss(Average) 5.4862 (4.7793)\t\n",
      "Epoch: [4/1000][50/140]\tBatchTime(Average) 1.259 (1.253)\tDataTime(Average) 1.072 (1.059)\tLoss(Average) 4.3648 (4.7315)\t\n",
      "Epoch: [4/1000][60/140]\tBatchTime(Average) 1.198 (1.251)\tDataTime(Average) 1.003 (1.057)\tLoss(Average) 4.3411 (4.7196)\t\n",
      "Epoch: [4/1000][70/140]\tBatchTime(Average) 1.274 (1.253)\tDataTime(Average) 1.077 (1.059)\tLoss(Average) 4.8140 (4.6800)\t\n",
      "Epoch: [4/1000][80/140]\tBatchTime(Average) 1.247 (1.252)\tDataTime(Average) 1.053 (1.058)\tLoss(Average) 4.0768 (4.6774)\t\n",
      "Epoch: [4/1000][90/140]\tBatchTime(Average) 1.223 (1.252)\tDataTime(Average) 1.031 (1.057)\tLoss(Average) 3.9458 (4.6696)\t\n",
      "Epoch: [4/1000][100/140]\tBatchTime(Average) 1.263 (1.252)\tDataTime(Average) 1.070 (1.059)\tLoss(Average) 4.9010 (4.6430)\t\n",
      "Epoch: [4/1000][110/140]\tBatchTime(Average) 1.270 (1.254)\tDataTime(Average) 1.077 (1.060)\tLoss(Average) 4.8627 (4.6504)\t\n",
      "Epoch: [4/1000][120/140]\tBatchTime(Average) 1.230 (1.255)\tDataTime(Average) 1.032 (1.061)\tLoss(Average) 3.6297 (4.6533)\t\n",
      "Epoch: [4/1000][130/140]\tBatchTime(Average) 1.248 (1.254)\tDataTime(Average) 1.058 (1.060)\tLoss(Average) 5.4034 (4.6520)\t\n",
      "Epoch: [5/1000][0/140]\tBatchTime(Average) 0.989 (0.989)\tDataTime(Average) 0.810 (0.810)\tLoss(Average) 5.2609 (5.2609)\t\n",
      "Epoch: [5/1000][10/140]\tBatchTime(Average) 1.263 (1.227)\tDataTime(Average) 1.068 (1.032)\tLoss(Average) 4.2329 (4.3434)\t\n",
      "Epoch: [5/1000][20/140]\tBatchTime(Average) 1.219 (1.243)\tDataTime(Average) 1.024 (1.048)\tLoss(Average) 3.3424 (4.4625)\t\n",
      "Epoch: [5/1000][30/140]\tBatchTime(Average) 1.248 (1.243)\tDataTime(Average) 1.054 (1.048)\tLoss(Average) 3.9096 (4.6011)\t\n",
      "Epoch: [5/1000][40/140]\tBatchTime(Average) 1.251 (1.248)\tDataTime(Average) 1.054 (1.054)\tLoss(Average) 4.1415 (4.5718)\t\n",
      "Epoch: [5/1000][50/140]\tBatchTime(Average) 1.259 (1.247)\tDataTime(Average) 1.059 (1.053)\tLoss(Average) 4.7714 (4.5796)\t\n",
      "Epoch: [5/1000][60/140]\tBatchTime(Average) 1.211 (1.245)\tDataTime(Average) 1.023 (1.052)\tLoss(Average) 3.3342 (4.5342)\t\n",
      "Epoch: [5/1000][70/140]\tBatchTime(Average) 1.257 (1.246)\tDataTime(Average) 1.070 (1.053)\tLoss(Average) 4.7968 (4.5687)\t\n",
      "Epoch: [5/1000][80/140]\tBatchTime(Average) 1.233 (1.245)\tDataTime(Average) 1.050 (1.052)\tLoss(Average) 4.9582 (4.5428)\t\n",
      "Epoch: [5/1000][90/140]\tBatchTime(Average) 1.251 (1.246)\tDataTime(Average) 1.058 (1.053)\tLoss(Average) 4.1424 (4.5822)\t\n",
      "Epoch: [5/1000][100/140]\tBatchTime(Average) 1.260 (1.247)\tDataTime(Average) 1.062 (1.054)\tLoss(Average) 4.1591 (4.5847)\t\n",
      "Epoch: [5/1000][110/140]\tBatchTime(Average) 1.271 (1.248)\tDataTime(Average) 1.079 (1.054)\tLoss(Average) 5.2074 (4.5934)\t\n",
      "Epoch: [5/1000][120/140]\tBatchTime(Average) 1.232 (1.247)\tDataTime(Average) 1.039 (1.053)\tLoss(Average) 5.0116 (4.5918)\t\n",
      "Epoch: [5/1000][130/140]\tBatchTime(Average) 1.372 (1.255)\tDataTime(Average) 1.182 (1.061)\tLoss(Average) 5.8707 (4.5830)\t\n",
      "New best score! Model saved as best_model.pkl\n",
      "Epoch: [6/1000][0/140]\tBatchTime(Average) 1.056 (1.056)\tDataTime(Average) 0.880 (0.880)\tLoss(Average) 4.5939 (4.5939)\t\n",
      "Epoch: [6/1000][10/140]\tBatchTime(Average) 1.267 (1.274)\tDataTime(Average) 1.075 (1.081)\tLoss(Average) 4.0804 (4.6920)\t\n",
      "Epoch: [6/1000][20/140]\tBatchTime(Average) 1.260 (1.268)\tDataTime(Average) 1.074 (1.074)\tLoss(Average) 4.9270 (4.4935)\t\n",
      "Epoch: [6/1000][30/140]\tBatchTime(Average) 1.220 (1.268)\tDataTime(Average) 1.026 (1.074)\tLoss(Average) 4.3216 (4.4912)\t\n",
      "Epoch: [6/1000][40/140]\tBatchTime(Average) 1.247 (1.275)\tDataTime(Average) 1.055 (1.080)\tLoss(Average) 5.3017 (4.5104)\t\n",
      "Epoch: [6/1000][50/140]\tBatchTime(Average) 1.314 (1.279)\tDataTime(Average) 1.125 (1.085)\tLoss(Average) 4.5475 (4.5049)\t\n",
      "Epoch: [6/1000][60/140]\tBatchTime(Average) 1.318 (1.279)\tDataTime(Average) 1.134 (1.084)\tLoss(Average) 4.3280 (4.5525)\t\n",
      "Epoch: [6/1000][70/140]\tBatchTime(Average) 1.266 (1.277)\tDataTime(Average) 1.076 (1.082)\tLoss(Average) 4.5368 (4.5347)\t\n",
      "Epoch: [6/1000][80/140]\tBatchTime(Average) 1.269 (1.273)\tDataTime(Average) 1.078 (1.078)\tLoss(Average) 4.3058 (4.5258)\t\n",
      "Epoch: [6/1000][90/140]\tBatchTime(Average) 1.246 (1.271)\tDataTime(Average) 1.051 (1.076)\tLoss(Average) 4.9347 (4.5315)\t\n",
      "Epoch: [6/1000][100/140]\tBatchTime(Average) 1.226 (1.270)\tDataTime(Average) 1.041 (1.075)\tLoss(Average) 5.2211 (4.5343)\t\n",
      "Epoch: [6/1000][110/140]\tBatchTime(Average) 1.250 (1.268)\tDataTime(Average) 1.049 (1.073)\tLoss(Average) 4.9839 (4.5548)\t\n",
      "Epoch: [6/1000][120/140]\tBatchTime(Average) 1.244 (1.267)\tDataTime(Average) 1.054 (1.073)\tLoss(Average) 5.4385 (4.5401)\t\n",
      "Epoch: [6/1000][130/140]\tBatchTime(Average) 1.295 (1.265)\tDataTime(Average) 1.108 (1.071)\tLoss(Average) 4.8162 (4.5407)\t\n",
      "Epoch: [7/1000][0/140]\tBatchTime(Average) 1.012 (1.012)\tDataTime(Average) 0.831 (0.831)\tLoss(Average) 4.5804 (4.5804)\t\n",
      "Epoch: [7/1000][10/140]\tBatchTime(Average) 1.245 (1.228)\tDataTime(Average) 1.051 (1.034)\tLoss(Average) 4.5047 (4.7855)\t\n",
      "Epoch: [7/1000][20/140]\tBatchTime(Average) 1.263 (1.238)\tDataTime(Average) 1.064 (1.044)\tLoss(Average) 5.1407 (4.4901)\t\n",
      "Epoch: [7/1000][30/140]\tBatchTime(Average) 1.250 (1.243)\tDataTime(Average) 1.055 (1.050)\tLoss(Average) 3.4522 (4.4905)\t\n",
      "Epoch: [7/1000][40/140]\tBatchTime(Average) 1.209 (1.249)\tDataTime(Average) 1.022 (1.056)\tLoss(Average) 4.2900 (4.4860)\t\n",
      "Epoch: [7/1000][50/140]\tBatchTime(Average) 1.298 (1.253)\tDataTime(Average) 1.099 (1.059)\tLoss(Average) 3.5647 (4.4831)\t\n",
      "Epoch: [7/1000][60/140]\tBatchTime(Average) 1.278 (1.255)\tDataTime(Average) 1.087 (1.061)\tLoss(Average) 4.0082 (4.4851)\t\n",
      "Epoch: [7/1000][70/140]\tBatchTime(Average) 1.250 (1.258)\tDataTime(Average) 1.062 (1.064)\tLoss(Average) 5.5869 (4.4913)\t\n",
      "Epoch: [7/1000][80/140]\tBatchTime(Average) 1.301 (1.260)\tDataTime(Average) 1.103 (1.066)\tLoss(Average) 4.2903 (4.4994)\t\n",
      "Epoch: [7/1000][90/140]\tBatchTime(Average) 1.262 (1.263)\tDataTime(Average) 1.068 (1.069)\tLoss(Average) 3.9429 (4.5057)\t\n",
      "Epoch: [7/1000][100/140]\tBatchTime(Average) 1.272 (1.265)\tDataTime(Average) 1.074 (1.071)\tLoss(Average) 4.8481 (4.4941)\t\n",
      "Epoch: [7/1000][110/140]\tBatchTime(Average) 1.270 (1.266)\tDataTime(Average) 1.074 (1.072)\tLoss(Average) 4.9260 (4.4991)\t\n",
      "Epoch: [7/1000][120/140]\tBatchTime(Average) 1.214 (1.267)\tDataTime(Average) 1.022 (1.073)\tLoss(Average) 3.9755 (4.4685)\t\n",
      "Epoch: [7/1000][130/140]\tBatchTime(Average) 1.250 (1.266)\tDataTime(Average) 1.052 (1.072)\tLoss(Average) 3.4676 (4.4775)\t\n",
      "Epoch: [8/1000][0/140]\tBatchTime(Average) 1.019 (1.019)\tDataTime(Average) 0.842 (0.842)\tLoss(Average) 4.9700 (4.9700)\t\n",
      "Epoch: [8/1000][10/140]\tBatchTime(Average) 1.273 (1.236)\tDataTime(Average) 1.075 (1.042)\tLoss(Average) 5.7369 (4.8364)\t\n",
      "Epoch: [8/1000][20/140]\tBatchTime(Average) 1.270 (1.239)\tDataTime(Average) 1.075 (1.046)\tLoss(Average) 3.5061 (4.5992)\t\n",
      "Epoch: [8/1000][30/140]\tBatchTime(Average) 1.252 (1.244)\tDataTime(Average) 1.058 (1.051)\tLoss(Average) 3.4113 (4.5541)\t\n",
      "Epoch: [8/1000][40/140]\tBatchTime(Average) 1.201 (1.242)\tDataTime(Average) 1.010 (1.048)\tLoss(Average) 3.7665 (4.4479)\t\n",
      "Epoch: [8/1000][50/140]\tBatchTime(Average) 1.269 (1.243)\tDataTime(Average) 1.076 (1.050)\tLoss(Average) 4.3449 (4.4204)\t\n",
      "Epoch: [8/1000][60/140]\tBatchTime(Average) 1.242 (1.243)\tDataTime(Average) 1.048 (1.050)\tLoss(Average) 5.1646 (4.5100)\t\n",
      "Epoch: [8/1000][70/140]\tBatchTime(Average) 1.257 (1.242)\tDataTime(Average) 1.057 (1.049)\tLoss(Average) 4.9349 (4.5159)\t\n",
      "Epoch: [8/1000][80/140]\tBatchTime(Average) 1.281 (1.243)\tDataTime(Average) 1.088 (1.050)\tLoss(Average) 4.4874 (4.4853)\t\n",
      "Epoch: [8/1000][90/140]\tBatchTime(Average) 1.264 (1.244)\tDataTime(Average) 1.066 (1.051)\tLoss(Average) 5.0096 (4.4660)\t\n",
      "Epoch: [8/1000][100/140]\tBatchTime(Average) 1.226 (1.244)\tDataTime(Average) 1.029 (1.050)\tLoss(Average) 4.5763 (4.4662)\t\n",
      "Epoch: [8/1000][110/140]\tBatchTime(Average) 1.269 (1.244)\tDataTime(Average) 1.072 (1.051)\tLoss(Average) 4.2672 (4.4759)\t\n",
      "Epoch: [8/1000][120/140]\tBatchTime(Average) 1.281 (1.245)\tDataTime(Average) 1.091 (1.051)\tLoss(Average) 5.4150 (4.4588)\t\n",
      "Epoch: [8/1000][130/140]\tBatchTime(Average) 1.251 (1.244)\tDataTime(Average) 1.059 (1.051)\tLoss(Average) 5.1923 (4.4561)\t\n",
      "Epoch: [9/1000][0/140]\tBatchTime(Average) 1.010 (1.010)\tDataTime(Average) 0.831 (0.831)\tLoss(Average) 5.2587 (5.2587)\t\n",
      "Epoch: [9/1000][10/140]\tBatchTime(Average) 1.276 (1.238)\tDataTime(Average) 1.082 (1.043)\tLoss(Average) 5.5606 (4.6355)\t\n",
      "Epoch: [9/1000][20/140]\tBatchTime(Average) 1.279 (1.245)\tDataTime(Average) 1.085 (1.050)\tLoss(Average) 4.2905 (4.5105)\t\n",
      "Epoch: [9/1000][30/140]\tBatchTime(Average) 1.262 (1.250)\tDataTime(Average) 1.070 (1.056)\tLoss(Average) 4.1144 (4.4025)\t\n",
      "Epoch: [9/1000][40/140]\tBatchTime(Average) 1.230 (1.252)\tDataTime(Average) 1.040 (1.058)\tLoss(Average) 3.7527 (4.3496)\t\n",
      "Epoch: [9/1000][50/140]\tBatchTime(Average) 1.276 (1.254)\tDataTime(Average) 1.084 (1.059)\tLoss(Average) 5.0740 (4.3508)\t\n",
      "Epoch: [9/1000][60/140]\tBatchTime(Average) 1.272 (1.257)\tDataTime(Average) 1.083 (1.062)\tLoss(Average) 3.8667 (4.3663)\t\n",
      "Epoch: [9/1000][70/140]\tBatchTime(Average) 1.290 (1.257)\tDataTime(Average) 1.093 (1.063)\tLoss(Average) 4.7565 (4.3544)\t\n",
      "Epoch: [9/1000][80/140]\tBatchTime(Average) 1.250 (1.258)\tDataTime(Average) 1.055 (1.063)\tLoss(Average) 4.9390 (4.3742)\t\n",
      "Epoch: [9/1000][90/140]\tBatchTime(Average) 1.261 (1.258)\tDataTime(Average) 1.067 (1.063)\tLoss(Average) 4.1903 (4.4039)\t\n",
      "Epoch: [9/1000][100/140]\tBatchTime(Average) 1.225 (1.256)\tDataTime(Average) 1.028 (1.062)\tLoss(Average) 4.4054 (4.4211)\t\n",
      "Epoch: [9/1000][110/140]\tBatchTime(Average) 1.273 (1.255)\tDataTime(Average) 1.078 (1.061)\tLoss(Average) 4.4491 (4.4249)\t\n",
      "Epoch: [9/1000][120/140]\tBatchTime(Average) 1.245 (1.254)\tDataTime(Average) 1.055 (1.060)\tLoss(Average) 3.1638 (4.4088)\t\n",
      "Epoch: [9/1000][130/140]\tBatchTime(Average) 1.233 (1.253)\tDataTime(Average) 1.047 (1.059)\tLoss(Average) 3.9529 (4.4184)\t\n",
      "Epoch: [10/1000][0/140]\tBatchTime(Average) 1.024 (1.024)\tDataTime(Average) 0.847 (0.847)\tLoss(Average) 4.9125 (4.9125)\t\n",
      "Epoch: [10/1000][10/140]\tBatchTime(Average) 1.247 (1.225)\tDataTime(Average) 1.049 (1.033)\tLoss(Average) 3.8325 (4.4335)\t\n",
      "Epoch: [10/1000][20/140]\tBatchTime(Average) 1.279 (1.237)\tDataTime(Average) 1.085 (1.044)\tLoss(Average) 4.7523 (4.2147)\t\n",
      "Epoch: [10/1000][30/140]\tBatchTime(Average) 1.230 (1.237)\tDataTime(Average) 1.034 (1.044)\tLoss(Average) 4.2485 (4.3394)\t\n",
      "Epoch: [10/1000][40/140]\tBatchTime(Average) 1.238 (1.240)\tDataTime(Average) 1.047 (1.047)\tLoss(Average) 3.3692 (4.3143)\t\n",
      "Epoch: [10/1000][50/140]\tBatchTime(Average) 1.276 (1.239)\tDataTime(Average) 1.084 (1.046)\tLoss(Average) 3.9175 (4.2646)\t\n",
      "Epoch: [10/1000][60/140]\tBatchTime(Average) 1.256 (1.241)\tDataTime(Average) 1.063 (1.048)\tLoss(Average) 3.4661 (4.2872)\t\n",
      "Epoch: [10/1000][70/140]\tBatchTime(Average) 1.288 (1.243)\tDataTime(Average) 1.093 (1.050)\tLoss(Average) 5.5137 (4.3562)\t\n",
      "Epoch: [10/1000][80/140]\tBatchTime(Average) 1.234 (1.243)\tDataTime(Average) 1.037 (1.050)\tLoss(Average) 4.7133 (4.3739)\t\n",
      "Epoch: [10/1000][90/140]\tBatchTime(Average) 1.231 (1.244)\tDataTime(Average) 1.039 (1.050)\tLoss(Average) 4.4398 (4.3775)\t\n",
      "Epoch: [10/1000][100/140]\tBatchTime(Average) 1.245 (1.244)\tDataTime(Average) 1.049 (1.051)\tLoss(Average) 4.5649 (4.3572)\t\n",
      "Epoch: [10/1000][110/140]\tBatchTime(Average) 1.395 (1.247)\tDataTime(Average) 1.192 (1.054)\tLoss(Average) 5.6662 (4.3510)\t\n",
      "Epoch: [10/1000][120/140]\tBatchTime(Average) 1.276 (1.252)\tDataTime(Average) 1.068 (1.059)\tLoss(Average) 3.9046 (4.3447)\t\n",
      "Epoch: [10/1000][130/140]\tBatchTime(Average) 1.328 (1.255)\tDataTime(Average) 1.120 (1.061)\tLoss(Average) 4.6180 (4.3522)\t\n",
      "Epoch: [11/1000][0/140]\tBatchTime(Average) 0.986 (0.986)\tDataTime(Average) 0.806 (0.806)\tLoss(Average) 3.8942 (3.8942)\t\n",
      "Epoch: [11/1000][10/140]\tBatchTime(Average) 1.321 (1.252)\tDataTime(Average) 1.123 (1.058)\tLoss(Average) 5.4971 (4.4630)\t\n",
      "Epoch: [11/1000][20/140]\tBatchTime(Average) 1.246 (1.264)\tDataTime(Average) 1.050 (1.068)\tLoss(Average) 4.4636 (4.3687)\t\n",
      "Epoch: [11/1000][30/140]\tBatchTime(Average) 1.347 (1.271)\tDataTime(Average) 1.146 (1.075)\tLoss(Average) 4.0484 (4.3419)\t\n",
      "Epoch: [11/1000][40/140]\tBatchTime(Average) 1.278 (1.273)\tDataTime(Average) 1.086 (1.077)\tLoss(Average) 4.6004 (4.3027)\t\n",
      "Epoch: [11/1000][50/140]\tBatchTime(Average) 1.273 (1.275)\tDataTime(Average) 1.081 (1.079)\tLoss(Average) 3.9888 (4.3171)\t\n",
      "Epoch: [11/1000][60/140]\tBatchTime(Average) 1.303 (1.277)\tDataTime(Average) 1.106 (1.082)\tLoss(Average) 4.3305 (4.3110)\t\n",
      "Epoch: [11/1000][70/140]\tBatchTime(Average) 1.276 (1.280)\tDataTime(Average) 1.083 (1.085)\tLoss(Average) 4.1302 (4.3393)\t\n",
      "Epoch: [11/1000][80/140]\tBatchTime(Average) 1.336 (1.281)\tDataTime(Average) 1.148 (1.085)\tLoss(Average) 5.3566 (4.3261)\t\n",
      "Epoch: [11/1000][90/140]\tBatchTime(Average) 1.308 (1.279)\tDataTime(Average) 1.112 (1.084)\tLoss(Average) 4.2036 (4.3106)\t\n",
      "Epoch: [11/1000][100/140]\tBatchTime(Average) 1.251 (1.278)\tDataTime(Average) 1.050 (1.083)\tLoss(Average) 3.8976 (4.3401)\t\n",
      "Epoch: [11/1000][110/140]\tBatchTime(Average) 1.303 (1.278)\tDataTime(Average) 1.109 (1.083)\tLoss(Average) 2.9984 (4.3155)\t\n",
      "Epoch: [11/1000][120/140]\tBatchTime(Average) 1.255 (1.277)\tDataTime(Average) 1.061 (1.082)\tLoss(Average) 4.5638 (4.3385)\t\n",
      "Epoch: [11/1000][130/140]\tBatchTime(Average) 1.258 (1.278)\tDataTime(Average) 1.067 (1.083)\tLoss(Average) 4.0530 (4.3328)\t\n",
      "New best score! Model saved as best_model.pkl\n",
      "Epoch: [12/1000][0/140]\tBatchTime(Average) 1.167 (1.167)\tDataTime(Average) 0.984 (0.984)\tLoss(Average) 3.8207 (3.8207)\t\n",
      "Epoch: [12/1000][10/140]\tBatchTime(Average) 1.318 (1.332)\tDataTime(Average) 1.117 (1.135)\tLoss(Average) 4.5630 (4.5963)\t\n",
      "Epoch: [12/1000][20/140]\tBatchTime(Average) 1.291 (1.308)\tDataTime(Average) 1.099 (1.112)\tLoss(Average) 3.9531 (4.4940)\t\n",
      "Epoch: [12/1000][30/140]\tBatchTime(Average) 1.325 (1.296)\tDataTime(Average) 1.129 (1.101)\tLoss(Average) 3.6656 (4.4386)\t\n",
      "Epoch: [12/1000][40/140]\tBatchTime(Average) 1.275 (1.285)\tDataTime(Average) 1.080 (1.090)\tLoss(Average) 4.2909 (4.3129)\t\n",
      "Epoch: [12/1000][50/140]\tBatchTime(Average) 1.272 (1.278)\tDataTime(Average) 1.080 (1.082)\tLoss(Average) 4.0545 (4.2898)\t\n",
      "Epoch: [12/1000][60/140]\tBatchTime(Average) 1.245 (1.274)\tDataTime(Average) 1.046 (1.079)\tLoss(Average) 4.4002 (4.3225)\t\n",
      "Epoch: [12/1000][70/140]\tBatchTime(Average) 1.237 (1.269)\tDataTime(Average) 1.043 (1.074)\tLoss(Average) 3.7590 (4.2985)\t\n",
      "Epoch: [12/1000][80/140]\tBatchTime(Average) 1.244 (1.269)\tDataTime(Average) 1.048 (1.074)\tLoss(Average) 4.4908 (4.2511)\t\n",
      "Epoch: [12/1000][90/140]\tBatchTime(Average) 1.266 (1.269)\tDataTime(Average) 1.068 (1.074)\tLoss(Average) 3.1561 (4.2694)\t\n",
      "Epoch: [12/1000][100/140]\tBatchTime(Average) 1.246 (1.266)\tDataTime(Average) 1.057 (1.072)\tLoss(Average) 3.7115 (4.2863)\t\n",
      "Epoch: [12/1000][110/140]\tBatchTime(Average) 1.260 (1.265)\tDataTime(Average) 1.062 (1.070)\tLoss(Average) 4.8438 (4.3149)\t\n",
      "Epoch: [12/1000][120/140]\tBatchTime(Average) 1.232 (1.263)\tDataTime(Average) 1.036 (1.069)\tLoss(Average) 4.1922 (4.3112)\t\n",
      "Epoch: [12/1000][130/140]\tBatchTime(Average) 1.248 (1.262)\tDataTime(Average) 1.054 (1.067)\tLoss(Average) 4.3562 (4.2998)\t\n",
      "Epoch: [13/1000][0/140]\tBatchTime(Average) 1.044 (1.044)\tDataTime(Average) 0.861 (0.861)\tLoss(Average) 4.2820 (4.2820)\t\n",
      "Epoch: [13/1000][10/140]\tBatchTime(Average) 1.255 (1.233)\tDataTime(Average) 1.059 (1.038)\tLoss(Average) 4.8456 (4.4472)\t\n",
      "Epoch: [13/1000][20/140]\tBatchTime(Average) 1.280 (1.245)\tDataTime(Average) 1.086 (1.050)\tLoss(Average) 4.3673 (4.3294)\t\n",
      "Epoch: [13/1000][30/140]\tBatchTime(Average) 1.230 (1.243)\tDataTime(Average) 1.038 (1.049)\tLoss(Average) 4.8041 (4.3367)\t\n",
      "Epoch: [13/1000][40/140]\tBatchTime(Average) 1.271 (1.244)\tDataTime(Average) 1.083 (1.050)\tLoss(Average) 4.0787 (4.2628)\t\n",
      "Epoch: [13/1000][50/140]\tBatchTime(Average) 1.243 (1.244)\tDataTime(Average) 1.048 (1.050)\tLoss(Average) 3.6683 (4.2843)\t\n",
      "Epoch: [13/1000][60/140]\tBatchTime(Average) 1.301 (1.247)\tDataTime(Average) 1.106 (1.053)\tLoss(Average) 4.3243 (4.3632)\t\n",
      "Epoch: [13/1000][70/140]\tBatchTime(Average) 1.255 (1.248)\tDataTime(Average) 1.061 (1.054)\tLoss(Average) 3.2027 (4.3275)\t\n",
      "Epoch: [13/1000][80/140]\tBatchTime(Average) 1.260 (1.248)\tDataTime(Average) 1.065 (1.054)\tLoss(Average) 5.2441 (4.3181)\t\n",
      "Epoch: [13/1000][90/140]\tBatchTime(Average) 1.285 (1.247)\tDataTime(Average) 1.095 (1.054)\tLoss(Average) 4.2482 (4.2707)\t\n",
      "Epoch: [13/1000][100/140]\tBatchTime(Average) 1.271 (1.248)\tDataTime(Average) 1.078 (1.054)\tLoss(Average) 4.2978 (4.2241)\t\n",
      "Epoch: [13/1000][110/140]\tBatchTime(Average) 1.243 (1.248)\tDataTime(Average) 1.054 (1.054)\tLoss(Average) 4.1422 (4.2385)\t\n",
      "Epoch: [13/1000][120/140]\tBatchTime(Average) 1.229 (1.248)\tDataTime(Average) 1.036 (1.055)\tLoss(Average) 5.0082 (4.2563)\t\n",
      "Epoch: [13/1000][130/140]\tBatchTime(Average) 1.265 (1.249)\tDataTime(Average) 1.071 (1.055)\tLoss(Average) 3.8036 (4.2750)\t\n",
      "Epoch: [14/1000][0/140]\tBatchTime(Average) 1.022 (1.022)\tDataTime(Average) 0.844 (0.844)\tLoss(Average) 3.5810 (3.5810)\t\n",
      "Epoch: [14/1000][10/140]\tBatchTime(Average) 1.244 (1.228)\tDataTime(Average) 1.047 (1.033)\tLoss(Average) 3.6178 (4.3501)\t\n",
      "Epoch: [14/1000][20/140]\tBatchTime(Average) 1.233 (1.239)\tDataTime(Average) 1.042 (1.045)\tLoss(Average) 4.5958 (4.3206)\t\n",
      "Epoch: [14/1000][30/140]\tBatchTime(Average) 1.217 (1.244)\tDataTime(Average) 1.021 (1.051)\tLoss(Average) 5.0822 (4.2576)\t\n",
      "Epoch: [14/1000][40/140]\tBatchTime(Average) 1.280 (1.243)\tDataTime(Average) 1.088 (1.050)\tLoss(Average) 4.3181 (4.3188)\t\n",
      "Epoch: [14/1000][50/140]\tBatchTime(Average) 1.223 (1.244)\tDataTime(Average) 1.033 (1.050)\tLoss(Average) 3.5441 (4.3204)\t\n",
      "Epoch: [14/1000][60/140]\tBatchTime(Average) 1.217 (1.246)\tDataTime(Average) 1.026 (1.053)\tLoss(Average) 4.1347 (4.3336)\t\n",
      "Epoch: [14/1000][70/140]\tBatchTime(Average) 1.195 (1.247)\tDataTime(Average) 1.009 (1.053)\tLoss(Average) 4.4268 (4.3396)\t\n",
      "Epoch: [14/1000][80/140]\tBatchTime(Average) 1.250 (1.246)\tDataTime(Average) 1.056 (1.053)\tLoss(Average) 3.6895 (4.3166)\t\n",
      "Epoch: [14/1000][90/140]\tBatchTime(Average) 1.212 (1.246)\tDataTime(Average) 1.018 (1.053)\tLoss(Average) 3.7356 (4.3132)\t\n",
      "Epoch: [14/1000][100/140]\tBatchTime(Average) 1.229 (1.245)\tDataTime(Average) 1.036 (1.052)\tLoss(Average) 4.6024 (4.3164)\t\n",
      "Epoch: [14/1000][110/140]\tBatchTime(Average) 1.246 (1.246)\tDataTime(Average) 1.051 (1.052)\tLoss(Average) 4.6926 (4.2991)\t\n",
      "Epoch: [14/1000][120/140]\tBatchTime(Average) 1.270 (1.246)\tDataTime(Average) 1.075 (1.053)\tLoss(Average) 4.4809 (4.2786)\t\n",
      "Epoch: [14/1000][130/140]\tBatchTime(Average) 1.239 (1.246)\tDataTime(Average) 1.053 (1.052)\tLoss(Average) 4.6465 (4.2493)\t\n",
      "Epoch: [15/1000][0/140]\tBatchTime(Average) 1.019 (1.019)\tDataTime(Average) 0.842 (0.842)\tLoss(Average) 3.9971 (3.9971)\t\n",
      "Epoch: [15/1000][10/140]\tBatchTime(Average) 1.265 (1.224)\tDataTime(Average) 1.073 (1.032)\tLoss(Average) 4.3008 (4.0978)\t\n",
      "Epoch: [15/1000][20/140]\tBatchTime(Average) 1.272 (1.232)\tDataTime(Average) 1.079 (1.040)\tLoss(Average) 4.1379 (4.2464)\t\n",
      "Epoch: [15/1000][30/140]\tBatchTime(Average) 1.323 (1.246)\tDataTime(Average) 1.131 (1.054)\tLoss(Average) 4.4525 (4.2943)\t\n",
      "Epoch: [15/1000][40/140]\tBatchTime(Average) 1.217 (1.250)\tDataTime(Average) 1.027 (1.057)\tLoss(Average) 4.6469 (4.3446)\t\n",
      "Epoch: [15/1000][50/140]\tBatchTime(Average) 1.236 (1.248)\tDataTime(Average) 1.048 (1.055)\tLoss(Average) 4.5088 (4.3199)\t\n",
      "Epoch: [15/1000][60/140]\tBatchTime(Average) 1.213 (1.248)\tDataTime(Average) 1.017 (1.055)\tLoss(Average) 5.0866 (4.2901)\t\n",
      "Epoch: [15/1000][70/140]\tBatchTime(Average) 1.205 (1.251)\tDataTime(Average) 1.014 (1.057)\tLoss(Average) 3.6440 (4.2524)\t\n",
      "Epoch: [15/1000][80/140]\tBatchTime(Average) 1.274 (1.253)\tDataTime(Average) 1.083 (1.060)\tLoss(Average) 4.0751 (4.2621)\t\n",
      "Epoch: [15/1000][90/140]\tBatchTime(Average) 1.268 (1.254)\tDataTime(Average) 1.074 (1.061)\tLoss(Average) 5.0796 (4.2224)\t\n",
      "Epoch: [15/1000][100/140]\tBatchTime(Average) 1.283 (1.256)\tDataTime(Average) 1.087 (1.063)\tLoss(Average) 5.1518 (4.2253)\t\n",
      "Epoch: [15/1000][110/140]\tBatchTime(Average) 1.245 (1.257)\tDataTime(Average) 1.053 (1.064)\tLoss(Average) 3.8642 (4.2488)\t\n",
      "Epoch: [15/1000][120/140]\tBatchTime(Average) 1.286 (1.259)\tDataTime(Average) 1.089 (1.065)\tLoss(Average) 3.2706 (4.2181)\t\n",
      "Epoch: [15/1000][130/140]\tBatchTime(Average) 1.305 (1.260)\tDataTime(Average) 1.108 (1.066)\tLoss(Average) 3.7519 (4.2169)\t\n",
      "Epoch: [16/1000][0/140]\tBatchTime(Average) 1.021 (1.021)\tDataTime(Average) 0.840 (0.840)\tLoss(Average) 3.4812 (3.4812)\t\n",
      "Epoch: [16/1000][10/140]\tBatchTime(Average) 1.233 (1.246)\tDataTime(Average) 1.037 (1.052)\tLoss(Average) 3.7965 (3.9178)\t\n",
      "Epoch: [16/1000][20/140]\tBatchTime(Average) 1.237 (1.258)\tDataTime(Average) 1.050 (1.064)\tLoss(Average) 3.8095 (4.1496)\t\n",
      "Epoch: [16/1000][30/140]\tBatchTime(Average) 1.275 (1.262)\tDataTime(Average) 1.082 (1.068)\tLoss(Average) 4.2893 (4.1468)\t\n",
      "Epoch: [16/1000][40/140]\tBatchTime(Average) 1.313 (1.266)\tDataTime(Average) 1.127 (1.072)\tLoss(Average) 3.5345 (4.1526)\t\n",
      "Epoch: [16/1000][50/140]\tBatchTime(Average) 1.277 (1.269)\tDataTime(Average) 1.089 (1.075)\tLoss(Average) 4.0536 (4.1224)\t\n",
      "Epoch: [16/1000][60/140]\tBatchTime(Average) 1.269 (1.270)\tDataTime(Average) 1.076 (1.076)\tLoss(Average) 4.6702 (4.1781)\t\n",
      "Epoch: [16/1000][70/140]\tBatchTime(Average) 1.248 (1.267)\tDataTime(Average) 1.065 (1.073)\tLoss(Average) 4.1589 (4.1834)\t\n",
      "Epoch: [16/1000][80/140]\tBatchTime(Average) 1.303 (1.272)\tDataTime(Average) 1.108 (1.078)\tLoss(Average) 4.5677 (4.2259)\t\n",
      "Epoch: [16/1000][90/140]\tBatchTime(Average) 1.295 (1.274)\tDataTime(Average) 1.100 (1.080)\tLoss(Average) 4.5667 (4.1897)\t\n",
      "Epoch: [16/1000][100/140]\tBatchTime(Average) 1.304 (1.276)\tDataTime(Average) 1.102 (1.082)\tLoss(Average) 2.9958 (4.1986)\t\n",
      "Epoch: [16/1000][110/140]\tBatchTime(Average) 1.263 (1.275)\tDataTime(Average) 1.061 (1.081)\tLoss(Average) 4.8176 (4.1986)\t\n",
      "Epoch: [16/1000][120/140]\tBatchTime(Average) 1.220 (1.274)\tDataTime(Average) 1.025 (1.080)\tLoss(Average) 5.3375 (4.2222)\t\n",
      "Epoch: [16/1000][130/140]\tBatchTime(Average) 1.301 (1.272)\tDataTime(Average) 1.108 (1.078)\tLoss(Average) 4.9680 (4.2176)\t\n",
      "Epoch: [17/1000][0/140]\tBatchTime(Average) 0.995 (0.995)\tDataTime(Average) 0.817 (0.817)\tLoss(Average) 4.3318 (4.3318)\t\n",
      "Epoch: [17/1000][10/140]\tBatchTime(Average) 1.241 (1.224)\tDataTime(Average) 1.044 (1.031)\tLoss(Average) 4.5983 (4.4699)\t\n",
      "Epoch: [17/1000][20/140]\tBatchTime(Average) 1.206 (1.224)\tDataTime(Average) 1.017 (1.032)\tLoss(Average) 2.8725 (4.1444)\t\n",
      "Epoch: [17/1000][30/140]\tBatchTime(Average) 1.255 (1.229)\tDataTime(Average) 1.064 (1.037)\tLoss(Average) 2.9648 (4.0976)\t\n",
      "Epoch: [17/1000][40/140]\tBatchTime(Average) 1.230 (1.228)\tDataTime(Average) 1.041 (1.037)\tLoss(Average) 4.1342 (4.1273)\t\n",
      "Epoch: [17/1000][50/140]\tBatchTime(Average) 1.234 (1.231)\tDataTime(Average) 1.036 (1.039)\tLoss(Average) 3.6906 (4.1356)\t\n",
      "Epoch: [17/1000][60/140]\tBatchTime(Average) 1.243 (1.232)\tDataTime(Average) 1.043 (1.040)\tLoss(Average) 4.2456 (4.1190)\t\n",
      "Epoch: [17/1000][70/140]\tBatchTime(Average) 1.216 (1.232)\tDataTime(Average) 1.020 (1.040)\tLoss(Average) 3.6930 (4.1352)\t\n",
      "Epoch: [17/1000][80/140]\tBatchTime(Average) 1.232 (1.233)\tDataTime(Average) 1.042 (1.041)\tLoss(Average) 3.7711 (4.1327)\t\n",
      "Epoch: [17/1000][90/140]\tBatchTime(Average) 1.210 (1.234)\tDataTime(Average) 1.019 (1.042)\tLoss(Average) 4.3933 (4.1539)\t\n",
      "Epoch: [17/1000][100/140]\tBatchTime(Average) 1.251 (1.235)\tDataTime(Average) 1.054 (1.043)\tLoss(Average) 4.3759 (4.1701)\t\n",
      "Epoch: [17/1000][110/140]\tBatchTime(Average) 1.215 (1.237)\tDataTime(Average) 1.017 (1.045)\tLoss(Average) 4.2561 (4.1645)\t\n",
      "Epoch: [17/1000][120/140]\tBatchTime(Average) 1.241 (1.237)\tDataTime(Average) 1.051 (1.045)\tLoss(Average) 4.0489 (4.1809)\t\n",
      "Epoch: [17/1000][130/140]\tBatchTime(Average) 1.263 (1.238)\tDataTime(Average) 1.069 (1.046)\tLoss(Average) 4.3434 (4.1871)\t\n",
      "New best score! Model saved as best_model.pkl\n",
      "Epoch: [18/1000][0/140]\tBatchTime(Average) 1.057 (1.057)\tDataTime(Average) 0.878 (0.878)\tLoss(Average) 3.6165 (3.6165)\t\n",
      "Epoch: [18/1000][10/140]\tBatchTime(Average) 1.252 (1.270)\tDataTime(Average) 1.053 (1.074)\tLoss(Average) 4.7239 (4.3248)\t\n",
      "Epoch: [18/1000][20/140]\tBatchTime(Average) 1.226 (1.266)\tDataTime(Average) 1.029 (1.071)\tLoss(Average) 5.1282 (4.3312)\t\n",
      "Epoch: [18/1000][30/140]\tBatchTime(Average) 1.241 (1.260)\tDataTime(Average) 1.046 (1.066)\tLoss(Average) 3.7201 (4.2626)\t\n",
      "Epoch: [18/1000][40/140]\tBatchTime(Average) 1.232 (1.262)\tDataTime(Average) 1.038 (1.068)\tLoss(Average) 5.0193 (4.2692)\t\n",
      "Epoch: [18/1000][50/140]\tBatchTime(Average) 1.270 (1.260)\tDataTime(Average) 1.071 (1.067)\tLoss(Average) 4.8373 (4.2084)\t\n",
      "Epoch: [18/1000][60/140]\tBatchTime(Average) 1.254 (1.262)\tDataTime(Average) 1.057 (1.068)\tLoss(Average) 4.4215 (4.1952)\t\n",
      "Epoch: [18/1000][70/140]\tBatchTime(Average) 1.279 (1.263)\tDataTime(Average) 1.085 (1.069)\tLoss(Average) 4.3561 (4.1858)\t\n",
      "Epoch: [18/1000][80/140]\tBatchTime(Average) 1.235 (1.263)\tDataTime(Average) 1.040 (1.069)\tLoss(Average) 4.3962 (4.1924)\t\n",
      "Epoch: [18/1000][90/140]\tBatchTime(Average) 1.241 (1.261)\tDataTime(Average) 1.048 (1.067)\tLoss(Average) 3.0940 (4.1638)\t\n",
      "Epoch: [18/1000][100/140]\tBatchTime(Average) 1.287 (1.261)\tDataTime(Average) 1.093 (1.066)\tLoss(Average) 3.9210 (4.1540)\t\n",
      "Epoch: [18/1000][110/140]\tBatchTime(Average) 1.254 (1.260)\tDataTime(Average) 1.061 (1.066)\tLoss(Average) 4.4802 (4.1670)\t\n",
      "Epoch: [18/1000][120/140]\tBatchTime(Average) 1.255 (1.261)\tDataTime(Average) 1.064 (1.066)\tLoss(Average) 5.1170 (4.1717)\t\n",
      "Epoch: [18/1000][130/140]\tBatchTime(Average) 1.267 (1.261)\tDataTime(Average) 1.064 (1.067)\tLoss(Average) 3.9134 (4.1420)\t\n",
      "Epoch: [19/1000][0/140]\tBatchTime(Average) 1.007 (1.007)\tDataTime(Average) 0.828 (0.828)\tLoss(Average) 3.6509 (3.6509)\t\n",
      "Epoch: [19/1000][10/140]\tBatchTime(Average) 1.268 (1.244)\tDataTime(Average) 1.073 (1.050)\tLoss(Average) 4.6546 (4.3388)\t\n",
      "Epoch: [19/1000][20/140]\tBatchTime(Average) 1.246 (1.250)\tDataTime(Average) 1.058 (1.057)\tLoss(Average) 3.3693 (4.3430)\t\n",
      "Epoch: [19/1000][30/140]\tBatchTime(Average) 1.239 (1.251)\tDataTime(Average) 1.046 (1.057)\tLoss(Average) 3.3028 (4.1282)\t\n",
      "Epoch: [19/1000][40/140]\tBatchTime(Average) 1.263 (1.254)\tDataTime(Average) 1.072 (1.060)\tLoss(Average) 4.0067 (4.1165)\t\n",
      "Epoch: [19/1000][50/140]\tBatchTime(Average) 1.284 (1.255)\tDataTime(Average) 1.089 (1.062)\tLoss(Average) 3.7468 (4.1466)\t\n",
      "Epoch: [19/1000][60/140]\tBatchTime(Average) 1.264 (1.255)\tDataTime(Average) 1.070 (1.061)\tLoss(Average) 3.9980 (4.1527)\t\n",
      "Epoch: [19/1000][70/140]\tBatchTime(Average) 1.277 (1.255)\tDataTime(Average) 1.078 (1.061)\tLoss(Average) 4.0986 (4.1265)\t\n",
      "Epoch: [19/1000][80/140]\tBatchTime(Average) 1.278 (1.256)\tDataTime(Average) 1.087 (1.062)\tLoss(Average) 2.7910 (4.1244)\t\n",
      "Epoch: [19/1000][90/140]\tBatchTime(Average) 1.308 (1.263)\tDataTime(Average) 1.104 (1.069)\tLoss(Average) 4.3410 (4.1197)\t\n",
      "Epoch: [19/1000][100/140]\tBatchTime(Average) 1.282 (1.269)\tDataTime(Average) 1.092 (1.075)\tLoss(Average) 3.3376 (4.1133)\t\n",
      "Epoch: [19/1000][110/140]\tBatchTime(Average) 1.322 (1.274)\tDataTime(Average) 1.121 (1.080)\tLoss(Average) 4.9159 (4.1241)\t\n",
      "Epoch: [19/1000][120/140]\tBatchTime(Average) 1.287 (1.276)\tDataTime(Average) 1.092 (1.082)\tLoss(Average) 4.5025 (4.1257)\t\n",
      "Epoch: [19/1000][130/140]\tBatchTime(Average) 1.257 (1.276)\tDataTime(Average) 1.057 (1.081)\tLoss(Average) 3.7684 (4.1418)\t\n",
      "Epoch: [20/1000][0/140]\tBatchTime(Average) 0.998 (0.998)\tDataTime(Average) 0.820 (0.820)\tLoss(Average) 5.3975 (5.3975)\t\n",
      "Epoch: [20/1000][10/140]\tBatchTime(Average) 1.255 (1.252)\tDataTime(Average) 1.059 (1.057)\tLoss(Average) 3.7932 (4.3138)\t\n",
      "Epoch: [20/1000][20/140]\tBatchTime(Average) 1.322 (1.271)\tDataTime(Average) 1.131 (1.077)\tLoss(Average) 4.1269 (4.3346)\t\n",
      "Epoch: [20/1000][30/140]\tBatchTime(Average) 1.310 (1.276)\tDataTime(Average) 1.118 (1.082)\tLoss(Average) 4.9846 (4.3196)\t\n",
      "Epoch: [20/1000][40/140]\tBatchTime(Average) 1.302 (1.281)\tDataTime(Average) 1.107 (1.087)\tLoss(Average) 4.1894 (4.2070)\t\n",
      "Epoch: [20/1000][50/140]\tBatchTime(Average) 1.250 (1.277)\tDataTime(Average) 1.059 (1.083)\tLoss(Average) 4.5913 (4.1782)\t\n",
      "Epoch: [20/1000][60/140]\tBatchTime(Average) 1.267 (1.274)\tDataTime(Average) 1.071 (1.080)\tLoss(Average) 4.8643 (4.1770)\t\n",
      "Epoch: [20/1000][70/140]\tBatchTime(Average) 1.262 (1.275)\tDataTime(Average) 1.070 (1.081)\tLoss(Average) 3.6668 (4.1411)\t\n",
      "Epoch: [20/1000][80/140]\tBatchTime(Average) 1.227 (1.273)\tDataTime(Average) 1.042 (1.079)\tLoss(Average) 3.7985 (4.1282)\t\n",
      "Epoch: [20/1000][90/140]\tBatchTime(Average) 1.234 (1.272)\tDataTime(Average) 1.043 (1.078)\tLoss(Average) 4.6584 (4.1356)\t\n",
      "Epoch: [20/1000][100/140]\tBatchTime(Average) 1.308 (1.273)\tDataTime(Average) 1.101 (1.079)\tLoss(Average) 3.6078 (4.1279)\t\n",
      "Epoch: [20/1000][110/140]\tBatchTime(Average) 1.263 (1.274)\tDataTime(Average) 1.070 (1.080)\tLoss(Average) 3.5557 (4.1408)\t\n",
      "Epoch: [20/1000][120/140]\tBatchTime(Average) 1.288 (1.274)\tDataTime(Average) 1.090 (1.080)\tLoss(Average) 4.3522 (4.1342)\t\n",
      "Epoch: [20/1000][130/140]\tBatchTime(Average) 1.305 (1.275)\tDataTime(Average) 1.111 (1.080)\tLoss(Average) 4.0100 (4.1156)\t\n",
      "Epoch: [21/1000][0/140]\tBatchTime(Average) 1.054 (1.054)\tDataTime(Average) 0.873 (0.873)\tLoss(Average) 5.5533 (5.5533)\t\n",
      "Epoch: [21/1000][10/140]\tBatchTime(Average) 1.248 (1.249)\tDataTime(Average) 1.051 (1.053)\tLoss(Average) 5.2125 (4.3258)\t\n",
      "Epoch: [21/1000][20/140]\tBatchTime(Average) 1.261 (1.246)\tDataTime(Average) 1.066 (1.052)\tLoss(Average) 3.7523 (4.2530)\t\n",
      "Epoch: [21/1000][30/140]\tBatchTime(Average) 1.215 (1.247)\tDataTime(Average) 1.026 (1.053)\tLoss(Average) 3.8551 (4.1242)\t\n",
      "Epoch: [21/1000][40/140]\tBatchTime(Average) 1.202 (1.246)\tDataTime(Average) 1.012 (1.052)\tLoss(Average) 4.4095 (4.1690)\t\n",
      "Epoch: [21/1000][50/140]\tBatchTime(Average) 1.231 (1.245)\tDataTime(Average) 1.038 (1.051)\tLoss(Average) 3.5837 (4.0638)\t\n",
      "Epoch: [21/1000][60/140]\tBatchTime(Average) 1.211 (1.244)\tDataTime(Average) 1.021 (1.051)\tLoss(Average) 3.9779 (4.1259)\t\n",
      "Epoch: [21/1000][70/140]\tBatchTime(Average) 1.224 (1.244)\tDataTime(Average) 1.033 (1.051)\tLoss(Average) 3.6701 (4.0727)\t\n",
      "Epoch: [21/1000][80/140]\tBatchTime(Average) 1.252 (1.244)\tDataTime(Average) 1.054 (1.051)\tLoss(Average) 3.8304 (4.0979)\t\n",
      "Epoch: [21/1000][90/140]\tBatchTime(Average) 1.218 (1.244)\tDataTime(Average) 1.022 (1.050)\tLoss(Average) 4.5343 (4.0891)\t\n",
      "Epoch: [21/1000][100/140]\tBatchTime(Average) 1.229 (1.245)\tDataTime(Average) 1.044 (1.051)\tLoss(Average) 3.8860 (4.0913)\t\n",
      "Epoch: [21/1000][110/140]\tBatchTime(Average) 1.259 (1.245)\tDataTime(Average) 1.062 (1.052)\tLoss(Average) 4.6399 (4.1054)\t\n",
      "Epoch: [21/1000][120/140]\tBatchTime(Average) 1.198 (1.245)\tDataTime(Average) 1.010 (1.052)\tLoss(Average) 3.7998 (4.1065)\t\n",
      "Epoch: [21/1000][130/140]\tBatchTime(Average) 1.255 (1.246)\tDataTime(Average) 1.058 (1.053)\tLoss(Average) 4.6620 (4.1284)\t\n",
      "Epoch: [22/1000][0/140]\tBatchTime(Average) 0.983 (0.983)\tDataTime(Average) 0.805 (0.805)\tLoss(Average) 3.8087 (3.8087)\t\n",
      "Epoch: [22/1000][10/140]\tBatchTime(Average) 1.211 (1.228)\tDataTime(Average) 1.024 (1.036)\tLoss(Average) 3.1342 (4.0133)\t\n",
      "Epoch: [22/1000][20/140]\tBatchTime(Average) 1.214 (1.235)\tDataTime(Average) 1.023 (1.042)\tLoss(Average) 4.6918 (3.9800)\t\n",
      "Epoch: [22/1000][30/140]\tBatchTime(Average) 1.245 (1.238)\tDataTime(Average) 1.050 (1.044)\tLoss(Average) 3.7226 (4.0688)\t\n",
      "Epoch: [22/1000][40/140]\tBatchTime(Average) 1.245 (1.240)\tDataTime(Average) 1.051 (1.047)\tLoss(Average) 4.6296 (4.0707)\t\n",
      "Epoch: [22/1000][50/140]\tBatchTime(Average) 1.219 (1.241)\tDataTime(Average) 1.024 (1.048)\tLoss(Average) 4.3817 (4.0946)\t\n",
      "Epoch: [22/1000][60/140]\tBatchTime(Average) 1.219 (1.242)\tDataTime(Average) 1.026 (1.049)\tLoss(Average) 5.0778 (4.0836)\t\n",
      "Epoch: [22/1000][70/140]\tBatchTime(Average) 1.253 (1.243)\tDataTime(Average) 1.063 (1.050)\tLoss(Average) 5.3782 (4.0847)\t\n",
      "Epoch: [22/1000][80/140]\tBatchTime(Average) 1.268 (1.242)\tDataTime(Average) 1.073 (1.049)\tLoss(Average) 3.8818 (4.0880)\t\n",
      "Epoch: [22/1000][90/140]\tBatchTime(Average) 1.220 (1.243)\tDataTime(Average) 1.028 (1.050)\tLoss(Average) 3.3625 (4.1086)\t\n",
      "Epoch: [22/1000][100/140]\tBatchTime(Average) 1.220 (1.243)\tDataTime(Average) 1.030 (1.050)\tLoss(Average) 3.4778 (4.0830)\t\n",
      "Epoch: [22/1000][110/140]\tBatchTime(Average) 1.269 (1.243)\tDataTime(Average) 1.069 (1.050)\tLoss(Average) 4.0184 (4.0834)\t\n",
      "Epoch: [22/1000][120/140]\tBatchTime(Average) 1.211 (1.243)\tDataTime(Average) 1.022 (1.050)\tLoss(Average) 3.4269 (4.0738)\t\n",
      "Epoch: [22/1000][130/140]\tBatchTime(Average) 1.262 (1.242)\tDataTime(Average) 1.075 (1.049)\tLoss(Average) 3.5601 (4.0677)\t\n",
      "Epoch: [23/1000][0/140]\tBatchTime(Average) 1.014 (1.014)\tDataTime(Average) 0.837 (0.837)\tLoss(Average) 3.3122 (3.3122)\t\n",
      "Epoch: [23/1000][10/140]\tBatchTime(Average) 1.227 (1.226)\tDataTime(Average) 1.031 (1.032)\tLoss(Average) 3.3182 (3.6723)\t\n",
      "Epoch: [23/1000][20/140]\tBatchTime(Average) 1.265 (1.241)\tDataTime(Average) 1.066 (1.046)\tLoss(Average) 3.3261 (3.9914)\t\n",
      "Epoch: [23/1000][30/140]\tBatchTime(Average) 1.250 (1.246)\tDataTime(Average) 1.056 (1.052)\tLoss(Average) 3.0217 (3.9278)\t\n",
      "Epoch: [23/1000][40/140]\tBatchTime(Average) 1.279 (1.244)\tDataTime(Average) 1.086 (1.051)\tLoss(Average) 4.6844 (3.9692)\t\n",
      "Epoch: [23/1000][50/140]\tBatchTime(Average) 1.238 (1.248)\tDataTime(Average) 1.052 (1.054)\tLoss(Average) 3.8697 (3.9629)\t\n",
      "Epoch: [23/1000][60/140]\tBatchTime(Average) 1.232 (1.246)\tDataTime(Average) 1.041 (1.052)\tLoss(Average) 4.3791 (3.9467)\t\n",
      "Epoch: [23/1000][70/140]\tBatchTime(Average) 1.254 (1.245)\tDataTime(Average) 1.063 (1.051)\tLoss(Average) 3.6032 (3.9942)\t\n",
      "Epoch: [23/1000][80/140]\tBatchTime(Average) 1.268 (1.246)\tDataTime(Average) 1.080 (1.053)\tLoss(Average) 3.1388 (4.0066)\t\n",
      "Epoch: [23/1000][90/140]\tBatchTime(Average) 1.247 (1.247)\tDataTime(Average) 1.053 (1.054)\tLoss(Average) 4.4191 (4.0180)\t\n",
      "Epoch: [23/1000][100/140]\tBatchTime(Average) 1.255 (1.247)\tDataTime(Average) 1.067 (1.053)\tLoss(Average) 4.1417 (4.0233)\t\n",
      "Epoch: [23/1000][110/140]\tBatchTime(Average) 1.240 (1.246)\tDataTime(Average) 1.053 (1.052)\tLoss(Average) 4.1337 (4.0200)\t\n",
      "Epoch: [23/1000][120/140]\tBatchTime(Average) 1.285 (1.246)\tDataTime(Average) 1.086 (1.053)\tLoss(Average) 4.7854 (4.0773)\t\n",
      "Epoch: [23/1000][130/140]\tBatchTime(Average) 1.262 (1.247)\tDataTime(Average) 1.063 (1.054)\tLoss(Average) 4.1683 (4.0825)\t\n",
      "New best score! Model saved as best_model.pkl\n",
      "Epoch: [24/1000][0/140]\tBatchTime(Average) 1.088 (1.088)\tDataTime(Average) 0.910 (0.910)\tLoss(Average) 3.6449 (3.6449)\t\n",
      "Epoch: [24/1000][10/140]\tBatchTime(Average) 1.301 (1.292)\tDataTime(Average) 1.102 (1.098)\tLoss(Average) 4.1501 (4.0551)\t\n",
      "Epoch: [24/1000][20/140]\tBatchTime(Average) 1.316 (1.296)\tDataTime(Average) 1.126 (1.101)\tLoss(Average) 3.5286 (3.8818)\t\n",
      "Epoch: [24/1000][30/140]\tBatchTime(Average) 1.260 (1.286)\tDataTime(Average) 1.062 (1.092)\tLoss(Average) 4.5517 (3.7931)\t\n",
      "Epoch: [24/1000][40/140]\tBatchTime(Average) 1.264 (1.282)\tDataTime(Average) 1.072 (1.088)\tLoss(Average) 3.5989 (3.9161)\t\n",
      "Epoch: [24/1000][50/140]\tBatchTime(Average) 1.249 (1.282)\tDataTime(Average) 1.055 (1.087)\tLoss(Average) 4.0270 (3.9561)\t\n",
      "Epoch: [24/1000][60/140]\tBatchTime(Average) 1.308 (1.284)\tDataTime(Average) 1.114 (1.090)\tLoss(Average) 3.3879 (4.0040)\t\n",
      "Epoch: [24/1000][70/140]\tBatchTime(Average) 1.289 (1.284)\tDataTime(Average) 1.091 (1.089)\tLoss(Average) 5.4410 (4.0165)\t\n",
      "Epoch: [24/1000][80/140]\tBatchTime(Average) 1.249 (1.285)\tDataTime(Average) 1.059 (1.090)\tLoss(Average) 4.3665 (4.0255)\t\n",
      "Epoch: [24/1000][90/140]\tBatchTime(Average) 1.263 (1.286)\tDataTime(Average) 1.058 (1.091)\tLoss(Average) 4.3753 (4.0596)\t\n",
      "Epoch: [24/1000][100/140]\tBatchTime(Average) 1.264 (1.286)\tDataTime(Average) 1.071 (1.091)\tLoss(Average) 4.4434 (4.0728)\t\n",
      "Epoch: [24/1000][110/140]\tBatchTime(Average) 1.257 (1.286)\tDataTime(Average) 1.066 (1.091)\tLoss(Average) 3.5122 (4.0845)\t\n",
      "Epoch: [24/1000][120/140]\tBatchTime(Average) 1.290 (1.285)\tDataTime(Average) 1.096 (1.090)\tLoss(Average) 3.2643 (4.0551)\t\n",
      "Epoch: [24/1000][130/140]\tBatchTime(Average) 1.254 (1.284)\tDataTime(Average) 1.057 (1.089)\tLoss(Average) 4.6150 (4.0876)\t\n",
      "Epoch: [25/1000][0/140]\tBatchTime(Average) 1.034 (1.034)\tDataTime(Average) 0.854 (0.854)\tLoss(Average) 3.8123 (3.8123)\t\n",
      "Epoch: [25/1000][10/140]\tBatchTime(Average) 1.249 (1.267)\tDataTime(Average) 1.051 (1.072)\tLoss(Average) 3.4305 (4.0093)\t\n",
      "Epoch: [25/1000][20/140]\tBatchTime(Average) 1.329 (1.275)\tDataTime(Average) 1.127 (1.079)\tLoss(Average) 4.0047 (3.9918)\t\n",
      "Epoch: [25/1000][30/140]\tBatchTime(Average) 1.262 (1.287)\tDataTime(Average) 1.067 (1.092)\tLoss(Average) 4.6743 (3.9822)\t\n",
      "Epoch: [25/1000][40/140]\tBatchTime(Average) 1.271 (1.287)\tDataTime(Average) 1.077 (1.091)\tLoss(Average) 3.1395 (3.9751)\t\n",
      "Epoch: [25/1000][50/140]\tBatchTime(Average) 1.283 (1.285)\tDataTime(Average) 1.092 (1.089)\tLoss(Average) 4.0853 (3.9879)\t\n",
      "Epoch: [25/1000][60/140]\tBatchTime(Average) 1.346 (1.290)\tDataTime(Average) 1.148 (1.094)\tLoss(Average) 4.2809 (4.0214)\t\n",
      "Epoch: [25/1000][70/140]\tBatchTime(Average) 1.260 (1.291)\tDataTime(Average) 1.063 (1.095)\tLoss(Average) 3.2302 (4.0356)\t\n",
      "Epoch: [25/1000][80/140]\tBatchTime(Average) 1.252 (1.288)\tDataTime(Average) 1.065 (1.092)\tLoss(Average) 4.3224 (4.0151)\t\n",
      "Epoch: [25/1000][90/140]\tBatchTime(Average) 1.267 (1.286)\tDataTime(Average) 1.073 (1.090)\tLoss(Average) 4.5021 (4.0253)\t\n",
      "Epoch: [25/1000][100/140]\tBatchTime(Average) 1.256 (1.284)\tDataTime(Average) 1.061 (1.088)\tLoss(Average) 3.5487 (4.0052)\t\n",
      "Epoch: [25/1000][110/140]\tBatchTime(Average) 1.230 (1.281)\tDataTime(Average) 1.035 (1.085)\tLoss(Average) 3.1448 (4.0346)\t\n",
      "Epoch: [25/1000][120/140]\tBatchTime(Average) 1.232 (1.277)\tDataTime(Average) 1.039 (1.082)\tLoss(Average) 4.9787 (4.0518)\t\n",
      "Epoch: [25/1000][130/140]\tBatchTime(Average) 1.224 (1.275)\tDataTime(Average) 1.032 (1.080)\tLoss(Average) 3.2396 (4.0753)\t\n",
      "New best score! Model saved as best_model.pkl\n",
      "Epoch: [26/1000][0/140]\tBatchTime(Average) 1.081 (1.081)\tDataTime(Average) 0.903 (0.903)\tLoss(Average) 5.0808 (5.0808)\t\n",
      "Epoch: [26/1000][10/140]\tBatchTime(Average) 1.244 (1.259)\tDataTime(Average) 1.049 (1.066)\tLoss(Average) 3.8443 (4.0267)\t\n",
      "Epoch: [26/1000][20/140]\tBatchTime(Average) 1.265 (1.254)\tDataTime(Average) 1.074 (1.061)\tLoss(Average) 4.6345 (4.0499)\t\n",
      "Epoch: [26/1000][30/140]\tBatchTime(Average) 1.271 (1.251)\tDataTime(Average) 1.081 (1.058)\tLoss(Average) 3.7229 (4.0276)\t\n",
      "Epoch: [26/1000][40/140]\tBatchTime(Average) 1.267 (1.249)\tDataTime(Average) 1.070 (1.055)\tLoss(Average) 4.2455 (3.9881)\t\n",
      "Epoch: [26/1000][50/140]\tBatchTime(Average) 1.283 (1.248)\tDataTime(Average) 1.088 (1.055)\tLoss(Average) 3.9071 (4.0095)\t\n",
      "Epoch: [26/1000][60/140]\tBatchTime(Average) 1.258 (1.247)\tDataTime(Average) 1.060 (1.054)\tLoss(Average) 3.9383 (4.0835)\t\n",
      "Epoch: [26/1000][70/140]\tBatchTime(Average) 1.236 (1.246)\tDataTime(Average) 1.042 (1.053)\tLoss(Average) 3.9833 (4.0599)\t\n",
      "Epoch: [26/1000][80/140]\tBatchTime(Average) 1.229 (1.245)\tDataTime(Average) 1.036 (1.052)\tLoss(Average) 4.0980 (4.0613)\t\n",
      "Epoch: [26/1000][90/140]\tBatchTime(Average) 1.231 (1.244)\tDataTime(Average) 1.038 (1.052)\tLoss(Average) 3.4328 (4.0362)\t\n",
      "Epoch: [26/1000][100/140]\tBatchTime(Average) 1.265 (1.245)\tDataTime(Average) 1.070 (1.052)\tLoss(Average) 4.0868 (4.0433)\t\n",
      "Epoch: [26/1000][110/140]\tBatchTime(Average) 1.252 (1.244)\tDataTime(Average) 1.057 (1.051)\tLoss(Average) 2.9601 (4.0303)\t\n",
      "Epoch: [26/1000][120/140]\tBatchTime(Average) 1.260 (1.244)\tDataTime(Average) 1.067 (1.051)\tLoss(Average) 4.2351 (4.0380)\t\n",
      "Epoch: [26/1000][130/140]\tBatchTime(Average) 1.238 (1.244)\tDataTime(Average) 1.048 (1.051)\tLoss(Average) 3.7689 (4.0566)\t\n",
      "Epoch: [27/1000][0/140]\tBatchTime(Average) 1.015 (1.015)\tDataTime(Average) 0.836 (0.836)\tLoss(Average) 3.9161 (3.9161)\t\n",
      "Epoch: [27/1000][10/140]\tBatchTime(Average) 1.290 (1.242)\tDataTime(Average) 1.098 (1.048)\tLoss(Average) 4.6132 (3.9925)\t\n",
      "Epoch: [27/1000][20/140]\tBatchTime(Average) 1.238 (1.250)\tDataTime(Average) 1.044 (1.056)\tLoss(Average) 4.3019 (3.9774)\t\n",
      "Epoch: [27/1000][30/140]\tBatchTime(Average) 1.249 (1.256)\tDataTime(Average) 1.058 (1.062)\tLoss(Average) 3.4880 (3.8729)\t\n",
      "Epoch: [27/1000][40/140]\tBatchTime(Average) 1.237 (1.255)\tDataTime(Average) 1.045 (1.062)\tLoss(Average) 5.4082 (3.8993)\t\n",
      "Epoch: [27/1000][50/140]\tBatchTime(Average) 1.288 (1.255)\tDataTime(Average) 1.093 (1.061)\tLoss(Average) 4.5806 (3.8982)\t\n",
      "Epoch: [27/1000][60/140]\tBatchTime(Average) 1.216 (1.253)\tDataTime(Average) 1.022 (1.060)\tLoss(Average) 4.0022 (3.9335)\t\n",
      "Epoch: [27/1000][70/140]\tBatchTime(Average) 1.236 (1.251)\tDataTime(Average) 1.052 (1.058)\tLoss(Average) 4.3509 (3.9753)\t\n",
      "Epoch: [27/1000][80/140]\tBatchTime(Average) 1.234 (1.250)\tDataTime(Average) 1.043 (1.057)\tLoss(Average) 3.0610 (3.9633)\t\n",
      "Epoch: [27/1000][90/140]\tBatchTime(Average) 1.198 (1.249)\tDataTime(Average) 1.005 (1.056)\tLoss(Average) 3.2471 (3.9922)\t\n",
      "Epoch: [27/1000][100/140]\tBatchTime(Average) 1.273 (1.247)\tDataTime(Average) 1.081 (1.054)\tLoss(Average) 3.4111 (3.9710)\t\n",
      "Epoch: [27/1000][110/140]\tBatchTime(Average) 1.246 (1.247)\tDataTime(Average) 1.054 (1.053)\tLoss(Average) 5.3854 (3.9708)\t\n",
      "Epoch: [27/1000][120/140]\tBatchTime(Average) 1.231 (1.247)\tDataTime(Average) 1.036 (1.054)\tLoss(Average) 3.9227 (3.9825)\t\n",
      "Epoch: [27/1000][130/140]\tBatchTime(Average) 1.254 (1.245)\tDataTime(Average) 1.056 (1.052)\tLoss(Average) 3.1296 (4.0170)\t\n",
      "Epoch: [28/1000][0/140]\tBatchTime(Average) 1.026 (1.026)\tDataTime(Average) 0.848 (0.848)\tLoss(Average) 3.6257 (3.6257)\t\n",
      "Epoch: [28/1000][10/140]\tBatchTime(Average) 1.267 (1.236)\tDataTime(Average) 1.073 (1.043)\tLoss(Average) 3.8648 (4.1448)\t\n",
      "Epoch: [28/1000][20/140]\tBatchTime(Average) 1.249 (1.242)\tDataTime(Average) 1.055 (1.049)\tLoss(Average) 3.8301 (4.0164)\t\n",
      "Epoch: [28/1000][30/140]\tBatchTime(Average) 1.237 (1.241)\tDataTime(Average) 1.040 (1.048)\tLoss(Average) 4.6026 (4.1606)\t\n",
      "Epoch: [28/1000][40/140]\tBatchTime(Average) 1.259 (1.240)\tDataTime(Average) 1.067 (1.047)\tLoss(Average) 3.5676 (4.1089)\t\n",
      "Epoch: [28/1000][50/140]\tBatchTime(Average) 1.265 (1.240)\tDataTime(Average) 1.070 (1.047)\tLoss(Average) 3.4203 (4.1152)\t\n",
      "Epoch: [28/1000][60/140]\tBatchTime(Average) 1.233 (1.239)\tDataTime(Average) 1.042 (1.046)\tLoss(Average) 4.7515 (4.1100)\t\n",
      "Epoch: [28/1000][70/140]\tBatchTime(Average) 1.261 (1.241)\tDataTime(Average) 1.070 (1.048)\tLoss(Average) 2.8768 (4.0975)\t\n",
      "Epoch: [28/1000][80/140]\tBatchTime(Average) 1.256 (1.243)\tDataTime(Average) 1.065 (1.050)\tLoss(Average) 3.4185 (4.1002)\t\n",
      "Epoch: [28/1000][90/140]\tBatchTime(Average) 1.254 (1.242)\tDataTime(Average) 1.056 (1.050)\tLoss(Average) 4.1592 (4.0496)\t\n",
      "Epoch: [28/1000][100/140]\tBatchTime(Average) 1.208 (1.242)\tDataTime(Average) 1.015 (1.049)\tLoss(Average) 4.2315 (4.0389)\t\n",
      "Epoch: [28/1000][110/140]\tBatchTime(Average) 1.241 (1.242)\tDataTime(Average) 1.050 (1.049)\tLoss(Average) 3.8898 (4.0211)\t\n",
      "Epoch: [28/1000][120/140]\tBatchTime(Average) 1.239 (1.242)\tDataTime(Average) 1.046 (1.049)\tLoss(Average) 4.5253 (4.0319)\t\n",
      "Epoch: [28/1000][130/140]\tBatchTime(Average) 1.279 (1.243)\tDataTime(Average) 1.089 (1.049)\tLoss(Average) 4.1627 (4.0267)\t\n",
      "Epoch: [29/1000][0/140]\tBatchTime(Average) 0.995 (0.995)\tDataTime(Average) 0.814 (0.814)\tLoss(Average) 4.0716 (4.0716)\t\n",
      "Epoch: [29/1000][10/140]\tBatchTime(Average) 1.252 (1.229)\tDataTime(Average) 1.057 (1.036)\tLoss(Average) 3.9540 (4.0991)\t\n",
      "Epoch: [29/1000][20/140]\tBatchTime(Average) 1.238 (1.239)\tDataTime(Average) 1.040 (1.046)\tLoss(Average) 3.3658 (4.2204)\t\n",
      "Epoch: [29/1000][30/140]\tBatchTime(Average) 1.277 (1.247)\tDataTime(Average) 1.083 (1.053)\tLoss(Average) 4.1582 (4.2042)\t\n",
      "Epoch: [29/1000][40/140]\tBatchTime(Average) 1.278 (1.249)\tDataTime(Average) 1.082 (1.055)\tLoss(Average) 4.2969 (4.1504)\t\n",
      "Epoch: [29/1000][50/140]\tBatchTime(Average) 1.262 (1.249)\tDataTime(Average) 1.073 (1.055)\tLoss(Average) 4.4845 (4.0742)\t\n",
      "Epoch: [29/1000][60/140]\tBatchTime(Average) 1.290 (1.250)\tDataTime(Average) 1.098 (1.057)\tLoss(Average) 3.0561 (4.0098)\t\n",
      "Epoch: [29/1000][70/140]\tBatchTime(Average) 1.297 (1.252)\tDataTime(Average) 1.100 (1.058)\tLoss(Average) 3.8660 (4.0147)\t\n",
      "Epoch: [29/1000][80/140]\tBatchTime(Average) 1.279 (1.253)\tDataTime(Average) 1.082 (1.060)\tLoss(Average) 3.5885 (3.9725)\t\n",
      "Epoch: [29/1000][90/140]\tBatchTime(Average) 1.244 (1.258)\tDataTime(Average) 1.050 (1.064)\tLoss(Average) 4.4384 (3.9658)\t\n",
      "Epoch: [29/1000][100/140]\tBatchTime(Average) 1.325 (1.263)\tDataTime(Average) 1.136 (1.068)\tLoss(Average) 3.7734 (3.9877)\t\n",
      "Epoch: [29/1000][110/140]\tBatchTime(Average) 1.273 (1.267)\tDataTime(Average) 1.082 (1.073)\tLoss(Average) 4.2977 (4.0398)\t\n",
      "Epoch: [29/1000][120/140]\tBatchTime(Average) 1.328 (1.271)\tDataTime(Average) 1.130 (1.076)\tLoss(Average) 3.8507 (4.0254)\t\n",
      "Epoch: [29/1000][130/140]\tBatchTime(Average) 1.283 (1.272)\tDataTime(Average) 1.093 (1.078)\tLoss(Average) 4.4375 (4.0250)\t\n",
      "Epoch: [30/1000][0/140]\tBatchTime(Average) 1.019 (1.019)\tDataTime(Average) 0.841 (0.841)\tLoss(Average) 4.2496 (4.2496)\t\n",
      "Epoch: [30/1000][10/140]\tBatchTime(Average) 1.333 (1.272)\tDataTime(Average) 1.135 (1.077)\tLoss(Average) 3.1276 (3.8378)\t\n",
      "Epoch: [30/1000][20/140]\tBatchTime(Average) 1.326 (1.283)\tDataTime(Average) 1.121 (1.088)\tLoss(Average) 4.3848 (4.0870)\t\n",
      "Epoch: [30/1000][30/140]\tBatchTime(Average) 1.260 (1.285)\tDataTime(Average) 1.052 (1.089)\tLoss(Average) 3.2259 (3.9550)\t\n",
      "Epoch: [30/1000][40/140]\tBatchTime(Average) 1.289 (1.287)\tDataTime(Average) 1.094 (1.092)\tLoss(Average) 4.1097 (3.9972)\t\n",
      "Epoch: [30/1000][50/140]\tBatchTime(Average) 1.366 (1.287)\tDataTime(Average) 1.169 (1.092)\tLoss(Average) 3.7299 (3.9605)\t\n",
      "Epoch: [30/1000][60/140]\tBatchTime(Average) 1.229 (1.285)\tDataTime(Average) 1.032 (1.089)\tLoss(Average) 3.4323 (3.9580)\t\n",
      "Epoch: [30/1000][70/140]\tBatchTime(Average) 1.327 (1.285)\tDataTime(Average) 1.141 (1.090)\tLoss(Average) 5.4052 (3.9992)\t\n",
      "Epoch: [30/1000][80/140]\tBatchTime(Average) 1.329 (1.286)\tDataTime(Average) 1.132 (1.091)\tLoss(Average) 4.6571 (3.9876)\t\n",
      "Epoch: [30/1000][90/140]\tBatchTime(Average) 1.234 (1.282)\tDataTime(Average) 1.039 (1.087)\tLoss(Average) 3.4348 (3.9735)\t\n",
      "Epoch: [30/1000][100/140]\tBatchTime(Average) 1.314 (1.283)\tDataTime(Average) 1.117 (1.087)\tLoss(Average) 3.6537 (3.9583)\t\n",
      "Epoch: [30/1000][110/140]\tBatchTime(Average) 1.262 (1.281)\tDataTime(Average) 1.073 (1.086)\tLoss(Average) 3.2944 (3.9558)\t\n",
      "Epoch: [30/1000][120/140]\tBatchTime(Average) 1.266 (1.279)\tDataTime(Average) 1.077 (1.084)\tLoss(Average) 3.6763 (3.9957)\t\n",
      "Epoch: [30/1000][130/140]\tBatchTime(Average) 1.310 (1.277)\tDataTime(Average) 1.121 (1.082)\tLoss(Average) 4.5581 (3.9986)\t\n",
      "Epoch: [31/1000][0/140]\tBatchTime(Average) 1.042 (1.042)\tDataTime(Average) 0.864 (0.864)\tLoss(Average) 3.7710 (3.7710)\t\n",
      "Epoch: [31/1000][10/140]\tBatchTime(Average) 1.309 (1.258)\tDataTime(Average) 1.119 (1.065)\tLoss(Average) 4.5437 (4.0332)\t\n",
      "Epoch: [31/1000][20/140]\tBatchTime(Average) 1.294 (1.259)\tDataTime(Average) 1.103 (1.066)\tLoss(Average) 4.4426 (3.9053)\t\n",
      "Epoch: [31/1000][30/140]\tBatchTime(Average) 1.276 (1.258)\tDataTime(Average) 1.081 (1.065)\tLoss(Average) 4.4209 (3.9406)\t\n",
      "Epoch: [31/1000][40/140]\tBatchTime(Average) 1.298 (1.263)\tDataTime(Average) 1.103 (1.069)\tLoss(Average) 3.8747 (3.9746)\t\n",
      "Epoch: [31/1000][50/140]\tBatchTime(Average) 1.283 (1.263)\tDataTime(Average) 1.092 (1.070)\tLoss(Average) 3.5356 (3.9102)\t\n",
      "Epoch: [31/1000][60/140]\tBatchTime(Average) 1.294 (1.269)\tDataTime(Average) 1.103 (1.076)\tLoss(Average) 3.4194 (3.8779)\t\n",
      "Epoch: [31/1000][70/140]\tBatchTime(Average) 1.275 (1.269)\tDataTime(Average) 1.083 (1.076)\tLoss(Average) 4.2745 (3.9498)\t\n",
      "Epoch: [31/1000][80/140]\tBatchTime(Average) 1.255 (1.265)\tDataTime(Average) 1.062 (1.071)\tLoss(Average) 5.0559 (3.9508)\t\n",
      "Epoch: [31/1000][90/140]\tBatchTime(Average) 1.260 (1.262)\tDataTime(Average) 1.066 (1.069)\tLoss(Average) 2.9203 (3.9504)\t\n",
      "Epoch: [31/1000][100/140]\tBatchTime(Average) 1.255 (1.260)\tDataTime(Average) 1.061 (1.067)\tLoss(Average) 3.6368 (3.9451)\t\n",
      "Epoch: [31/1000][110/140]\tBatchTime(Average) 1.237 (1.258)\tDataTime(Average) 1.042 (1.065)\tLoss(Average) 2.9674 (3.9484)\t\n",
      "Epoch: [31/1000][120/140]\tBatchTime(Average) 1.247 (1.256)\tDataTime(Average) 1.054 (1.063)\tLoss(Average) 4.9385 (3.9908)\t\n",
      "Epoch: [31/1000][130/140]\tBatchTime(Average) 1.219 (1.253)\tDataTime(Average) 1.029 (1.061)\tLoss(Average) 4.2661 (3.9735)\t\n",
      "Epoch: [32/1000][0/140]\tBatchTime(Average) 0.989 (0.989)\tDataTime(Average) 0.812 (0.812)\tLoss(Average) 4.5768 (4.5768)\t\n",
      "Epoch: [32/1000][10/140]\tBatchTime(Average) 1.264 (1.216)\tDataTime(Average) 1.071 (1.025)\tLoss(Average) 4.0001 (3.9289)\t\n",
      "Epoch: [32/1000][20/140]\tBatchTime(Average) 1.255 (1.228)\tDataTime(Average) 1.061 (1.035)\tLoss(Average) 2.7611 (3.8073)\t\n",
      "Epoch: [32/1000][30/140]\tBatchTime(Average) 1.231 (1.231)\tDataTime(Average) 1.037 (1.039)\tLoss(Average) 4.0488 (3.8761)\t\n",
      "Epoch: [32/1000][40/140]\tBatchTime(Average) 1.215 (1.233)\tDataTime(Average) 1.026 (1.041)\tLoss(Average) 3.6405 (3.9378)\t\n",
      "Epoch: [32/1000][50/140]\tBatchTime(Average) 1.229 (1.233)\tDataTime(Average) 1.036 (1.041)\tLoss(Average) 3.3374 (3.9436)\t\n",
      "Epoch: [32/1000][60/140]\tBatchTime(Average) 1.230 (1.234)\tDataTime(Average) 1.036 (1.041)\tLoss(Average) 3.3251 (3.9407)\t\n",
      "Epoch: [32/1000][70/140]\tBatchTime(Average) 1.242 (1.235)\tDataTime(Average) 1.049 (1.043)\tLoss(Average) 4.8317 (3.9572)\t\n",
      "Epoch: [32/1000][80/140]\tBatchTime(Average) 1.301 (1.239)\tDataTime(Average) 1.106 (1.047)\tLoss(Average) 4.4475 (3.9922)\t\n",
      "Epoch: [32/1000][90/140]\tBatchTime(Average) 1.263 (1.242)\tDataTime(Average) 1.070 (1.049)\tLoss(Average) 4.3659 (3.9989)\t\n",
      "Epoch: [32/1000][100/140]\tBatchTime(Average) 1.276 (1.243)\tDataTime(Average) 1.082 (1.050)\tLoss(Average) 4.0870 (4.0122)\t\n",
      "Epoch: [32/1000][110/140]\tBatchTime(Average) 1.283 (1.243)\tDataTime(Average) 1.095 (1.050)\tLoss(Average) 2.9365 (3.9920)\t\n",
      "Epoch: [32/1000][120/140]\tBatchTime(Average) 1.226 (1.243)\tDataTime(Average) 1.034 (1.050)\tLoss(Average) 4.4801 (4.0076)\t\n",
      "Epoch: [32/1000][130/140]\tBatchTime(Average) 1.268 (1.242)\tDataTime(Average) 1.072 (1.049)\tLoss(Average) 3.4553 (3.9943)\t\n",
      "Epoch: [33/1000][0/140]\tBatchTime(Average) 1.011 (1.011)\tDataTime(Average) 0.833 (0.833)\tLoss(Average) 3.9346 (3.9346)\t\n",
      "Epoch: [33/1000][10/140]\tBatchTime(Average) 1.246 (1.234)\tDataTime(Average) 1.054 (1.041)\tLoss(Average) 5.0188 (3.8447)\t\n",
      "Epoch: [33/1000][20/140]\tBatchTime(Average) 1.248 (1.245)\tDataTime(Average) 1.054 (1.050)\tLoss(Average) 4.2917 (3.8317)\t\n",
      "Epoch: [33/1000][30/140]\tBatchTime(Average) 1.245 (1.244)\tDataTime(Average) 1.060 (1.050)\tLoss(Average) 3.5557 (3.7952)\t\n",
      "Epoch: [33/1000][40/140]\tBatchTime(Average) 1.260 (1.244)\tDataTime(Average) 1.067 (1.050)\tLoss(Average) 4.0856 (3.9464)\t\n",
      "Epoch: [33/1000][50/140]\tBatchTime(Average) 1.265 (1.244)\tDataTime(Average) 1.070 (1.050)\tLoss(Average) 3.3985 (3.8560)\t\n",
      "Epoch: [33/1000][60/140]\tBatchTime(Average) 1.278 (1.253)\tDataTime(Average) 1.085 (1.059)\tLoss(Average) 2.6890 (3.8633)\t\n",
      "Epoch: [33/1000][70/140]\tBatchTime(Average) 1.271 (1.255)\tDataTime(Average) 1.080 (1.060)\tLoss(Average) 4.6350 (3.9227)\t\n",
      "Epoch: [33/1000][80/140]\tBatchTime(Average) 1.214 (1.252)\tDataTime(Average) 1.022 (1.058)\tLoss(Average) 3.8795 (3.9425)\t\n",
      "Epoch: [33/1000][90/140]\tBatchTime(Average) 1.173 (1.251)\tDataTime(Average) 0.982 (1.058)\tLoss(Average) 4.1307 (3.9676)\t\n",
      "Epoch: [33/1000][100/140]\tBatchTime(Average) 1.245 (1.251)\tDataTime(Average) 1.051 (1.058)\tLoss(Average) 2.6502 (3.9649)\t\n",
      "Epoch: [33/1000][110/140]\tBatchTime(Average) 1.237 (1.251)\tDataTime(Average) 1.045 (1.058)\tLoss(Average) 3.5160 (3.9292)\t\n",
      "Epoch: [33/1000][120/140]\tBatchTime(Average) 1.248 (1.251)\tDataTime(Average) 1.055 (1.057)\tLoss(Average) 4.4521 (3.9590)\t\n",
      "Epoch: [33/1000][130/140]\tBatchTime(Average) 1.215 (1.251)\tDataTime(Average) 1.024 (1.057)\tLoss(Average) 5.5681 (3.9815)\t\n",
      "Epoch: [34/1000][0/140]\tBatchTime(Average) 1.008 (1.008)\tDataTime(Average) 0.829 (0.829)\tLoss(Average) 3.5397 (3.5397)\t\n",
      "Epoch: [34/1000][10/140]\tBatchTime(Average) 1.263 (1.247)\tDataTime(Average) 1.070 (1.054)\tLoss(Average) 3.9486 (3.7992)\t\n",
      "Epoch: [34/1000][20/140]\tBatchTime(Average) 1.246 (1.258)\tDataTime(Average) 1.051 (1.064)\tLoss(Average) 4.1119 (4.1100)\t\n",
      "Epoch: [34/1000][30/140]\tBatchTime(Average) 1.289 (1.255)\tDataTime(Average) 1.101 (1.062)\tLoss(Average) 4.5771 (4.1270)\t\n",
      "Epoch: [34/1000][40/140]\tBatchTime(Average) 1.228 (1.255)\tDataTime(Average) 1.035 (1.062)\tLoss(Average) 4.2601 (3.9813)\t\n",
      "Epoch: [34/1000][50/140]\tBatchTime(Average) 1.254 (1.256)\tDataTime(Average) 1.061 (1.062)\tLoss(Average) 3.4802 (3.9839)\t\n",
      "Epoch: [34/1000][60/140]\tBatchTime(Average) 1.256 (1.256)\tDataTime(Average) 1.063 (1.063)\tLoss(Average) 3.3748 (3.9482)\t\n",
      "Epoch: [34/1000][70/140]\tBatchTime(Average) 1.255 (1.256)\tDataTime(Average) 1.069 (1.063)\tLoss(Average) 3.9252 (3.9531)\t\n",
      "Epoch: [34/1000][80/140]\tBatchTime(Average) 1.288 (1.258)\tDataTime(Average) 1.088 (1.064)\tLoss(Average) 3.8127 (3.9350)\t\n",
      "Epoch: [34/1000][90/140]\tBatchTime(Average) 1.243 (1.257)\tDataTime(Average) 1.046 (1.064)\tLoss(Average) 4.2226 (3.9731)\t\n",
      "Epoch: [34/1000][100/140]\tBatchTime(Average) 1.266 (1.258)\tDataTime(Average) 1.077 (1.065)\tLoss(Average) 4.0883 (3.9809)\t\n",
      "Epoch: [34/1000][110/140]\tBatchTime(Average) 1.316 (1.260)\tDataTime(Average) 1.125 (1.067)\tLoss(Average) 3.5852 (3.9756)\t\n",
      "Epoch: [34/1000][120/140]\tBatchTime(Average) 1.342 (1.263)\tDataTime(Average) 1.141 (1.070)\tLoss(Average) 5.0244 (3.9904)\t\n",
      "Epoch: [34/1000][130/140]\tBatchTime(Average) 1.265 (1.265)\tDataTime(Average) 1.071 (1.072)\tLoss(Average) 3.6869 (3.9765)\t\n",
      "Epoch: [35/1000][0/140]\tBatchTime(Average) 1.015 (1.015)\tDataTime(Average) 0.837 (0.837)\tLoss(Average) 3.1819 (3.1819)\t\n",
      "Epoch: [35/1000][10/140]\tBatchTime(Average) 1.286 (1.256)\tDataTime(Average) 1.098 (1.061)\tLoss(Average) 4.0523 (3.5257)\t\n",
      "Epoch: [35/1000][20/140]\tBatchTime(Average) 1.246 (1.263)\tDataTime(Average) 1.058 (1.069)\tLoss(Average) 4.0750 (3.7524)\t\n",
      "Epoch: [35/1000][30/140]\tBatchTime(Average) 1.272 (1.265)\tDataTime(Average) 1.080 (1.071)\tLoss(Average) 3.0026 (3.9460)\t\n",
      "Epoch: [35/1000][40/140]\tBatchTime(Average) 1.325 (1.268)\tDataTime(Average) 1.135 (1.074)\tLoss(Average) 5.3228 (3.9007)\t\n",
      "Epoch: [35/1000][50/140]\tBatchTime(Average) 1.269 (1.268)\tDataTime(Average) 1.074 (1.074)\tLoss(Average) 4.3220 (3.9216)\t\n"
     ]
    }
   ],
   "source": [
    "## Training the model here by calling main() which will run the training loop\n",
    "dset_root = DatasetPath\n",
    "encoder = ColorfulColorizer()\n",
    "criterion = MultinomialCELoss()\n",
    "optimizer = torch.optim.SGD(encoder.parameters(), lr=learningRate)\n",
    "main(dset_root, batch_size, epochs, print_freq, encoder, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zGeGA5w_Acjo"
   },
   "source": [
    "## Testing Images from Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cellView": "code",
    "colab": {},
    "colab_type": "code",
    "id": "YfrzCJG_-s-p"
   },
   "outputs": [],
   "source": [
    "rescale = Rescale(imageSize)\n",
    "test_dataset = CustomImages(\n",
    "    root=DatasetPath, train=False, transform=rescale)\n",
    "\n",
    "location = 'cuda'\n",
    "test_cases = np.floor(np.random.rand(5) * len(test_dataset)).astype(int)\n",
    "test_cases = np.append(test_cases, [0], 0)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "encoder = ColorfulColorizer()\n",
    "encoder.load_state_dict(torch.load(StatePath+'/states/colorizer.pkl'))\n",
    "if 'cuda' in location:\n",
    "    print('Using:', torch.cuda.get_device_name(torch.cuda.current_device()))\n",
    "    encoder.cuda()\n",
    "\n",
    "encoder.eval()\n",
    "# encoder.parameters()\n",
    "\n",
    "outputs = []\n",
    "images = []\n",
    "labels = []\n",
    "for c in test_cases:\n",
    "    print('Encoding image number:', c)\n",
    "    image,_, label = test_dataset[c]\n",
    "    image = image.unsqueeze(0)\n",
    "    with torch.no_grad():\n",
    "        if 'cuda' in location:\n",
    "          image = image.cuda()\n",
    "          label = label.cuda()\n",
    "        images.append(image)\n",
    "        labels.append(label)\n",
    "        print(image.shape)\n",
    "        output = encoder(image)\n",
    "        outputs.append(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ml8hMiRaGncd"
   },
   "outputs": [],
   "source": [
    "T = 0.38\n",
    "q = 313  # number of colours\n",
    "nnenc = NNEncode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Iii1Mup0Hc1I"
   },
   "outputs": [],
   "source": [
    "bin_index = np.arange(q)\n",
    "print('Getting ab_list')\n",
    "ab_list = nnenc.bin2color(bin_index)   # q, 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3RxxaHup_6A0"
   },
   "outputs": [],
   "source": [
    "f, axarr = plt.subplots(len(test_cases), 3)\n",
    "for i in range(len(test_cases)):\n",
    "    l_layer = images[i].data[0].cpu().numpy()\n",
    "    bin_probabilities = outputs[i].data[0].cpu().numpy()  # bin_probabilities dim: q, h, w\n",
    "    ab_label = labels[i].data.cpu().numpy().astype('float64')\n",
    "\n",
    "    # convert bin_probab -> ab_pred\n",
    "    bin_probabilities = np.exp(np.log(bin_probabilities)/T)\n",
    "    bin_sum = bin_probabilities.sum(0)\n",
    "    bin_sum = bin_sum.reshape((1, bin_sum.shape[0], bin_sum.shape[1]))\n",
    "    bin_probabilities /= bin_sum\n",
    "\n",
    "    # ab_pred dim: 2, h, w\n",
    "    ab_pred = (bin_probabilities[:, np.newaxis, :, :] * ab_list[:, :, np.newaxis, np.newaxis]).sum(0)\n",
    "\n",
    "    img_input = l_layer[0]\n",
    "#     img_input = np.concatenate((l_layer, torch.zeros([2,128,128])), axis=0)\n",
    "    img_pred = np.concatenate((l_layer, ab_pred), axis=0)\n",
    "    img_actual = np.concatenate((l_layer, ab_label), axis=0)\n",
    "    \n",
    "#     img_input = lab2rgb(img_input.transpose(1, 2, 0))\n",
    "    img_pred = lab2rgb(img_pred.transpose(1, 2, 0))\n",
    "    img_actual = lab2rgb(img_actual.transpose(1, 2, 0))\n",
    "    \n",
    "    axarr[i][0].imshow(img_input)\n",
    "    axarr[i][1].imshow(img_pred)\n",
    "    axarr[i][2].imshow(img_actual)\n",
    "    sample_image(img_input, img_pred, img_actual, 1, i)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of PytorchVersion.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}