Unet-Resnet34-Open Solution-FastAI.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "from fastai.conv_learner import *\n",
    "from fastai.dataset import *\n",
    "from fastai.models.resnet import vgg_resnet50\n",
    "from fastai.models.senet import *\n",
    "from skimage.transform import resize\n",
    "import json\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold , KFold\n",
    "from sklearn.metrics import jaccard_similarity_score\n",
    "from pycocotools import mask as cocomask\n",
    "from utils import my_eval,intersection_over_union_thresholds,RLenc\n",
    "from lovasz_losses import lovasz_hinge\n",
    "print(torch.__version__)\n",
    "torch.cuda.is_available()\n",
    "torch.backends.cudnn.benchmark=True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "PATH = Path('data/tgs/')\n",
    "TRN_MASKS = 'trn_masks'\n",
    "TRN_IMG = 'trn_images'\n",
    "TRN_MSK = 'trn_masks'\n",
    "TST_IMG = 'tst_images'\n",
    "trn = pd.read_csv(PATH/'train.csv')\n",
    "dpth = pd.read_csv(PATH/'depths.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def show_img(im, figsize=None, ax=None, alpha=None):\n",
    "    if not ax: fig,ax = plt.subplots(figsize=figsize)\n",
    "    ax.imshow(im, alpha=alpha)\n",
    "    ax.set_axis_off()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class DepthDataset(Dataset):\n",
    "    def __init__(self,ds,dpth_dict):\n",
    "        self.dpth = dpth_dict\n",
    "        self.ds = ds\n",
    "        \n",
    "    def __getitem__(self,i):\n",
    "        val = self.ds[i]\n",
    "        return val[0],self.dpth[self.ds.fnames[i].split('/')[1][:-4]],val[1]\n",
    "    \n",
    "class MatchedFilesDataset(FilesDataset):\n",
    "    def __init__(self, fnames, y, transform, path):\n",
    "        self.y=y\n",
    "        assert(len(fnames)==len(y))\n",
    "        super().__init__(fnames, transform, path)\n",
    "        \n",
    "    def get_x(self, i): \n",
    "        return open_image(os.path.join(self.path, self.fnames[i]))\n",
    "    \n",
    "    def get_y(self, i):\n",
    "        return open_image(os.path.join(str(self.path), str(self.y[i])))\n",
    "\n",
    "    def get_c(self): return 0\n",
    "    \n",
    "class TestFilesDataset(FilesDataset):\n",
    "    def __init__(self, fnames, y, transform,flip, path):\n",
    "        self.y=y\n",
    "        self.flip = flip\n",
    "        super().__init__(fnames, transform, path)\n",
    "        \n",
    "    def get_x(self, i): \n",
    "        im = open_image(os.path.join(self.path, self.fnames[i]))\n",
    "        return np.fliplr(im) if self.flip else im\n",
    "        \n",
    "    def get_y(self, i):\n",
    "        im = open_image(os.path.join(str(self.path), str(self.y[i])))\n",
    "        return np.fliplr(im) if self.flip else im\n",
    "    def get_c(self): return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Creating K-Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_names = np.array([f'{TRN_IMG}/{o.name}' for o in (PATH/TRN_MASKS).iterdir()])\n",
    "y_names = np.array([f'{TRN_MASKS}/{o.name}' for o in (PATH/TRN_MASKS).iterdir()])\n",
    "tst_x = np.array([f'{TST_IMG}/{o.name}' for o in (PATH/TST_IMG).iterdir()])\n",
    "f_name = [o.split('/')[-1] for o in x_names]\n",
    "\n",
    "c = dpth.set_index('id')\n",
    "dpth_dict = c['z'].to_dict()\n",
    "\n",
    "kf = 5\n",
    "kfold = KFold(n_splits=kf, shuffle=True, random_state=42)\n",
    "\n",
    "train_folds = []\n",
    "val_folds = []\n",
    "for idxs in kfold.split(f_name):\n",
    "    train_folds.append([f_name[idx] for idx in idxs[0]])\n",
    "    val_folds.append([f_name[idx] for idx in idxs[1]])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_folds = pickle.load(open('train_folds.pkl',mode='rb'))\n",
    "val_folds = pickle.load(open('val_folds.pkl',mode='rb'))\n",
    "tst_x = pickle.load(open('tst_x.pkl',mode='rb'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Unet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class SaveFeatures():\n",
    "    features=None\n",
    "    def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)\n",
    "    def hook_fn(self, module, input, output): self.features = output\n",
    "    def remove(self): self.hook.remove()\n",
    "    \n",
    "class UnetBlock(nn.Module):\n",
    "    def __init__(self, up_in, x_in, n_out):\n",
    "        super().__init__()\n",
    "        up_out = x_out = n_out//2\n",
    "        self.x_conv  = nn.Conv2d(x_in,  x_out,  1)\n",
    "        self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)\n",
    "        self.bn = nn.BatchNorm2d(n_out)\n",
    "        \n",
    "    def forward(self, up_p, x_p):\n",
    "        up_p = self.tr_conv(up_p)\n",
    "        x_p = self.x_conv(x_p)\n",
    "        cat_p = torch.cat([up_p,x_p], dim=1)\n",
    "        return self.bn(F.relu(cat_p))\n",
    "    \n",
    "class Unet34(nn.Module):\n",
    "    def __init__(self, rn):\n",
    "        super().__init__()\n",
    "        self.rn = rn\n",
    "        self.sfs = [SaveFeatures(rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(512,256,128)\n",
    "        self.up2 = UnetBlock(128,128,128)\n",
    "        self.up3 = UnetBlock(128,64,128)\n",
    "        self.up4 = UnetBlock(128,64,128)\n",
    "        self.up5 = nn.ConvTranspose2d(128, 1, 2, stride=2)\n",
    "        \n",
    "    def forward(self,img,depth):\n",
    "        x = F.relu(self.rn(img))\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x)\n",
    "        return x[:,0]\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()\n",
    "\n",
    "\n",
    "class UnetModel():\n",
    "    def __init__(self,model,lr_cut,name='unet'):\n",
    "        self.model,self.name = model,name\n",
    "        self.lr_cut = lr_cut\n",
    "\n",
    "    def get_layer_groups(self, precompute):\n",
    "        lgs = list(split_by_idxs(children(self.model.rn), [self.lr_cut]))\n",
    "        return lgs + [children(self.model)[1:]]\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_tgs_model():\n",
    "    f = resnet34\n",
    "    cut,lr_cut = model_meta[f]\n",
    "    m_base = get_base(f,cut)\n",
    "    m = to_gpu(Unet34(m_base))\n",
    "    models = UnetModel(m,lr_cut)\n",
    "    learn = ConvLearner(md, models)\n",
    "    return learn\n",
    "\n",
    "def get_base(f,cut):\n",
    "    layers = cut_model(f(True), cut)\n",
    "    return nn.Sequential(*layers)            \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[256, 64]\n",
      "fold_id0\n",
      "5_fold_simple_resnet34_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73bf451b359c4808be097145564a500b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval                  \n",
      "    0      1.325157   0.93006    0.765038  \n",
      "    1      1.03828    0.913555   0.778321                 \n",
      "    2      0.939755   0.824341   0.801629                  \n",
      "    3      0.88104    0.808754   0.807644                  \n",
      "    4      0.852355   0.849393   0.798747                  \n",
      "    5      0.825028   0.805423   0.806266                  \n",
      "    6      0.799593   0.807329   0.808271                  \n",
      "    7      0.753976   0.780195   0.813283                  \n",
      "    8      0.731826   0.771001   0.813033                  \n",
      "    9      0.702984   0.773757   0.815539                  \n",
      "    10     0.702384   0.875103   0.803258                  \n",
      "    11     0.799662   0.888119   0.784085                  \n",
      "    12     0.833936   0.828985   0.801754                  \n",
      "    13     0.817738   0.80451    0.80802                   \n",
      "    14     0.780242   0.826695   0.796491                  \n",
      "    15     0.720219   0.791626   0.816541                  \n",
      "    16     0.694303   0.796775   0.809398                  \n",
      "    17     0.671596   0.772202   0.817544                  \n",
      "    18     0.639161   0.788361   0.821053                  \n",
      "    19     0.621297   0.773167   0.819799                  \n",
      "    20     0.620868   0.905281   0.780952                  \n",
      "    21     0.702979   0.856898   0.80401                   \n",
      "    22     0.737761   0.812372   0.809524                  \n",
      "    23     0.736216   0.825973   0.808271                  \n",
      "    24     0.735905   0.814299   0.815915                  \n",
      "    25     0.680798   0.77566    0.813033                  \n",
      "    26     0.637793   0.781759   0.820426                  \n",
      "    27     0.604662   0.776221   0.817544                  \n",
      "    28     0.575798   0.775428   0.817794                  \n",
      "    29     0.54858    0.78509    0.82005                   \n",
      "\n",
      "0.8210526315789473\n",
      "fold_id1\n",
      "5_fold_simple_resnet34_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5bad59687504bd8ba5ce0dae7b6b194",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval                  \n",
      "    0      1.348665   1.033372   0.747494  \n",
      "    1      1.078376   0.959368   0.769674                 \n",
      "    2      0.967243   0.929143   0.750376                  \n",
      "    3      0.907108   0.909645   0.76391                   \n",
      "    4      0.845982   0.840461   0.79198                   \n",
      "    5      0.822131   0.858117   0.775689                  \n",
      "    6      0.80265    0.841035   0.784712                  \n",
      "    7      0.736551   0.830294   0.793233                  \n",
      "    8      0.71209    0.826889   0.788722                  \n",
      "    9      0.68642    0.818437   0.792732                  \n",
      "    10     0.67733    0.87536    0.769298                  \n",
      "    11     0.768952   0.91329    0.77619                   \n",
      "    12     0.772509   0.873983   0.787845                  \n",
      "    13     0.76916    0.835945   0.802256                  \n",
      "    14     0.733515   0.822953   0.781579                  \n",
      "    15     0.687887   0.840586   0.768045                  \n",
      "    16     0.653878   0.812393   0.787845                  \n",
      "    17     0.622058   0.802763   0.799123                  \n",
      "    18     0.590854   0.782723   0.800627                  \n",
      "    19     0.564545   0.780446   0.805138                  \n",
      "    20     0.584671   0.92863    0.771679                  \n",
      "    21     0.711783   0.856759   0.791729                  \n",
      "    22     0.704852   0.854031   0.782456                  \n",
      "    23     0.687806   0.917681   0.760526                  \n",
      "    24     0.681609   0.808088   0.796115                  \n",
      "    25     0.621207   0.820036   0.793108                  \n",
      "    26     0.59361    0.843446   0.791855                  \n",
      "    27     0.558614   0.78089    0.805388                  \n",
      "    28     0.526056   0.795237   0.813409                  \n",
      "    29     0.487891   0.759982   0.806767                  \n",
      "\n",
      "0.813408521303258\n",
      "fold_id2\n",
      "5_fold_simple_resnet34_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fea58c3fd9a4c46ad2201f7173c0250",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval                  \n",
      "    0      1.284321   1.039837   0.740226  \n",
      "    1      1.036117   0.917657   0.779073                 \n",
      "    2      0.939302   0.862901   0.787469                  \n",
      "    3      0.888223   0.843559   0.79198                   \n",
      "    4      0.832163   0.804011   0.803383                  \n",
      "    5      0.787491   0.81721    0.802256                  \n",
      "    6      0.726067   0.760629   0.810902                  \n",
      "    7      0.690015   0.747631   0.817794                  \n",
      "    8      0.643923   0.724834   0.822306                  \n",
      "    9      0.597543   0.736826   0.812281                  \n",
      "    10     0.610031   0.85177    0.80614                   \n",
      "    11     0.684985   0.956533   0.777569                  \n",
      "    12     0.749024   0.841364   0.79787                   \n",
      "    13     0.745144   0.886766   0.792857                  \n",
      "    14     0.702938   0.766046   0.815539                  \n",
      "    15     0.645297   0.744435   0.816416                  \n",
      "    16     0.591229   0.78739    0.818922                  \n",
      "    17     0.551063   0.742451   0.820677                  \n",
      "    18     0.500171   0.755703   0.8099                    \n",
      "    19     0.462674   0.748881   0.813659                  \n",
      "    20     0.488585   0.876356   0.803133                  \n",
      "    21     0.56171    0.868642   0.809398                  \n",
      "    22     0.602444   0.842901   0.802632                  \n",
      "    23     0.589505   0.927599   0.77193                   \n",
      "    24     0.580468   0.85071    0.809649                  \n",
      "    25     0.539967   0.775132   0.819048                  \n",
      "    26     0.489004   0.795378   0.816667                  \n",
      "    27     0.456576   0.778352   0.819674                  \n",
      "    28     0.40038    0.766835   0.813534                  \n",
      "    29     0.368988   0.795965   0.818421                  \n",
      "\n",
      "0.8223057644110277\n",
      "fold_id3\n",
      "5_fold_simple_resnet34_3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3229e9889bac41528d18ac2bcfdfcd81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval                  \n",
      "    0      1.372224   1.006186   0.755013  \n",
      "    1      1.05301    0.937049   0.776817                 \n",
      "    2      0.92111    0.897983   0.779323                  \n",
      "    3      0.855703   0.91295    0.77594                   \n",
      "    4      0.807139   0.875474   0.789474                  \n",
      "    5      0.757861   0.825887   0.796491                  \n",
      "    6      0.715815   0.825675   0.802005                  \n",
      "    7      0.668804   0.803978   0.805138                  \n",
      "    8      0.633585   0.779532   0.806266                  \n",
      "    9      0.598439   0.775525   0.812406                  \n",
      "    10     0.595149   0.944765   0.779699                  \n",
      "    11     0.708947   1.008999   0.752005                  \n",
      "    12     0.73316    0.865063   0.790727                  \n",
      "    13     0.717747   0.815308   0.809273                  \n",
      "    14     0.681317   0.86026    0.795489                  \n",
      "    15     0.622366   0.784242   0.808647                  \n",
      "    16     0.590551   0.795878   0.811404                  \n",
      "    17     0.547718   0.788322   0.814411                  \n",
      "    18     0.510185   0.791338   0.812155                  \n",
      "    19     0.481769   0.784741   0.820175                  \n",
      "    20     0.482904   0.940439   0.796366                  \n",
      "    21     0.541447   0.916361   0.790476                  \n",
      "    22     0.592643   0.869368   0.799248                  \n",
      "    23     0.604125   0.884436   0.795363                  \n",
      "    24     0.569964   0.86229    0.805514                  \n",
      "    25     0.531787   0.899627   0.805013                  \n",
      "    26     0.50264    0.862456   0.806767                  \n",
      "    27     0.470732   0.825488   0.806015                  \n",
      "    28     0.438868   0.843719   0.824185                  \n",
      "    29     0.406581   0.835985   0.816667                  \n",
      "\n",
      "0.8241854636591479\n",
      "fold_id4\n",
      "5_fold_simple_resnet34_4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b56e1d8fde146138f4f3b16098fbb4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval                  \n",
      "    0      1.361298   1.10476    0.728356  \n",
      "    1      1.036111   1.038042   0.748934                 \n",
      "    2      0.891879   0.983283   0.768632                  \n",
      "    3      0.834489   1.03786    0.754078                  \n",
      "    4      0.785943   0.919298   0.777039                  \n",
      "    5      0.729437   0.951818   0.775784                  \n",
      "    6      0.687682   0.937895   0.784191                  \n",
      "    7      0.658635   0.902859   0.78143                   \n",
      "    8      0.601494   0.886      0.796738                  \n",
      "    9      0.571334   0.881849   0.792974                  \n",
      "    10     0.575541   0.916668   0.784065                  \n",
      "    11     0.667784   1.034864   0.754078                  \n",
      "    12     0.690661   1.014947   0.768507                  \n",
      "    13     0.674954   0.988458   0.767378                  \n",
      "    14     0.627315   0.892071   0.783438                  \n",
      "    15     0.58735    0.920014   0.783689                  \n",
      "    16     0.553768   0.931009   0.787202                  \n",
      "    17     0.498617   0.918791   0.794103                  \n",
      "    18     0.470062   0.88031    0.799373                  \n",
      "    19     0.435687   0.879542   0.804141                  \n",
      "    20     0.444049   0.994929   0.784191                  \n",
      "    21     0.557501   1.043981   0.758093                  \n",
      "    22     0.586135   1.058152   0.76675                   \n",
      "    23     0.566919   0.973012   0.78005                   \n",
      "    24     0.53908    0.920541   0.791217                  \n",
      "    25     0.479652   0.948885   0.784316                  \n",
      "    26     0.433641   1.033326   0.789335                  \n",
      "    27     0.402444   0.959513   0.793601                  \n",
      "    28     0.374302   0.965276   0.793099                  \n",
      "    29     0.346213   0.96585    0.793852                  \n",
      "\n",
      "0.8041405269761606\n"
     ]
    }
   ],
   "source": [
    "model = 'simple_resnet34'\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(256,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'trn_images/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'trn_masks/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'trn_images/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'trn_masks/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(resnet34, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=tst_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "        learn.fit(lrs/2,3, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "\n",
    "\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.8210526315789473,\n",
       "  0.813408521303258,\n",
       "  0.8223057644110277,\n",
       "  0.8241854636591479,\n",
       "  0.8041405269761606],\n",
       " 0.8170185815857083)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bst_acc,np.mean(bst_acc)#With 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.818922305764411,\n",
       "  0.793734335839599,\n",
       "  0.8147869674185464,\n",
       "  0.8031328320802005,\n",
       "  0.7912170639899623],\n",
       " 0.8043587010185437)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bst_acc,np.mean(bst_acc) #With 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Submission - TTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45d11eaa2f124312897f36ec705e9fe6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5_fold_simple_resnet34_0\n",
      "5_fold_simple_resnet34_1\n",
      "5_fold_simple_resnet34_2\n",
      "5_fold_simple_resnet34_3\n",
      "5_fold_simple_resnet34_4\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a904b594ed7a4238bedc9f73164937ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5_fold_simple_resnet34_0\n",
      "5_fold_simple_resnet34_1\n",
      "5_fold_simple_resnet34_2\n",
      "5_fold_simple_resnet34_3\n",
      "5_fold_simple_resnet34_4\n",
      "\n"
     ]
    }
   ],
   "source": [
    "preds = np.zeros(shape = (18000,sz,sz))\n",
    "for o in [True,False]:\n",
    "    md.test_dl.dataset = TestFilesDataset(tst_x,tst_x,tfms[1],flip=o,path=PATH)\n",
    "    md.test_dl.dataset = DepthDataset(md.test_dl.dataset,dpth_dict)\n",
    "    \n",
    "    for i in tqdm_notebook(range(kf)):\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "        learn.load(pa)\n",
    "        pred = learn.predict(is_test=True)\n",
    "        pred = to_np(torch.sigmoid(torch.from_numpy(pred)))    \n",
    "        for im_idx,im in enumerate(pred):\n",
    "                preds[im_idx] += np.fliplr(im) if o else im\n",
    "        del pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f6d75e6fe10>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAD8CAYAAACVSwr3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADY9JREFUeJzt3V2sXWWdx/HvbwoyEZlIeUspnQFNJxm8sJITJGFinJAR6E3xwglcaGNI6gUkmjgXVS/k0pmMmpDMkNRIrBMHhvgSesGMYmNi5kKgGCxvA1Rk5NiG+hYkY4JQ/3Nx1hk2ffbh7J6z19lrt99PsrPWfvaz9v53ted3nmetvVZTVUjSqD+ZdQGShsdgkNQwGCQ1DAZJDYNBUsNgkNToLRiS3JDkmSRHkuzt63MkTV/6+B5Dkk3As8DfAovAI8AtVfXU1D9M0tT1NWK4GjhSVc9X1R+Ae4FdPX2WpCk7q6f33Qq8OPJ8EXj/Sp0v3LypLt92dk+lnLmePfz2WZegAXmF3/6qqi6apG9fwZAxbW+asyTZA+wB+POtZ/Hwd7f1VMqZ6/pLd8y6BA3I9+ub/zNp376mEovA6E/6ZcDR0Q5Vta+qFqpq4aILNvVUxpnLUNB69BUMjwDbk1yR5G3AzcCBnj5L0pT1MpWoqteT3A58F9gE3F1VT/bxWZKmr69jDFTVA8ADfb2/VuY0QuvlNx8lNQyG04yjBU2DwSCpYTBIahgMpxGnEZoWg0FSw2CQ1DAYJDUMBkkNg0FSw2CQ1DAYJDUMBkkNg0FSw2CQ1DAYJDUMBkkNg0FSw2A4TXhlpabJYJDUMBhOA44WNG0Gw5wzFNQHg0FSw2CYY44W1BeDQVLDYJDUMBjmlNMI9clgkNQwGCQ1DAZJDYNBUsNgkNQwGCQ1DIY55KlK9e2s9Wyc5AXgFeAE8HpVLSTZDPw7cDnwAvB3VfXb9ZWpZYaCNsI0Rgx/U1U7qmqhe74XOFhV24GD3XNNgaGgjdLHVGIXsL9b3w/c1MNnSOrReoOhgO8leTTJnq7tkqo6BtAtLx63YZI9SQ4lOfTLX59YZxmSpmldxxiAa6vqaJKLgQeT/PekG1bVPmAfwMJ7/7TWWcdpz2mENtK6RgxVdbRbHge+A1wNvJRkC0C3PL7eIiVtrDUHQ5Jzk5y3vA58CHgCOADs7rrtBu5fb5GSNtZ6phKXAN9Jsvw+/1ZV/5nkEeC+JLcCPwc+sv4yJW2kNQdDVT0PvHdM+6+B69ZTlKTZ8puPc8ADj9poBoOkhsEgqWEwSGoYDJIaBoOkhsEgqWEwSGoYDJIaBsPA+eUmzYLBIKlhMAyYowXNisEgqWEwSGoYDAPlNEKzZDBIahgMA+RoQbNmMEhqGAySGgaDpIbBIKlhMAyMBx41BAaDpIbBMCCOFjQUBoOkhsEgqWEwDITTCA2JwSCpYTBIaqz5f7vWdDiF0BA5YpDUMBhmyNGChspgmBFDQUO2ajAkuTvJ8SRPjLRtTvJgkue65flde5LcmeRIksNJruqzeEn9mGTE8DXghpPa9gIHq2o7cLB7DnAjsL177AHumk6ZpxdHCxq6VYOhqn4I/Oak5l3A/m59P3DTSPvXa8mPgHcm2TKtYiVtjLUeY7ikqo4BdMuLu/atwIsj/Ra7NklzZNoHHzOmrcZ2TPYkOZTk0C9/fWLKZQyX0wjNg7UGw0vLU4RuebxrXwS2jfS7DDg67g2qal9VLVTVwkUXbFpjGfPFUNC8WGswHAB2d+u7gftH2j/WnZ24Bnh5ecohaX6s+pXoJPcAHwQuTLIIfB74AnBfkluBnwMf6bo/AOwEjgC/Bz7eQ81zx5GC5s2qwVBVt6zw0nVj+hZw23qLmtT1l+7gu0cf26iPWxNDQfPIbz5Kasz91ZXjfiMPZRThaEHzyhGDpMZcjhhW+028/PqsRg6OFDTvBhkM0/rBmoeDk9IQDW4qMe3ftv72lk7dYEYMff4Ab+TUwiDS6WAQI4ZnD7991iVIGjGIYNgofY9KHC3odDGYqcQ8Mgh0ujqjRgySJuOI4RQ5StCZwBGDpMYZN2JY66lLRwo6k5yxIwZ/0KWVnbHBAG+cYlwOCcNCWnLGTSVWYjhIbzijRwySxjMYJDUMBkkNg0FSw2CQ1DAYJDUMBkkNg0FSw2CQ1DAYJDUMBkkNg0FSw2CQ1DAYJDUMBkkNg0FSw2CQ1Fg1GJLcneR4kidG2u5I8oskj3WPnSOvfSbJkSTPJLm+r8Il9WeSEcPXgBvGtH+5qnZ0jwcAklwJ3Ay8p9vmX5JsmlaxkjbGqsFQVT8EfjPh++0C7q2qV6vqZ8AR4Op11CdpBtZzjOH2JIe7qcb5XdtW4MWRPotdWyPJniSHkhx6jVfXUYakaVtrMNwFvBvYARwDvti1Z0zfGvcGVbWvqhaqauFszlljGZL6sKZgqKqXqupEVf0R+ApvTBcWgW0jXS8Djq6vREkbbU3BkGTLyNMPA8tnLA4ANyc5J8kVwHbg4fWVKGmjrfofziS5B/ggcGGSReDzwAeT7GBpmvAC8AmAqnoyyX3AU8DrwG1VdaKf0iX1JVVjDwFsqD/L5np/rpt1GdJp7fv1zUeramGSvn7zUVLDYJDUMBgkNQwGSQ2DQVLDYJDUMBgkNQwGSQ2DQVLDYJDUMBgkNQwGSQ2DQVLDYJDUMBgkNQwGSQ2DQVLDYJDUMBgkNQwGSQ2DQVLDYJDUMBgkNQwGSQ2DQVLDYJDUMBgkNQwGSQ2DQVLDYJDUMBgkNQwGSQ2DQVJj1WBIsi3JD5I8neTJJJ/s2jcneTDJc93y/K49Se5MciTJ4SRX9f2HkDRdk4wYXgc+XVV/BVwD3JbkSmAvcLCqtgMHu+cANwLbu8ce4K6pVy2pV6sGQ1Udq6ofd+uvAE8DW4FdwP6u237gpm59F/D1WvIj4J1Jtky9ckm9OaVjDEkuB94HPARcUlXHYCk8gIu7bluBF0c2W+zaJM2JiYMhyTuAbwGfqqrfvVXXMW015v32JDmU5NBrvDppGZI2wETBkORslkLhG1X17a75peUpQrc83rUvAttGNr8MOHrye1bVvqpaqKqFszlnrfVL6sEkZyUCfBV4uqq+NPLSAWB3t74buH+k/WPd2YlrgJeXpxyS5sNZE/S5Fvgo8HiSx7q2zwJfAO5Lcivwc+Aj3WsPADuBI8DvgY9PtWJJvVs1GKrqvxh/3ADgujH9C7htnXVJmiG/+SipYTBIahgMkhoGg6SGwSCpYTBIahgMkhoGg6SGwSCpYTBIahgMkhoGg6SGwSCpYTBIahgMkhoGg6SGwSCpYTBIahgMkhoGg6SGwSCpYTBIahgMkhoGg6SGwSCpYTBIahgMkhoGg6SGwSCpYTBIahgMkhoGg6SGwSCpYTBIaqwaDEm2JflBkqeTPJnkk137HUl+keSx7rFzZJvPJDmS5Jkk1/f5B5A0fWdN0Od14NNV9eMk5wGPJnmwe+3LVfVPo52TXAncDLwHuBT4fpK/rKoT0yxcUn9WHTFU1bGq+nG3/grwNLD1LTbZBdxbVa9W1c+AI8DV0yhW0sY4pWMMSS4H3gc81DXdnuRwkruTnN+1bQVeHNlskTFBkmRPkkNJDr3Gq6dcuKT+TBwMSd4BfAv4VFX9DrgLeDewAzgGfHG565jNq2mo2ldVC1W1cDbnnHLhkvozUTAkOZulUPhGVX0boKpeqqoTVfVH4Cu8MV1YBLaNbH4ZcHR6JUvq2yRnJQJ8FXi6qr400r5lpNuHgSe69QPAzUnOSXIFsB14eHolS+rbJGclrgU+Cjye5LGu7bPALUl2sDRNeAH4BEBVPZnkPuApls5o3OYZCWm+pKqZ/m98Eckvgf8FfjXrWiZwIfNRJ8xPrdY5feNq/YuqumiSjQcRDABJDlXVwqzrWM281AnzU6t1Tt96a/Ur0ZIaBoOkxpCCYd+sC5jQvNQJ81OrdU7fumodzDEGScMxpBGDpIGYeTAkuaG7PPtIkr2zrudkSV5I8nh3afmhrm1zkgeTPNctz1/tfXqo6+4kx5M8MdI2tq4subPbx4eTXDWAWgd32f5b3GJgUPt1Q26FUFUzewCbgJ8C7wLeBvwEuHKWNY2p8QXgwpPa/hHY263vBf5hBnV9ALgKeGK1uoCdwH+wdB3LNcBDA6j1DuDvx/S9svt3cA5wRffvY9MG1bkFuKpbPw94tqtnUPv1Leqc2j6d9YjhauBIVT1fVX8A7mXpsu2h2wXs79b3AzdtdAFV9UPgNyc1r1TXLuDrteRHwDtP+kp7r1aodSUzu2y/Vr7FwKD261vUuZJT3qezDoaJLtGesQK+l+TRJHu6tkuq6hgs/SUBF8+sujdbqa6h7uc1X7bft5NuMTDY/TrNWyGMmnUwTHSJ9oxdW1VXATcCtyX5wKwLWoMh7ud1XbbfpzG3GFix65i2Dat12rdCGDXrYBj8JdpVdbRbHge+w9IQ7KXlIWO3PD67Ct9kpboGt59roJftj7vFAAPcr33fCmHWwfAIsD3JFUnextK9Ig/MuKb/l+Tc7j6XJDkX+BBLl5cfAHZ33XYD98+mwsZKdR0APtYdRb8GeHl5aDwrQ7xsf6VbDDCw/bpSnVPdpxtxFHWVI6w7WTqq+lPgc7Ou56Ta3sXS0dyfAE8u1wdcABwEnuuWm2dQ2z0sDRdfY+k3wq0r1cXSUPKfu338OLAwgFr/tavlcPcPd8tI/891tT4D3LiBdf41S0Psw8Bj3WPn0PbrW9Q5tX3qNx8lNWY9lZA0QAaDpIbBIKlhMEhqGAySGgaDpIbBIKlhMEhq/B+1vZtDcyHTvQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(((preds[16]/10)>0.5).astype(np.uint8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "p = [cv2.resize(o/10,dsize=(101,101)) for o in preds]\n",
    "p = [(o>0.5).astype(np.uint8) for o in p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c76d2f1f04484c3f92fb502dde60a920",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "pred_dict = {id_[11:-4]:RLenc(p[i]) for i,id_ in tqdm_notebook(enumerate(tst_x))}\n",
    "sub = pd.DataFrame.from_dict(pred_dict,orient='index')\n",
    "sub.index.names = ['id']\n",
    "sub.columns = ['rle_mask']\n",
    "sub.to_csv('simple_k_fold_flipped.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}