TGS_Salt_resnext50_unet_5Fold_scSE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 440
    },
    "colab_type": "code",
    "id": "0mTKWCsxXo4r",
    "outputId": "87668956-8fa4-4bb0-9247-015a74a51caa"
   },
   "outputs": [],
   "source": [
    "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
    "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
    "!apt-get update -qq 2>&1 > /dev/null\n",
    "!curl --header \"Host: launchpadlibrarian.net\" --header \"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36\" --header \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\" --header \"Accept-Language: zh,zh-CN;q=0.9,zh-TW;q=0.8,en-US;q=0.7,en;q=0.6\" \"https://launchpadlibrarian.net/386846978/google-drive-ocamlfuse_0.7.0-0ubuntu1_amd64.deb\" -o \"google-drive-ocamlfuse_0.7.0-0ubuntu1_amd64.deb\" -L\n",
    "!apt-get -y install -qq fuse\n",
    "!dpkg -i google-drive-ocamlfuse_0.7.0-0ubuntu1_amd64.deb\n",
    "\n",
    "from googleapiclient.discovery import build\n",
    "import io, os\n",
    "from googleapiclient.http import MediaIoBaseDownload\n",
    "from google.colab import auth\n",
    "\n",
    "auth.authenticate_user()\n",
    "\n",
    "drive_service = build('drive', 'v3')\n",
    "results = drive_service.files().list(\n",
    "        q=\"name = 'kaggle.json'\", fields=\"files(id)\").execute()\n",
    "kaggle_api_key = results.get('files', [])\n",
    "\n",
    "filename = \"/content/.kaggle/kaggle.json\"\n",
    "os.makedirs(os.path.dirname(filename), exist_ok=True)\n",
    "\n",
    "request = drive_service.files().get_media(fileId=kaggle_api_key[0]['id'])\n",
    "fh = io.FileIO(filename, 'wb')\n",
    "downloader = MediaIoBaseDownload(fh, request)\n",
    "done = False\n",
    "while done is False:\n",
    "    status, done = downloader.next_chunk()\n",
    "    print(\"Download %d%%.\" % int(status.progress() * 100))\n",
    "os.chmod(filename, 600)\n",
    "!mkdir ~/.kaggle\n",
    "!cp /content/.kaggle/kaggle.json ~/.kaggle/kaggle.json\n",
    "\n",
    "from oauth2client.client import GoogleCredentials\n",
    "creds = GoogleCredentials.get_application_default()\n",
    "import getpass\n",
    "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
    "vcode = getpass.getpass()\n",
    "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "K102BSsYDGMy"
   },
   "outputs": [],
   "source": [
    "!mkdir -p drive\n",
    "!google-drive-ocamlfuse -o nonempty drive\n",
    "!ls drive > /dev/null"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "teccmM5G-YOk"
   },
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 134
    },
    "colab_type": "code",
    "id": "EaDX0j2Kt4W1",
    "outputId": "0b410fa0-4198-4f63-ac96-ab67db380594"
   },
   "outputs": [],
   "source": [
    "!mkdir src -p && cd src && git clone https://github.com/fastai/fastai.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "698sgxtBuXZO",
    "outputId": "78e4c0d3-0361-43c7-90f4-7412461d2592"
   },
   "outputs": [],
   "source": [
    "!pip3 install -q bcolz graphviz sklearn_pandas isoweek pandas_summary ipywidgets torch torchvision torchtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D6T70YIYyuKp"
   },
   "outputs": [],
   "source": [
    "!git config --global user.email 'nikshuang@163.com'\n",
    "!git config --global user.name 'Alex Huang'\n",
    "!pip install -q kaggle\n",
    "!pip install -q -U git+https://github.com/albu/albumentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 118
    },
    "colab_type": "code",
    "id": "kv9wLQUwc5Vd",
    "outputId": "89b20b5f-d89f-4759-eb4a-6efc2e41e4dc"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/svishnu88/TGS-SaltIdentification-Open-Solution-fastai.git\n",
    "!cp TGS-SaltIdentification-Open-Solution-fastai/*.py .\n",
    "!pip install -q Cython --upgrade\n",
    "!pip install -q pycocotools --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "ib2O97lzt4cG",
    "outputId": "5304877b-0461-4af1-c124-51a5eccf8b1c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4.1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(True, True)"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append(\"/content/src/fastai/old\") # on windows use \\'s instead\n",
    "\n",
    "from fastai.conv_learner import *\n",
    "from fastai.dataset import *\n",
    "from skimage.transform import resize\n",
    "import gc\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold , KFold\n",
    "from sklearn.metrics import jaccard_similarity_score\n",
    "from pycocotools import mask as cocomask\n",
    "from utils import my_eval,intersection_over_union_thresholds,RLenc\n",
    "from lovasz_losses import *\n",
    "print(torch.__version__)\n",
    "torch.backends.cudnn.benchmark=True\n",
    "torch.cuda.is_available(), torch.backends.cudnn.enabled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zkqghZ5gOF1L"
   },
   "outputs": [],
   "source": [
    "PATH = Path('data/salt')\n",
    "TRN_PATH = PATH/'train'\n",
    "TEST_PATH = PATH/'test'\n",
    "DISK_PATH = 'drive/tgs_salt'\n",
    "os.makedirs(DISK_PATH, exist_ok=True)\n",
    "os.makedirs(PATH, exist_ok=True)\n",
    "os.makedirs(TRN_PATH, exist_ok=True)\n",
    "os.makedirs(TEST_PATH, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W8sZ5U5xp4kO"
   },
   "outputs": [],
   "source": [
    "from albumentations import (\n",
    "    Resize,\n",
    "    HorizontalFlip,\n",
    "    VerticalFlip,\n",
    "    Compose,\n",
    "    RandomRotate90,\n",
    "    RandomSizedCrop,\n",
    "    OneOf,\n",
    "    RandomSizedCrop,\n",
    "    Rotate,\n",
    "    RandomContrast,\n",
    "    RandomGamma,\n",
    "    ElasticTransform,\n",
    "    GridDistortion, \n",
    "    OpticalDistortion,\n",
    "    RandomBrightness\n",
    ")   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 235
    },
    "colab_type": "code",
    "id": "9CJQ66L9dELB",
    "outputId": "4ead758a-b897-4118-983e-19e2d2e43923"
   },
   "outputs": [],
   "source": [
    "!wget http://files.fast.ai/models/weights.tgz\n",
    "!tar xf weights.tgz -C /content/src/fastai/old/fastai/\n",
    "!ls /content/src/fastai/old/fastai/weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 269
    },
    "colab_type": "code",
    "id": "Q5YLtNMJzSyS",
    "outputId": "a5930376-e517-4a8a-cea3-4215152ab2c1"
   },
   "outputs": [],
   "source": [
    "!kaggle competitions download -c tgs-salt-identification-challenge -p {PATH}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E5j5p85o0DD6"
   },
   "outputs": [],
   "source": [
    "!cd {TRN_PATH} && unzip -q /content/{PATH}/'train.zip'\n",
    "!cd {TEST_PATH} && unzip -q /content/{PATH}/'test.zip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "Pmbd4oZ10mXH",
    "outputId": "8fc58192-70fc-45e0-962e-082360d4f164"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "images\tmasks\n",
      "images\n"
     ]
    }
   ],
   "source": [
    "!ls {TRN_PATH}\n",
    "!ls {TEST_PATH}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Dz4H7ksT0tMo"
   },
   "source": [
    "### EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WpM9V6Pmzc3o"
   },
   "outputs": [],
   "source": [
    "depth = pd.read_csv(PATH/'depths.csv')\n",
    "train = pd.read_csv(PATH/'train.csv')\n",
    "submission = pd.read_csv(PATH/'sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U25DaYEF1L4U"
   },
   "outputs": [],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZUNgd4s50-dd"
   },
   "outputs": [],
   "source": [
    "depth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NjEG1A6F5V9k"
   },
   "outputs": [],
   "source": [
    "depth.set_index('id').z.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w4TItUxR5ND0"
   },
   "outputs": [],
   "source": [
    "depth.z.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y6-sKto01EPu"
   },
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2AgjhosP1G2s"
   },
   "outputs": [],
   "source": [
    "!ls {TRN_PATH}/images | head -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fFXbh_0F1Awx"
   },
   "outputs": [],
   "source": [
    "IMG_ID = '008a50a2ec'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3PLLX_oV1i9Z"
   },
   "outputs": [],
   "source": [
    "Image.open(TRN_IMG_PATH/f'{IMG_ID}.png').resize((32, 32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pjCMbgZk2Eu4"
   },
   "outputs": [],
   "source": [
    "Image.open(TRN_MASK_PATH/f'{IMG_ID}.png').resize((32, 32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "B-yOgm0x0QbQ"
   },
   "outputs": [],
   "source": [
    "rn34 = torchvision.models.resnet34()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GRmr-npg3JpT"
   },
   "outputs": [],
   "source": [
    "rn34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5SmuAbQh0Qfz"
   },
   "outputs": [],
   "source": [
    "rn34.layer2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eLYRkoraNbfi"
   },
   "outputs": [],
   "source": [
    "m_base = get_base_model(arch, cut, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QlUi3xT5jlxA"
   },
   "outputs": [],
   "source": [
    "m_base[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "teVt8T7aNTnE"
   },
   "outputs": [],
   "source": [
    "pretrained_state_dict = torch.load('/root/.torch/models/resnet34-333f7ec4.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qDY9tpPZNT0R"
   },
   "outputs": [],
   "source": [
    "pretrained_state_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "prL_3f_JN4F2"
   },
   "outputs": [],
   "source": [
    "arch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fkL9G_kG07dN"
   },
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eG_MV_FOOfh9"
   },
   "outputs": [],
   "source": [
    "TRN_DN = 'train/images'\n",
    "MASK_DN = 'train/masks'\n",
    "TEST_DN = 'test/images'\n",
    "TRN_PATH = PATH/TRN_DN\n",
    "MASK_PATH = PATH/MASK_DN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5Sj0FM7X_5gC"
   },
   "outputs": [],
   "source": [
    "today = datetime.datetime.now().strftime('%Y%m%d')\n",
    "\n",
    "def save_model_weights(path=DISK_PATH):\n",
    "    targ_dir = f'{path}/{today}/models'\n",
    "    os.makedirs(targ_dir, exist_ok=True)\n",
    "    for f in list(PATH.glob('**/*.h5')):\n",
    "        shutil.copy('/'.join(f.parts), targ_dir)\n",
    "        \n",
    "def load_model_weights(path=PATH, disk_path=DISK_PATH, subdir=None):\n",
    "    if subdir == None: subdir = today\n",
    "    os.makedirs(f'{path}/models', exist_ok=True)\n",
    "    src_dir = f'{disk_path}/{subdir}/models'\n",
    "    targ_dir = f'{path}/models'\n",
    "    for f in os.listdir(src_dir):\n",
    "        shutil.copy(os.path.join(src_dir, f), targ_dir)\n",
    "        \n",
    "def save_dataset(path=DISK_PATH):\n",
    "    targ_dir = f'{path}/{today}'\n",
    "    os.makedirs(targ_dir, exist_ok=True)\n",
    "    shutil.copy('test_x.pkl', targ_dir)\n",
    "    shutil.copy('train_folds.pkl', targ_dir)\n",
    "    shutil.copy('val_folds.pkl', targ_dir)\n",
    "    \n",
    "def load_dataset(path=PATH, disk_path=DISK_PATH, time_stamp=None):\n",
    "    if time_stamp == None: time_stamp = today\n",
    "    src_dir = f'{disk_path}/{time_stamp}'\n",
    "#     assert os.path.exists(src_dir) == True\n",
    "    shutil.copy(f'{src_dir}/test_x.pkl', '.')\n",
    "    shutil.copy(f'{src_dir}/train_folds.pkl', '.')\n",
    "    shutil.copy(f'{src_dir}/val_folds.pkl', '.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kVd-dF4ZO8s7"
   },
   "outputs": [],
   "source": [
    "class DepthDataset(Dataset):\n",
    "    def __init__(self,ds,dpth_dict):\n",
    "        self.dpth = dpth_dict\n",
    "        self.ds = ds\n",
    "        \n",
    "    def __getitem__(self,i):\n",
    "        val = self.ds[i]\n",
    "        return val[0],self.dpth[self.ds.fnames[i].split('/')[-1][:-4]],val[1]\n",
    "    \n",
    "class DepthDatasetV2(Dataset):\n",
    "    def __init__(self,ds,dpth_dict):\n",
    "        self.dpth = dpth_dict\n",
    "        self.ds = ds\n",
    "        \n",
    "    def __getitem__(self,i):\n",
    "        val = self.ds[i]\n",
    "        return val[0],self.dpth[self.ds.fnames[i].name[:-4]],val[1]\n",
    "      \n",
    "class MatchedFilesDataset(FilesDataset):\n",
    "    def __init__(self, fnames, y, transform, path):\n",
    "        self.y=y\n",
    "        assert(len(fnames)==len(y))\n",
    "        super().__init__(fnames, transform, path)\n",
    "        \n",
    "    def get_x(self, i): \n",
    "        return open_image(os.path.join(self.path, self.fnames[i]))\n",
    "    \n",
    "    def get_y(self, i):\n",
    "        return open_image(os.path.join(str(self.path), str(self.y[i])))\n",
    "\n",
    "    def get_c(self): return 0\n",
    "    \n",
    "class TestFilesDataset(FilesDataset):\n",
    "    def __init__(self, fnames, y, transform,flip, path):\n",
    "        self.y=y\n",
    "        self.flip = flip\n",
    "        super().__init__(fnames, transform, path)\n",
    "        \n",
    "    def get_x(self, i): \n",
    "        im = open_image(os.path.join(self.path, self.fnames[i]))\n",
    "        return np.fliplr(im) if self.flip else im\n",
    "        \n",
    "    def get_y(self, i):\n",
    "        im = open_image(os.path.join(str(self.path), str(self.y[i])))\n",
    "        return np.fliplr(im) if self.flip else im\n",
    "    def get_c(self): return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SYwzDb6CPqEw"
   },
   "outputs": [],
   "source": [
    "# trn = pd.read_csv(PATH/'train.csv')\n",
    "dpth = pd.read_csv(PATH/'depths.csv')\n",
    "c = dpth.set_index('id')\n",
    "dpth_dict = c['z'].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4mlccVfddfRm"
   },
   "outputs": [],
   "source": [
    "kf = 5\n",
    "# kf = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1qSG--oWPJNS"
   },
   "outputs": [],
   "source": [
    "x_names = np.array([Path('/'.join([TRN_DN, f])) for f in list(os.listdir(TRN_PATH))])\n",
    "y_names = np.array([Path('/'.join([MASK_DN, f])) for f in list(os.listdir(TRN_PATH))])\n",
    "test_x = np.array([Path('/'.join([TEST_DN, f.name])) for f in list((PATH/TEST_DN).iterdir())])\n",
    "f_name = [o.name for o in x_names]\n",
    "\n",
    "c = dpth.set_index('id')\n",
    "dpth_dict = c['z'].to_dict()\n",
    "\n",
    "kfold = KFold(n_splits=kf, shuffle=True, random_state=42)\n",
    "\n",
    "train_folds = []\n",
    "val_folds = []\n",
    "for idxs in kfold.split(f_name):\n",
    "    train_folds.append([f_name[idx] for idx in idxs[0]])\n",
    "    val_folds.append([f_name[idx] for idx in idxs[1]])\n",
    "\n",
    "with open('train_folds.pkl', 'wb') as fp:\n",
    "  pickle.dump(train_folds, fp)\n",
    "with open('val_folds.pkl', 'wb') as fp:\n",
    "  pickle.dump(val_folds, fp)\n",
    "with open('test_x.pkl', 'wb') as fp:\n",
    "  pickle.dump(test_x, fp)\n",
    "\n",
    "save_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XgpkySKgYiMw"
   },
   "outputs": [],
   "source": [
    "load_dataset(time_stamp='20181008') # 5 folds\n",
    "# load_dataset(time_stamp='20181016') # 10 folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E9EHO3w5SZVD"
   },
   "outputs": [],
   "source": [
    "train_folds = pickle.load(open('train_folds.pkl',mode='rb'))\n",
    "val_folds = pickle.load(open('val_folds.pkl',mode='rb'))\n",
    "test_x = pickle.load(open('test_x.pkl',mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ys7cmvFiSDy3"
   },
   "outputs": [],
   "source": [
    "def open_image(fn):\n",
    "    \"\"\" Opens an image using OpenCV given the file path.\n",
    "\n",
    "    Arguments:\n",
    "        fn: the file path of the image\n",
    "\n",
    "    Returns:\n",
    "        The image in RGB format as numpy array of floats normalized to range between 0.0 - 1.0\n",
    "    \"\"\"\n",
    "    flags = cv2.IMREAD_UNCHANGED+cv2.IMREAD_ANYDEPTH+cv2.IMREAD_ANYCOLOR\n",
    "    if not os.path.exists(fn) and not str(fn).startswith(\"http\"):\n",
    "        raise OSError('No such file or directory: {}'.format(fn))\n",
    "    elif os.path.isdir(fn) and not str(fn).startswith(\"http\"):\n",
    "        raise OSError('Is a directory: {}'.format(fn))\n",
    "#     elif isdicom(fn):\n",
    "#         slice = pydicom.read_file(fn)\n",
    "#         if slice.PhotometricInterpretation.startswith('MONOCHROME'):\n",
    "#             # Make a fake RGB image\n",
    "#             im = np.stack([slice.pixel_array]*3,-1)\n",
    "#             return im / ((1 << slice.BitsStored)-1)\n",
    "#         else:\n",
    "#             # No support for RGB yet, as it involves various color spaces.\n",
    "#             # It shouldn't be too difficult to add though, if needed.\n",
    "#             raise OSError('Unsupported DICOM image with PhotometricInterpretation=={}'.format(slice.PhotometricInterpretation))\n",
    "    else:\n",
    "        #res = np.array(Image.open(fn), dtype=np.float32)/255\n",
    "        #if len(res.shape)==2: res = np.repeat(res[...,None],3,2)\n",
    "        #return res\n",
    "        try:\n",
    "            if str(fn).startswith(\"http\"):\n",
    "                req = urllib.urlopen(str(fn))\n",
    "                image = np.asarray(bytearray(req.read()), dtype=\"uint8\")\n",
    "                im = cv2.imdecode(image, flags).astype(np.float32)/255\n",
    "            else:\n",
    "                im = cv2.imread(str(fn), flags).astype(np.float32)/255\n",
    "            if im is None: raise OSError(f'File not recognized by opencv: {fn}')\n",
    "            return cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n",
    "        except Exception as e:\n",
    "            raise OSError('Error handling image at: {}'.format(fn)) from e"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0vkz5LH-NF6i"
   },
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3Kl7WMiC9aCK"
   },
   "outputs": [],
   "source": [
    "class SCSEModule(nn.Module):\n",
    "    def __init__(self, ch, re=16):\n",
    "        super().__init__()\n",
    "        self.cSE = nn.Sequential(nn.AdaptiveAvgPool2d(1),\n",
    "                                 nn.Conv2d(ch,ch//re,1),\n",
    "                                 nn.ReLU(inplace=True),\n",
    "                                 nn.Conv2d(ch//re,ch,1),\n",
    "                                 nn.Sigmoid())\n",
    "        self.sSE = nn.Sequential(nn.Conv2d(ch,ch,1),\n",
    "                                 nn.Sigmoid())\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x * self.cSE(x) + x * self.sSE(x)\n",
    "\n",
    "\n",
    "class SaveFeatures():\n",
    "    features=None\n",
    "    def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)\n",
    "    def hook_fn(self, module, input, output): self.features = output\n",
    "    def remove(self): self.hook.remove()\n",
    "\n",
    "\n",
    "class UnetBlock(nn.Module):\n",
    "    def __init__(self, up_in, x_in, n_out):\n",
    "        super().__init__()\n",
    "        up_out = x_out = n_out//2\n",
    "        self.x_conv  = nn.Conv2d(x_in,  x_out,  1)\n",
    "        self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)\n",
    "        self.x_scse = SCSEModule(x_in)\n",
    "        self.up_scse = SCSEModule(up_in)\n",
    "        self.bn = nn.BatchNorm2d(n_out)\n",
    "        \n",
    "    def forward(self, up_p, x_p):\n",
    "        up_p = self.tr_conv(self.up_scse(up_p))\n",
    "        x_p = self.x_conv(self.x_scse(x_p))\n",
    "        cat_p = torch.cat([up_p,x_p], dim=1)\n",
    "        return self.bn(F.relu(cat_p))\n",
    "\n",
    "\n",
    "class HyperColumn(nn.Module):\n",
    "    def __init__(self, nf):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(nf, nf // 16, 3, 1, 1)\n",
    "        self.conv2 = nn.Conv2d(nf // 16, 1, 1)\n",
    "        self.bn = nn.BatchNorm2d(nf // 16)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.conv2(self.bn(self.relu(self.conv1(x))))\n",
    "\n",
    "\n",
    "class Unet34(nn.Module):\n",
    "    def __init__(self, rn):\n",
    "        super().__init__()\n",
    "        self.rn = rn\n",
    "        self.sfs = [SaveFeatures(rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(512,256,128)\n",
    "        self.up2 = UnetBlock(128,128,128)\n",
    "        self.up3 = UnetBlock(128,64,128)\n",
    "        self.up4 = UnetBlock(128,64,128)\n",
    "#         self.up5 = nn.ConvTranspose2d(128, 1, 2, stride=2)\n",
    "        self.up5 = nn.ConvTranspose2d(128, 16, 2, stride=2)\n",
    "        self.dc = nn.Conv2d(128, 16, 1)\n",
    "        self.drop = nn.Dropout2d(p=0.5, inplace=True)\n",
    "        self.logit = nn.Sequential(\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(80, 1, 1)\n",
    "        )\n",
    "        \n",
    "    def forward(self,img,depth):\n",
    "        x = F.relu(self.rn(img))\n",
    "        x1 = self.up1(x, self.sfs[3].features)\n",
    "        x2 = self.up2(x1, self.sfs[2].features)\n",
    "        x3 = self.up3(x2, self.sfs[1].features)\n",
    "        x4 = self.up4(x3, self.sfs[0].features)\n",
    "#         x = self.up5(x4)\n",
    "        x5 = self.up5(x4)\n",
    "        x = torch.cat((x5,\n",
    "                       F.interpolate(self.dc(x4),scale_factor=2,align_corners=False,mode='bilinear'),\n",
    "                       F.interpolate(self.dc(x3),scale_factor=4,align_corners=False,mode='bilinear'),\n",
    "                       F.interpolate(self.dc(x2),scale_factor=8,align_corners=False,mode='bilinear'),\n",
    "                       F.interpolate(self.dc(x1),scale_factor=16,align_corners=False,mode='bilinear')),1)\n",
    "        x = self.logit(self.drop(x))\n",
    "        return x[:,0]\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()\n",
    "\n",
    "\n",
    "class UnetModel():\n",
    "    def __init__(self,model,lr_cut,name='unet'):\n",
    "        self.model,self.name = model,name\n",
    "        self.lr_cut = lr_cut\n",
    "\n",
    "    def get_layer_groups(self, precompute):\n",
    "        lgs = list(split_by_idxs(children(self.model.rn), [self.lr_cut]))\n",
    "        return lgs + [children(self.model)[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8Me7dRQVzdKB"
   },
   "outputs": [],
   "source": [
    "def dechannels(nf):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(nf, 32, 1),\n",
    "        nn.ReLU(inplace=True)\n",
    "    )\n",
    "  \n",
    "\n",
    "class Unet50(nn.Module):\n",
    "    def __init__(self, rn):\n",
    "        super().__init__()\n",
    "        self.rn = rn\n",
    "        self.sfs = [SaveFeatures(rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(2048, 1024, 512)\n",
    "        self.up2 = UnetBlock(512, 512, 256)\n",
    "        self.up3 = UnetBlock(256, 256, 128)\n",
    "        self.up4 = UnetBlock(128, 64, 128)\n",
    "        self.up5 = nn.ConvTranspose2d(128, 1, 2, stride=2)\n",
    "#         self.up5 = nn.ConvTranspose2d(128, 128, 2, stride=2)\n",
    "#         self.d1 = dechannels(512)\n",
    "#         self.d2 = dechannels(256)\n",
    "#         self.d3 = dechannels(128)\n",
    "#         self.d4 = dechannels(128)\n",
    "#         self.d5 = dechannels(128)\n",
    "#         self.drop = nn.Dropout2d(p=0.5, inplace=True)\n",
    "#         self.logit = nn.Conv2d(160, 1, 1)\n",
    "#         self.logit = nn.Sequential(\n",
    "#             nn.Conv2d(1152, 128, 3, 1, 1),\n",
    "#             nn.ReLU(inplace=True),\n",
    "#             nn.Conv2d(128, 128, 1)\n",
    "#         )\n",
    "        \n",
    "    def forward(self,img,depth):\n",
    "        x = F.relu(self.rn(img))\n",
    "        x1 = self.up1(x, self.sfs[3].features)\n",
    "        x2 = self.up2(x1, self.sfs[2].features)\n",
    "        x3 = self.up3(x2, self.sfs[1].features)\n",
    "        x4 = self.up4(x3, self.sfs[0].features)\n",
    "        x = self.up5(x4)\n",
    "#         x5 = self.up5(x4)\n",
    "# #         x = torch.cat((x5,\n",
    "# #                        F.interpolate(x4,scale_factor=2,align_corners=False,mode='bilinear'),\n",
    "# #                        F.interpolate(x3,scale_factor=4,align_corners=False,mode='bilinear'),\n",
    "# #                        F.interpolate(x2,scale_factor=8,align_corners=False,mode='bilinear'),\n",
    "# #                        F.interpolate(x1,scale_factor=16,align_corners=False,mode='bilinear')),1)\n",
    "#         x = torch.cat((self.d5(x5),\n",
    "#                        F.interpolate(self.d4(x4),scale_factor=2,align_corners=False,mode='bilinear'),\n",
    "#                        F.interpolate(self.d3(x3),scale_factor=4,align_corners=False,mode='bilinear'),\n",
    "#                        F.interpolate(self.d2(x2),scale_factor=8,align_corners=False,mode='bilinear'),\n",
    "#                        F.interpolate(self.d1(x1),scale_factor=16,align_corners=False,mode='bilinear')),1)\n",
    "#         x = self.logit(self.drop(x))\n",
    "        return x[:,0]\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6KHDtXQ4WESz"
   },
   "outputs": [],
   "source": [
    "def get_tgs_model():\n",
    "#     f = resnet34\n",
    "    f = resnext50\n",
    "    cut,lr_cut = model_meta[f]\n",
    "    m_base = get_base(f,cut)\n",
    "#     m = to_gpu(Unet34(m_base))\n",
    "    m = to_gpu(Unet50(m_base))\n",
    "    models = UnetModel(m,lr_cut)\n",
    "    learn = ConvLearner(md, models)\n",
    "    return learn\n",
    "\n",
    "def get_base(f,cut):\n",
    "    layers = cut_model(f(True), cut)\n",
    "    return nn.Sequential(*layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "lWcwr6hJFFHU",
    "outputId": "ddca2693-ffc7-4b8b-a8e9-967e76e8de35"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "800"
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PIwVeVA9xv04"
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lZcBehwhkNb7"
   },
   "outputs": [],
   "source": [
    "load_model_weights(subdir='tags/score_8.41')\n",
    "# load_model_weights(subdir='20181016')\n",
    "# load_model_weights(subdir='20181018')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "srDUaJW0fhTn"
   },
   "outputs": [],
   "source": [
    "def lovasz_hinge_flat(logits, labels):\n",
    "    if len(labels) == 0:\n",
    "        # only void pixels, the gradients should be 0\n",
    "        return logits.sum() * 0.\n",
    "    signs = 2. * labels.float() - 1.\n",
    "    errors = (1. - logits * Variable(signs.data))\n",
    "    errors_sorted, perm = torch.sort(errors, dim=0, descending=True)\n",
    "    perm = perm.data\n",
    "    gt_sorted = labels[perm]\n",
    "    grad = lovasz_grad(gt_sorted)\n",
    "    loss = torch.dot(F.elu(errors_sorted) +1, Variable(grad.data))\n",
    "    return loss\n",
    "\n",
    "  \n",
    "def lovasz_hinge(logits, labels, per_image=True, ignore=None):\n",
    "    if per_image:\n",
    "        loss = mean(lovasz_hinge_flat(*flatten_binary_scores(log.unsqueeze(0), lab.unsqueeze(0), ignore))\n",
    "                          for log, lab in zip(logits, labels))\n",
    "    else:\n",
    "        loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "haFc4sz-PIk3"
   },
   "outputs": [],
   "source": [
    "sz = 128\n",
    "# sz = 192\n",
    "\n",
    "aug = Compose([\n",
    "    OneOf([RandomSizedCrop(min_max_height=(50, 101), height=sz, width=sz, p=0.5),\n",
    "          Resize(sz, sz, p=0.5)], p=1),\n",
    "    OneOf([VerticalFlip(p=0.5), RandomRotate90(p=0.5), HorizontalFlip(p=0.5)], p=0.5),\n",
    "#     OneOf([\n",
    "#         ElasticTransform(p=0.5, alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),\n",
    "#         GridDistortion(p=0.5),\n",
    "#         OpticalDistortion(p=1, distort_limit=2, shift_limit=0.5)                  \n",
    "#         ], p=0.8),\n",
    "#     RandomContrast(limit=0.1, p=0.5),\n",
    "    RandomBrightness(limit=0.1, p=0.5),\n",
    "#     RandomGamma(p=0.5)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AZB93ZfeO1Kz"
   },
   "outputs": [],
   "source": [
    "class Transforms():\n",
    "    def __init__(self, sz, tfms, normalizer, denorm, crop_type=CropType.CENTER,\n",
    "                 tfm_y=TfmType.NO, sz_y=None, is_trn=True):\n",
    "        if sz_y is None: sz_y = sz\n",
    "        self.sz,self.denorm,self.norm,self.sz_y = sz,denorm,normalizer,sz_y\n",
    "        crop_tfm = crop_fn_lu[crop_type](sz, tfm_y, sz_y)\n",
    "        self.tfms = tfms\n",
    "        self.tfms.append(crop_tfm)\n",
    "        if normalizer is not None: self.tfms.append(normalizer)\n",
    "        self.tfms.append(ChannelOrder(tfm_y))\n",
    "        self.channel_order = ChannelOrder(tfm_y)\n",
    "        self.trn_tfms = aug if is_trn == True else None\n",
    "\n",
    "#     def __call__(self, im, y=None): return compose(im, y, self.trn_tfms, self.tfms, self.norm)\n",
    "    def __call__(self, im, y=None): return self.compose(im, y)\n",
    "    def __repr__(self): return str(self.trn_tfms) if self.trn_tfms else str(self.tfms)\n",
    "    \n",
    "    def compose(self, im, y):\n",
    "        if self.trn_tfms:\n",
    "            augmented = self.trn_tfms(image=im, mask=y)\n",
    "            im, y = self.norm(augmented['image'], augmented['mask'])\n",
    "            return self.channel_order(im, y)\n",
    "        else:\n",
    "            for fn in self.tfms:\n",
    "                im, y =fn(im, y)\n",
    "            return im if y is None else (im, y)\n",
    "\n",
    "\n",
    "def image_gen(normalizer, denorm, sz, tfms=None, max_zoom=None, pad=0, crop_type=None,\n",
    "              tfm_y=None, sz_y=None, pad_mode=cv2.BORDER_REFLECT, scale=None, is_trn=True):\n",
    "    if tfm_y is None: tfm_y=TfmType.NO\n",
    "    if tfms is None: tfms=[]\n",
    "    elif not isinstance(tfms, collections.Iterable): tfms=[tfms]\n",
    "    if sz_y is None: sz_y = sz\n",
    "    if scale is None:\n",
    "        scale = [RandomScale(sz, max_zoom, tfm_y=tfm_y, sz_y=sz_y) if max_zoom is not None\n",
    "                 else Scale(sz, tfm_y, sz_y=sz_y)]\n",
    "    elif not is_listy(scale): scale = [scale]\n",
    "    if pad: scale.append(AddPadding(pad, mode=pad_mode))\n",
    "    if crop_type!=CropType.GOOGLENET: tfms=scale+tfms\n",
    "    return Transforms(sz, tfms, normalizer, denorm, crop_type,\n",
    "                      tfm_y=tfm_y, sz_y=sz_y, is_trn=is_trn)\n",
    "  \n",
    "  \n",
    "def tfms_from_stats(stats, sz, aug_tfms=None, max_zoom=None, pad=0, crop_type=CropType.RANDOM,\n",
    "                    tfm_y=None, sz_y=None, pad_mode=cv2.BORDER_REFLECT, norm_y=True, scale=None):\n",
    "    if aug_tfms is None: aug_tfms=[]\n",
    "    tfm_norm = Normalize(*stats, tfm_y=tfm_y if norm_y else TfmType.NO) if stats is not None else None\n",
    "    tfm_denorm = Denormalize(*stats) if stats is not None else None\n",
    "    val_crop = CropType.CENTER if crop_type in (CropType.RANDOM,CropType.GOOGLENET) else crop_type\n",
    "    val_tfm = image_gen(tfm_norm, tfm_denorm, sz, pad=pad, crop_type=val_crop,\n",
    "            tfm_y=tfm_y, sz_y=sz_y, scale=scale, is_trn=False)\n",
    "    trn_tfm = image_gen(tfm_norm, tfm_denorm, sz, pad=pad, crop_type=crop_type,\n",
    "            tfm_y=tfm_y, sz_y=sz_y, tfms=aug_tfms, max_zoom=max_zoom, pad_mode=pad_mode, scale=scale, is_trn=True)\n",
    "    return trn_tfm, val_tfm\n",
    "  \n",
    "  \n",
    "def tfms_from_model(f_model, sz, aug_tfms=None, max_zoom=None, pad=0, crop_type=CropType.RANDOM,\n",
    "                    tfm_y=None, sz_y=None, pad_mode=cv2.BORDER_REFLECT, norm_y=True, scale=None):\n",
    "    stats = inception_stats if f_model in inception_models else imagenet_stats\n",
    "    return tfms_from_stats(stats, sz, aug_tfms, max_zoom=max_zoom, pad=pad, crop_type=crop_type,\n",
    "                           tfm_y=tfm_y, sz_y=sz_y, pad_mode=pad_mode, norm_y=norm_y, scale=scale)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "b53Co8tFUCZh"
   },
   "source": [
    "### Training: resnext50 + scSE + hypercolumns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1442
    },
    "colab_type": "code",
    "id": "vaZO9Kz44xYu",
    "outputId": "98d1552c-a4a1-41b4-85b9-cfac455fb289"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 64]\n",
      "fold_id0\n",
      "5_fold_resnext50_128_scse_hcol_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b582b36f65e04eb380f8bfa3d7a05a74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.296024   0.229948   0.6635    \n",
      "    1      0.226501   0.167776   0.7175    \n",
      "    2      0.197844   0.174305   0.70425   \n",
      "    3      0.178601   0.157858   0.734     \n",
      "    4      0.159284   0.141243   0.758375  \n",
      "    5      0.156893   0.144844   0.7555    \n",
      "    6      0.139704   0.14005    0.76825   \n",
      "    7      0.129483   0.135873   0.782125  \n",
      "    8      0.117466   0.133637   0.77575   \n",
      "    9      0.107742   0.148499   0.7775    \n",
      "\n",
      "0.7821250000000001\n",
      "fold_id1\n",
      "5_fold_resnext50_128_scse_hcol_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e95165aba22348e1a9e374735a41960f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.298795   0.330592   0.577375  \n",
      "    1      0.232322   0.154958   0.684125  \n",
      "    2      0.202833   0.152708   0.708     \n",
      "    3      0.182856   0.165284   0.688875  \n",
      "    4      0.160326   0.142061   0.73875   \n",
      "    5      0.149245   0.133409   0.737375  \n",
      "    6      0.142726   0.129582   0.747375  \n",
      "    7      0.136257   0.128909   0.754375  \n",
      "    8      0.121961   0.14172    0.746     \n",
      "    9      0.111444   0.138969   0.755375  \n",
      "\n",
      "0.7553750000000001\n",
      "fold_id2\n",
      "5_fold_resnext50_128_scse_hcol_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64c3fc40887d48a6923fa8811195da65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.302976   0.230204   0.6545    \n",
      "    1      0.236865   0.176783   0.70275   \n",
      "    2      0.200898   0.158097   0.706625  \n",
      "    3      0.182162   0.147802   0.752     \n",
      "    4      0.1738     0.146175   0.73025   \n",
      "    5      0.157935   0.135695   0.758875  \n",
      "    6      0.148242   0.143257   0.76125   \n",
      "    7      0.135907   0.130015   0.768625  \n",
      "    8      0.125547   0.12942    0.761     \n",
      "    9      0.114384   0.135982   0.76275   \n",
      "\n",
      "0.7686250000000001\n",
      "fold_id3\n",
      "5_fold_resnext50_128_scse_hcol_3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a2960908675441aa81622bf7f1373c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.298731   0.179404   0.649875  \n",
      "    1      0.236641   0.150297   0.6965    \n",
      "    2      0.208887   0.130067   0.725375  \n",
      "    3      0.187371   0.116859   0.7545    \n",
      "    4      0.170393   0.127697   0.749     \n",
      "    5      0.160323   0.117837   0.75725   \n",
      "    6      0.142183   0.105238   0.78125   \n",
      "    7      0.135057   0.109958   0.776875  \n",
      "    8      0.120477   0.110117   0.794     \n",
      "    9      0.113762   0.101085   0.7865    \n",
      "\n",
      "0.794\n",
      "fold_id4\n",
      "5_fold_resnext50_128_scse_hcol_4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c65c3a78192409ea3cee73d3ad370ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.302227   0.204683   0.65575   \n",
      "    1      0.229877   0.183582   0.71175   \n",
      "    2      0.199228   0.171225   0.68475   \n",
      "    3      0.192234   0.1548     0.734625  \n",
      "    4      0.167416   0.140409   0.74525   \n",
      "    5      0.149521   0.126696   0.743125  \n",
      "    6      0.136639   0.126258   0.774875  \n",
      "    7      0.125201   0.119398   0.774875  \n",
      "    8      0.120503   0.138388   0.762875  \n",
      "    9      0.111816   0.117212   0.782875  \n",
      "\n",
      "0.782875\n"
     ]
    }
   ],
   "source": [
    "model = 'resnext50_128_scse_hcol'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=4, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = nn.BCEWithLogitsLoss()\n",
    "        if os.path.exists(pa):\n",
    "            learn.load(pa)\n",
    "        learn.fit(lrs/2,1, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])\n",
    "        \n",
    "        del learn\n",
    "        del md\n",
    "        gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 4346
    },
    "colab_type": "code",
    "id": "ROi0HiEA4xeT",
    "outputId": "3edaabab-5f31-4435-ee92-a9ad8d121d57"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 64]\n",
      "fold_id0\n",
      "5_fold_resnext50_128_scse_hcol_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc2e39db82e44742bc11cb86df4f4ac8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.977655   0.9829     0.76025   \n",
      "    1      0.984492   1.024706   0.763625  \n",
      "    2      0.94598    0.968942   0.768125  \n",
      "    3      0.91825    0.867097   0.782625  \n",
      "    4      0.842798   0.835175   0.7835    \n",
      "    5      0.811193   0.819348   0.79875   \n",
      "    6      0.784356   0.763691   0.8005    \n",
      "    7      0.763975   0.773445   0.800375  \n",
      "    8      0.707037   0.741929   0.79875   \n",
      "    9      0.676857   0.763426   0.80925   \n",
      "    10     0.686296   0.84298    0.7955    \n",
      "    11     0.778831   0.926706   0.788625  \n",
      "    12     0.823366   0.929922   0.78175   \n",
      "    13     0.846463   0.840977   0.768375  \n",
      "    14     0.80643    0.766691   0.79525   \n",
      "    15     0.785104   0.772868   0.793375  \n",
      "    16     0.771523   0.761224   0.803625  \n",
      "    17     0.73849    0.711444   0.80725   \n",
      "    18     0.698676   0.748108   0.807125  \n",
      "    19     0.664383   0.732889   0.804625  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38873a4d51c9454a903188c1d9af5995",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.642692   0.774102   0.79425   \n",
      "    1      0.670723   0.791728   0.797125  \n",
      "    2      0.701122   0.799743   0.794125  \n",
      "    3      0.695085   0.809558   0.8015    \n",
      "    4      0.709821   0.699042   0.801     \n",
      "    5      0.686103   0.869248   0.78125   \n",
      "    6      0.668086   0.807663   0.79025   \n",
      "    7      0.708722   0.761571   0.786875  \n",
      "    8      0.695549   0.760038   0.809125  \n",
      "    9      0.710776   0.817635   0.794625  \n",
      "    10     0.667159   0.799424   0.79725   \n",
      "    11     0.674622   0.791136   0.797375  \n",
      "    12     0.653431   0.732944   0.809125  \n",
      "    13     0.618604   0.733997   0.803     \n",
      "    14     0.614818   0.781419   0.8115    \n",
      "    15     0.592244   0.753463   0.8135    \n",
      "    16     0.593055   0.76313    0.818875  \n",
      "    17     0.591503   0.800476   0.791875  \n",
      "    18     0.592983   0.749201   0.816125  \n",
      "    19     0.586945   0.764278   0.814375  \n",
      "\n",
      "0.818875\n",
      "fold_id1\n",
      "5_fold_resnext50_128_scse_hcol_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d9c315bb12748429c63e7d655c9407d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.871871   1.046633   0.740375  \n",
      "    1      0.953284   1.179364   0.718875  \n",
      "    2      0.927234   0.987735   0.73175   \n",
      "    3      0.891905   0.919984   0.759875  \n",
      "    4      0.824233   0.970662   0.76025   \n",
      "    5      0.779335   0.885995   0.766125  \n",
      "    6      0.750891   0.903214   0.774125  \n",
      "    7      0.726601   0.843466   0.783625  \n",
      "    8      0.684857   0.833532   0.789125  \n",
      "    9      0.668015   0.820718   0.791125  \n",
      "    10     0.638963   0.86629    0.77425   \n",
      "    11     0.764584   1.057496   0.69975   \n",
      "    12     0.819989   1.005252   0.747375  \n",
      "    13     0.824506   0.927726   0.76375   \n",
      "    14     0.803146   0.867914   0.77575   \n",
      "    15     0.80264    0.916781   0.7715    \n",
      "    16     0.771997   0.827193   0.786625  \n",
      "    17     0.727303   0.858733   0.788625  \n",
      "    18     0.686257   0.846492   0.779625  \n",
      "    19     0.661692   0.800186   0.79375   \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55f8389c6cd8486ebb04a737a18b32c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.680326   0.828174   0.7895    \n",
      "    1      0.68143    0.84101    0.775875  \n",
      "    2      0.716512   0.921755   0.780375  \n",
      "    3      0.716995   0.865955   0.780875  \n",
      "    4      0.733966   0.841387   0.754875  \n",
      "    5      0.726975   0.876049   0.77775   \n",
      "    6      0.702523   0.901705   0.77425   \n",
      "    7      0.672928   0.878522   0.7805    \n",
      "    8      0.668419   0.859559   0.7695    \n",
      "    9      0.645093   0.857982   0.788     \n",
      "    10     0.646567   0.948428   0.7555    \n",
      "    11     0.659159   0.892936   0.783125  \n",
      "    12     0.626061   0.893832   0.787125  \n",
      "    13     0.633051   0.822176   0.787     \n",
      "    14     0.60531    0.902848   0.788625  \n",
      "    15     0.598807   0.832705   0.7875    \n",
      "    16     0.571184   0.847312   0.787625  \n",
      "    17     0.563408   0.871148   0.795625  \n",
      "    18     0.571028   0.850332   0.791875  \n",
      "    19     0.538525   0.868956   0.798375  \n",
      "\n",
      "0.7983750000000001\n",
      "fold_id2\n",
      "5_fold_resnext50_128_scse_hcol_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4c02576e160455984579f98cf21250c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.930581   1.076116   0.750875  \n",
      "    1      0.977241   0.991857   0.748     \n",
      "    2      0.963887   0.864013   0.77875   \n",
      "    3      0.921879   0.839139   0.7825    \n",
      "    4      0.891747   0.80363    0.787875  \n",
      "    5      0.842367   0.788842   0.7915    \n",
      "    6      0.809913   0.766017   0.80325   \n",
      "    7      0.745459   0.74085    0.81025   \n",
      "    8      0.717501   0.751182   0.80775   \n",
      "    9      0.672494   0.721534   0.81775   \n",
      "    10     0.687882   0.777983   0.802125  \n",
      "    11     0.835816   0.843362   0.768375  \n",
      "    12     0.861769   0.882111   0.774125  \n",
      "    13     0.916062   0.933928   0.7635    \n",
      "    14     0.860758   0.787877   0.783875  \n",
      "    15     0.817398   0.785797   0.791625  \n",
      "    16     0.799109   0.764803   0.792125  \n",
      "    17     0.763636   0.776863   0.801875  \n",
      "    18     0.734554   0.735275   0.809875  \n",
      "    19     0.69847    0.73502    0.808125  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "794f42c7ffbb49a4a48d5599479b1550",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.671766   0.759295   0.791375  \n",
      "    1      0.731133   0.811743   0.789875  \n",
      "    2      0.763409   0.791452   0.789625  \n",
      "    3      0.774678   0.789636   0.792     \n",
      "    4      0.758107   0.77808    0.772     \n",
      "    5      0.735919   0.742705   0.796625  \n",
      "    6      0.720245   0.771582   0.782125  \n",
      "    7      0.710452   0.743036   0.7825    \n",
      "    8      0.710112   0.777343   0.790875  \n",
      "    9      0.736112   0.743986   0.7875    \n",
      "    10     0.70377    0.773617   0.7935    \n",
      "    11     0.66605    0.770097   0.803     \n",
      "    12     0.662434   0.712938   0.808625  \n",
      "    13     0.652016   0.743833   0.807625  \n",
      "    14     0.649992   0.737581   0.811125  \n",
      "    15     0.638489   0.707261   0.81575   \n",
      "    16     0.596832   0.710357   0.818     \n",
      "    17     0.595883   0.704559   0.804625  \n",
      "    18     0.572071   0.705111   0.82025   \n",
      "    19     0.563588   0.69612    0.82325   \n",
      "\n",
      "0.8232499999999999\n",
      "fold_id3\n",
      "5_fold_resnext50_128_scse_hcol_3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4961bde0db542448bf37775dbc087d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.910043   0.931082   0.774375  \n",
      "    1      0.962166   0.841506   0.7835    \n",
      "    2      0.966172   0.884748   0.76575   \n",
      "    3      0.919205   0.797827   0.792     \n",
      "    4      0.887789   0.838587   0.782125  \n",
      "    5      0.833859   0.758443   0.79875   \n",
      "    6      0.787326   0.748243   0.7995    \n",
      "    7      0.768268   0.734664   0.80275   \n",
      "    8      0.721092   0.701125   0.81575   \n",
      "    9      0.678771   0.707262   0.817625  \n",
      "    10     0.686963   0.761993   0.807     \n",
      "    11     0.797779   0.845824   0.78325   \n",
      "    12     0.836203   0.781706   0.79575   \n",
      "    13     0.807488   0.871408   0.7955    \n",
      "    14     0.786293   0.773449   0.801375  \n",
      "    15     0.779008   0.795548   0.800875  \n",
      "    16     0.756668   0.742518   0.80525   \n",
      "    17     0.705088   0.703899   0.81325   \n",
      "    18     0.677209   0.693749   0.819125  \n",
      "    19     0.639317   0.679222   0.82      \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46e4957616e84cd499a4f53b689cc927",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.653376   0.719673   0.812375  \n",
      "    1      0.701078   0.751476   0.80475   \n",
      "    2      0.728884   0.854194   0.79975   \n",
      "    3      0.752242   0.724502   0.8095    \n",
      "    4      0.704703   0.742548   0.815     \n",
      "    5      0.716587   0.734281   0.811875  \n",
      "    6      0.683265   0.743123   0.806875  \n",
      "    7      0.678836   0.815633   0.793625  \n",
      "    8      0.689473   0.700409   0.81575   \n",
      "    9      0.665838   0.669478   0.82075   \n",
      "    10     0.633243   0.730008   0.816125  \n",
      "    11     0.655839   0.701937   0.816125  \n",
      "    12     0.632317   0.649175   0.82675   \n",
      "    13     0.598759   0.674421   0.83225   \n",
      "    14     0.596253   0.666588   0.829875  \n",
      "    15     0.586747   0.670523   0.827125  \n",
      "    16     0.567371   0.686732   0.824375  \n",
      "    17     0.56206    0.673798   0.8225    \n",
      "    18     0.545179   0.67912    0.82925   \n",
      "    19     0.526347   0.698485   0.827     \n",
      "\n",
      "0.8322499999999999\n",
      "fold_id4\n",
      "5_fold_resnext50_128_scse_hcol_4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "161724e6ee244e4c9dc4fc87b9a7c5f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.92871    0.959824   0.784125  \n",
      "    1      0.949663   0.872785   0.776375  \n",
      "    2      0.909849   0.817117   0.790375  \n",
      "    3      0.896153   0.804488   0.794125  \n",
      "    4      0.83174    0.835692   0.79425   \n",
      "    5      0.814119   0.780774   0.797125  \n",
      "    6      0.755571   0.759111   0.801875  \n",
      "    7      0.729774   0.78065    0.808     \n",
      "    8      0.674973   0.741045   0.813375  \n",
      "    9      0.660971   0.72706    0.8115    \n",
      "    10     0.689892   0.819716   0.786375  \n",
      "    11     0.789879   0.855918   0.789375  \n",
      "    12     0.783312   0.796657   0.79      \n",
      "    13     0.781245   0.913932   0.78375   \n",
      "    14     0.787869   0.830769   0.782625  \n",
      "    15     0.799666   0.75475    0.805375  \n",
      "    16     0.750443   0.712465   0.81      \n",
      "    17     0.708587   0.7156     0.812875  \n",
      "    18     0.680404   0.758386   0.805625  \n",
      "    19     0.647      0.703068   0.813125  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "882d2ce3e11448a7897138b5879d9d77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.625973   0.726331   0.820375  \n",
      "    1      0.670526   0.805685   0.80375   \n",
      "    2      0.701503   0.781464   0.788875  \n",
      "    3      0.69349    0.791473   0.801625  \n",
      "    4      0.682512   0.745718   0.81325   \n",
      "    5      0.69381    0.70612    0.806625  \n",
      "    6      0.665137   0.696765   0.816375  \n",
      "    7      0.641031   0.729714   0.8195    \n",
      "    8      0.632783   0.706463   0.80375   \n",
      "    9      0.634154   0.687877   0.81325   \n",
      "    10     0.626251   0.690043   0.809375  \n",
      "    11     0.591347   0.709688   0.815125  \n",
      "    12     0.575432   0.702248   0.816625  \n",
      "    13     0.587349   0.694433   0.81625   \n",
      "    14     0.573425   0.694701   0.820375  \n",
      "    15     0.584016   0.66213    0.8265    \n",
      "    16     0.547527   0.703967   0.824625  \n",
      "    17     0.548586   0.674334   0.82825   \n",
      "    18     0.534016   0.680121   0.829125  \n",
      "    19     0.507791   0.67429    0.823375  \n",
      "\n",
      "0.8291249999999999\n"
     ]
    }
   ],
   "source": [
    "model = 'resnext50_128_scse_hcol'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "        learn.fit(lrs/2,2, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "        learn.fit(lrs/3,1, wds=wd, cycle_len=20,use_clr=(20,16),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])\n",
    "        \n",
    "        del learn\n",
    "        del md\n",
    "        gc.collect()\n",
    "\n",
    "save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 3898
    },
    "colab_type": "code",
    "id": "mFTMgQkX6VtI",
    "outputId": "9c990f72-dbe6-4ece-96f6-97e445e91f25"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 64]\n",
      "fold_id0\n",
      "5_fold_resnext50_128_scse_hcol_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c64641c0ef945a182d60ab806fcfdc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.580299   0.837661   0.80625   \n",
      "    1      0.628745   0.803514   0.798375  \n",
      "    2      0.619702   0.796008   0.80575   \n",
      "    3      0.628366   0.750468   0.81175   \n",
      "    4      0.623308   0.748861   0.802     \n",
      "    5      0.650031   0.795647   0.803     \n",
      "    6      0.646449   0.763365   0.81125   \n",
      "    7      0.636074   0.781185   0.811375  \n",
      "    8      0.623231   0.742726   0.809875  \n",
      "    9      0.602579   0.753169   0.813375  \n",
      "    10     0.585192   0.776042   0.817375  \n",
      "    11     0.576949   0.790247   0.8175    \n",
      "    12     0.562295   0.796752   0.813875  \n",
      "    13     0.56244    0.795689   0.788875  \n",
      "    14     0.575644   0.806945   0.8145    \n",
      "    15     0.58317    0.765245   0.81525   \n",
      "    16     0.560825   0.782946   0.805875  \n",
      "    17     0.562333   0.797728   0.79975   \n",
      "    18     0.562816   0.761856   0.816375  \n",
      "    19     0.553854   0.775206   0.8135    \n",
      "    20     0.546581   0.714756   0.82125   \n",
      "    21     0.541287   0.770479   0.8095    \n",
      "    22     0.518532   0.805129   0.813625  \n",
      "    23     0.521006   0.732549   0.806625  \n",
      "    24     0.495842   0.853394   0.811375  \n",
      "    25     0.493223   0.838361   0.81125   \n",
      "    26     0.482846   0.831322   0.814375  \n",
      "    27     0.483749   0.788495   0.820125  \n",
      "    28     0.496672   0.792871   0.82125   \n",
      "    29     0.497796   0.762518   0.823875  \n",
      "    30     0.49775    0.79859    0.81975   \n",
      "    31     0.491748   0.779934   0.822375  \n",
      "    32     0.475641   0.790661   0.816625  \n",
      "    33     0.4794     0.78795    0.82175   \n",
      "    34     0.475329   0.764116   0.81825   \n",
      "    35     0.455563   0.822148   0.815     \n",
      "    36     0.451707   0.795527   0.821375  \n",
      "    37     0.444007   0.772276   0.8155    \n",
      "    38     0.435822   0.779683   0.817375  \n",
      "    39     0.445946   0.780952   0.814     \n",
      "\n",
      "0.823875\n",
      "fold_id1\n",
      "5_fold_resnext50_128_scse_hcol_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2faf773816a409e9d909ded6b7f2313",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.565228   0.871326   0.785875  \n",
      "    1      0.601723   0.882529   0.79425   \n",
      "    2      0.642919   0.897614   0.78075   \n",
      "    3      0.637186   0.878214   0.777875  \n",
      "    4      0.609784   0.895229   0.79075   \n",
      "    5      0.609371   0.926091   0.7865    \n",
      "    6      0.601144   0.836737   0.788125  \n",
      "    7      0.630633   0.871732   0.7835    \n",
      "    8      0.611182   0.890666   0.7895    \n",
      "    9      0.592919   0.831647   0.80075   \n",
      "    10     0.567077   0.936684   0.786875  \n",
      "    11     0.553323   0.966901   0.78875   \n",
      "    12     0.559024   0.944055   0.77      \n",
      "    13     0.590418   0.850013   0.796     \n",
      "    14     0.573935   0.856166   0.7945    \n",
      "    15     0.567616   0.860534   0.79075   \n",
      "    16     0.569173   0.960924   0.781     \n",
      "    17     0.558296   0.893143   0.791     \n",
      "    18     0.542129   0.934519   0.794125  \n",
      "    19     0.547132   0.855531   0.80075   \n",
      "    20     0.53702    0.854191   0.796875  \n",
      "    21     0.538296   0.912877   0.787625  \n",
      "    22     0.526649   0.917397   0.800375  \n",
      "    23     0.502509   0.896211   0.7885    \n",
      "    24     0.521145   0.898083   0.796375  \n",
      "    25     0.491927   0.923525   0.802625  \n",
      "    26     0.501542   0.864817   0.794     \n",
      "    27     0.494032   0.884823   0.796375  \n",
      "    28     0.490451   0.899924   0.795875  \n",
      "    29     0.481898   0.880166   0.784     \n",
      "    30     0.472968   0.895765   0.797375  \n",
      "    31     0.462385   0.878445   0.793625  \n",
      "    32     0.455645   0.896859   0.795     \n",
      "    33     0.469805   0.894238   0.7995    \n",
      "    34     0.455159   0.913979   0.801375  \n",
      "    35     0.457995   0.898335   0.8015    \n",
      "    36     0.453348   0.900837   0.804875  \n",
      "    37     0.446832   0.902242   0.793     \n",
      "    38     0.453402   0.907094   0.801375  \n",
      "    39     0.434806   0.921633   0.80875   \n",
      "\n",
      "0.80875\n",
      "fold_id2\n",
      "5_fold_resnext50_128_scse_hcol_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2914de385caf4b7d879001cf1ca135af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.577075   0.786943   0.80825   \n",
      "    1      0.609053   0.768417   0.804375  \n",
      "    2      0.656391   0.73859    0.812125  \n",
      "    3      0.652841   0.757531   0.797     \n",
      "    4      0.655581   0.736351   0.809875  \n",
      "    5      0.623059   0.69868    0.812375  \n",
      "    6      0.627167   0.77681    0.7915    \n",
      "    7      0.623288   0.750636   0.802125  \n",
      "    8      0.620909   0.76361    0.807125  \n",
      "    9      0.628333   0.755546   0.793625  \n",
      "    10     0.63122    0.744164   0.79475   \n",
      "    11     0.63828    0.720994   0.8095    \n",
      "    12     0.607952   0.712268   0.804125  \n",
      "    13     0.588277   0.729939   0.80725   \n",
      "    14     0.585946   0.705454   0.8255    \n",
      "    15     0.58386    0.691177   0.81425   \n",
      "    16     0.561116   0.713686   0.812625  \n",
      "    17     0.553486   0.771892   0.818875  \n",
      "    18     0.553116   0.707916   0.817875  \n",
      "    19     0.565598   0.728426   0.815625  \n",
      "    20     0.554347   0.766951   0.819625  \n",
      "    21     0.536056   0.723741   0.820125  \n",
      "    22     0.536135   0.724242   0.821625  \n",
      "    23     0.554331   0.767052   0.8075    \n",
      "    24     0.550846   0.718899   0.82025   \n",
      "    25     0.539729   0.703877   0.81175   \n",
      "    26     0.517849   0.694621   0.820125  \n",
      "    27     0.504886   0.708674   0.821625  \n",
      "    28     0.502828   0.726042   0.82175   \n",
      "    29     0.498927   0.701111   0.81025   \n",
      "    30     0.495155   0.715683   0.8185    \n",
      "    31     0.507423   0.730976   0.82175   \n",
      "    32     0.51031    0.709125   0.814125  \n",
      "    33     0.504614   0.7094     0.81775   \n",
      "    34     0.500099   0.694097   0.825375  \n",
      "    35     0.482972   0.689517   0.824625  \n",
      "    36     0.484635   0.679306   0.82225   \n",
      "    37     0.481951   0.681534   0.828     \n",
      "    38     0.475027   0.686403   0.825625  \n",
      "    39     0.483564   0.695852   0.82175   \n",
      "\n",
      "0.828\n",
      "fold_id3\n",
      "5_fold_resnext50_128_scse_hcol_3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "019ac79c6d2e4e56ac325d5d3dd9a97a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.575313   0.695703   0.823625  \n",
      "    1      0.624102   0.780891   0.813625  \n",
      "    2      0.648184   0.697116   0.81825   \n",
      "    3      0.651478   0.729888   0.80325   \n",
      "    4      0.633732   0.689429   0.816875  \n",
      "    5      0.6391     0.707463   0.821875  \n",
      "    6      0.608808   0.734975   0.821     \n",
      "    7      0.583592   0.727842   0.8085    \n",
      "    8      0.607585   0.754664   0.806125  \n",
      "    9      0.602211   0.745517   0.82925   \n",
      "    10     0.58275    0.690118   0.826     \n",
      "    11     0.577038   0.690472   0.816875  \n",
      "    12     0.585021   0.703931   0.820875  \n",
      "    13     0.573149   0.737506   0.822875  \n",
      "    14     0.593045   0.701834   0.823125  \n",
      "    15     0.576374   0.722945   0.8195    \n",
      "    16     0.578671   0.718012   0.824125  \n",
      "    17     0.576396   0.737476   0.825     \n",
      "    18     0.562349   0.740334   0.82075   \n",
      "    19     0.562596   0.703988   0.83375   \n",
      "    20     0.545302   0.742709   0.824875  \n",
      "    21     0.536242   0.730253   0.821375  \n",
      "    22     0.538357   0.726912   0.834375  \n",
      "    23     0.524991   0.705785   0.82975   \n",
      "    24     0.539796   0.718887   0.821625  \n",
      "    25     0.528741   0.74735    0.825125  \n",
      "    26     0.52838    0.697965   0.833875  \n",
      "    27     0.515824   0.80159    0.82975   \n",
      "    28     0.519962   0.729874   0.813625  \n",
      "    29     0.506576   0.729921   0.8205    \n",
      "    30     0.488176   0.70798    0.823     \n",
      "    31     0.487228   0.747718   0.824625  \n",
      "    32     0.477002   0.720598   0.829625  \n",
      "    33     0.468923   0.748624   0.833625  \n",
      "    34     0.487961   0.757478   0.833125  \n",
      "    35     0.470121   0.742304   0.813875  \n",
      "    36     0.48037    0.745087   0.831875  \n",
      "    37     0.476722   0.772283   0.8305    \n",
      "    38     0.456916   0.735131   0.8195    \n",
      "    39     0.461469   0.748882   0.8215    \n",
      "\n",
      "0.834375\n",
      "fold_id4\n",
      "5_fold_resnext50_128_scse_hcol_4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6269952b5694a12a836f74645dd9bbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.546006   0.717454   0.8235    \n",
      "    1      0.563938   0.751986   0.812375  \n",
      "    2      0.598719   0.75113    0.815     \n",
      "    3      0.588652   0.798206   0.822     \n",
      "    4      0.599703   0.805366   0.8095    \n",
      "    5      0.591598   0.734947   0.82075   \n",
      "    6      0.596523   0.733508   0.811375  \n",
      "    7      0.597403   0.742025   0.816125  \n",
      "    8      0.588319   0.729897   0.80425   \n",
      "    9      0.57466    0.743337   0.808375  \n",
      "    10     0.590704   0.786838   0.8205    \n",
      "    11     0.557619   0.687391   0.81425   \n",
      "    12     0.539545   0.704461   0.82875   \n",
      "    13     0.543599   0.810272   0.820625  \n",
      "    14     0.549854   0.70434    0.819     \n",
      "    15     0.541127   0.726442   0.820625  \n",
      "    16     0.547006   0.710707   0.805625  \n",
      "    17     0.556314   0.704044   0.825     \n",
      "    18     0.533182   0.687741   0.81375   \n",
      "    19     0.511729   0.7148     0.834     \n",
      "    20     0.509799   0.690751   0.8375    \n",
      "    21     0.496054   0.709561   0.83325   \n",
      "    22     0.499565   0.745653   0.8235    \n",
      "    23     0.506426   0.723755   0.824375  \n",
      "    24     0.487424   0.73846    0.828125  \n",
      "    25     0.485281   0.725011   0.822     \n",
      "    26     0.466737   0.791306   0.827125  \n",
      "    27     0.46954    0.759082   0.827625  \n",
      "    28     0.479826   0.683398   0.8315    \n",
      "    29     0.463535   0.724489   0.831     \n",
      "    30     0.460917   0.712501   0.831     \n",
      "    31     0.441873   0.723946   0.829     \n",
      "    32     0.435788   0.724188   0.82975   \n",
      "    33     0.438317   0.718923   0.821375  \n",
      "    34     0.442253   0.724379   0.82475   \n",
      "    35     0.425489   0.734597   0.824125  \n",
      "    36     0.44013    0.728877   0.820125  \n",
      "    37     0.443834   0.744366   0.828     \n",
      "    38     0.426803   0.729525   0.827875  \n",
      "    39     0.435825   0.735564   0.822     \n",
      "\n",
      "0.8375\n"
     ]
    }
   ],
   "source": [
    "model = 'resnext50_128_scse_hcol'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "#         learn.fit(lrs/4,1, wds=wd, cycle_len=20,use_clr=(20,16),best_save_name=pa)\n",
    "        learn.fit(lrs/4,1, wds=wd, cycle_len=40,use_clr=(20,32),best_save_name=pa)\n",
    "  \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])\n",
    "        \n",
    "        del learn\n",
    "        del md\n",
    "        gc.collect()\n",
    "\n",
    "save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1996
    },
    "colab_type": "code",
    "id": "H7YZnN_HiiuR",
    "outputId": "ee40d675-4739-4c54-f4dd-a0a55119aad3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 64]\n",
      "fold_id0\n",
      "5_fold_resnext50_128_scse_hcol_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56e79297e25847f0a15aff07aa418769",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.464316   0.82814    0.816375  \n",
      "    1      0.503966   0.973039   0.808125  \n",
      "    2      0.524512   0.875324   0.802125  \n",
      "    3      0.552585   0.802356   0.8155    \n",
      "    4      0.524045   0.898251   0.8165    \n",
      "    5      0.519004   0.884085   0.815     \n",
      "    6      0.512701   0.761209   0.816625  \n",
      "    7      0.520394   0.781913   0.80575   \n",
      "    8      0.501048   0.748156   0.815875  \n",
      "    9      0.504658   0.800892   0.808875  \n",
      "    10     0.491141   0.764409   0.813375  \n",
      "    11     0.480288   0.845715   0.823     \n",
      "    12     0.482994   0.864056   0.817125  \n",
      "    13     0.474893   0.844127   0.818     \n",
      "    14     0.475831   0.82016    0.822375  \n",
      "    15     0.462766   0.80165    0.816     \n",
      "    16     0.459729   0.801812   0.822625  \n",
      "    17     0.452857   0.825043   0.821125  \n",
      "    18     0.432667   0.798102   0.819125  \n",
      "    19     0.448399   0.799831   0.82425   \n",
      "\n",
      "0.8242499999999999\n",
      "fold_id1\n",
      "5_fold_resnext50_128_scse_hcol_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8aea5543371447dbafac57bda9d368dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.460871   0.923814   0.807875  \n",
      "    1      0.463133   0.902192   0.804     \n",
      "    2      0.487417   0.895001   0.79675   \n",
      "    3      0.49072    0.918243   0.7825    \n",
      "    4      0.51323    0.852118   0.80025   \n",
      "    5      0.496024   0.872492   0.789875  \n",
      "    6      0.506847   0.856714   0.802125  \n",
      "    7      0.506169   0.917013   0.785     \n",
      "    8      0.480601   0.872236   0.7935    \n",
      "    9      0.478031   0.859447   0.79525   \n",
      "    10     0.465379   0.893305   0.79625   \n",
      "    11     0.455413   0.920751   0.802125  \n",
      "    12     0.450254   0.921361   0.797125  \n",
      "    13     0.437148   0.914842   0.802     \n",
      "    14     0.445062   0.855681   0.797125  \n",
      "    15     0.438345   0.867074   0.79825   \n",
      "    16     0.445464   0.852298   0.80075   \n",
      "    17     0.439749   0.874568   0.79525   \n",
      "    18     0.434257   0.861378   0.809375  \n",
      "    19     0.41873    0.868665   0.806125  \n",
      "\n",
      "0.809375\n",
      "fold_id2\n",
      "5_fold_resnext50_128_scse_hcol_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "758e5881ec094853afe6f93c9588adcd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.46378    0.7404     0.81825   \n",
      "    1      0.523088   0.696791   0.81175   \n",
      "    2      0.561582   0.741509   0.806625  \n",
      "    3      0.559633   0.743049   0.79175   \n",
      " 70%|███████   | 35/50 [00:57<00:22,  1.49s/it, loss=0.537]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-2c0f2f03bdbb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0;31m#         learn.fit(lrs/4,1, wds=wd, cycle_len=20,use_clr=(20,16),best_save_name=pa)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcycle_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0muse_clr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbest_save_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/src/fastai/old/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    300\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 302\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/src/fastai/old/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    247\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 249\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/src/fastai/old/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, visualize, **kwargs)\u001b[0m\n\u001b[1;32m    139\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/src/fastai/old/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_scale\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32massert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_scale\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxtra\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mupdate_fp32_grads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp32_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_scale\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \"\"\"\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = 'resnext50_128_scse_hcol'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "#         learn.fit(lrs/4,1, wds=wd, cycle_len=20,use_clr=(20,16),best_save_name=pa)\n",
    "        learn.fit(lrs/5,1, wds=wd, cycle_len=20,use_clr=(20,16),best_save_name=pa)\n",
    "  \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])\n",
    "        \n",
    "        del learn\n",
    "        del md\n",
    "        gc.collect()\n",
    "\n",
    "# save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R2qbSFsNYJqO"
   },
   "outputs": [],
   "source": [
    "save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "TagiWKJ16V2z",
    "outputId": "4c898c92-63b4-4bb0-af97-306f6113ac9a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.823875, 0.80875, 0.828, 0.834375, 0.8375], 0.8265)"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bst_acc,np.mean(bst_acc)#With 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FswF_Yw14xk2"
   },
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YxJSW5-QTk8b"
   },
   "source": [
    "### Training: resnext50 + scSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jXY7FrEcrLHj"
   },
   "outputs": [],
   "source": [
    "model = 'resnext50_128_scse'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=4, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = nn.BCEWithLogitsLoss()\n",
    "        if os.path.exists(pa):\n",
    "            learn.load(pa)\n",
    "        learn.fit(lrs/2,1, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6mFbdBCuWbgc"
   },
   "outputs": [],
   "source": [
    "model = 'resnext50_128_scse'\n",
    "arch = resnet34\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "#         learn.fit(lrs/3,2, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "        learn.fit(lrs/3,3, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UpWadtxBDMYv"
   },
   "outputs": [],
   "source": [
    "model = 'resnext50_128_scse'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "#         learn.fit(lrs/3,1, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "        learn.fit(lrs/3,1, wds=wd, cycle_len=20,use_clr=(20,10),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 5830
    },
    "colab_type": "code",
    "id": "qrPUrDI6pieX",
    "outputId": "8c65fb9e-b0e7-422a-f6ab-0e82cce4b87b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 64]\n",
      "fold_id0\n",
      "5_fold_resnext50_128_scse_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a222f50d2b1423db51268c7a6edcbcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.533707   0.76256    0.8175    \n",
      "    1      0.594565   0.858352   0.798875  \n",
      "    2      0.601546   0.788414   0.812125  \n",
      "    3      0.602312   0.763777   0.816625  \n",
      "    4      0.597194   0.756633   0.816125  \n",
      "    5      0.56907    0.767038   0.81325   \n",
      "    6      0.564023   0.746384   0.82175   \n",
      "    7      0.579532   0.831695   0.81125   \n",
      "    8      0.551373   0.760372   0.810875  \n",
      "    9      0.54197    0.845872   0.809625  \n",
      "    10     0.539457   0.726901   0.813625  \n",
      "    11     0.528602   0.7233     0.81875   \n",
      "    12     0.520925   0.772535   0.822875  \n",
      "    13     0.518783   0.764549   0.8235    \n",
      "    14     0.513134   0.735043   0.830125  \n",
      "    15     0.510657   0.727242   0.826625  \n",
      "    16     0.493573   0.704724   0.82725   \n",
      "    17     0.481923   0.73212    0.829125  \n",
      "    18     0.471987   0.698919   0.828     \n",
      "    19     0.463161   0.711147   0.82725   \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "076489da5da74916b79331f943e5b625",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.479967   0.827555   0.821125  \n",
      "    1      0.506115   0.7792     0.807     \n",
      "    2      0.521105   0.704616   0.823375  \n",
      "    3      0.531661   0.759747   0.813625  \n",
      "    4      0.527358   0.76719    0.8165    \n",
      "    5      0.599692   0.78576    0.8095    \n",
      "    6      0.579218   0.723072   0.814375  \n",
      "    7      0.553373   0.735349   0.81025   \n",
      "    8      0.522601   0.74302    0.82075   \n",
      "    9      0.51634    0.725892   0.8255    \n",
      "    10     0.507244   0.736419   0.810875  \n",
      "    11     0.495299   0.747552   0.8235    \n",
      "    12     0.517736   0.709832   0.8205    \n",
      "    13     0.52503    0.702308   0.82      \n",
      "    14     0.517949   0.763225   0.822875  \n",
      "    15     0.481496   0.760829   0.825875  \n",
      "    16     0.467406   0.75723    0.818     \n",
      "    17     0.48397    0.719035   0.821125  \n",
      "    18     0.480835   0.74827    0.81975   \n",
      "    19     0.464327   0.758697   0.822625  \n",
      "    20     0.460972   0.714269   0.8275    \n",
      "    21     0.451309   0.763296   0.824375  \n",
      "    22     0.443526   0.740101   0.82525   \n",
      "    23     0.432715   0.74261    0.829875  \n",
      "    24     0.459113   0.808962   0.8255    \n",
      "    25     0.449143   0.793144   0.827     \n",
      "    26     0.460752   0.774263   0.824     \n",
      "    27     0.448464   0.768298   0.823625  \n",
      "    28     0.429408   0.746062   0.826875  \n",
      "    29     0.418083   0.767724   0.830375  \n",
      "    30     0.420722   0.716629   0.82625   \n",
      "    31     0.426644   0.73742    0.829625  \n",
      "    32     0.425052   0.748321   0.82975   \n",
      "    33     0.419258   0.759544   0.829125  \n",
      "    34     0.411915   0.743406   0.82825   \n",
      "    35     0.4213     0.755378   0.82675   \n",
      "    36     0.404763   0.741769   0.831125  \n",
      "    37     0.398306   0.767223   0.825625  \n",
      "    38     0.396879   0.733644   0.825625  \n",
      "    39     0.413759   0.745653   0.826     \n",
      "\n",
      "0.831125\n",
      "fold_id1\n",
      "5_fold_resnext50_128_scse_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "756be4b59566488e876b7880d74f2849",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.467102   0.833521   0.792125  \n",
      "    1      0.514805   0.866801   0.770625  \n",
      "    2      0.512858   0.888006   0.78125   \n",
      "    3      0.523391   0.872697   0.785625  \n",
      "    4      0.541148   0.89274    0.7915    \n",
      "    5      0.534021   0.817045   0.794     \n",
      "    6      0.511217   0.827669   0.80725   \n",
      "    7      0.504867   0.870196   0.796375  \n",
      "    8      0.482823   0.813677   0.80225   \n",
      "    9      0.470734   0.859631   0.79925   \n",
      "    10     0.50188    0.805922   0.790625  \n",
      "    11     0.50007    0.865748   0.790125  \n",
      "    12     0.490715   0.808949   0.79925   \n",
      "    13     0.467356   0.822104   0.80175   \n",
      "    14     0.459628   0.823243   0.8005    \n",
      "    15     0.442727   0.81513    0.789125  \n",
      "    16     0.433787   0.801445   0.803375  \n",
      "    17     0.428531   0.821544   0.79725   \n",
      "    18     0.427835   0.775848   0.80675   \n",
      "    19     0.41361    0.812282   0.805     \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc7d0f393faf4c2ea88f2dee53b90a55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.443954   0.821174   0.795625  \n",
      "    1      0.473285   0.845159   0.792     \n",
      "    2      0.465905   0.985209   0.785     \n",
      "    3      0.456353   0.974172   0.786875  \n",
      "    4      0.467155   0.920761   0.79475   \n",
      "    5      0.470211   0.900441   0.79725   \n",
      "    6      0.455301   0.96447    0.792     \n",
      "    7      0.47351    0.90089    0.78875   \n",
      "    8      0.463646   0.856061   0.794875  \n",
      "    9      0.443074   0.860522   0.792     \n",
      "    10     0.429617   0.824053   0.798375  \n",
      "    11     0.445916   0.875289   0.796375  \n",
      "    12     0.453059   0.872382   0.798125  \n",
      "    13     0.447237   0.872192   0.801125  \n",
      "    14     0.44465    0.867648   0.800875  \n",
      "    15     0.447324   0.889119   0.795     \n",
      "    16     0.449389   0.843531   0.800875  \n",
      "    17     0.438213   0.823923   0.8075    \n",
      "    18     0.434376   0.847579   0.7975    \n",
      "    19     0.443935   0.843054   0.801625  \n",
      "    20     0.422225   0.849103   0.80025   \n",
      "    21     0.414883   0.866637   0.7965    \n",
      "    22     0.415798   0.935687   0.791625  \n",
      "    23     0.401541   0.8712     0.800375  \n",
      "    24     0.408011   0.873514   0.793375  \n",
      "    25     0.38908    0.84675    0.798875  \n",
      "    26     0.392715   0.857604   0.79125   \n",
      "    27     0.376516   0.896894   0.792125  \n",
      "    28     0.369762   0.973169   0.786875  \n",
      "    29     0.385661   0.912236   0.798875  \n",
      "    30     0.38374    0.864333   0.801625  \n",
      "    31     0.36842    0.901064   0.806     \n",
      "    32     0.378856   0.892412   0.800125  \n",
      "    33     0.377577   0.88206    0.801     \n",
      "    34     0.35965    0.856498   0.79675   \n",
      "    35     0.365454   0.8875     0.798875  \n",
      "    36     0.371453   0.905397   0.80125   \n",
      "    37     0.365299   0.876433   0.7985    \n",
      "    38     0.358871   0.897993   0.799625  \n",
      "    39     0.360378   0.904567   0.797375  \n",
      "\n",
      "0.8075\n",
      "fold_id2\n",
      "5_fold_resnext50_128_scse_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a25f7884982344d68ae68729bf31f33e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.485156   0.756766   0.82075   \n",
      "    1      0.517438   0.832651   0.814375  \n",
      "    2      0.518396   0.735615   0.815     \n",
      "    3      0.522302   0.761968   0.806625  \n",
      "    4      0.540298   0.807242   0.81675   \n",
      "    5      0.536279   0.714934   0.81575   \n",
      "    6      0.510553   0.791743   0.797375  \n",
      "    7      0.511985   0.739001   0.817625  \n",
      "    8      0.50142    0.754067   0.81825   \n",
      "    9      0.496233   0.749199   0.818     \n",
      "    10     0.479415   0.739346   0.821375  \n",
      "    11     0.4808     0.692525   0.826     \n",
      "    12     0.473838   0.731722   0.823     \n",
      "    13     0.467366   0.737328   0.82275   \n",
      "    14     0.476735   0.719575   0.81975   \n",
      "    15     0.448495   0.726288   0.820375  \n",
      "    16     0.434496   0.749208   0.82025   \n",
      "    17     0.426012   0.765359   0.825     \n",
      "    18     0.435108   0.743209   0.8205    \n",
      "    19     0.433077   0.765477   0.82475   \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "354e58e006fe4d679dab82ffe007f623",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.443196   0.750643   0.826125  \n",
      "    1      0.463696   0.795019   0.81775   \n",
      "    2      0.475888   0.753367   0.819375  \n",
      "    3      0.478167   0.75828    0.814875  \n",
      "    4      0.471563   0.778656   0.813625  \n",
      "    5      0.483763   0.742092   0.81475   \n",
      "    6      0.479101   0.748993   0.81875   \n",
      "    7      0.476277   0.780733   0.810875  \n",
      "    8      0.498361   0.69888    0.82975   \n",
      "    9      0.486142   0.739697   0.81925   \n",
      "    10     0.457164   0.801588   0.825375  \n",
      "    11     0.455916   0.745798   0.815875  \n",
      "    12     0.457613   0.76608    0.818625  \n",
      "    13     0.45103    0.773046   0.8205    \n",
      "    14     0.447433   0.764471   0.813     \n",
      "    15     0.436356   0.770631   0.82025   \n",
      "    16     0.440452   0.798791   0.824875  \n",
      "    17     0.43147    0.753223   0.825125  \n",
      "    18     0.432604   0.828235   0.8245    \n",
      "    19     0.435804   0.800897   0.81925   \n",
      "    20     0.416318   0.782468   0.82325   \n",
      "    21     0.425427   0.760889   0.82525   \n",
      "    22     0.414975   0.779994   0.817375  \n",
      "    23     0.41498    0.785758   0.82025   \n",
      "    24     0.423222   0.784227   0.81475   \n",
      "    25     0.417059   0.753324   0.823875  \n",
      "    26     0.406137   0.769448   0.825125  \n",
      "    27     0.398413   0.820555   0.824125  \n",
      "    28     0.393823   0.790955   0.82025   \n",
      "    29     0.404996   0.793106   0.827375  \n",
      "    30     0.388835   0.761253   0.827     \n",
      "    31     0.38426    0.748922   0.829625  \n",
      "    32     0.380705   0.752186   0.829125  \n",
      "    33     0.378552   0.722271   0.827     \n",
      "    34     0.378998   0.761381   0.83075   \n",
      "    35     0.380338   0.760387   0.828625  \n",
      "    36     0.375267   0.79428    0.825375  \n",
      "    37     0.37371    0.777205   0.830375  \n",
      "    38     0.374267   0.77222    0.8245    \n",
      "    39     0.372311   0.783963   0.824625  \n",
      "\n",
      "0.83075\n",
      "fold_id3\n",
      "5_fold_resnext50_128_scse_3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f568474ab13a402f984119d9c0490b8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.496033   0.709309   0.81325   \n",
      "    1      0.521143   0.718663   0.82275   \n",
      "    2      0.54441    0.835224   0.81875   \n",
      "    3      0.549383   0.731907   0.821625  \n",
      "    4      0.539594   0.64432    0.82425   \n",
      "    5      0.534994   0.674456   0.82175   \n",
      "    6      0.529376   0.71022    0.82975   \n",
      "    7      0.502961   0.68506    0.824875  \n",
      "    8      0.498022   0.686693   0.830625  \n",
      "    9      0.500132   0.725364   0.822125  \n",
      "    10     0.50281    0.675001   0.831875  \n",
      "    11     0.512494   0.684257   0.836875  \n",
      "    12     0.484919   0.696663   0.83475   \n",
      "    13     0.469144   0.709017   0.837875  \n",
      "    14     0.469601   0.659997   0.834375  \n",
      "    15     0.46119    0.692171   0.834125  \n",
      "    16     0.453164   0.69287    0.83025   \n",
      "    17     0.443224   0.70075    0.834625  \n",
      "    18     0.44307    0.694192   0.83      \n",
      "    19     0.428427   0.728258   0.83575   \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5eaf0a7d85f94e4aac0efe9e6d732595",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.422124   0.726504   0.8295    \n",
      "    1      0.462113   0.731205   0.8245    \n",
      "    2      0.486406   0.728322   0.829125  \n",
      "    3      0.487896   0.765298   0.828875  \n",
      "    4      0.484002   0.75814    0.8245    \n",
      "    5      0.491141   0.697839   0.827125  \n",
      "    6      0.477768   0.709401   0.822625  \n",
      "    7      0.483948   0.766132   0.82175   \n",
      "    8      0.490673   0.695626   0.830375  \n",
      "    9      0.472208   0.698568   0.81975   \n",
      "    10     0.466838   0.67741    0.827375  \n",
      "    11     0.463749   0.797842   0.81275   \n",
      "    12     0.448588   0.679358   0.830125  \n",
      "    13     0.469459   0.723771   0.828375  \n",
      "    14     0.452238   0.672009   0.830875  \n",
      "    15     0.447767   0.722987   0.830875  \n",
      "    16     0.43008    0.666037   0.831     \n",
      "    17     0.430931   0.738625   0.827625  \n",
      "    18     0.427639   0.6799     0.83275   \n",
      "    19     0.430004   0.645604   0.82625   \n",
      "    20     0.440587   0.710382   0.825625  \n",
      "    21     0.427687   0.758092   0.8225    \n",
      "    22     0.424524   0.714262   0.830125  \n",
      "    23     0.415856   0.734838   0.829375  \n",
      "    24     0.420022   0.709158   0.834375  \n",
      "    25     0.40909    0.71966    0.838125  \n",
      "    26     0.392587   0.719539   0.836625  \n",
      "    27     0.393285   0.732876   0.82375   \n",
      "    28     0.389524   0.695115   0.83525   \n",
      "    29     0.391245   0.734108   0.829375  \n",
      "    30     0.384423   0.73317    0.833625  \n",
      "    31     0.373081   0.731024   0.830875  \n",
      "    32     0.389755   0.754126   0.83025   \n",
      "    33     0.38323    0.752325   0.83575   \n",
      "    34     0.375477   0.733968   0.83      \n",
      "    35     0.371067   0.743891   0.83375   \n",
      "    36     0.382864   0.747606   0.833875  \n",
      "    37     0.370416   0.745533   0.83125   \n",
      "    38     0.362293   0.751041   0.834     \n",
      "    39     0.365368   0.740053   0.833625  \n",
      "\n",
      "0.8381250000000001\n",
      "fold_id4\n",
      "5_fold_resnext50_128_scse_4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6ae02badebd42bcbf21aaabae7cd6f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.47414    0.722221   0.827     \n",
      "    1      0.491114   0.712578   0.82425   \n",
      "    2      0.51879    0.73661    0.815875  \n",
      "    3      0.540288   0.692429   0.826375  \n",
      "    4      0.534986   0.700633   0.827     \n",
      "    5      0.518888   0.720056   0.826625  \n",
      "    6      0.510556   0.819519   0.814     \n",
      "    7      0.498047   0.740013   0.821125  \n",
      "    8      0.467538   0.726275   0.821875  \n",
      "    9      0.465198   0.740374   0.82125   \n",
      "    10     0.454122   0.729831   0.822125  \n",
      "    11     0.465724   0.69371    0.827625  \n",
      "    12     0.459633   0.711312   0.82275   \n",
      "    13     0.44714    0.709925   0.8275    \n",
      "    14     0.450566   0.755889   0.81475   \n",
      "    15     0.433707   0.689775   0.836125  \n",
      "    16     0.419188   0.687338   0.8345    \n",
      "    17     0.419773   0.675787   0.834     \n",
      "    18     0.422193   0.681697   0.83225   \n",
      "    19     0.414564   0.676884   0.833125  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95d90582589d4934be7383e615f1dbd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=40), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.416673   0.743005   0.82225   \n",
      "    1      0.451756   0.796281   0.822     \n",
      "    2      0.47234    0.772019   0.824125  \n",
      "    3      0.479158   0.759053   0.807875  \n",
      "    4      0.46425    0.769957   0.815375  \n",
      "    5      0.464162   0.772572   0.815875  \n",
      "    6      0.460326   0.737922   0.820625  \n",
      "    7      0.468473   0.750393   0.818375  \n",
      "    8      0.458967   0.704921   0.831375  \n",
      "    9      0.439696   0.740977   0.832625  \n",
      "    10     0.445156   0.723011   0.824875  \n",
      "    11     0.434864   0.725554   0.833625  \n",
      "    12     0.422101   0.704498   0.828375  \n",
      "    13     0.429801   0.699662   0.82975   \n",
      "    14     0.432414   0.763216   0.81775   \n",
      "    15     0.423432   0.72516    0.822875  \n",
      "    16     0.425951   0.732884   0.832625  \n",
      "    17     0.41282    0.699522   0.830125  \n",
      "    18     0.40507    0.741123   0.8335    \n",
      "    19     0.406175   0.729666   0.8355    \n",
      "    20     0.423782   0.71638    0.82775   \n",
      "    21     0.407595   0.724523   0.82375   \n",
      "    22     0.400728   0.743059   0.82825   \n",
      "    23     0.397718   0.712238   0.828375  \n",
      "    24     0.400653   0.771733   0.81775   \n",
      "    25     0.406241   0.706003   0.83425   \n",
      "    26     0.384568   0.726671   0.830625  \n",
      "    27     0.395504   0.695435   0.83125   \n",
      "    28     0.392318   0.724178   0.831125  \n",
      "    29     0.381853   0.684155   0.83      \n",
      "    30     0.375725   0.688363   0.83775   \n",
      "    31     0.385391   0.703829   0.836875  \n",
      "    32     0.362482   0.684262   0.84275   \n",
      "    33     0.354302   0.701925   0.83675   \n",
      "    34     0.365955   0.715952   0.83425   \n",
      "    35     0.367506   0.688223   0.8365    \n",
      "    36     0.361231   0.706321   0.839     \n",
      "    37     0.35691    0.693329   0.835625  \n",
      "    38     0.363043   0.709944   0.832125  \n",
      "    39     0.350134   0.699815   0.834625  \n",
      "\n",
      "0.8427500000000001\n"
     ]
    }
   ],
   "source": [
    "model = 'resnext50_128_scse'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/100,lr/10,lr])\n",
    "\n",
    "        learn.unfreeze()        \n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "#         learn.fit(lrs/3,1, wds=wd, cycle_len=10,use_clr=(20,8),best_save_name=pa)\n",
    "        learn.fit(lrs/4,1, wds=wd, cycle_len=20,use_clr=(20,16),best_save_name=pa)\n",
    "        learn.fit(lrs/5,1, wds=wd, cycle_len=40,use_clr=(20,32),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])\n",
    "\n",
    "save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 2218
    },
    "colab_type": "code",
    "id": "ga4JYphvCPyd",
    "outputId": "b589a96a-cf8d-40d5-b11f-a32c260d5aa5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 64]\n",
      "fold_id0\n",
      "5_fold_resnext50_128_scse_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f19b564db994e6687c8b43ab8f917ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.352572   0.727342   0.824     \n",
      "    1      0.369447   0.676443   0.83      \n",
      "    2      0.387457   0.791503   0.825     \n",
      "    3      0.351119   0.766687   0.827625  \n",
      "    4      0.36598    0.731143   0.83275   \n",
      "    5      0.364853   0.723793   0.8195    \n",
      "    6      0.381143   0.841033   0.829125  \n",
      "    7      0.364594   0.680827   0.83075   \n",
      "    8      0.349329   0.828338   0.817375  \n",
      "    9      0.35967    0.746246   0.834     \n",
      "    10     0.355637   0.749717   0.83125   \n",
      "    11     0.353103   0.816089   0.83225   \n",
      "    12     0.33925    0.821926   0.83525   \n",
      "    13     0.347948   0.751935   0.83625   \n",
      "    14     0.325283   0.842732   0.833625  \n",
      "    15     0.332459   0.769492   0.83075   \n",
      "    16     0.329128   0.773248   0.83475   \n",
      "    17     0.326993   0.837361   0.837     \n",
      "    18     0.33641    0.736122   0.83775   \n",
      "    19     0.32058    0.791872   0.8365    \n",
      "\n",
      "0.8377500000000001\n",
      "fold_id1\n",
      "5_fold_resnext50_128_scse_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68bf61a771034be3aa0b6105e3a7b93f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.329473   0.895153   0.791     \n",
      "    1      0.345151   0.857987   0.802125  \n",
      "    2      0.346842   0.957962   0.787625  \n",
      "    3      0.359232   0.914885   0.78725   \n",
      "    4      0.35904    0.945004   0.79475   \n",
      "    5      0.364206   0.90916    0.79025   \n",
      "    6      0.352084   0.907089   0.803125  \n",
      "    7      0.340385   0.884313   0.79975   \n",
      "    8      0.331111   0.935045   0.80525   \n",
      "    9      0.327103   0.912742   0.80725   \n",
      "    10     0.334112   0.903979   0.799875  \n",
      "    11     0.339094   0.997254   0.79625   \n",
      "    12     0.336058   0.903867   0.802     \n",
      "    13     0.330904   0.945125   0.79575   \n",
      "    14     0.328544   0.888129   0.798625  \n",
      "    15     0.322872   0.931454   0.7975    \n",
      "    16     0.320054   0.917544   0.803375  \n",
      "    17     0.318902   0.909964   0.8055    \n",
      "    18     0.318903   0.864422   0.80375   \n",
      "    19     0.319137   0.889656   0.803875  \n",
      "\n",
      "0.8072499999999999\n",
      "fold_id2\n",
      "5_fold_resnext50_128_scse_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01b404522f664b47ad0a31f6acfa7b1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.334564   0.907237   0.827     \n",
      "    1      0.356609   0.77755    0.821125  \n",
      "    2      0.359052   0.832501   0.83225   \n",
      "    3      0.365679   0.80743    0.821     \n",
      "    4      0.361891   0.790572   0.814625  \n",
      "    5      0.343242   0.863644   0.823625  \n",
      "    6      0.339013   0.821969   0.828     \n",
      "    7      0.334319   0.807286   0.818625  \n",
      "    8      0.327691   0.864193   0.828125  \n",
      "    9      0.328807   0.806045   0.81325   \n",
      "    10     0.34561    0.859754   0.82925   \n",
      "    11     0.334322   0.871514   0.828     \n",
      "    12     0.320915   0.854894   0.823     \n",
      "    13     0.30964    0.863667   0.8175    \n",
      "    14     0.3198     0.776783   0.81925   \n",
      "    15     0.312285   0.829952   0.83175   \n",
      "    16     0.315182   0.793502   0.8215    \n",
      "    17     0.304829   0.833349   0.82575   \n",
      "    18     0.305454   0.814216   0.833375  \n",
      "    19     0.309406   0.807178   0.82525   \n",
      "\n",
      "0.8333750000000001\n",
      "fold_id3\n",
      "5_fold_resnext50_128_scse_3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e9d4a962e6644e7a36ccbb7aa2958ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.303919   0.798046   0.829125  \n",
      "    1      0.330936   0.707061   0.83625   \n",
      "    2      0.344996   0.750156   0.829125  \n",
      "    3      0.344157   0.924753   0.833125  \n",
      "    4      0.326378   0.760323   0.833875  \n",
      "    5      0.328699   0.870832   0.825625  \n",
      "    6      0.331057   0.820294   0.83625   \n",
      "    7      0.34331    0.716414   0.838625  \n",
      "    8      0.335372   0.924066   0.83275   \n",
      "    9      0.323994   0.739209   0.83925   \n",
      "    10     0.334121   0.761376   0.834     \n",
      "    11     0.322592   0.740011   0.83975   \n",
      "    12     0.330691   0.819215   0.837125  \n",
      "    13     0.313002   0.78467    0.842875  \n",
      "    14     0.318578   0.795953   0.83725   \n",
      "    15     0.316326   0.765305   0.842625  \n",
      "    16     0.310763   0.723363   0.84525   \n",
      "    17     0.296692   0.825693   0.84075   \n",
      "    18     0.29445    0.823472   0.842625  \n",
      "    19     0.297867   0.751366   0.83925   \n",
      "\n",
      "0.8452500000000001\n",
      "fold_id4\n",
      "5_fold_resnext50_128_scse_4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a025c00a503477aa9cfcba1f1ab3cd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.32856    0.730436   0.834625  \n",
      "    1      0.335845   0.899728   0.837125  \n",
      "    2      0.348538   0.913843   0.8295    \n",
      "    3      0.33758    0.769679   0.8235    \n",
      "    4      0.346654   0.802165   0.835625  \n",
      "    5      0.339364   0.746379   0.83825   \n",
      "    6      0.343552   0.756106   0.828125  \n",
      "    7      0.331363   0.792459   0.8335    \n",
      "    8      0.31865    0.735218   0.836125  \n",
      "    9      0.325964   0.810198   0.839     \n",
      "    10     0.311348   0.795061   0.841875  \n",
      "    11     0.321848   0.720627   0.83475   \n",
      "    12     0.311088   0.884234   0.8375    \n",
      "    13     0.304974   0.749801   0.836     \n",
      "    14     0.305366   0.71613    0.83375   \n",
      "    15     0.314715   0.693329   0.84125   \n",
      "    16     0.318737   0.787573   0.83675   \n",
      "    17     0.304981   0.747158   0.83425   \n",
      "    18     0.302457   0.767533   0.834     \n",
      "    19     0.299126   0.759485   0.833625  \n",
      "\n",
      "0.841875\n"
     ]
    }
   ],
   "source": [
    "model = 'resnext50_128_scse'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/150,lr/20,lr])\n",
    "\n",
    "        learn.unfreeze()\n",
    "        learn.bn_freeze(True)\n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "        learn.fit(lrs/10,1, wds=wd, cycle_len=20,use_clr=(20,20),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])\n",
    "\n",
    "        if i < kf - 1:\n",
    "            del learn\n",
    "            del md\n",
    "            gc.collect()\n",
    "        \n",
    "save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1730
    },
    "colab_type": "code",
    "id": "ITXgqK7jjdZL",
    "outputId": "0b712303-2fb6-4477-9985-1c67849c6563"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 64]\n",
      "fold_id0\n",
      "5_fold_resnext50_128_scse_0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80b17c7b21dd4f91b3af5c1ad0da18a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.352473   0.759049   0.829625  \n",
      "    1      0.363571   0.880663   0.826     \n",
      "    2      0.35344    0.807012   0.824875  \n",
      "    3      0.344199   0.811331   0.82425   \n",
      "    4      0.341503   0.780056   0.826375  \n",
      "    5      0.349747   0.841056   0.820375  \n",
      "    6      0.350928   0.883612   0.823375  \n",
      "    7      0.346245   0.738591   0.819875  \n",
      "    8      0.339464   0.790437   0.826375  \n",
      "    9      0.339998   0.844214   0.826875  \n",
      "    10     0.347131   0.780476   0.83      \n",
      "    11     0.341404   0.798782   0.829     \n",
      "    12     0.339125   0.789289   0.829875  \n",
      "    13     0.334993   0.868688   0.827625  \n",
      "    14     0.328118   0.880147   0.83275   \n",
      "    15     0.306373   0.860532   0.831625  \n",
      "    16     0.304504   0.829747   0.830875  \n",
      "    17     0.300264   0.837326   0.832625  \n",
      "    18     0.317846   0.774235   0.836375  \n",
      "    19     0.310489   0.787925   0.833875  \n",
      "\n",
      "0.8363749999999999\n",
      "fold_id1\n",
      "5_fold_resnext50_128_scse_1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40209053452647b2858ae4b6ad13be2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.327861   0.839905   0.7995    \n",
      "    1      0.332155   0.878513   0.80625   \n",
      "    2      0.327904   0.842675   0.78325   \n",
      "    3      0.323279   1.045407   0.800125  \n",
      "    4      0.34445    0.838635   0.794375  \n",
      "    5      0.336768   0.906733   0.8       \n",
      "    6      0.325074   0.921411   0.804625  \n",
      "    7      0.331317   0.867638   0.802125  \n",
      "    8      0.336123   0.883447   0.796875  \n",
      "    9      0.331685   0.851472   0.80425   \n",
      "    10     0.315916   0.857878   0.790625  \n",
      "    11     0.317591   1.001778   0.802125  \n",
      "    12     0.322167   0.868535   0.79975   \n",
      "    13     0.324085   0.90506    0.800625  \n",
      "    14     0.309266   0.923913   0.795375  \n",
      "    15     0.311752   0.973415   0.79975   \n",
      "    16     0.300871   0.953171   0.8045    \n",
      "    17     0.290202   0.962913   0.800125  \n",
      "    18     0.297091   0.981357   0.80025   \n",
      "    19     0.299129   0.941654   0.802625  \n",
      "\n",
      "0.80625\n",
      "fold_id2\n",
      "5_fold_resnext50_128_scse_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91f55feb2d0541cd800bc6b78ff08643",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.337081   0.823228   0.83475   \n",
      "    1      0.344549   0.913375   0.831625  \n",
      "    2      0.329596   0.828047   0.831125  \n",
      "    3      0.325009   0.777783   0.8285    \n",
      "    4      0.33951    0.821626   0.808375  \n",
      "    5      0.338317   0.940252   0.820875  \n",
      "    6      0.330713   0.791693   0.83      \n",
      "    7      0.332787   0.776885   0.82775   \n",
      "    8      0.332073   0.806915   0.82675   \n",
      "    9      0.329484   0.799763   0.817375  \n",
      "    10     0.329214   0.797196   0.82475   \n",
      "    11     0.31536    0.82193    0.832625  \n",
      "    12     0.321162   0.843901   0.827375  \n",
      "    13     0.310198   0.90338    0.828375  \n",
      "    14     0.292332   0.859527   0.8295    \n",
      "    15     0.291901   0.843471   0.8235    \n",
      "    16     0.296143   0.85044    0.819875  \n",
      "    17     0.302732   0.847719   0.827     \n",
      "    18     0.303203   0.819016   0.827875  \n",
      "    19     0.303959   0.815134   0.8255    \n",
      "\n",
      "0.8347500000000001\n",
      "fold_id3\n",
      "5_fold_resnext50_128_scse_3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28a3862558874de39a8aed9c8a9bf792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=20, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   my_eval    \n",
      "    0      0.316421   0.866826   0.83525   \n",
      "    1      0.337529   0.992894   0.828     \n",
      "    2      0.355061   0.758108   0.83175   \n",
      "    3      0.341312   0.793649   0.82925   \n",
      "    4      0.3637     0.741804   0.83575   \n",
      "    5      0.330458   0.743388   0.839375  \n",
      "    6      0.334183   0.73421    0.841375  \n",
      "    7      0.30901    0.724088   0.83675   \n",
      "    8      0.322369   0.710755   0.830625  \n",
      "    9      0.312626   0.907613   0.841     \n",
      "    10     0.314347   0.75424    0.838875  \n",
      "    11     0.287641   0.762639   0.836125  \n",
      "    12     0.291472   0.738226   0.844875  \n",
      "    13     0.294669   0.741112   0.841375  \n",
      "    14     0.294856   0.747115   0.842375  \n",
      "    15     0.295397   0.75156    0.843125  \n",
      "    16     0.287345   0.807438   0.841875  \n",
      "    17     0.295449   0.754707   0.843625  \n",
      " 82%|████████▏ | 41/50 [00:54<00:10,  1.22s/it, loss=0.29] "
     ]
    }
   ],
   "source": [
    "model = 'resnext50_128_scse'\n",
    "arch = resnext50\n",
    "bst_acc=[]\n",
    "use_clr_min=20\n",
    "use_clr_div=10\n",
    "aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "\n",
    "szs = [(128,64)]\n",
    "for sz,bs in szs:\n",
    "    print([sz,bs])\n",
    "    for i in range(kf) :\n",
    "        print(f'fold_id{i}')\n",
    "        \n",
    "        trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[i]])\n",
    "        trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[i]])\n",
    "        val_x = [f'{TRN_DN}/{o}' for o in val_folds[i]]\n",
    "        val_y = [f'{MASK_DN}/{o}' for o in val_folds[i]]\n",
    "        \n",
    "        tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "        datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "        md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "        denorm = md.trn_ds.denorm\n",
    "        md.trn_dl.dataset = DepthDataset(md.trn_ds,dpth_dict)\n",
    "        md.val_dl.dataset = DepthDataset(md.val_ds,dpth_dict)\n",
    "        md.test_dl.dataset = DepthDataset(md.test_ds,dpth_dict)\n",
    "        learn = get_tgs_model() \n",
    "        learn.opt_fn = optim.Adam\n",
    "        learn.metrics=[my_eval]\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "\n",
    "        lr=1e-2\n",
    "        wd=1e-7\n",
    "        lrs = np.array([lr/150,lr/20,lr])\n",
    "\n",
    "        learn.unfreeze()\n",
    "        learn.bn_freeze(True)\n",
    "        learn.crit = lovasz_hinge\n",
    "        learn.load(pa)\n",
    "        learn.fit(lrs/10,1, wds=wd, cycle_len=20,use_clr=(20,20),best_save_name=pa)\n",
    "               \n",
    "        learn.load(pa)        \n",
    "        #Calcuating mean iou score\n",
    "        v_targ = md.val_ds.ds[:][1]\n",
    "        v_preds = np.zeros((len(v_targ),sz,sz))     \n",
    "        v_pred = learn.predict()\n",
    "        v_pred = to_np(torch.sigmoid(torch.from_numpy(v_pred)))\n",
    "        p = ((v_pred)>0.5).astype(np.uint8)\n",
    "        bst_acc.append(intersection_over_union_thresholds(v_targ,p))\n",
    "        print(bst_acc[-1])\n",
    "\n",
    "        if i < kf - 1:\n",
    "            del learn\n",
    "            del md\n",
    "            gc.collect()\n",
    "        \n",
    "save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lTpda6ZVbF0b"
   },
   "outputs": [],
   "source": [
    "save_model_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xP7Kn5wYs3NG"
   },
   "outputs": [],
   "source": [
    "load_model_weights(time_stamp='20181010')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "lz43bF8JWcKC",
    "outputId": "3efd88a3-094c-448b-b5f2-5b6ae26e2a4e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.835, 0.8065000000000001, 0.8315, 0.8447499999999999, 0.8440000000000001],\n",
       " 0.8323500000000001)"
      ]
     },
     "execution_count": 74,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bst_acc,np.mean(bst_acc)#With 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rF8h3iawT6PI"
   },
   "source": [
    "### Submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LqbkAvBMMKR6"
   },
   "outputs": [],
   "source": [
    "# sz = 128\n",
    "# bs = 64\n",
    "# model = 'resnext50_128_scse'\n",
    "\n",
    "# # trn_x = np.array([f'{TRN_DN}/{o}' for o in train_folds[0]])\n",
    "# # trn_y = np.array([f'{MASK_DN}/{o}' for o in train_folds[0]])\n",
    "# # val_x = [f'{TRN_DN}/{o}' for o in val_folds[0]]\n",
    "# # val_y = [f'{MASK_DN}/{o}' for o in val_folds[0]]\n",
    "        \n",
    "# aug_tfms = [RandomFlip(tfm_y=TfmType.CLASS)]\n",
    "# tfms = tfms_from_model(arch, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "# datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "# md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "# # tfms = tfms_from_model(resnet34, sz=sz, pad=0,crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)\n",
    "# # datasets = ImageData.get_ds(MatchedFilesDataset, (trn_x,trn_y), (val_x,val_y), tfms,test=test_x,path=PATH)\n",
    "# # md = ImageData(PATH, datasets, bs, num_workers=16, classes=None)\n",
    "\n",
    "# learn = get_tgs_model() \n",
    "# learn.opt_fn = optim.Adam\n",
    "# learn.crit = lovasz_hinge\n",
    "# learn.metrics=[my_eval]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 252
    },
    "colab_type": "code",
    "id": "ms-zAHP2WcGb",
    "outputId": "e1af7781-86a4-4154-82cf-4d6eb7e55897"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef25c63e5456401eb8edde4089530451",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5_fold_resnext50_128_scse_0\n",
      "5_fold_resnext50_128_scse_1\n",
      "5_fold_resnext50_128_scse_2\n",
      "5_fold_resnext50_128_scse_3\n",
      "5_fold_resnext50_128_scse_4\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c581f68713c40a090b48c2b02908785",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5_fold_resnext50_128_scse_0\n",
      "5_fold_resnext50_128_scse_1\n",
      "5_fold_resnext50_128_scse_2\n",
      "5_fold_resnext50_128_scse_3\n",
      "5_fold_resnext50_128_scse_4\n",
      "\n"
     ]
    }
   ],
   "source": [
    "preds = np.zeros(shape = (18000,sz,sz))\n",
    "for o in [True,False]:\n",
    "    md.test_dl.dataset = TestFilesDataset(test_x,test_x,tfms[1],flip=o,path=PATH)\n",
    "    md.test_dl.dataset = DepthDatasetV2(md.test_dl.dataset,dpth_dict)\n",
    "    \n",
    "    for i in tqdm_notebook(range(kf)):\n",
    "        pa = f'{kf}_fold_{model}_{i}'\n",
    "        print(pa)\n",
    "        learn.load(pa)\n",
    "        pred = learn.predict(is_test=True)\n",
    "        pred = to_np(torch.sigmoid(torch.from_numpy(pred)))    \n",
    "        for im_idx,im in enumerate(pred):\n",
    "                preds[im_idx] += np.fliplr(im) if o else im\n",
    "        del pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZHKkmKrqWb7W"
   },
   "outputs": [],
   "source": [
    "# plt.imshow(((preds[16]/10)>0.5).astype(np.uint8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pe59_TSGWb4K"
   },
   "outputs": [],
   "source": [
    "p = [cv2.resize(o/10,dsize=(101,101)) for o in preds]\n",
    "p = [(o>0.5).astype(np.uint8) for o in p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qu8Gin-NTMef",
    "outputId": "acaf9176-75b0-4b53-d2f5-ece8890b9862"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(101, 101)"
      ]
     },
     "execution_count": 67,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "7ZRdo1lAWb08",
    "outputId": "615f7553-793c-4f59-d6a1-22ccdedbbf6d"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a28697675af4cbc90c8dcd4afba37d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "pred_dict = {id_.name[:-4]:RLenc(p[i]) for i,id_ in tqdm_notebook(enumerate(test_x))}\n",
    "sub = pd.DataFrame.from_dict(pred_dict,orient='index')\n",
    "sub.index.names = ['id']\n",
    "sub.columns = ['rle_mask']\n",
    "sub.to_csv('submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "et8aAjFVWbxh",
    "outputId": "3af2bb8f-dcdc-4187-a48e-9dabfaebf9f4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully submitted to TGS Salt Identification Challenge"
     ]
    }
   ],
   "source": [
    "!kaggle competitions submit -c tgs-salt-identification-challenge -f submission.csv -m \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 386
    },
    "colab_type": "code",
    "id": "38KczbM-WbuM",
    "outputId": "0a1ab856-ec21-45ad-803e-fa3dc06ec9dc"
   },
   "outputs": [],
   "source": [
    "!kaggle competitions submissions -c tgs-salt-identification-challenge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7rbnFVohWbqd"
   },
   "outputs": [],
   "source": [
    "!fusermount -u drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "emlLQBDaj6fk"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "Dz4H7ksT0tMo"
   ],
   "name": "TGS_Salt_Unet50_5-Fold_scSE_score_8.42.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}