research/basic_usage.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic usage\n",
    "This notebook shows the process of finding point of radial mode."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import warnings\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "sys.path.insert(0, os.path.join('..'))\n",
    "\n",
    "from radial import calculate_metrics\n",
    "\n",
    "from radial.batchflow import Dataset, FilesIndex\n",
    "from radial.batchflow.models.tf import ResNet\n",
    "\n",
    "from radial.core import RadialBatch\n",
    "from radial.core import calculate_results\n",
    "from radial.pipelines import create_preprocess_pipeline,\\\n",
    "                             create_train_pipeline,\\\n",
    "                             create_predict_pipeline\n",
    "from radial.preprocessing import drop_outliers, xls_to_npz\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "plt.style.use('ggplot')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Preprocessing.\n",
    "#### 1.1. Raw data to NPZ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Files with train/test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "xls_to_npz(path_from=['./test_90.xlsx', './train_311.xlsx'],\n",
    "           path_to=['./data_splited/test', './data_splited/train'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The same files without splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "xls_to_npz(path_from=['./test_90.xlsx', './train_311.xlsx'],\n",
    "           path_to=['./data'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2. Removing outliers.\n",
    "We are dropping the outliers using Isolation Forest algorithm. ```path_from``` is a path to data you want to filter out. ```path_to``` is a path where result will be saved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 311/311 [00:49<00:00,  6.71it/s]\n",
      "100%|██████████| 90/90 [00:13<00:00,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "drop_outliers(path_from=['data_splited/train', 'data_splited/test'],\n",
    "              path_to=['data_splited/train_isol', 'data_splited/test_isol'], rewrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 401/401 [01:07<00:00,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "drop_outliers(path_from=['data'],\n",
    "              path_to=['data_isol'], rewrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Training process\n",
    "\n",
    "Config creation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_SAMPLES = 100\n",
    "\n",
    "model_config = {\n",
    "    'inputs': dict(points=dict(shape=(N_SAMPLES, 2)),\n",
    "                   targets=dict(name='target', shape=1)),\n",
    "    'initial_block/inputs': 'points',\n",
    "    'head': dict(layout='f',\n",
    "                 units=1),\n",
    "    'body/num_blocks': [1, 1, 1],\n",
    "    'body/filters': [2, 4, 8],\n",
    "    'initial_block/filters': 8,\n",
    "    'loss': 'mse',\n",
    "    'optimizer': 'Adam'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the data and create Dataset objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_train = Dataset(FilesIndex(path=\"./data_splited/train_isol/*\"), batch_class=RadialBatch)\n",
    "ds_test = Dataset(FilesIndex(path=\"./data_splited/test_isol/*\"), batch_class=RadialBatch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create two pipelines and run training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "prep_pipeline = create_preprocess_pipeline(N_SAMPLES, np.random.random)\n",
    "train_pipeline = create_train_pipeline(ResNet, model_config, prep_pipeline) << ds_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800/1800 [13:57<00:00,  2.21it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<radial.batchflow.batchflow.pipeline.Pipeline at 0x7f05bd043d68>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pipeline.run(50, n_epochs=300, drop_last=True, shuffle=True, bar=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Evaluation\n",
    "Firstly create a predict pipeline and run it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pipeline = create_predict_pipeline(prep_pipeline, train_pipeline) << ds_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 899/900 [00:16<00:00, 53.86it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<radial.batchflow.batchflow.pipeline.Pipeline at 0x7f05b90f4b70>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pipeline.run(1, n_epochs=10, drop_last=True, bar=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate statistics and plot the histogram of MAPE error."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "results, _ = calculate_results(test_pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "true = []\n",
    "pred = []\n",
    "for key in results.keys():\n",
    "    results[key]['pred'] = np.mean(results[key]['pred'])\n",
    "    true.append(results[key]['true'][0])\n",
    "    pred.append(results[key]['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 0.148\n",
      "MAPE: 0.09533\n",
      "Percentage of error less than 30%: 97.78%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAF2CAYAAAC/AOuxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcFPWd//HXt6cBGRpBaK4ZEFAUBBQVNJ6I4oG32TXfqDGJ15LLrJtj3ZjkFzeuSdxort24SVhjNIcmX4nriXgrxogiyqEoHijKJYwcOgzXTH1/f1Qj49DDNNNHVXe/n49HP5iq/nbV51vTw7uruupbxnuPiIiIRCsRdQEiIiKiQBYREYkFBbKIiEgMKJBFRERiQIEsIiISAwpkERGRGFAgi8hHjDGTjDHeGDO4g3beGHNhqeoSqQYKZJEiMsbckgmvO7M8d3bmueZ2XvuAMabFGHN6lucuyrx2++M9Y8x9xpgDW7V5ok2b7Y+XCtC1QcD0XBoaYy40xmjAA5EOKJBFiu8d4AxjzIA2878ALM32AmPMMGAScAMwtZ3lthAG4yDgHKA/8KAxplerNre1arP9cVwn+vAx3vtV3vvN+S5HRHZQIIsU3+vAbOCi7TOMMXsDJwG/a+c1lwEzgF8Apxhj6rM1ygTjKu/9M8DXCAP3iFZNNrVqs/3xfg41H2CMmWWMaTLGLDLGnNr6ybaHrI0xlxljXjHGbDbGrM28drAxZhLwh1av8caYW3JYv0jVUSCLlMY04DJjjMlMXwY8SpY9ZGNMErgEuMV7vwJ4HLg0h3VsyvzbJf9yuQH4ITAOeBb4izFmr2wNjTHjgV8DPwJGEu6B/z7z9N+ByzM/b99Dv6IA9YlUHAWySGlMB/oAk4wxNYSBO62dtmcS/m0+kJm+FbjUGNPu36sxph/wfeAD4LlWT33eGNPY5vHLHOr9vvd+pvf+deBbQE/g8Hba7g1sBO7y3i/13i/03t/kvV/mvd8KbICP7c1vyGH9IlVHgSxSApnvW/8A/BNwOpAE7m2n+VTgT9777Sd73QX0Ak5t065me8gCq4ERwLne+9Wt2vwfcHCbx/dzKHleq9rfI/y+uu134Ns9DCwB3jLG/NkYM9UYk85hHSLSSjLqAkSqyDTgBWAI8Dvv/bYdR7BDmZO5TgZONsa0PrRbQxjU97ea10IYsB5Y7b3/MMs6P/Dev9GJWrdmmZf1A7z3vtEYMwE4GjgR+CLwY2PMZO/93E6sW6QqKZBFSsR7v8gYM4cwuC5qp9llwCvAeW3mjwH+ZIyp994vb7XMzoRtwXnvW4BZwCxjzNXAIuACYC6ZcDfG1GTaiUgWCmSR0joF2MN7v7btE61O5vqV9/6lNs+9DPyE8OSua3Zjfd2NMQPbzAvaHNbOizHmbGAfwkBeA4wnPAqwKNPkrcy/Zxlj/kZ45ndjodYvUin0HbJICXnvm7KFccaZhGchuyyv88AddHByVxYXACvbPJbsVtEdW0dY+0zgNeDHwLXe+98CeO/nEF6+9RvC77pzOalMpOqY8O9cREREoqQ9ZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJgVJfh6xTukVEpNqYjptEMDDIihUrirLcdDpNQ0NDUZZdDtR/9V/9r97+g7ZBXPtfV1eXc1sdshYREYkBBbKIiEgMKJBFRERiQIEsIiISAwpkERGRGFAgi4iIxECHlz1Za28GzgBWO+fGtpr/VeArQAtwv3PuyqJVKSIiUuFy2UO+BZjSeoa19njgbGCcc24McEPhSxMREakeHQayc24W0PaG6l8CrnPObcm0WV2E2kRERKpGZ0fq2h841lr7A2Az8E3n3JxsDa21U4GpAM450ul0J1e5a8lksmjLLgfqv/qv/ldv/0HboBL639lATgJ9gCOAwwBnrd3HObfTWNXOuWnAtMykL9bQZnEdNq1U1H/1X/2v3v6DtkFc+1+KoTOXAXc657xz7jkgAMr7o4mIiEiEOhvIdwHHA1hr9we6AvH7aCIiIlImcrns6XZgEpC21i4DrgZuBm621r4EbAU+n+1wtZRWMGvmTvMSE6dkaVle6xIRqQYdBrJz7vx2nrqwwLWIiIhULY3UJSIiEgMKZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJAQWyiIhIDCiQRUREYkCBLCIiEgMKZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJAQWyiIhIDCiQRUREYkCBLCIiEgMKZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJAQWyiIhIDCiQRUREYkCBLCIiEgMKZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJgWRHDay1NwNnAKudc2PbPPcN4Aagn3OuoTglioiIVL5c9pBvAaa0nWmtHQKcDLxT4JpERESqToeB7JybBazN8tTPgCsBX+iiREREqk2Hh6yzsdaeDSx3zs231nbUdiowFcA5Rzqd7swqO5RMJou27HKQTCZJpVI7za8t0jZpKuG6cqHfv/pfzf0HbYNK6P9uB7K1thb4NuHh6g4556YB0zKTvqGhOF81p9NpirXscpBOp2lsbNxpflORtklQwnXlQr9/9b+a+w/aBnHtf11dXc5tO3OW9b7AcGC+tfZtYDDwgrV2YCeWJSIiInRiD9k5txDov306E8oTdJa1iIhI53W4h2ytvR14BhhprV1mrb20+GWJiIhUlw73kJ1z53fw/LCCVSMiIlKlNFKXiIhIDCiQRUREYkCBLCIiEgMKZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJAQWyiIhIDCiQRUREYkCBLCIiEgMKZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJAQWyiIhIDCiQRUREYkCBLCIiEgMKZBERkRhQIIuIiMSAAllERCQGFMgiIiIxoEAWERGJAQWyiIhIDCiQRUREYkCBLCIiEgMKZBERkRhQIIuIiMSAAllERCQGkh01sNbeDJwBrHbOjc3Mux44E9gKvAlc7JxbX8xCRUREKlkue8i3AFPazHsYGOucOwh4DbiqwHWJiIhUlQ4D2Tk3C1jbZt5DzrnmzORsYHARahMREakahfgO+RLggQIsR0REpGp1+B3yrlhrvwM0A3/aRZupwFQA5xzpdDqfVbYrmUwWbdnlIJlMkkqldppfW6Rt0lTCdeVCv3/1v5r7D9oGldD/TgeytfYiwpO9JjvnfHvtnHPTgGmZSd/Q0NDZVe5SOp2mWMsuB+l0msbGxp3mNxVpmwQlXFcu9PtX/6u5/6BtENf+19XV5dy2U4FsrZ0CXAkc55xr6swyREREZIdcLnu6HZgEpK21y4CrCc+q7gY8bK0FmO2c+2IR6xQREaloHQayc+78LLN/W4RaREREqpZG6hIREYkBBbKIiEgMKJBFRERiQIEsIiISAwpkERGRGFAgi4iIxIACWUREJAYUyCIiIjGgQBYREYkBBbKIiEgMKJBFRERiQIEsIiISAwpkERGRGFAgi4iIxIACWUREJAYUyCIiIjGgQBYREYkBBbKIiEgMKJBFRERiQIEsIiISAwpkERGRGFAgi4iIxIACWUREJAYUyCIiIjGQjLoAKb1g1sys8xMTp5S4EhER2U57yCIiIjGgQBYREYkBBbKIiEgMKJBFRERiQIEsIiISAwpkERGRGOjwsidr7c3AGcBq59zYzLw+wF+AYcDbgHXOrStemSIiIpUtlz3kW4C2F6h+C3jUObcf8GhmWkRERDqpw0B2zs0C1raZfTZwa+bnW4FzClyXiIhIVensSF0DnHMrMz+vAga019BaOxWYCuCcI51Od3KVu5ZMJou27HKQTCZJpVI7za/Nsk2asrRrr217si1jd15faPr9q//V3H/QNqiE/uc9dKZzzltr/S6enwZMy0z6hoaGfFeZVTqdpljLLgfpdJrGxsad5jdl2SZBlnbttW1PtmXszusLTb9/9b+a+w/aBnHtf11dXc5tO3uW9XvW2kEAmX9Xd3I5IiIiQucD+R7g85mfPw/cXZhyREREqlMulz3dDkwC0tbaZcDVwHWAs9ZeCiwFbDGLFBERqXQdBrJz7vx2nppc4FpERESqlkbqEhERiQEFsoiISAwokEVERGJAgSwiIhIDCmQREZEYUCCLiIjEgAJZREQkBhTIIiIiMaBAFhERiQEFsoiISAwokEVERGJAgSwiIhIDCmQREZEYUCCLiIjEgAJZREQkBhTIIiIiMaBAFhERiQEFsoiISAwokEVERGJAgSwiIhIDCmQREZEYUCCLiIjEgAJZREQkBhTIIiIiMaBAFhERiQEFsoiISAwokEVERGJAgSwiIhIDCmQREZEYUCCLiIjEQDKfF1trvwZcBnhgIXCxc25zIQoTERGpJp3eQ7bW1gP/DExwzo0FaoDzClWYiIhINcn3kHUS6G6tTQK1wIr8SxIREak+nT5k7Zxbbq29AXgH2AQ85Jx7qG07a+1UYGrmNaTT6c6ucpeSyWTRll0OkskkqVRqp/m1WbZJU5Z27bVtT7Zl7M7rC02/f/W/mvsP2gaV0H/jve/UC621ewF/BT4NrAfuAKY75/64i5f5FSuKsxOdTqdpaGgoyrLLQTqdZvWdO2/6xMQpO80LZs3MuoxsbduTbRm78/pC0+9f/a/m/oO2QVz7X1dXB2ByaZvPSV0nAm8559YAWGvvBI4CdhXIImWnEB9gREQ6kk8gvwMcYa2tJTxkPRl4viBViYiIVJlOn9TlnHsWmA68QHjJUwKYVqC6REREqkpe1yE7564Gri5QLSIiIlVLI3WJiIjEgAJZREQkBhTIIiIiMaBAFhERiQEFsoiISAwokEVERGJAgSwiIhIDCmQREZEYUCCLiIjEgAJZREQkBhTIIiIiMaBAFhERiQEFsoiISAwokEVERGJAgSwiIhIDCmQREZEYUCCLiIjEgAJZREQkBhTIIiIiMaBAFhERiQEFsoiISAwokEVERGIgGXUBUjmCWTOzzk9MnFL0dTWlUgSNjUVZl4hIKWgPWUREJAYUyCIiIjGgQBYREYkBBbKIiEgMKJBFRERiQIEsIiISAwpkERGRGMjrOmRrbW/gJmAs4IFLnHPPFKIwERGRapLvHvIvgJnOuVHAOOCV/EsSERGpPp3eQ7bW9gImAhcBOOe2AlsLU5aIiEh1yeeQ9XBgDfA7a+04YC5whXNuY0EqExERqSL5BHISOBT4qnPuWWvtL4BvAf+vdSNr7VRgKoBzjnQ6nccqd1FMMlm0ZZeDZDJJKpXaaX5tlm3SlKVde23b094y8l1uZ9dfk6ghlUqVZF3bFWNdnaX3f3X3H7QNKqH/+QTyMmCZc+7ZzPR0wkD+GOfcNGBaZtI3NDTkscr2pdNpirXscpBOp2lsbNxpflOWbRJkadde2/a0t4x8l9vZ9adSKRobG0uyru2Ksa7O0vu/uvsP2gZx7X9dXV3ObTt9UpdzbhXwrrV2ZGbWZGBRZ5cnIiJSzfK9/eJXgT9Za7sCS4CL8y9JRESk+uQVyM65ecCEAtUiIiJStTRSl4iISAwokEVERGJAgSwiIhIDCmQREZEYyPcsa6lAfvMmeOs1/JLF+KVvYPaohQF10H8QZv+xmF57RV2iiEjFUSDLR/zmTfj7/oJ/5B5oaQ5n9q/Db90MzzwWtkkmMUdNxvdJY3r2irBaEZHKokAWvPfwzhL8fQ7WNWCOPB5z+EQYPhLTIxw20m/ZDKuW4596EP/0I9DSgt9nJBx2DKZL14h7ICJS/hTIVc57D3P+BosXwuBhJKZ+EzNi9E7tTLc9YOi+mKFfxp9xHsHNP4NXF8CaVfiJp2D26htB9SIilUMndVUx7z08NysM4wPGkfjuz7KGcVumdx/MhKPhxLNg21Z4YDr+Dd0KW0QkH9pDrlLee5j9JLyxCMYcAoccgX/6YXybdomJU9pdhhlYjz/dwt8ehmcex2/ehBl7aHELL7Jg1syoSxCRKqU95Go19+9hGI8dD4ccgTGmU4sx3Wth8pkwbD94cTZ+wZwCFyoiUh20h1yF/DtL4JX5MPJAOPjwTofxdiaRwB89GRIJmD8HHwQwLv/liohUEwVylfHvr4FnHoc+/WD8UQULTZNI4I86IQzlhXMh2QXK/PC1iEgpKZCriG9uJvjf68EHMPFkTE1NQZdvjMEfMQmam8PD17UpzD77F3QdIiKVSoFcRfy9t8Obr8IxJxVtUA9jTLinvGkjPPMYvnttUdYjIlJpdFJXlfDLl+Jn/hVz9GTM8P2Kui5TUwOTToWeveHJmfjlS4u6PhGRSqBArgLee4Lbp0H3HphzLy7JOk3XbjD5dEgmCX55Lf7DD0qyXhGRcqVArgL++XAkLnPOhZjUniVbr+nRE447Fda/T/Cb/8Q3N5ds3SIi5UaBXOH85k14dzPsvQ9m4sklX7/pNwDz2a/A4oX4O24u+fpFRMqFArnC+RkO1r9P4vwvYBKFPas6V4mjJmNOPBv/2H0ETz0USQ0iInGnQK5gvvFD/EN3h3dvGnFApLWYcy+C0Qfj//Rr/BuLIq1FRCSOFMiVbOFcMGDOuTDqSjA1NSSmXgl9+xH8z4/wa9dEXZKISKwokCuU/3ADvPkq5thTMH36RV0OAKZHisTl34VtWwlu/CF+y5aoSxIRiQ0FcqVaOBeMwZx2btSVfIwZNITEZd+Ed5fgb/2v8K5TIiKiQK5E/oP1sGQx7D8G07tv1OXsxIw7DHPOhfg5T+Fn/jXqckREYkGBXIkWPA+Jmljf3MGcei7msGPx//cH3bJRRAQFcsXxG9bB26/DyLHhvYpjyhiD+fw/w5DhBDf9BL/y3ahLEhGJlAK50iyaF94CcfTBUVfSIdOtG4kvfweSXQh++QMNrykiVU2BXEH8pqbwu+N9R8V677g107cfiS9fBWvXENx4rc68FpGqpUCuJK8ugCCAA8ZFXcluMSNGk7jsG7BkMcFNN+CDlqhLEhEpOQVyhQg2bYTXXg7HrN6zd9Tl7DYz/ijMp/8J5j2Lv32aLocSkaqTzHcB1toa4HlguXPujPxLks7Y9Mh9sHVLWXx33J7E5DMI1jXgH7wTutfCJz+HMSbqskRESiLvQAauAF4BSndfP/kY39xM071/hv6DMP0GRl1OXsw/fA42N+Ef+Ctg4JOfVSiLSFXI65C1tXYwcDpwU2HKkc7wc58mWPMejD4k6lLyZhIJzAVfxEw8Bf/AdPxdf9ThaxGpCvnuIf8cuBLo2V4Da+1UYCqAc450Op3nKrNLJpNFW3bcrX3qQYJBQ+g+asxOe5O1WbZJUyqV87Kzvb5Qy9gVf8X/48Nu3dg04w66Jwypi76Kqdlx+8i2669J1JBKpTq1rtaK3a9iqeb3P6j/oG1QCf3vdCBba88AVjvn5lprJ7XXzjk3DZiWmfQNDQ2dXeUupdNpirXsOPNL3yRY/BKpS65g48aNOz3flGWbBI2NOS+/8c4/5lVfezVkE8ya+bFpP6AeRh1E032OpmVLSVz2DUy3PcK2bfqQSqVobGzMeV3Z1re7dmddxVat7//tqr3/oG0Q1/7X1dXl3DafQ9ZHA2dZa98G/gycYK3N/39v2S3+8fuhaze6n3Ba1KUUnDEGc9gxmPOnwvw5BNd/G79+bdRliYgURaf3kJ1zVwFXAWT2kL/pnIv+xrtVxDd+gH9uFubI40n0aPdbg7KXOOEMfN/+BNOuJ7jmChIXXxF1SSIiBafrkMuYf/oR2LYVc3zl7R23ZcYdTuLbP4FeexH81zX4557CNzdHXZaISMEU4rInnHNPAE8UYlmSGx+04J94ILzF4uDhUZdTEqZ+bxLfvgF/5x/wj9wNK97Bjz8aBg+NujQRkbxpD7lcLXwBGt4jcfzpUVdSUqZLVxKfvhROPBOMgSdmwCP30vL+6qhLExHJS0H2kKX0gsfvg9594OAjoi4lEmbQEPyZnw6HC50/hyZ3C9TtjU8PhAPGaTARESk7CuQy5N9bAS+/iDnrAkyyen+FJlEDow7CD9+frm+9xtaFcwl+9j2oH4o58njMhGMwfftHXaaISE6q93/zMuafmAE1SczEU6IuJRZMtz3oNuEotu43BtN1D/yTD+Cn34Kffkt4K8oDJ2AOGAdDR3xscBERkThRIJcZv3kT/ulHw7sj9dor6nJixdTUkDh6Mhw9Gb9mFX7OU/i5T4fDb971R+jeA0YeiBk9DjNqHN57HdoWkdhQIJcZ/+yTsGkjpspO5tpdpt9AzGmfgtM+hf9gPX7xQnhlPn7RPPy82XiA2h74gfUwcDAMrMdU8LXcIhJ/CuQy4r0PR+YaMhz2HRV1OWXD7Nkbc9ixcNixAOHe8yvz8LMeguXvwpLXwvl79YX6YTBkGPTtr71nESkpBXI5ef1lWL4U87nLFRZ5MP0GYvpNISD8kMP6tbDiHVi2FF5+AV6aCz33xO8zEoaPxPTUnUVFpPgUyGXEP3Y/1KYwhx8XdSkVwxgDe/UNH2MOwW/ZDMvehjcXw/w5MH8Ovn7v8NaWA+r0QUhEikaBXCb8+vfx82ZjJp+J6dYt6nIqlum2R/h1wL6j8I0fwpuvwOKX4OG7oW9//EETwsuqFMwiUmAK5DLhZz0IQYA57tSoS6kaJtUTxh2OH3MoLHkVFs2Dx2fAgHr8+KOiLk9EKowCuQz45ubwBKQxh2L6D4q6nKpjkknYfyx+xAHw2iJYMAdm3EGwYR3mU5dgeqSiLlFEKoDGsi4H82bDhrUkJlX+XZ3izCRqMKMOhHM+A6MPxj/zGMHVl+PnPxd1aSJSARTIZSB4fAb07Q8HHhp1KQKYrt0w448i8e0bINWT4JfXEvz2p/jNTVGXJiJlTIEcc375UnjtJcykU8OxmyU2zNARJL77U8wZ5+GfnUVw7Tfwy96KuiwRKVMK5JjzT8yAZBfM0SdFXYpkYZJdSJx9AYlvXAubNxH88F8Jnnoo6rJEpAwpkGPMb2rCP/ME5rBjNThFzJmRY0l872cw4gD8739JcNuv8S0tUZclImVEgRxjfvbjsGUT5nidzFUOzJ57kfiXf8ec/En84zMI/usafNPGqMsSkTKhQI6pcNzqGeEtA4fvH3U5kiOTqCHxqYsxn7scFi8guO5KfMN7UZclImVAgRxXixfCynd1V6cylTj2ZBJfuwY2rCP4z3/Dr3gn6pJEJOYUyDEVPD4DevTEHHZM1KVIJ5mRB5K48kfgIfjxVfi3Xou6JBGJMY3UFUN+3fswbzbmpLMxXfMbtzqYNbNAVZWvYm2DbMtNTJzysWlTP5TEv11H8MNvEvz4WzDpVMygIbu9rrbLLbX2tmHUdYlUEu0hx5Cf9SB4r3GrK4TpNxBO+SSk9oTH7se/82bUJYlIDCmQY8Y3N+OfehDGjg//I5eKYGp7wMnnQN9+MOsh/BuvRF2SiMSMAjlm/NynYcM6ErrUqeKYbnvA5LNg0GB45nH8onlRlyQiMaJAjhHvPf7hu2FgPYzRuNWVyHTpApNOg6H7wty/4+c/h/c+6rJEJAZ0UlecvL4Ilr6B+cyXMAl9VqpUpqYGf8xJkOwCC56H5m34Q4/CGBN1aSISIQVyjAQP3x1e6nTkCVGXIkVmEgn8kcdDMgmL5kNzM/7wiQplkSqmQI4Jv3olzH8Wc+q5mG75Xeok5cEYgz/s2HBP+eUXwz3lI0/Q0RGRKqVAjgn/6L2QqNG41VXGGIM/5IgwlOc/B80t+GNOxNToVpsi1abTgWytHQL8HhgAeGCac+4XhSqsmvimRvzTj4R3derdN+pypMSMMXDQBHyyC8x9Gp7chp84BZPU52WRapLPsbFm4BvOudHAEcBXrLWjC1NWdfFPPghbNmNOOivqUiRCZvQ4+MRxsPydcACRbduiLklESqjTgeycW+mceyHz84fAK0B9oQqrFn7rFvzDd8HoQzB77xt1ORIxs/8YOHoyrF4Bj96D37ol6pJEpEQKckzMWjsMOAR4NstzU4GpAM450ul0IVa5k2QyWbRlF1PTjOl8+OEG9rrgMrrmUX8ymSSVShWwssKpzbFfTe3Un+31bdvWJGpIpVI5tS2m9vq6WzUcNJ5tqZ5sfvgeEo/eS/fTP0Wqg21Y7Pf/7vxuolCuf/+FVO3boBL6n3cgW2tTwF+Bf3HOfdD2eefcNGBaZtI3NDTku8qs0uk0xVp2sfjmbQR//T2MOIAN/Qdj8qg/nU7T2NhYwOoKpynHfgXt1J/t9W3bplIpGhsbc2pbTO31dbdr6F8Hk04jePIBNt75BzbtfyCmf127zYv9/t+d300UyvHvv9CqfRvEtf91de3/3baV1/UV1touhGH8J+fcnfksqxr5Z2fB2gYSp31K15/KTkz93nDS2bB1K8GPrsQvWRx1SSJSRJ0OZGutAX4LvOKc+2nhSqoOPmjBz5wOg4fD2PFRlyMxZfoNhCn/AN1rCX7yHfwLz0RdkogUST57yEcDnwVOsNbOyzx0EW2uXpwNq5ZjtHcsHTB79ibxrR9D/TCCX/2I4L4/a/xrkQrU6e+QnXN/A5QkneCDFoJ7bocB9ZjxR0ZdjpQBs2dvEt/8Af4PN+Lvvg2WLYWLrwjvICUiFUFj9EXAPzsLVryDOfszmIRGZJLcmK7dMJd8DXPuxfgXniH40b/iV74bdVkiUiAK5BLzzdvw99wGQ4Zjxh8VdTlSZowxJE75JIkrroYP1hNc+3WCvz8adVkiUgAK5BLzTz0MDe+R+OTndBMB6TQz5hAS3/s5DN8f/7tfsOHn38dvjOdlbyKSGyVCCfktW/D3/wVGjIaxh0ZdjpQ507svia9fgznzfDY/9QjB1V/Bz5sddVki0kkK5BLyj90HG9aR+ORndWa1FIRJ1JA463z6XH8T9OxNcOMPCaZdj1//ftSlichuUiCXiP9gHf6BO+DACeF4xSIF1GWfkSS+8xPM2RfgX3yG4LtfIrjfaSxskTKiQC4Rf+fvYetWEvaSqEuRCmWSSRJnnEfimv+BMYfg7/ojwfe+QvC3h/HNzVGXJyIdUCCXgF+yGP/0o5gTz8IMHBx1OVLhTL+B1HzpKhLfuBZSe+Jv/W+C736RYNaDuqXtH1jiAAANV0lEQVSjSIzpDuhF5oOA4LbfQK8+mDNs1OVEIpg1c6d5iYlTIqik+LL1tVjL7WgbmlEHkfjOT2Dh8wS3/SYcVOSOm2G/0bDfGEyP7Hdw2p3fze70N9ty23t9rm1L+T7anVrjqhL6UMkUyEXmn34Elr6BufTrmD1qoy5HqowxBg46DNY1wMplsHghLJwLL72Arx8Kw/eHwcMwSf1XIBI1/RUWkf9gffjd8YjRmE8cF3U5UsWMMVA3BOqG4D/8AF5/GZYshmVvQ5cu+CHDYchwGLR31KWKVC0FcpF47wn+8D+weROJC7+sy5wkNkzPPeHQI/EHfwLeWwFvvQbvvgVLXoNEDS0vzcWMGocZdRAMGabhXUVKRIFcJH72EzBvNubci8P72orEjEkkYNBgGDQYH7TA6lXhHnPDe/jpv8MD1PaA/cdCsgsMrIfeffThUqRIFMhF4Nc24G+fBiMOwJx0VtTliHTIJGrCwB1YT2LiFPz69/GvLoTFC/GLF8KaVWHDrt3w/QZC/0Hho29/TI32oEUKQYFcYN57glv/G1qaSVx8hQ73SVkyvftijpgER0wCoGXGHfDecli9MnwsXxo2TNTg0/1he0j3G6hbQop0kgK5wPwD02HRi5gLvojpXxd1OSIFYVI9ITUK9h0FgN+8KQzmNSvDQ92L5sPLL4bP9e6TCedwL9qkekZZukjZUCAXkF84F3/XHzGHT8RMOjXqckSKxuzRHfbeJ3wQ3laUhtU7Qvqt1+C1l8Pneu4JAwfje+wJow7E9FBAi2SjQC4Qv3oFwU03QP0wzOe+qhNfpKqY7Sd9DawHwgFxWL82PMy9ajm8/TrBr68DY2DvfTGjx2HGjMcHLfpaRyRDgVwAfvMmght/CCZB4stXYbp1i7okkUiZRAL6pMPHAePwQQuJwcPxi+bjX5mPf+gu/AN/hS5d8XV7w+ChULd3uOctUqUUyHnyW7cQ/PJaWLWMxBX/juk3MOqSRGLHJGowI0ZjRoyGs87Hb2qCV+YRPHRXeILY0jfAGHx6ANQPxe8zCuqH6kiTVBUFch78tm0Ev/oRvPZSODTm6IOjLkmkLJjutXDoUZjGD/Dew/urw2BethTmPUsw71no0w9z0GGYgybAyAOjLlmk6BTIneSbmwmm/RheegHz+a+S0NCYIp1ijIH0gPAx7nB800ZMj574Bc/j//4o/okZ0LUr9K+D+qHhnnM7N8YQKWcK5E7wmzcR/O8NsGAO5oIvkDjmpKhLEqkYprYHiWNPhmNPxm/bCotfwi+Yg39uVjiSGOD36gv1w/D1w2D4fpHWK1IoCuTd5NeuIfjva2HFUsxnvkhi0mlRlyRSsUyXrjD2UMzYQ2mp3xs2rNtxaPvlFwhemgupPdkw/kiC4aMwBxyE6dMv6rJFOkWBvBv8W68R3PgD2LqFxFe/hxl7aNQliVQNYwz07hM+xhyC37IZ06sPLJjDlhefxT/5YDj+dv+68LKqUeN03bOUFQVyDnzzNvyMO/Az7oDefUl87T90wwiRiJlue5A4fCIcPpG+ffrQsGAu/pUF4WVVzzyBf+KBsOGgIZh9RsI+IzH7joJBg3Xts8SSArkDftnbBL/7ObyzBHPEJMx5/6RP3CIxYxIJzODhmMHD4aSz8c3N8Pbr+MUL8W++ip//LDz9SLgH3b0Whu2HGToCBg/DDB4GA+oxSf13KNHSO7Adfu0a/L1/xv/9UejRk8SXrsIcemTUZYlIDkwyGd5tbcQBQHjTF1avxL/5Kix5Fb9kMf7hu6GlOQzpZBIGDgnDuX7vcBx63SxDSkyB3IZfswr/2H2Zw10ec/zpmNM/Hd7UXUTKkjEGBtRhBtTBUScAmfG3Vy3HL3sblr2NX74U/+oCmP14GNLb9eoD/QdCEECPFNRmHj1SUNsD770GMJGCUCATDvDBwucJZs0M71hjEpijjseceT6mb/+oyxORIjDJLjsOWbfimzbCmlX41Sth9QpYszL8ecU7sKlpp+UEd90GvfaCnntCz16Ynr2gZ69wOtULU9sDuveA2h7h4fLaHtCtu0JcdpJXIFtrpwC/AGqAm5xz1xWkqhLwa9fgX12An/8cvDwPtmyCvdJhCB9zEqZPOuoSRSQCprYHDN0XM3Tfj80PZs3Et7SEodzUmHlsxPTuCx+sx3+4IQzyJYuh8YNwjxo+vrf90UoS0LUbdOkCXbqG/yZb/dyla2a6C5AJbgPmo59N+GhlfbduBFu2ZO+UD6vwa1Z+vIaaGkgkCFa8CzXJcJ3JVv926QJ71IYjq+1RC927Z/4Np02XLru5dWVXOh3I1toa4EbgJGAZMMdae49zblGhiisE39wMa1bBqmX4le/il74JSxbD+vfDBr37Yo44DjPuEzD6YEyNzr4UkexMTQ2keoaPjMTEKTu180EAmzbChxsyAb4RNm0M9763T2/bAtu2wbatsG0bvnnrjumtW2BjIzRv+yhMIRPu3vNRzPvMtDE019SEHxg+Knan6sMPETuKhJYAghb88qXhupqbP/og8bH+tLdBkskdAV2byhwF6BGOpLb9qMD2ebU9drSp7QHdU9C1q44UtJLPHvLhwBvOuSUA1to/A2cDJQlkv2YVrHw38wbfSCOeoGFN+Kbf+CGsex/WNcCG9eEbb7u+/TH7j4F9RmH2Gw1DhusNISIFZRIJ6NEzfLSeX8R1ptNpGhoadtkmmDUz6/zWHyp80BIGc3Nz+OFg8ybY3BR+kNjchN/08enwA0YTftPGMPDXr93x89atO5abbcU1yUxA7whzunfHdO0G3fYIjyJkeZhu3aCmS7iHX5OEmhq2reuL/7Dxo+mPHh3xQNASfhBpaYaaZHiuQQTyCeR64N1W08uAT+RXTu7804/g73cfTW+EcLzb2swns959MGMOgb3S4ZmSg4bAwMHhoRcREcnKJGqga00YfvQIvx9v/fxuLMtv2xYeKWja+NEhfv/R9I554dGDzM9r14RDpm7dAls2h/+2XW6Wda3drV7uwpDh1HzvF4Va2m4x3rd7MGKXrLXnAlOcc5dlpj8LfMI5d3mbdlOBqQDOufH5lSsiIlJ2cvock8hjBcuBIa2mB2fmfYxzbppzboJzbkKmqKI8rLVzi7n8uD/Uf/U/6hrUf20D9b/dR07yOWQ9B9jPWjucMIjPAy7IY3kiIiJVq9N7yM65ZuBy4EHglXCWe7lQhYmIiFSTvK5Dds7NAGYUqJZ8TYu6gIip/9VN/Zdq3wZl3/9On9QlIiIihZPPSV0iIiJSIGU7lrW1tg/wF2AY8DZgnXPrsrSbCRwB/M05d0YpayyGjoYrtdZ2A34PjAfeBz7tnHu71HUWSw79nwj8HDgIOM85N730VRZPDv3/OnAZ0AysAS5xzi0teaFFkkP/vwh8BWgBGoGpcRs9MB+5Dldsrf1HYDpwmHPu+RKWWHQ5vAcuAq5nx1U/v3TO3VTSIjupnPeQvwU86pzbD3g0M53N9cBnS1ZVEbUarvRUYDRwvrV2dJtmlwLrnHMjgJ8B/1naKosnx/6/A1wE3Fba6oovx/6/CExwzh1E+B/yj0tbZfHk2P/bnHMHOucOJuz7T0tcZtHk2H+stT2BK4BnS1th8eW6DYC/OOcOzjzKIoyhvAP5bODWzM+3Audka+ScexT4sFRFFdlHw5U657YC24crba31dpkOTLbW5nwdXMx12H/n3NvOuQXAzgPylr9c+v+4c277LYlmE44PUCly6f8HrSZ7sIthmMtQLn//AP9B+EF8cymLK5Fct0FZKudAHuCc237rklXAgCiLKZFsw5XWt9cmc2naBqBvSaorvlz6X8l2t/+XAg8UtaLSyqn/1tqvWGvfJNxD/ucS1VYKHfbfWnsoMMQ5d38pCyuhXP8G/tFau8BaO91aOyTL87EU6++QrbWPAAOzPPWd1hPOOW+traRPwiJ5sdZeCEwAjou6llJzzt0I3GitvQD4LvD5iEsqCWttgvAQ/UURlxK1e4HbnXNbrLVfIDxieELENeUk1oHsnDuxveeste9Zawc551ZaawcBq0tYWlRyGa50e5tl1tok0Ivw5K5KkNNwrRUsp/5ba08k/NB6nHOunRvklqXd/f3/GfhVUSsqrY763xMYCzxhrYVwZ+Yea+1ZFXRiV4fvAedc6//vbqKMzqOIdSB34B7CT77XZf69O9pySiKX4Uq3b5dngHOBx5xzlXL0oNqHa+2w/9baQ4DfEN74pdI+pObS//2cc69nJk8HXqdy7LL/zrkNQHr7tLX2CeCbFRTGkNt7YFCrrzPPIhxJsiyU83fI1wEnWWtfB07MTGOtnWCt/eisOmvtU8AdhCc3LbPWnhJJtQXQ3nCl1tprrLVnZZr9FuhrrX0D+Drtn31ednLpv7X2MGvtMuBTwG+stRUznGuOv//rgRRwh7V2nrX2nojKLbgc+3+5tfZla+08wvd/xRyuzrH/FS3HbfDPmffAfMJzCC6Kptrdp5G6REREYqCc95BFREQqhgJZREQkBhTIIiIiMaBAFhERiQEFsoiISAwokEVERGJAgSwiIhIDCmQREZEY+P9jrC3oUlfVZQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "calculate_metrics(np.array(true), np.array(pred), 50)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}