superclass_ensemble.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "TH4I7h_FVgZS",
        "pxR7HxjuVlAo",
        "8kxaObKrswJQ",
        "b_-qSgf8hOEc"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "dc972ee1a9624d40b27cf427a179d54e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_080a94e85bc74a2e921ba02a725cabd9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7b1589966cc940f8bc4a8e19beebc5e0",
              "IPY_MODEL_14b1d49493ba42d4ab0955e110322217",
              "IPY_MODEL_c655a12ddc4749faa44cebe6690b06f6"
            ]
          }
        },
        "080a94e85bc74a2e921ba02a725cabd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b1589966cc940f8bc4a8e19beebc5e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_352375d2357543baad91b9870bb3a979",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 80%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4bde10fba3c04bddad20b8d1973a3082"
          }
        },
        "14b1d49493ba42d4ab0955e110322217": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_244ed8f789b14d129ce572a332db9b6b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_085ef2cef50144b6af753403afe1d20e"
          }
        },
        "c655a12ddc4749faa44cebe6690b06f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a5b61e78bddb4d47af2fb43e90b0e4ea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8/10 [2:14:17&lt;33:35, 1007.66s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2cdd13c67461484984a0fa27e15ae172"
          }
        },
        "352375d2357543baad91b9870bb3a979": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4bde10fba3c04bddad20b8d1973a3082": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "244ed8f789b14d129ce572a332db9b6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "085ef2cef50144b6af753403afe1d20e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a5b61e78bddb4d47af2fb43e90b0e4ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2cdd13c67461484984a0fa27e15ae172": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ncorriveau/ood_computervision/blob/main/superclass_ensemble.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Packages"
      ],
      "metadata": {
        "id": "TH4I7h_FVgZS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade torchvision==0.13"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UYtdsA3UVI5c",
        "outputId": "65a1e3ed-1ddb-48c1-e955-ce6f79e01193"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torchvision==0.13 in /usr/local/lib/python3.7/dist-packages (0.13.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchvision==0.13) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision==0.13) (1.21.6)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torchvision==0.13) (3.10.0.2)\n",
            "Requirement already satisfied: torch==1.12.0 in /usr/local/lib/python3.7/dist-packages (from torchvision==0.13) (1.12.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision==0.13) (7.1.2)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision==0.13) (1.25.11)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision==0.13) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision==0.13) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision==0.13) (2.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Adxw-_vaTXSM"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import torch\n",
        "from torch.utils.data import DataLoader\n",
        "import torchvision\n",
        "from torchvision import datasets, transforms\n",
        "\n",
        "import numpy as np \n",
        "import pandas as pd\n",
        "import os\n",
        "from pathlib import Path\n",
        "import random\n",
        "from PIL import Image\n",
        "from matplotlib import pyplot as plt\n",
        "import requests\n",
        "import zipfile\n",
        "import shutil "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "  import timm\n",
        "except:\n",
        "  !pip install timm\n",
        "  import timm"
      ],
      "metadata": {
        "id": "Dsp_5jTaToW5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "    from torchinfo import summary\n",
        "except:\n",
        "    print(\"[INFO] Couldn't find torchinfo... installing it.\")\n",
        "    !pip install -q torchinfo\n",
        "    from torchinfo import summary\n",
        "\n",
        "# Try to import the going_modular directory, download it from GitHub if it doesn't work\n",
        "try:\n",
        "    from going_modular.going_modular import data_setup, engine\n",
        "except:\n",
        "    # Get the going_modular scripts\n",
        "    print(\"[INFO] Couldn't find going_modular scripts... downloading them from GitHub.\")\n",
        "    !git clone https://github.com/mrdbourke/pytorch-deep-learning\n",
        "    !mv pytorch-deep-learning/going_modular .\n",
        "    !rm -rf pytorch-deep-learning\n",
        "    from going_modular.going_modular import data_setup, engine\n"
      ],
      "metadata": {
        "id": "noK5GtYCXBwi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "device"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "3GsUV6xTVza1",
        "outputId": "9399ac85-bbfb-4b1b-9e8f-aee97fbeff2c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'cuda'"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "R6M-t428JEEl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Set up data and helper functions"
      ],
      "metadata": {
        "id": "pxR7HxjuVlAo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "  from google.colab import drive\n",
        "  drive.mount('/content/drive') \n",
        "  train_path = '/content/drive/MyDrive/NN and DL/projects/Released_Data/train'\n",
        "  test_path = '/content/drive/MyDrive/NN and DL/projects/Released_Data/test'\n",
        "  base_path = '/content/drive/MyDrive/NN and DL/projects/Released_Data/models/'\n",
        "\n",
        "except:\n",
        "  train_path = '/content/train'\n",
        "  test_path = '/content/test'\n",
        "  base_path = '/content/models'"
      ],
      "metadata": {
        "id": "3brPhq5oVeVa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''process below follows:\n",
        "https://www.learnpytorch.io/06_pytorch_transfer_learning/'''\n",
        "\n",
        "manual_transforms = transforms.Compose([\n",
        "    transforms.Resize((224, 224)), # 1. Reshape all images to 224x224 (though some models may require different sizes)\n",
        "    transforms.ToTensor(), # 2. Turn image values to between 0 & 1 \n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406], # 3. A mean of [0.485, 0.456, 0.406] (across each colour channel)\n",
        "                         std=[0.229, 0.224, 0.225]) # 4. A standard deviation of [0.229, 0.224, 0.225] (across each colour channel),\n",
        "])"
      ],
      "metadata": {
        "id": "BBRsUPmzWioZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# !unzip /content/train.zip\n",
        "# !unzip /content/test.zip\n",
        "# !unzip /content/models.zip"
      ],
      "metadata": {
        "id": "Hdq_59yRXpBH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "weights_resnet = torchvision.models.ResNet50_Weights.DEFAULT.DEFAULT\n",
        "vit_weights = torchvision.models.ViT_L_16_Weights.IMAGENET1K_SWAG_E2E_V1.DEFAULT # .DEFAULT = best available weights from pretraining on ImageNet\n",
        "# weights\n",
        "#weights = torchvision.models.ViT_B_16_Weights.IMAGENET1K_SWAG_E2E_V1.DEFAULT\n",
        "res_t = weights_resnet.transforms()\n",
        "vit_t = vit_weights.transforms()\n",
        "print(f\"Resnet transforms: {res_t}\\n Vit transforms {vit_t}\")"
      ],
      "metadata": {
        "id": "SPyJeRMjRB3L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f7fa800-9fce-4b49-ba64-b0817a69c84f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Resnet transforms: ImageClassification(\n",
            "    crop_size=[224]\n",
            "    resize_size=[232]\n",
            "    mean=[0.485, 0.456, 0.406]\n",
            "    std=[0.229, 0.224, 0.225]\n",
            "    interpolation=InterpolationMode.BILINEAR\n",
            ")\n",
            " Vit transforms ImageClassification(\n",
            "    crop_size=[224]\n",
            "    resize_size=[242]\n",
            "    mean=[0.485, 0.456, 0.406]\n",
            "    std=[0.229, 0.224, 0.225]\n",
            "    interpolation=InterpolationMode.BILINEAR\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataloader, test_dataloader, class_names = data_setup.create_dataloaders(train_dir=train_path,\n",
        "                                                                               test_dir=test_path,\n",
        "                                                                               transform=res_t, # resize, convert images to between 0 & 1 and normalize them\n",
        "                                                                               batch_size=32) # set mini-batch size to 32\n",
        "\n",
        "train_dataloader, test_dataloader, class_names"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "blUjeKdSW9NL",
        "outputId": "6a8e7963-68a7-4b63-e88e-7e1072092713"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<torch.utils.data.dataloader.DataLoader at 0x7fe1a6523210>,\n",
              " <torch.utils.data.dataloader.DataLoader at 0x7fe1a6523390>,\n",
              " ['bird', 'dog', 'reptile'])"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import Tuple\n",
        "def test_model(model: torch.nn.Module, \n",
        "              dataloader: torch.utils.data.DataLoader, \n",
        "              loss_fn: torch.nn.Module,\n",
        "              device: torch.device) -> Tuple[float, float]:\n",
        "    \"\"\"Tests a PyTorch model for a single epoch.\n",
        "\n",
        "    Turns a target PyTorch model to \"eval\" mode and then performs\n",
        "    a forward pass on a testing dataset.\n",
        "\n",
        "    Args:\n",
        "    model: A PyTorch model to be tested.\n",
        "    dataloader: A DataLoader instance for the model to be tested on.\n",
        "    loss_fn: A PyTorch loss function to calculate loss on the test data.\n",
        "    device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
        "\n",
        "    Returns:\n",
        "    A tuple of testing loss and testing accuracy metrics.\n",
        "    In the form (test_loss, test_accuracy). For example:\n",
        "\n",
        "    (0.0223, 0.8985)\n",
        "    \"\"\"\n",
        "    # Setup test loss and test accuracy values\n",
        "    test_loss, test_acc = 0, 0\n",
        "\n",
        "    # Turn on inference context manager\n",
        "    with torch.inference_mode():\n",
        "        # Loop through DataLoader batches\n",
        "        for batch, (X, y) in enumerate(dataloader):\n",
        "            # Send data to target device\n",
        "            X, y = X.to(device), y.to(device)\n",
        "\n",
        "            # 1. Forward pass\n",
        "            test_pred_logits = model(X)\n",
        "\n",
        "            # 2. Calculate and accumulate loss\n",
        "            loss = loss_fn(test_pred_logits, y)\n",
        "            test_loss += loss.item()\n",
        "\n",
        "            # Calculate and accumulate accuracy\n",
        "            test_pred_labels = test_pred_logits.argmax(dim=1)\n",
        "            test_acc += ((test_pred_labels == y).sum().item()/len(test_pred_labels))\n",
        "\n",
        "    # Adjust metrics to get average loss and accuracy per batch \n",
        "    test_loss = test_loss / len(dataloader)\n",
        "    test_acc = test_acc / len(dataloader)\n",
        "    return test_loss, test_acc"
      ],
      "metadata": {
        "id": "PrbF3gUaa7cN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "from tqdm.auto import tqdm\n",
        "from typing import Dict, List, Tuple\n",
        "\n",
        "def train_step(model: torch.nn.Module, \n",
        "               dataloader: torch.utils.data.DataLoader, \n",
        "               loss_fn: torch.nn.Module, \n",
        "               optimizer: torch.optim.Optimizer,\n",
        "               device: torch.device) -> Tuple[float, float]:\n",
        "    \"\"\"Trains a PyTorch model for a single epoch.\n",
        "\n",
        "    Turns a target PyTorch model to training mode and then\n",
        "    runs through all of the required training steps (forward\n",
        "    pass, loss calculation, optimizer step).\n",
        "\n",
        "    Args:\n",
        "    model: A PyTorch model to be trained.\n",
        "    dataloader: A DataLoader instance for the model to be trained on.\n",
        "    loss_fn: A PyTorch loss function to minimize.\n",
        "    optimizer: A PyTorch optimizer to help minimize the loss function.\n",
        "    device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
        "\n",
        "    Returns:\n",
        "    A tuple of training loss and training accuracy metrics.\n",
        "    In the form (train_loss, train_accuracy). For example:\n",
        "\n",
        "    (0.1112, 0.8743)\n",
        "    \"\"\"\n",
        "    # Put model in train mode\n",
        "    model.train()\n",
        "\n",
        "    # Setup train loss and train accuracy values\n",
        "    train_loss, train_acc = 0, 0\n",
        "\n",
        "    # Loop through data loader data batches\n",
        "    for batch, (X, y) in enumerate(dataloader):\n",
        "        # Send data to target device\n",
        "        X, y = X.to(device), y.to(device)\n",
        "\n",
        "        # 1. Forward pass\n",
        "        y_pred = model(X)\n",
        "\n",
        "        # 2. Calculate  and accumulate loss\n",
        "        loss = loss_fn(y_pred, y)\n",
        "        train_loss += loss.item() \n",
        "\n",
        "        # 3. Optimizer zero grad\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # 4. Loss backward\n",
        "        loss.backward()\n",
        "\n",
        "        # 5. Optimizer step\n",
        "        optimizer.step()\n",
        "\n",
        "        # Calculate and accumulate accuracy metric across all batches\n",
        "        y_pred_class = torch.argmax(torch.softmax(y_pred, dim=1), dim=1)\n",
        "        train_acc += (y_pred_class == y).sum().item()/len(y_pred)\n",
        "\n",
        "    # Adjust metrics to get average loss and accuracy per batch \n",
        "    train_loss = train_loss / len(dataloader)\n",
        "    train_acc = train_acc / len(dataloader)\n",
        "    return train_loss, train_acc\n",
        "\n",
        "def test_step(model: torch.nn.Module, \n",
        "              dataloader: torch.utils.data.DataLoader, \n",
        "              loss_fn: torch.nn.Module,\n",
        "              device: torch.device) -> Tuple[float, float]:\n",
        "    \"\"\"Tests a PyTorch model for a single epoch.\n",
        "\n",
        "    Turns a target PyTorch model to \"eval\" mode and then performs\n",
        "    a forward pass on a testing dataset.\n",
        "\n",
        "    Args:\n",
        "    model: A PyTorch model to be tested.\n",
        "    dataloader: A DataLoader instance for the model to be tested on.\n",
        "    loss_fn: A PyTorch loss function to calculate loss on the test data.\n",
        "    device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
        "\n",
        "    Returns:\n",
        "    A tuple of testing loss and testing accuracy metrics.\n",
        "    In the form (test_loss, test_accuracy). For example:\n",
        "\n",
        "    (0.0223, 0.8985)\n",
        "    \"\"\"\n",
        "    # Put model in eval mode\n",
        "    model.eval() \n",
        "\n",
        "    # Setup test loss and test accuracy values\n",
        "    test_loss, test_acc = 0, 0\n",
        "\n",
        "    # Turn on inference context manager\n",
        "    with torch.inference_mode():\n",
        "        # Loop through DataLoader batches\n",
        "        for batch, (X, y) in enumerate(dataloader):\n",
        "            # Send data to target device\n",
        "            X, y = X.to(device), y.to(device)\n",
        "\n",
        "            # 1. Forward pass\n",
        "            test_pred_logits = model(X)\n",
        "\n",
        "            # 2. Calculate and accumulate loss\n",
        "            loss = loss_fn(test_pred_logits, y)\n",
        "            test_loss += loss.item()\n",
        "\n",
        "            # Calculate and accumulate accuracy\n",
        "            test_pred_labels = test_pred_logits.argmax(dim=1)\n",
        "            test_acc += ((test_pred_labels == y).sum().item()/len(test_pred_labels))\n",
        "\n",
        "    # Adjust metrics to get average loss and accuracy per batch \n",
        "    test_loss = test_loss / len(dataloader)\n",
        "    test_acc = test_acc / len(dataloader)\n",
        "    return test_loss, test_acc\n",
        "\n",
        "class EarlyStopper:\n",
        "  '''creates a class that tracks validation loss -\n",
        "  the parameters 'patience' determines how long you can have a validation loss\n",
        "  that is greater than min validation loss + delta'''\n",
        "  def __init__(self, patience=1, min_delta=0):\n",
        "        self.patience = patience\n",
        "        self.min_delta = min_delta\n",
        "        self.counter = 0\n",
        "        self.min_validation_loss = np.inf\n",
        "\n",
        "  def early_stop(self, validation_loss):\n",
        "      if validation_loss < self.min_validation_loss:\n",
        "          self.min_validation_loss = validation_loss\n",
        "          self.counter = 0\n",
        "      elif validation_loss > (self.min_validation_loss + self.min_delta):\n",
        "          self.counter += 1\n",
        "          if self.counter >= self.patience:\n",
        "              return True\n",
        "      return False\n",
        "\n",
        "def train(model: torch.nn.Module, \n",
        "          train_dataloader: torch.utils.data.DataLoader, \n",
        "          test_dataloader: torch.utils.data.DataLoader, \n",
        "          optimizer: torch.optim.Optimizer,\n",
        "          loss_fn: torch.nn.Module,\n",
        "          epochs: int,\n",
        "          device: torch.device,\n",
        "          save_path:str,\n",
        "          early_stop: False) -> Dict[str, List]:\n",
        "    \"\"\"Trains and tests a PyTorch model.\n",
        "\n",
        "    Passes a target PyTorch models through train_step() and test_step()\n",
        "    functions for a number of epochs, training and testing the model\n",
        "    in the same epoch loop.\n",
        "\n",
        "    Calculates, prints and stores evaluation metrics throughout.\n",
        "\n",
        "    Args:\n",
        "    model: A PyTorch model to be trained and tested.\n",
        "    train_dataloader: A DataLoader instance for the model to be trained on.\n",
        "    test_dataloader: A DataLoader instance for the model to be tested on.\n",
        "    optimizer: A PyTorch optimizer to help minimize the loss function.\n",
        "    loss_fn: A PyTorch loss function to calculate loss on both datasets.\n",
        "    epochs: An integer indicating how many epochs to train for.\n",
        "    device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
        "\n",
        "    Returns:\n",
        "    A dictionary of training and testing loss as well as training and\n",
        "    testing accuracy metrics. Each metric has a value in a list for \n",
        "    each epoch.\n",
        "    In the form: {train_loss: [...],\n",
        "              train_acc: [...],\n",
        "              test_loss: [...],\n",
        "              test_acc: [...]} \n",
        "    For example if training for epochs=2: \n",
        "             {train_loss: [2.0616, 1.0537],\n",
        "              train_acc: [0.3945, 0.3945],\n",
        "              test_loss: [1.2641, 1.5706],\n",
        "              test_acc: [0.3400, 0.2973]} \n",
        "    \n",
        "    adding that the training loop saves the best performing weights and \n",
        "    early stopping \n",
        "    \"\"\"\n",
        "    import copy\n",
        "\n",
        "    # Create empty results dictionary\n",
        "    results = {\"train_loss\": [],\n",
        "               \"train_acc\": [],\n",
        "               \"test_loss\": [],\n",
        "               \"test_acc\": []\n",
        "    }\n",
        "    \n",
        "    # Make sure model on target device\n",
        "    model.to(device)\n",
        "    min_loss = np.Inf\n",
        "    \n",
        "    #instantiate early stopper \n",
        "    if early_stop:\n",
        "      early_stopper = EarlyStopper(patience=3, min_delta=0.2)\n",
        "    # Loop through training and testing steps for a number of epochs\n",
        "    for epoch in tqdm(range(epochs)):\n",
        "        train_loss, train_acc = train_step(model=model,\n",
        "                                          dataloader=train_dataloader,\n",
        "                                          loss_fn=loss_fn,\n",
        "                                          optimizer=optimizer,\n",
        "                                          device=device)\n",
        "        \n",
        "        test_loss, test_acc = test_step(model=model,\n",
        "                                        dataloader=test_dataloader,\n",
        "                                        loss_fn=loss_fn,\n",
        "                                        device=device)\n",
        "        \n",
        "\n",
        "        if early_stop:\n",
        "          if early_stopper.early_stop(test_loss):\n",
        "            break\n",
        "\n",
        "        if test_loss < min_loss:\n",
        "          min_loss = test_loss\n",
        "          best_model_state = copy.deepcopy(model.classifier.state_dict()) \n",
        "          torch.save(model.classifier.state_dict(), save_path) #model.heads is specific to ViT\n",
        "\n",
        "        # Print out what's happening\n",
        "        print(\n",
        "          f\"Epoch: {epoch+1} | \"\n",
        "          f\"train_loss: {train_loss:.4f} | \"\n",
        "          f\"train_acc: {train_acc:.4f} | \"\n",
        "          f\"test_loss: {test_loss:.4f} | \"\n",
        "          f\"test_acc: {test_acc:.4f}\"\n",
        "        )\n",
        "\n",
        "        # Update results dictionary\n",
        "        results[\"train_loss\"].append(train_loss)\n",
        "        results[\"train_acc\"].append(train_acc)\n",
        "        results[\"test_loss\"].append(test_loss)\n",
        "        results[\"test_acc\"].append(test_acc)\n",
        "\n",
        "    # Return the filled results at the end of the epochs\n",
        "    return results"
      ],
      "metadata": {
        "id": "K_SjYGqQg78_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_loss_curves(loss_hist, acc_hist):\n",
        "    \"\"\"Plots training curves of a loss and accuracy dicts. \n",
        "    Args:\n",
        "       loss_hist = {train_loss:[], val_loss:[]}\n",
        "       train_hist = {train_acc:[], val_acc:[]}\n",
        "    \"\"\"\n",
        "    \n",
        "    # Get the loss values of the results dictionary (training and test)\n",
        "    train_loss = loss_hist['train']\n",
        "    test_loss = loss_hist['val']\n",
        "\n",
        "    # Get the accuracy values of the results dictionary (training and test)\n",
        "    train_accuracy = acc_hist['train']\n",
        "    test_accuracy = acc_hist['val']\n",
        "\n",
        "    # Figure out how many epochs there were\n",
        "    epochs = range(len(loss_hist['train']))\n",
        "\n",
        "    # Setup a plot \n",
        "    plt.figure(figsize=(15, 7))\n",
        "\n",
        "    # Plot loss\n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.plot(epochs, train_loss, label='train_loss')\n",
        "    plt.plot(epochs, test_loss, label='val_loss')\n",
        "    plt.title('Loss')\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.legend()\n",
        "\n",
        "    # Plot accuracy\n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.plot(epochs, train_accuracy, label='train_accuracy')\n",
        "    plt.plot(epochs, test_accuracy, label='val_accuracy')\n",
        "    plt.title('Accuracy')\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.legend();"
      ],
      "metadata": {
        "id": "FTPl5BRrF_UC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Create and run ensemble classifier"
      ],
      "metadata": {
        "id": "Y49qa2_dVphf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class EnsembleModelAll(nn.Module):   \n",
        "  '''\n",
        "  ensemble class will take three models and use their input to classify\n",
        "  the superclass by learning the right weights between the three models\n",
        "  in our case, this wil be a ResNet50, ViT, and EfficientNet as the three \n",
        "  CV models. \n",
        "\n",
        "  So each incoming feature vector will be a 3x1, and we will have three of them.\n",
        "  The output will be a 3x1 \n",
        "  softmax vector of the super classes \n",
        "  '''\n",
        "\n",
        "  def __init__(self, modelA, modelB, modelC):\n",
        "    super().__init__()\n",
        "    self.modelA = modelA\n",
        "    self.modelB = modelB\n",
        "    self.modelC = modelC\n",
        "    self.classifier = nn.Linear(3 * 3, 3)\n",
        "        \n",
        "  def forward(self, x):\n",
        "    x1 = self.modelA(x)\n",
        "    x2 = self.modelB(x)\n",
        "    x3 = self.modelC(x)\n",
        "    x = torch.cat((x1, x2, x3), dim=1)\n",
        "    out = self.classifier(x)\n",
        "    return out\n",
        "    "
      ],
      "metadata": {
        "id": "4d1xai_gT0XD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#initialize ensemble modles \n",
        "weights_resnet = torchvision.models.ResNet50_Weights.DEFAULT.DEFAULT\n",
        "resnet_model = torchvision.models.resnet50(weights=weights_resnet)\n",
        "in_ftrs = resnet_model.fc.in_features\n",
        "out_fts = 3\n",
        "resnet_model.fc = nn.Linear(in_ftrs, out_fts)\n",
        "resnet_model.load_state_dict(torch.load('/content/models/resnet_50.pth'))\n",
        "resnet_model.eval()\n",
        "\n",
        "resnet_model.to(device)\n"
      ],
      "metadata": {
        "id": "5HxEvdtORUy5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "weights_vit = torchvision.models.ViT_L_16_Weights.IMAGENET1K_SWAG_E2E_V1.DEFAULT # .DEFAULT = best available weights from pretraining on ImageNet\n",
        "\n",
        "vit_model = torchvision.models.vit_l_16(weights=weights_vit)\n",
        "vit_model.heads = torch.nn.Sequential(\n",
        "    #torch.nn.Dropout(p=0.2, inplace=True), \n",
        "    torch.nn.Linear(in_features=1024, \n",
        "                    out_features=3, # same number of output units as our number of classes\n",
        "                    bias=True))\n",
        "vit_model.heads.load_state_dict(torch.load('/content/models/vit_l_16_classifier.pth'))\n",
        "vit_model.eval()\n",
        "\n",
        "vit_model.to(device)\n",
        "#test_model(vit_model, test_dataloader, nn.CrossEntropyLoss(), device=device)"
      ],
      "metadata": {
        "id": "1hsWRBl1SrXK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "weights_en = torchvision.models.EfficientNet_V2_S_Weights.IMAGENET1K_V1\n",
        "en_model = torchvision.models.efficientnet_v2_s(weights=weights_en)\n",
        "en_model.classifier[1] = nn.Linear(1280, 3)\n",
        "en_model.load_state_dict(torch.load('/content/models/efficientnet_v2_s.pth'))\n",
        "en_model.eval() \n",
        "\n",
        "en_model.to(device)\n",
        "#test_model(en_model, test_dataloader, nn.CrossEntropyLoss(), device=device)"
      ],
      "metadata": {
        "id": "usVDw8oF0PEW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#train ensemble \n",
        "ensemble_model = EnsembleModelAll(resnet_model, vit_model, en_model)\n",
        "\n",
        "for param in ensemble_model.parameters():\n",
        "    param.requires_grad = False\n",
        "\n",
        "for param in ensemble_model.classifier.parameters():\n",
        "    param.requires_grad = True    \n",
        "\n",
        "ensemble_model = ensemble_model.to(device)"
      ],
      "metadata": {
        "id": "-UEk4g53Vu6d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary(model=ensemble_model, \n",
        "        input_size=(32, 3, 224, 224), # make sure this is \"input_size\", not \"input_shape\"\n",
        "        col_names=[\"input_size\", \"output_size\", \"num_params\", \"trainable\"],\n",
        "        col_width=20,\n",
        "        row_settings=[\"var_names\"]\n",
        "        )"
      ],
      "metadata": {
        "id": "UJyU_huCQLhW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(ensemble_model.classifier.parameters(), lr=0.001)"
      ],
      "metadata": {
        "id": "5FhceR4kf_1p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#train ensemble model \n",
        "from timeit import default_timer as timer \n",
        "start_time = timer()\n",
        "\n",
        "# Setup training and save the results\n",
        "# Setup training and save the results\n",
        "results = train(model=ensemble_model,\n",
        "                train_dataloader=train_dataloader,\n",
        "                test_dataloader=test_dataloader,\n",
        "                optimizer=optimizer,\n",
        "                loss_fn=loss_fn,\n",
        "                epochs=50,\n",
        "                device=device,\n",
        "                save_path='/content/models/ensemble_3_best.pth',\n",
        "                early_stop=True)\n",
        "\n",
        "# End the timer and print out how long it took\n",
        "end_time = timer()\n",
        "print(f\"[INFO] Total training time: {end_time-start_time:.3f} seconds\")"
      ],
      "metadata": {
        "id": "aG3WmPolfGhq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_loss_curves(loss_hist, acc_hist):\n",
        "    \"\"\"Plots training curves of a loss and accuracy dicts. \n",
        "    Args:\n",
        "       loss_hist = {train_loss:[], val_loss:[]}\n",
        "       train_hist = {train_acc:[], val_acc:[]}\n",
        "    \"\"\"\n",
        "    \n",
        "    # Get the loss values of the results dictionary (training and test)\n",
        "    train_loss = loss_hist['train']\n",
        "    test_loss = loss_hist['val']\n",
        "\n",
        "    # Get the accuracy values of the results dictionary (training and test)\n",
        "    train_accuracy = acc_hist['train']\n",
        "    test_accuracy = acc_hist['val']\n",
        "\n",
        "    # Figure out how many epochs there were\n",
        "    epochs = range(len(loss_hist['train']))\n",
        "\n",
        "    # Setup a plot \n",
        "    plt.figure(figsize=(15, 7))\n",
        "\n",
        "    # Plot loss\n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.plot(epochs, train_loss, label='train_loss')\n",
        "    plt.plot(epochs, test_loss, label='val_loss')\n",
        "    plt.title('Loss')\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.legend()\n",
        "\n",
        "    # Plot accuracy\n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.plot(epochs, train_accuracy, label='train_accuracy')\n",
        "    plt.plot(epochs, test_accuracy, label='val_accuracy')\n",
        "    plt.title('Accuracy')\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.legend();"
      ],
      "metadata": {
        "id": "T4u6uwEpiUTj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_hist = {\"train\": ensemble_results[\"train_loss\"], \"val\": ensemble_results[\"test_loss\"]}\n",
        "acc_hist = {\"train\": ensemble_results[\"train_acc\"], \"val\": ensemble_results[\"test_acc\"]}\n",
        "                              \n",
        "plot_loss_curves(loss_hist, acc_hist)"
      ],
      "metadata": {
        "id": "ny9IcsWWiUxz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#test_model(vit_model, test_dataloader, nn.CrossEntropyLoss(), device=device)"
      ],
      "metadata": {
        "id": "14kz64HeNYMX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#test_model(resnet_model, test_dataloader, nn.CrossEntropyLoss(), device=device)"
      ],
      "metadata": {
        "id": "LxgqgT0fb62V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Reduce Ensemble to two"
      ],
      "metadata": {
        "id": "dUFIRt_PNTRF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class EnsembleModelTwo(nn.Module):   \n",
        "  '''\n",
        "  ensemble class will take two models this time and use their input to classify\n",
        "  the superclass by learning the right weights between the two models\n",
        "  in our case, this wil be a ResNet50 and Vit.\n",
        "\n",
        "  So each incoming feature vector will be a 3x1, and we will have two of them.\n",
        "  The output will be a 3x1 \n",
        "  softmax vector of the super classes \n",
        "  '''\n",
        "\n",
        "  def __init__(self, modelA, modelB):\n",
        "    super().__init__()\n",
        "    self.modelA = modelA\n",
        "    self.modelB = modelB\n",
        "    self.classifier = nn.Linear(2 * 3, 3)\n",
        "        \n",
        "  def forward(self, x):\n",
        "    x1 = self.modelA(x)\n",
        "    x2 = self.modelB(x)\n",
        "    x = torch.cat((x1, x2), dim=1)\n",
        "    out = self.classifier(x)\n",
        "    return out"
      ],
      "metadata": {
        "id": "jMMeYMhBNbO1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#train ensemble \n",
        "ensemble_model_2 = EnsembleModelTwo(resnet_model, vit_model)\n",
        "\n",
        "for param in ensemble_model_2.parameters():\n",
        "    param.requires_grad = False\n",
        "\n",
        "for param in ensemble_model_2.classifier.parameters():\n",
        "    param.requires_grad = True    \n",
        "\n",
        "ensemble_model_2 = ensemble_model_2.to(device)"
      ],
      "metadata": {
        "id": "kq6_WNtQNsnV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(ensemble_model_2.classifier.parameters(), lr=0.001)"
      ],
      "metadata": {
        "id": "BkB38aw7N1Kt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#train ensemble model \n",
        "from timeit import default_timer as timer \n",
        "start_time = timer()\n",
        "\n",
        "# Setup training and save the results\n",
        "# Setup training and save the results\n",
        "results = train(model=ensemble_model_2,\n",
        "                train_dataloader=train_dataloader,\n",
        "                test_dataloader=test_dataloader,\n",
        "                optimizer=optimizer,\n",
        "                loss_fn=loss_fn,\n",
        "                epochs=10,\n",
        "                device=device,\n",
        "                save_path='/content/models/ensemble_2_best.pth',\n",
        "                early_stop=True)\n",
        "\n",
        "# End the timer and print out how long it took\n",
        "end_time = timer()\n",
        "print(f\"[INFO] Total training time: {end_time-start_time:.3f} seconds\")\n",
        "\n",
        "loss_hist = {\"train\": results[\"train_loss\"], \"val\": results[\"test_loss\"]}\n",
        "acc_hist = {\"train\": results[\"train_acc\"], \"val\": results[\"test_acc\"]}"
      ],
      "metadata": {
        "id": "VNOdqj33N5Vw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 240,
          "referenced_widgets": [
            "dc972ee1a9624d40b27cf427a179d54e",
            "080a94e85bc74a2e921ba02a725cabd9",
            "7b1589966cc940f8bc4a8e19beebc5e0",
            "14b1d49493ba42d4ab0955e110322217",
            "c655a12ddc4749faa44cebe6690b06f6",
            "352375d2357543baad91b9870bb3a979",
            "4bde10fba3c04bddad20b8d1973a3082",
            "244ed8f789b14d129ce572a332db9b6b",
            "085ef2cef50144b6af753403afe1d20e",
            "a5b61e78bddb4d47af2fb43e90b0e4ea",
            "2cdd13c67461484984a0fa27e15ae172"
          ]
        },
        "outputId": "6fbb563a-3bda-4193-a311-ce01e27a58f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dc972ee1a9624d40b27cf427a179d54e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/10 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1 | train_loss: 1.0129 | train_acc: 0.4983 | test_loss: 0.9572 | test_acc: 0.5717\n",
            "Epoch: 2 | train_loss: 0.9565 | train_acc: 0.5551 | test_loss: 0.9035 | test_acc: 0.6073\n",
            "Epoch: 3 | train_loss: 0.9324 | train_acc: 0.5659 | test_loss: 0.8808 | test_acc: 0.6145\n",
            "Epoch: 4 | train_loss: 0.9088 | train_acc: 0.5816 | test_loss: 0.8571 | test_acc: 0.6185\n",
            "Epoch: 5 | train_loss: 0.9060 | train_acc: 0.5711 | test_loss: 0.8478 | test_acc: 0.6192\n",
            "Epoch: 6 | train_loss: 0.9024 | train_acc: 0.5776 | test_loss: 0.8491 | test_acc: 0.6150\n",
            "Epoch: 7 | train_loss: 0.9001 | train_acc: 0.5750 | test_loss: 0.8464 | test_acc: 0.6136\n",
            "Epoch: 8 | train_loss: 0.8980 | train_acc: 0.5859 | test_loss: 0.8342 | test_acc: 0.6241\n",
            "Epoch: 9 | train_loss: 0.9033 | train_acc: 0.5737 | test_loss: 0.8361 | test_acc: 0.6196\n",
            "Epoch: 10 | train_loss: 0.8895 | train_acc: 0.5851 | test_loss: 0.8307 | test_acc: 0.6228\n",
            "[INFO] Total training time: 10075.311 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plot_loss_curves(loss_hist, acc_hist)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 458
        },
        "id": "fk4P_7iC6Bs7",
        "outputId": "ae9dc5f8-70f2-4302-b836-ac3cde42b2d4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3cAAAG5CAYAAAAtTs37AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hVVdbH8e9ODxAICSWF3lsgoRcbVgSEYAEBKY7C2Ov4WkbHMuo4jmVmFMQuOIoiCqJiQaUICBIgoYXeQugEQk3f7x/nIgEpAW5ycpPf53nuk5tzzj1n3aA5WXevvbax1iIiIiIiIiK+zc/tAEREREREROT8KbkTEREREREpA5TciYiIiIiIlAFK7kRERERERMoAJXciIiIiIiJlgJI7ERERERGRMkDJnYiIiIiISBmg5E6khBljNhpjLnc7DhERkeJkjJlhjNlrjAl2OxaR8kLJnYiIiIh4lTGmHnAhYIE+JXjdgJK6lkhppOROpBQwxgQbY/5tjNnqefz76CedxphqxpivjTH7jDEZxphfjDF+nn0PG2PSjTEHjDGrjDGXuftOREREABgKzAM+AIYd3WiMqW2M+cIYs8sYs8cY83qhfSOMMamee9oKY0xbz3ZrjGlU6LgPjDHPep5fYozZ4rkfbgfeN8ZU9dw3d3lGDr82xtQq9PoIY8z7nvvtXmPMZM/2ZcaYawodF2iM2W2MSSi2n5KIlym5Eykd/gp0BuKBNkBH4HHPvgeBLUB1oCbwGGCNMU2Bu4AO1tow4CpgY8mGLSIiclJDgY88j6uMMTWNMf7A18AmoB4QC3wCYIy5AXjK87rKOKN9e4p4rSggAqgLjMT5+/Z9z/d1gCPA64WO/xCoALQEagCveraPA24qdFxPYJu1dnER4xBxnYauRUqHwcDd1tqdAMaYp4E3gSeAXCAaqGutXQv84jkmHwgGWhhjdllrN7oRuIiISGHGmAtwEqsJ1trdxph1wCCckbwY4CFrbZ7n8Nmer7cCL1prF3i+X3sWlywAnrTWZnu+PwJ8Xiie54DpnufRwNVApLV2r+eQmZ6v/wOeMMZUttbuB4bgJIIiPkMjdyKlQwzOJ5lHbfJsA/gXzk3uB2PMemPMIwCeRO8+nE86dxpjPjHGxCAiIuKuYcAP1trdnu8/9myrDWwqlNgVVhtYd47X22WtzTr6jTGmgjHmTWPMJmPMfmAWEO4ZOawNZBRK7H5nrd0KzAGuM8aE4ySBH51jTCKuUHInUjpsxfmU86g6nm1Yaw9Yax+01jbAKVN54OjcOmvtx9bao5+QWuCfJRu2iIjIMcaYUKA/cLExZrtnHtz9OFMOdgB1TtH0JA1oeIrTHsYpozwq6oT99oTvHwSaAp2stZWBi46G57lOhCd5O5mxOKWZNwC/WmvTT3GcSKmk5E7EHYHGmJCjD2A88LgxproxphrwN5zyEIwxvY0xjYwxBsgE8oECY0xTY8ylnsYrWThlKAXuvB0REREAEnHuUy1w5pHHA81xphQkAtuAF4wxFT33wG6e170D/MUY0844Ghljjn7omQwMMsb4G2N6ABefIYYwnHviPmNMBPDk0R3W2m3At8BoT+OVQGPMRYVeOxloC9yLMwdPxKcouRNxx1ScG8/RRwiQBCwBlgKLgGc9xzYGfgQOAr8Co62103Hm270A7Aa240wKf7Tk3oKIiMgfDAPet9ZuttZuP/rAaWgyELgGaARsxmkWNgDAWvsZ8BxOCecBnCQrwnPOez2v24czR33yGWL4NxCKc3+cB3x3wv4hOPPZVwI7caY44Inj6Hy9+sAXZ/neRVxnrD1xJFtEREREpHwyxvwNaGKtvemMB4uUMuqWKSIiIiKCswYecAvO6J6Iz1FZpoiIiIiUe8aYETgNV7611s5yOx6Rc6GyTBERERERkTJAI3ciIiIiIiJlgE/NuatWrZqtV6+e22GIiEgxW7hw4W5rbXW34/AVuj+KiJQfp7tH+lRyV69ePZKSktwOQ0REipkxZpPbMfgS3R9FRMqP090jVZYpIiIiIiJSBii5ExERERERKQOU3ImIiIiIiJQBPjXnTkSktMjNzWXLli1kZWW5HYpPCwkJoVatWgQGBrodioiIiM9Tcicicg62bNlCWFgY9erVwxjjdjg+yVrLnj172LJlC/Xr13c7HBEREZ+nskwRkXOQlZVFZGSkErvzYIwhMjJSo58iIiJeouROROQcKbE7f/oZioiIeI+SOxERERERkTJAyZ2IiIiIiEgZoORORMQH7du3j9GjR5/163r27Mm+ffvO+nXDhw9n4sSJZ/06ERERKTlK7kREfNCpkru8vLzTvm7q1KmEh4cXV1giIiLiIi2FICJynp7+ajkrtu736jlbxFTmyWtannL/I488wrp164iPjycwMJCQkBCqVq3KypUrWb16NYmJiaSlpZGVlcW9997LyJEjAahXrx5JSUkcPHiQq6++mgsuuIC5c+cSGxvLl19+SWho6Blj++mnn/jLX/5CXl4eHTp04I033iA4OJhHHnmEKVOmEBAQwJVXXslLL73EZ599xtNPP42/vz9VqlRh1qxZXvsZiYiIyPGU3ImI+KAXXniBZcuWkZyczIwZM+jVqxfLli37fb249957j4iICI4cOUKHDh247rrriIyMPO4ca9asYfz48bz99tv079+fzz//nJtuuum0183KymL48OH89NNPNGnShKFDh/LGG28wZMgQJk2axMqVKzHG/F76+cwzz/D9998TGxt7TuWgIiIiUnRK7kREztPpRthKSseOHY9bCPy///0vkyZNAiAtLY01a9b8IbmrX78+8fHxALRr146NGzee8TqrVq2ifv36NGnSBIBhw4YxatQo7rrrLkJCQrjlllvo3bs3vXv3BqBbt24MHz6c/v37c+2113rjrYqIiMgplKs5d4ey81i+NdPtMEREvK5ixYq/P58xYwY//vgjv/76KykpKSQkJJx0ofDg4ODfn/v7+59xvt7pBAQE8Ntvv3H99dfz9ddf06NHDwDGjBnDs88+S1paGu3atWPPnj3nfA0RESlFDmdAfq7bUcgJytXI3f2fJrMsPZPZD1+Kn58WzhUR3xUWFsaBAwdOui8zM5OqVatSoUIFVq5cybx587x23aZNm7Jx40bWrl1Lo0aN+PDDD7n44os5ePAghw8fpmfPnnTr1o0GDRoAsG7dOjp16kSnTp349ttvSUtL+8MIooiI+JCCfPjlFZjxPARXhqY9oXlvaHgpBJ553rYUr3KV3PVqHc0PK3bw28YMOjfQHxci4rsiIyPp1q0brVq1IjQ0lJo1a/6+r0ePHowZM4bmzZvTtGlTOnfu7LXrhoSE8P7773PDDTf83lDltttuIyMjg759+5KVlYW1lldeeQWAhx56iDVr1mCt5bLLLqNNmzZei0VERErYwZ3wxQhYPwNaJDrJ3KpvIOVjCKwIjS+H5n2g8RUQUsXtaMslY611O4Yia9++vU1KSjrn1x/Jyaf9s9O4pk0ML1zX2ouRiUh5k5qaSvPmzd0Oo0w42c/SGLPQWtvepZB8zvneH0VEzmjDLPj8VsjKhKtfhLZDwRinNHPjL5D6Naz8Gg7uAL9AaHAJNL/GGdmrVN3t6MuU090jy9Wcu9Agf65qFcU3S7eRlZvvdjgiIiIiIqVbQT7MfBHG9YXgMLj1J2g3zEnsAPwDnZLM3q/AAyvhTz9Apz/D7tXw1T3wchN4vxfMewP2pbn7XtyQmQ7LPoepD8GYC2FbSrFerlyVZQL0S4jli0XpzFi1kx6tot0OR0SkVLnzzjuZM2fOcdvuvfdebr75ZpciEhER1xQuw4zr7yRwwWGnPt7PD+p0ch5XPgs7lkHqV87ju0ecR0yCM6LX7Bqo3qTE3kqJKMiHnStg8zznkTYfMj0JbWBFqNWu2JvQlLvkrmvDalQPC2bS4nQldyIiJxg1apTbIYiISGlQuAyzz2uQMOTYaF1RGANRcc6j+2OwZ92xRO+nZ5xHtaZOotf8Gohuc3bnLw1yDsGWJCeJ2/yr8zx7v7MvLBpqd4Iud0KdzlAzDvyLP/Uqd8mdv5+hb5sYxv26iX2HcwivEOR2SCIiIiIipUNBPsx6CWa+AJGNYMgkqOmF9VwjG8IF9zmPzHRY+Q2kToHZr8IvL0GVOp5Er7eTFPn5n/81vW3/Nkibd2xkbvtSsPmAgRotIO56qN3ZSebC67iSrJa75A4gMSGWd2ZvYOrS7QzqVMftcERERERE3HdwpzNat2EmtB4AvV6B4Erev06VWOg00nkc2gOrv3VG9Ba8DfNGQcXq0KyXU7pZ/yIIcGEwpqAAdqUWKrGcB/s2O/sCQqFWe7jgfieRq9UBQsNLPsaTKJfJXcuYyjSqUYnJi9OV3ImIiIiIrJ/pzK/LyoQ+r0PCTSUz8lQx0rlWwk2QfQDW/OB03lw6ERZ+AMFVoMlVzqheo8sgqGLxxJFzGNIXHhuZS1sA2ZnOvko1ndHETrc5I3PRrZ1GMqVQuUzujDH0S4jlX9+vIi3jMLUjKrgdkoiIlDHGmB7AfwB/4B1r7QsnOaY/8BRggRRr7SBjTDzwBlAZyAees9Z+WmKBi0j5UpAPs/4FM16Aao29V4Z5LoLDoNV1ziM3y2nkkvoVrJoKSyc4I2aNLnMSvSZXQWjVc7/WgR2eRM4zX277EijIc/ZVbw6t+h0rsaxaz2fmA5bL5A6gT5sY/vX9KqakbOXO7o3cDkdEpFhVqlSJgwcPnnTfxo0b6d27N8uWLSvhqMouY4w/MAq4AtgCLDDGTLHWrih0TGPgUaCbtXavMaaGZ9dhYKi1do0xJgZYaIz53lq7r4TfhoiUdQd2wBe3Os1TWt8IvV4unjLMcxEYAk17OI/8PNg819OQxbOenl+AU7LZ/Bpo2gvCap76XAUFsHvVsQ6Wm3+FvRudfQEhENsOut5zrMSyQkSJvMXiUG6Tu9oRFehYL4IvFm3hjksaYnwkGxcREZ/QEVhrrV0PYIz5BOgLrCh0zAhglLV2L4C1dqfn6+qjB1hrtxpjdgLVASV3IuI962c68+uyD0DfURA/uPSOTvl7Ern6F0GPf8LWRZ5Ebwp8fT98/YBTNnm0IUulmpC+6NjIXNp8yPL8Cq1Y3Tm2w62eEss27szpKyblNrkDp7HKY5OWsnzrflrFVnE7HBHxVd8+4nTM8qaoOLj6D1V8v3vkkUeoXbs2d955JwBPPfUUAQEBTJ8+nb1795Kbm8uzzz5L3759z+qyWVlZ3H777SQlJREQEMArr7xC9+7dWb58OTfffDM5OTkUFBTw+eefExMTQ//+/dmyZQv5+fk88cQTDBgw4LzedhkSCxRerXcL0OmEY5oAGGPm4JRuPmWt/a7wAcaYjkAQsO7ECxhjRgIjAerU0fxxESmiE8swh34JNVu4HVXR+fk5zUxqtYfLn4KdqU6it/Ir+OGvzsMv4FiJZbWm0KLPsRLLiAalN4n1gnKd3PWKi+apKcuZtDhdyZ2I+JQBAwZw3333/Z7cTZgwge+//5577rmHypUrs3v3bjp37kyfPn3OqjJh1KhRGGNYunQpK1eu5Morr2T16tWMGTOGe++9l8GDB5OTk0N+fj5Tp04lJiaGb775BoDMzMxiea9lWADQGLgEqAXMMsbEHS2/NMZEAx8Cw6y1BSe+2Fr7FvAWQPv27W1JBS0COAsx5xyC3MNOI4qcg8ee5x5y9v2+v9BzjDP60vBSCKns9rsofwqXYbYZCD1fKj1lmOfCGCcxrdkCLnkYMjY4JZuHM6B2R2eEzodLLM9FuU7uqlQIpHuz6kxJ2cqjVzcjwN/P7ZBExBedZoStuCQkJLBz5062bt3Krl27qFq1KlFRUdx///3MmjULPz8/0tPT2bFjB1FRUUU+7+zZs7n77rsBaNasGXXr1mX16tV06dKF5557ji1btnDttdfSuHFj4uLiePDBB3n44Yfp3bs3F154YXG9XV+UDtQu9H0tz7bCtgDzrbW5wAZjzGqcZG+BMaYy8A3wV2vtvJIIWMqgggInocr1JF85h/+YbJ34/Ez7jz7Pzzm7WAIrOI+8bEh6F/wCoV43aNLDaYwR0aB4fgZyzPoZ8PkITxnmaEgY7HZE3hdRH7re7XYUrirXyR1Av4RYvl++g7nr9nBRk+puhyMiUmQ33HADEydOZPv27QwYMICPPvqIXbt2sXDhQgIDA6lXrx5ZWVleudagQYPo1KkT33zzDT179uTNN9/k0ksvZdGiRUydOpXHH3+cyy67jL/97W9euV4ZsABobIypj5PU3QgMOuGYycBA4H1jTDWcMs31xpggYBIwzlo7sQRjFl+Wl+00iVgzDdb+6KzHlXv47M7hH+QkYEGVIMiTjAVVhEo1jj0Pqnjm5yduC6zglNKB0xhjy2+w+jtY/T1894jzqNbESfKa9HDK5/zL/Z+o3lOQDzNfhJn/dH7Ow6ZAjeZuRyXFpNz/n3NJ0xpUDglg8uJ0JXci4lMGDBjAiBEj2L17NzNnzmTChAnUqFGDwMBApk+fzqZNm876nBdeeCEfffQRl156KatXr2bz5s00bdqU9evX06BBA+655x42b97MkiVLaNasGREREdx0002Eh4fzzjvvFMO79E3W2jxjzF3A9zjz6d6z1i43xjwDJFlrp3j2XWmMWYGz5MFD1to9xpibgIuASGPMcM8ph1trk0v+nUipti8N1k5zErr1M51ySP8gqNsVGl1+igSsYqHE7YQkriTW7fIPcOKr2xWueAYy1sPqH5xkb94YmPsahFSBRlc4iV6jy8pdWZ1XHdgBn98CG3+BNoOg10vFt06clArlPrkLCfSnV+topiRv5XBOHhWCyv2PRER8RMuWLTlw4ACxsbFER0czePBgrrnmGuLi4mjfvj3NmjU763Pecccd3H777cTFxREQEMAHH3xAcHAwEyZM4MMPPyQwMJCoqCgee+wxFixYwEMPPYSfnx+BgYG88cYbxfAufZe1diow9YRtfyv03AIPeB6Fj/kf8L+SiFF8TF6OMzp3NKHbtdLZXqUOtBkAja+Eehf61hyqiAbQ+TbnkX0A1k13RvTWfA/LJoLxc0bymvZwkr1qTcp0MwyvWjfdWZQ8+2DZLcOUPzDOvcU3tG/f3iYlJXn9vPPX72HAW/P4z43x9I2P9fr5RaTsSU1NpXlzlbV4w8l+lsaYhdba9i6F5HOK6/4opUDmlmOllutnOHPn/AKdka/GVzgJXVlMeAoKnHb3q79zHkc7Eletd2yeXt1uEBDsapilUkG+U4I580Wo3hRu+EBlmGXM6e6RGqYCOtSLIDY8lMmL05XciYiIiHvycpy1uY4mdDs9SyNWqQ1xNzgJXf2LIDjM3TiLW+F295c+7klyf3BG9RZ+APPHOGWlDS91kr3GV0IlTa/hwHZn7bqNvzjr1vX8l8owyxkld4Cfn6FvfAxvzlrP7oPZVKukT4FEpOxZunQpQ4YMOW5bcHAw8+fPdykiEQEgM/34uXM5Bzyjc13gir87CV31ZmVvdO5sVKkF7f/kPHIOO638jzZlSZ0CGIhtd2xULyqu/P28jpZh5hyCxDcg/sQeTlIeKLnz6JcQy+gZ6/g6ZSvDu9V3OxwR8QHW2rNaQ85tcXFxJCeXrp4cvjQ1QMRr8nMhbb4zErXmR9i53NleuRbEXec0E2lwcdkfnTtXQRWcOXhNe4C1Tsnm0fLN6c86j8qxx7pv1r8IAkPdjrr4FOQ7C5LP+pdThjnsa6hx9nOupWwoUnJnjHkP6A3stNa2Osl+A/wH6AkcxunqtcizbxjwuOfQZ621Yz3b2wEfAKE4E87vtS7e5RvXDKNFdGUmJSu5E5EzCwkJYc+ePURGRvpUgleaWGvZs2cPISEhbociUvz2b3XKLNf84IzOZe8HvwCo08XpGtnoCmdelH6fnB1jILq187j4/5zukGunOYnekgmQ9B4EhDrJ8tFRvcoxbkftPceVYd4EPV9UGWY5V9SRuw+A14Fxp9h/Nc7Cq42BTsAbQCdjTATwJNAesMBCY8wUa+1ezzEjgPk4yV0P4Ntzexve0S8hluemprJ+10EaVPehTlMiUuJq1arFli1b2LVrl9uh+LSQkBBq1arldhgi3pefC2m/HSu33LHM2R4WAy37eebOXQwhld2Ns6wJqwkJNzmPvGzYONsp3Vz9rZPwAUS19iR6PSAm4dgafL5m3c/wxUiVYcpxipTcWWtnGWPqneaQvjiLrVpgnjEm3BgTDVwCTLPWZgAYY6YBPYwxM4DK1tp5nu3jgERcTu76xMfw/LepTE7eygNXNHEzFBEp5QIDA6lfX6P8IlLI/m2FRudmHBudq90ZLn/aSehqtNDoXEkJCHbWyWt0GVz9T9i16lj55i8vwawXoWINpxlL0x7OPL1KURBYyqsJ8vNg5gsw6yVnLqbKMKUQb825iwXSCn2/xbPtdNu3nGT7HxhjRgIjAerUqeOlcE+uZuUQujWsxuTF6dx/eWOVWomIiMip5efBlt88nS2nHWvXHxYNLfo6SUODi51FucVdxjgJUI1mcMF9cDjDScRXfwepX0FyoaUlQ8IhLAoq1XT+LcM8XyvVdLaHRTlJYFCFkn8f+7c5ZZibZjujk1f/y504pNQq9Q1VrLVvAW+Bs45PcV8vMSGWv3yWwqLN+2hXt2pxX05ERER8TdZ+mPcGzH8DjuwF4w91OsNlTzoJXc2WGp0r7SpEQOv+ziM/F7YsgIz1cGCbM2/v4HZnPtumuc7z/Jw/niO4ipP4FU4CK0UdSwCPJoTeWlR+7U9OGWbuYUgcA/EDvXNeKVO8ldylA7ULfV/Lsy0dpzSz8PYZnu21TnK8665qWZPHJ/sxeXG6kjsRERE5JucwLHgbZv8bjmRA015OctDgEggNdzs6OVf+nkXh63Y9+X5rnST+wPZjSd/Rx8HtTjKYNh8O7oC8rD++PqjSsdG+U40ChkU53VFP9qFAfh7M+Af88rJThtl/rNMVU+QkvJXcTQHuMsZ8gtNQJdNau80Y8z3wvDHmaJZ0JfCotTbDGLPfGNMZp6HKUOA1L8VyXsJCArmiRRRfL9nKE71bEBTgo5NsRURExDvysmHhWGee1sEd0PAyZ2Ht2LZuRyYlwRhnpK9CBNRscerjrIWszD8mgQd3HBsR3LoYDnzrjL6dKLDCyUcB10zzlGEOgatfVBmmnFZRl0IYjzMCV80YswWnA2YggLV2DE63y57AWpylEG727MswxvwdWOA51TNHm6sAd3BsKYRvcbmZSmH9EmL4KmUrs1bv4vIWNd0OR0RERNyQnwcp42HmPyEzDep2gxs+OPUIj5RvxjgjuKHhp29wYi1kHzh+5O/ANk8S6EkIty+FA9Mg5yAEVoR+b0KbG0vuvYjPKmq3zNMW9Xq6ZN55in3vAe+dZHsS8Ic180qDCxtXJ6JiEJOS05XciYiIlDcFBbD8C5j+PGSsg5i20Oe/0KC75tLJ+TPGWQIjpDJUP0N39uyDzvFau06KqNQ3VHFDoL8f17SO5pMFaRzIyiUsJNDtkERERKS4WQsrv4Hpz8HOFVCjJdz4MTTtqaRO3OGtZixSbmhC2SkkJsSSnVfAd8u2ux2KiIiIFCdrnbb4b3eHTwc7nRGvexdumw3NeimxExGfoZG7U4ivHU69yApMTk7nhva1z/wCERER8T0b58DPz8LmuVClDvQdBa1vBH/9iSQivke/uU7BGENiQiz/+WkN2zOziKoS4nZIIiIi4i3pC52kbt3PTlfCni9B22EQEOR2ZCIi50xlmaeRGB+LtTAlpVQswSciIiLna8dyGD8I3r4UtqXAlc/CPYuh4wgldiLi8zRydxr1qlUkoU44kxZvZeRFDd0OR0RERM7V7rUw43lY9gUEV4buj0Pn25yFo0VEyggld2eQGB/Lk1OWs3L7fppFVXY7HBERETkbezfBzBch5WMICIEL7oeudzsLUouIlDEqyzyD3q2j8fczTF681e1QREREpKj2b4NvHoTX2sHSz6DT7XDvErj8SSV2IlJmaeTuDCIrBXNxk+p8mZzO/13VFD8/tUMWEREptQ7tgTmvwm9vQ0EeJAyBix6CKrFuRyYiUuyU3BVBYkIsP6/cyfwNGXRpGOl2OCIiInKiI/vg11EwbzTkHobWA+DihyGivtuRiYiUGCV3RXBF85pUDPJn8uJ0JXciIiKlSc4hmD8G5vwXsvZBi0To/hhUb+p2ZCIiJU7JXRGEBvnTo1U0U5du4+m+LQkJ9Hc7JBERkfItNwuS3oPZr8ChXdCkB3T/K0S3djsyERHXqKFKEfVLiOVAdh4/r9zpdigiIiLlV36uk9T9NwG+fxRqtIBbpsGgT5XYiUi5p5G7IurSMJIaYcFMWpxOz7hot8MREREpXwryYckEmPkC7N0ItTrCtW9C/YvcjkxEpNRQcldE/n6GvvExfDB3I/sO5xBeIcjtkERERMq+ggJInQLTn4fdqyCqNQz6DBpfAUYdrEVEClNZ5llITIglN9/yzdJtbociIiJSPnz3CHw2zHnefxyMnAlNrlRiJyJyEkruzkKL6Mo0qVmJyYvT3Q5FRESk7DuyDxaNg7j+cMev0KIv+OlPFxGRU9FvyLNgjCExIZYFG/eSlnHY7XBERETKtqWfQd4R6Hw7+KlTtYjImSi5O0t942MB+DJZo3ciIiLFatE4iIqDmAS3IxER8QlK7s5SbHgonepHMGlxOtZat8MREREpm7Yuhu1LoO0wza8TESkiJXfnoF9CLOt2HWJZ+n63QxERESmbFo2DgFCIu8HtSEREfIaSu3NwdVw0Qf5+TFJjFREREe/LOQRLPoOWiRAa7nY0IiI+Q8ndOagSGsilzWowJWUrefkFbocjIiJStiyfBDkHoO1QtyMREfEpSu7OUWJCLLsPZjNn3R63QxERESlbFo2DyMZQp4vbkYiI+BQld+eoe7PqVA4J0Jp3IiIi3rQzFdLmO6N2aqQiInJWlNydo+AAf3q1juG7Zds5lJ3ndjgiIiJlw6IPwS8Q4ge5HYmIiM9Rcnce+iXEciQ3n2krdrgdioiIiO/Ly4aU8dCsF1Ss5nY0IiI+R8ndeWhftyqx4aHqmiJdSmAAACAASURBVCkiIuINK7+GIxlqpCIico6U3J0HPz9DYkIMv6zZxa4D2W6HIyIi4tsWjoUqdaBBd7cjERHxSUruzlNifCwFFr5estXtUERERHxXxnrYMBPaDgE//XkiInIu9NvzPDWuGUar2MrqmikiInI+Fv8PjB/ED3Y7EhERn6XkzgsS42NJ2ZLJul0H3Q5FRETE9+TnweKPoPGVUCXW7WhERHyWkjsv6NMmBj8DX2r0TkRE5Oyt+QEOblcjFRGR86TkzgtqVA6hW6NqTEpOx1rrdjgiIlIKGGN6GGNWGWPWGmMeOcUx/Y0xK4wxy40xHxfaPswYs8bzGFZyUbtk0VioFAWNr3I7EhERn6bkzkv6JcSSlnGERZv3uh2KiIi4zBjjD4wCrgZaAAONMS1OOKYx8CjQzVrbErjPsz0CeBLoBHQEnjTGVC3B8EtWZrozchc/CPwD3I5GRMSnKbnzkqtaRhEa6K8170REBJykbK21dr21Ngf4BOh7wjEjgFHW2r0A1tqdnu1XAdOstRmefdOAHiUUd8lL/hhsgdMlU0REzouSOy+pGBzAlS1r8vWSbeTkFbgdjoiIuCsWSCv0/RbPtsKaAE2MMXOMMfOMMT3O4rUYY0YaY5KMMUm7du3yYuglqKAAFo+D+hdDRAO3oxER8XlK7rwoMT6WfYdzmbnaR2+yIiJSkgKAxsAlwEDgbWNMeFFfbK19y1rb3lrbvnr16sUUYjHbMAP2bVYjFRERLylScnemSeHGmLrGmJ+MMUuMMTOMMbU827sbY5ILPbKMMYmefR8YYzYU2hfv3bdW8i5oXI3IikFa805ERNKB2oW+r+XZVtgWYIq1NtdauwFYjZPsFeW1ZcPCsRBaFZpf43YkIiJlwhmTu6JMCgdeAsZZa1sDzwD/ALDWTrfWxltr44FLgcPAD4Ve99DR/dba5PN/O+4K9PfjmjYxTEvdwf6sXLfDERER9ywAGhtj6htjgoAbgSknHDMZZ9QOY0w1nDLN9cD3wJXGmKqeRipXeraVLYd2w8pvoM1ACAh2OxoRkTKhKCN3RZkU3gL42fN8+kn2A1wPfGutPXyuwfqCxIRYcvIK+G7pdrdDERERl1hr84C7cJKyVGCCtXa5MeYZY0wfz2HfA3uMMStw7p0PWWv3WGszgL/jJIgLgGc828qWlPFQkKuSTBERLypKcleUid0pwLWe5/2AMGNM5AnH3AiMP2Hbc55SzleNMWXiY7s2tapQv1pFdc0UESnnrLVTrbVNrLUNrbXPebb9zVo7xfPcWmsfsNa2sNbGWWs/KfTa96y1jTyP9916D8XGWqcks1ZHqNHc7WhERMoMbzVU+QtwsTFmMXAxztyA/KM7jTHRQBzHl5U8CjQDOgARwMMnO7GvdQMzxpAYH8u8DXvYlnnE7XBERERKn83zYM8aaFf212cXESlJRUnuzjix21q71Vp7rbU2AfirZ9u+Qof0ByZZa3MLvWab51PLbOB9nPLPP/DFbmCJCTFYC1OSt7odioiISOmzaCwEhUHLfm5HIiJSphQluTvjpHBjTDVjzNFzPQq8d8I5BnJCSaZnNA9jjAESgWVnH37pVDeyIm3rhKs0U0RE5ERH9sHyyRB3PQRVdDsaEZEy5YzJXREnhV8CrDLGrAZqAs8dfb0xph7OyN/ME079kTFmKbAUqAY8e17vpJTplxDLyu0HSN223+1QRERESo+ln0HeEZVkiogUg4CiHGStnQpMPWHb3wo9nwhMPMVrN/LHBixYay89m0B9Ta/WMTz91QomJ6fTPLqy2+GIiIi4z1qnJDMqDqJ9fnlbEZFSx1sNVeQEERWDuKRpdb5cvJWCAut2OCIiIu7blgzbl0LbYWCM29GIiJQ5Su6KUWJCLNv3ZzFvwx63QxEREXHfwrEQEApxN7gdiYhImaTkrhhd3rwmlYIDmKzGKiIiUt7lHIKlE6FlIoSGux2NiEiZpOSuGIUE+nN1qyi+XbqdrNz8M79ARESkrFo+CXIOOCWZIiJSLJTcFbN+CbEcyM7jp9SdbociIiLinkXjoFoTqNPZ7UhERMosJXfFrFODSGpWDtaadyIiUn7tTIW0+dB2qBqpiIgUIyV3xczfz9A3PpYZq3aScSjH7XBERERK3qJx4BcIbQa6HYmISJmm5K4EJMbHkldg+WbpNrdDERERKVl52ZAyHpr1gorV3I5GRKRMU3JXAppHh9G0Zpi6ZoqISPmT+hUc2Qvt1EhFRKS4KbkrAcYYEhNiWbhpL5v3HHY7HBERkZKzaByE14H6l7gdiYhImafkroT0jY8B4Mtkjd6JiEg5kbEeNsyEhKHgpz85RESKm37TlpCY8FA6N4hgUnI61lq3wxERESl+i/8Hxg/iB7kdiYhIuaDkrgT1S4hl/a5DLE3PdDsUERGR4pWfB4s/gsZXQpVYt6MRESkXlNyVoB6togkK8NOadyIiUvat+R4Oboe2aqQiIlJSlNyVoCqhgVzevAZfpWwlL7/A7XBERESKz6JxUCnKGbkTEZESoeSuhCXGx7L7YA6z1+52OxQREZHikZkOa36AhMHgH+B2NCIi5YaSuxJ2SdMahFcI1Jp3IiJSdiV/DLYAEm5yOxIRkXJFyV0JCwrwo1dcNN8v38Gh7Dy3wxEREfGuggJYPA7qXwwRDdyORkSkXFFy54J+CbEcyc3nhxXb3Q5FRETEu9ZPh32boZ0aqYiIlDQldy5oV7cqtaqGMmnxVrdDERER8a5F4yA0Apr1djsSEZFyR8mdC4wxJMbHMnvNLnYeyHI7HBEREe84tBtWfgNtBkJAsNvRiIiUO0ruXJKYEEOBha9StrkdioiIiHekjIeCXGg71O1IRETKJSV3LmlUI4y42CrqmikiImWDtbBwLNTuBDWauR2NiEi5pOTORYkJsSxNz2TtzoNuhyIiInJ+Ns+DPWugrRqpiIi4Rcmdi65pE42fgS+TNXonIiI+btFYCK4MLRPdjkREpNxScueiGmEhXNC4OpMWp2OtdTscERGRc3NkHyyfDHHXQ1BFt6MRESm3lNy5rF9CDFv2HmHhpr1uhyIiInJuln4GeUfUSEWkHMk4lKPBiVKofCV366bDb2+7HcVxrmwRRWigP5PUWEVERHyRtU5JZlRriElwOxoRKWbWWt6cuY72z07jzx8u5EhOvtshSSHlK7lbPgmmPQnZpaeBScXgAK5qWZOvl2wjJ6/A7XBERETOzrZk2L5Uo3Yi5cDB7Dzu+GgR//h2JfG1w5mWuoMb357H7oPZbofmE9IyDhf7NcpXchc/GHIPwYrJbkdynMSEWDKP5PLenA1uhyIiInJ2Fo6FgFBo3d/tSESkGK3deYC+r8/mhxU7+GvP5nx+e1fevKkdq7bvp9/oOazbVXoGT0obay3vz9lA95dm8OOKHcV6rfKV3NXuCJGNIPljtyM5zkWNq9MzLop/freSn1KL9x9cRETEa7IPwtKJ0LIfhFRxOxoRKSbfLNlG39fnkHkkl//d0okRFzXAGMOVLaP4ZGQXjuTkc+3oufy2IcPtUEud7Lx8Hv58CU9/tYLuzWrQuWFksV6vfCV3xkD8INg0BzLWux3N7/z8DC/fEE+rmCrcM34xK7fvdzskERGRM1sxGXIOqCRTpIzKyy/g+amp3PnxIppEhfHV3RfQ5YTkJL52OJPu6EZkpSBuemc+U1K2uhRt6bPzQBYD35rHhKQt3HNZY968qR2VggOK9ZrlK7kDaH0jGD9IHu92JMcJDfLn7aHtqRQSwC0fJKl2WURESr+FY6FaE6jT2e1IRMTLdh3I5qZ35/PWrPUM7VKXT0d2IbpK6EmPrR1RgS9u70p8nXDuGb+Y0TPWlvtOmku27KPPa3NI3XaA0YPb8sAVTfDzM8V+3fKX3FWJhQbdIWU8FJSuBiZRVUJ4e2h79hzK5s8fLiQrV92HRESklNqZClt+c0btTPH/wSIiJWfhpr30fu0XktP28Ur/NjzTtxVBAadPG8IrBPHhLR3pGx/Di9+t4rFJS8nLL11/a5eUyYvTuWHMr/j7GT6/vSs946JL7NrlL7kDpzQzMw02znI7kj9oXSucl2+IZ+GmvTz2xdJy/6mHiIiUUovGgV8gtBnodiQi4iXWWj78dSM3vvUrwQH+fHF7N65tW6vIrw8O8OfV/vHc2b0h439L45axSRzMziu+gEuZ/ALLP6amct+nycTXDmfKXd1oEVO5RGMon8lds94QXKXUNVY5qlfraB64oglfLE5n9Ix1bocjIiJyvLxspwKmeW+oWM3taETEC47k5PPghBSe+HI5Fzauzld3XXBOiYmfn+Ghq5rxj2vjmL12N/3H/Mr2zKxiiLh0yTySyy1jF/DmrPUM6VyX/93aichKwSUeR/lM7gJDIO46WDEFsjLdjuak7r60EX3axPCv71fx3bJtbocjIiJyTOpXcGSvGqmIlBGb9hyi3+g5TEpO54ErmvDO0PZUqRB4Xucc2LEO7w5r//u5y3LDwHW7DtJv1Bxmr9nN8/3i+HtiKwL93UmzymdyBxB/E+QdgeWla827o4wxvHh9a+Jrh3P/pyksSy+dSaiIiJRDi8ZCeB2of4nbkYjIefopdQe9X5vNtsws3h/egXsua+y1xh+XNK3BhNu6UGAt17/xK7+s2eWV85Ym01fuJNGzTMTHIzozqFMdV+Mpv8ldbFuo1hSSP3I7klMKCfTnraHtqFohkBHjkti5v+wPaYuISCmXsR42zIKEoeBXfv+MKM3SMg6zLfOI22FIKZdfYHll2mpuGZtEnYgKfH33BVzStIbXr9MypgqT7uhGraqh3Pz+AiYkpXn9Gm6w1jJm5jr+NHYBdSIrMOXuC+hYP8LtsIqW3BljehhjVhlj1hpjHjnJ/rrGmJ+MMUuMMTOMMbUK7cs3xiR7HlMKba9vjJnvOeenxpgg77ylIjIGEgZD2nzYvbZEL302aoSF8M6wDmQeyWXEuCR10BQREXct+tBZUihhsNuRyEms3L6fnv/9hctfnqn1xuSU9h3O4U8fLOC/P63h+na1+Pz2rtSOqFBs14sJD+Wz27rQpWEk/zdxCa/8sMqnmwZm5eZz36fJvPDtSnrFRTPxtq7Ehp98mYiSdsbkzhjjD4wCrgZaAAONMS1OOOwlYJy1tjXwDPCPQvuOWGvjPY8+hbb/E3jVWtsI2Avcch7v49y0HgDGv1SP3gG0iKnMqwPiWZKeyV8+S/Hp/xlERMSH5ec598zGV0HlGLejkRNs3XeE4e8toEKQP02jwrhn/GIem7RUHwzLcZalZ9L7tdn8um4Pz/Vrxb+ub01IoH+xXzcsJJD3hnegf/ta/PfntTw4IYWcPN9bKmFb5hFuGPMrU1K28tBVTXltYAKhQcX/8yuqoozcdQTWWmvXW2tzgE+Avicc0wL42fN8+kn2H8cYY4BLgYmeTWOBxKIG7TVhUdDockj5BApK9y++q1pG8X9XNePrJdv4z09r3A5HRETOoAhVL8ONMbsKVbfcWmjfi8aY5caYVGPMfz33Tfet+R4O7lAjlVIo80guw9//jUPZeXxwc0c+/XMX/nxxAz6ev5l+o+eyftdBt0OUUuCzpDSue2Mu+QWWCbd1YXCnupTkr5dAfz/+eV1rHvR0hR/23m9kHs4tseufr4WbMrjmtTls2H2It4e0587ujUr051cURUnuYoHCxbFbPNsKSwGu9TzvB4QZYyI934cYY5KMMfOMMUcTuEhgn7X26MIXJzsnAMaYkZ7XJ+3aVQyTMOMHwYGtsH6698/tZbdd3IDr2tbi3z+u4SuVWoiIlFpFrHoB+LRQdcs7ntd2BboBrYFWQAfg4pKJ/AwWjYNKUdD4SrcjkUKycvMZOS6JDbsP8eaQdjSPrkygvx+PXt2c94a3Z1vmEa55bTZfJqe7Haq4JDsvn8cmLeWhiUtoV7cqX999AfG1w12JxRjD3Zc15tUBbUjalMF1Y+aSlnHYlVjOxqcLNnPjW/OoFOzPpDu6cnmLmm6HdFLemgn9F+BiY8xinBtQOnB0KKyutbY9MAj4tzGm4dmc2Fr7lrW2vbW2ffXq1b0UbiFNr4bQqqV2zbvCjDE8f20r2tetyl8+SyElbZ/bIYmIyMkVperlVCwQAgQBwUAgsKNYojwbmemw5gdnrp1/gNvRiEdBgeXBCSnM35DBSze0oWuj49cdvLRZTabecyHNoitz7yfJPPqFyjTLm637jtB/zK98PH8zt1/SkHF/6ujK+msn6pdQi3F/6sTO/Vn0Gz2XJVtK59+1ufkFPDVlOQ9/vpTODSL58s4LaFwzzO2wTqkoyV06ULvQ97U8235nrd1qrb3WWpsA/NWzbZ/na7rn63pgBpAA7AHCjTEBpzpniQkIhrgbIPVrZ82eUi44wJ83h7SjelgwI8YlqRuWiEjpVJSqF4DrPM3IJhpjagNYa3/FmeKwzfP43lqbeuILi72y5UTJH4EtgIQhxX8tKRJrLX//ZgXfLN3GYz2b0Tf+pEVQxISH8snIzvz54gaM/20ziaPmsE5lmuXCnLW76f3abNbtOsSYm9rxcI9mBLi0/trJdGkYyee3dyU4wI8Bb87jp1T3P8cqbO+hHIa99xsfzN3IrRfU5/3hHc57/b/iVpR/3QVAY093yyDgRmBK4QOMMdWMMUfP9Sjwnmd7VWNM8NFjcMpMVlinI8h04HrPa4YBX57vmzln8YMgPxuWfeFaCGcjslIw7w7rwOGcfG4dm8ThnLwzv0hEREqbr4B6nmZk03Dmn2OMaQQ0x/ngMxa41Bhz4YkvLvbKlsIKCpwumfUvhoj6xXstKbJ3ftnA+3M2cnO3eoy4sMFpjz1apvn+8A7s2J9FH5VplmnWWt6YsY4h784nsmIQX97VjR6totwO66Qa1wxj0p1daVSjEiPGJfHhrxvdDglwOs/2GTWbpE17efmGNjzeu0WpSoxP5YwReubF3QV8D6QCE6y1y40xzxhjjna/vARYZYxZDdQEnvNsbw4kGWNScJK5F6y1Kzz7HgYeMMasxZmD966X3tPZi46HGi1LfdfMwppGhfHawARSt+3ngU9TKChQB00RkVKkKFUve6y12Z5v3wHaeZ73A+ZZaw9aaw8C3wJdijne01s/HTI3Q7throYhx0xJ2cpzU1PpFRfNE71aFLmpQ/dmNfjmngtprjLNMutAVi63/W8h//xuJT3jopl8ZzcaVq/kdlinVSMshE//3JlLm9XgiS+X8/zUVFf/tv1u2XauHT2X7NwCPh3Zmeva1Trzi0qJIqWf1tqp1tom1tqG1trnPNv+Zq2d4nk+0Vrb2HPMrUdvVtbaudbaOGttG8/Xdwudc721tqO1tpG19oZCN7iSZ4wzepe+EHaudC2Ms9W9WQ0e69mc75Zv5+Vpq9wOR0REjilK1Ut0oW/74HyACrAZZx57gDEmEGcu+x/KMkvUonEQGgHNersahjjmrt3NgxOS6Vg/gpf7t8HP7+y69cWEhzJ+ZGduv6ShyjTLmNU7DtD39Tn8mLqTJ3q34LWBCVQM9o05shWCAnhzSHuGdqnLW7PWc9f4RSX+wUNBgeU/P67htv8tpHHNML66+wIS6lQt0RjOV+kfWywprQeAXwCklP7GKoXdckF9Bnaszajp65i0eIvb4YiICEWuernHs9xBCnAPMNyzfSKwDliK0406xVr7VYm+gcIO7YaV30Cbgc48dXFV6rb9/PnDhdSvVpG3h7Q/5/XJAv39eLhHs9/LNNVN0/d9lbKVxFFz2J+Vx8e3duKWC+qXujb9Z+LvZ3i6T0se79Wcb5dtZ/A788k4lFMi1z6UncedHy/i1R9Xc23bWD4d2ZmalUNK5NreZHxpQez27dvbpKSk4rvA+IHO6N39K3yqE1hOXgFD35vPok37GD+yM+3q+tYnDCIiJzLGLPR0WpYiKNb745z/wrQn4I75UKNZ8VxDiiR93xGuHT0Hg+GLO7oSEx7qlfNuyzzC3R8vJmnTXgZ2rM2T17QskUWtSwtrLcu37scYaFwjjKAA3xr7yM0v4IVvV/Lu7A20q1uV0YPb+mRScqJvl27jvk+Tia4Swgc3d6RetYrFdq20jMOMGJfE6h0HeKxn81KfGJ/uHqnkrrDUr+HTwTDoM2jiW2v47D2UQ7/RcziYncfkO7tRq2oFt0MSETlnSu7OTrHdH62F1ztAhQi45Qfvn1+KLPNwLteNmcuOzCw+u70LzaIqe/X8efkFvDxtNW/MWEezqDBeH9SWRjVK9zyt81VQYJmWuoMxM9exeLPThj/Q39CoRhgtoivTIqay8zW6cqntkLjzQBZ3fbSY3zZmMLxrPR7r2dznktPTWbhpLyPGJWGt5Z1h7WlXN8Lr15i7bjd3frSI/ALL64PaclGTYm5Q5QVK7ooqLwdeaQb1LoD+44rvOsVk7c6D9Bs9h9jwUCbe3pVKPlJjLSJyIiV3Z6fY7o+b5sL7V0Pf0c76duKKrNx8hr77G8lp+/jgTx3o2rDamV90jqav2skDnyaTnVfA8/3iSEw4+fIKviw7L59Ji9J565f1rN91iNoRoYy4sAERFYNYsXU/K7btZ8XW/ew8cKwdRGx4KM0LJXwtYypTq2qoq6M7SRszuOOjRezPyuWFa1uXyX8rgI27DzH8/d/YmpnFq/3j6dU6+swvKgJrLR/O28TTX61wypyHtqd+MY4OetPp7pH667+wgCBn7t2Cd+BwhvNJpQ9pVKMSowe3Zfj7C7h3/GLeGtoe/7OcZC0iIvK7ReMguDK0THQ7knIrv8DywIRkftuYwWsDE4o1sQPo3rQGU++9kHvGL+a+T5OZt34PT/UpG2Wa+7Ny+WjeZt6bs4FdB7JpGVOZ1wYmcHWrqN9b3PduHfP78bsOZJO67Viyt2Lbfn5euYOjTRzDQgKchK9Q0te4ZiWCA4r3Z2WtZezcjTz7TSqxVUMZ+6eONI/27khuaVKvWkW+uKMbI8YlcefHi0jf14wRFzY4r8Q6J6+AJ6csY/xvaVzevAavDognLKR0js6eLSV3J4ofBPNGw9KJ0Gmk29GctQsbV+epa1rwxJfL+ed3K3msZ3O3QxIREV90ZB8snwzxAyHINz7NLmustfz96xVMXbqdx3s155o2MWd+kRdEVwll/IjOvDJtNaNnrCM5bZ9Pl2nu2J/Fe7M38NH8zRzMzuPCxtV4tX883RpFnjZBqB4WTPWw6seV6R3JyWf1jgPHJXwTktI4nON0dQzwMzSqUem4ks4WMZUJrxDklfdyOCePx75YyuTkrVzevAYv94+nSmjZSEpOJ6JiEB/d2okHJ6Tw/NSVpGUc4clrzm3duV0Hsrn9fwtJ2rSXO7s35MErmp51x9nSTMndiaLiIKq1s+adDyZ3AEO61GPNzoO8NWs9japXon+H2md+kYiISGFLP4O8I9BWa9u55a1Z6/lg7kZuuaA+t55hkXJvC/D34/96NKNj/QgemJBCn9dn81y/VvRL8J31vtbuPMhbs9YxaXE6+QWWXq1j+PNFDWgVW+Wczxka5E+b2uG0qR3++7aCAsumjMOeZC+TFVv3M2ftbr5YdKz7aEyVkGMJX0xlWkRXoXbE2ZV1btx9iNv+t5BVOw7wlyubcMcljcpUUnImIYH+vDYwgVpVQ3lz1nq27jvCa4MSqBBU9HRmWXomI8clkXE4h9cGJpTYByYlScndycQPhu8ehh3LoWZLt6M5J3/r3YINuw/x18lLqRtZgU4NIt0OSUREfIW1sGis82FnTLzb0ZRLkxen849vV9KrdTR/dbEK55KmNZh6j1Omef+nKcxbl8FTfVoSGlR6yzQXbtrLmJnrmLZiB8EBfgzsWIdbL2hAncjiaTbn52eoX60i9atVPG4+2O6DnrLOQvP4fl6581hZZ3DAcfP4WsRUplGNSictgf1xxQ7un5CMv59h7M0dfaLpR3Hw8zM82rM5taqG8uSU5Qx4cx7vDm9PjbAzdwedkrKV/5uYQkSFICbe1vW8kvzSTA1VTubQHni5KXT6M1z1XPFfr5hkHsml3+g57D2Uw+Q7u1E3UmU1IuIb1FDl7Hj9/pi+CN7uDr1ehg63eu+8UiRz1u5m+Pu/0a5uVcb+qWOxz+Eqirz8Al79cTWjpq+jac0wRg0uXWWaBQWWn1fu5M1Z61iwcS/hFQIZ2qUew7rUJbJS6VmfMSs3n1Xbjy/rTN22/w9lnYXn8v26bg+vT19LXGwVRg9uS+0IdUQH+Cl1B3d9vJiIikF8cHMHGtcMO+lx+QWWl35YxRsz1tGhXlXeuKkd1UrRfxPnQt0yz8WnN8HmefBAKvj7bi3zxt2H6DtqDtXDgvnijq5ULiOTRUWkbFNyd3a8fn/86j5I+QT+sgpCyuan26XViq376f/mr8SGhzLhti6lbj7VzNW7uP/TZLJy83k2sRXXtnW3TDMnr4Avk9N5a9Z61uw8SGx4KLdeWJ/+7WtT0Ue6hp+srHPFtv3s2H+sW+eA9rV5um/ZaGzjTUu3ZPKnsQvIys3nzSHt/tBwaH9WLvd9kszPK3cysGMdnu7TskwsFaHk7lys+hbG3wg3jodmPUvmmsXk13V7GPLufLo2qsZ7w9qf0+RTEZGSpOTu7Hj1/ph9EF5uBs2vgX5veOecUiRb9h7m2tFz8fdzFin/f/buO7rq+v7j+PNzs3fIgLAT9iaQMAQR60BERRGVJQruba2t1Q5trbX92dbWuhEVEAEF68bVqrWChCFDhigkjDBDQgbZ4/P7414gRpQASb733rwe59yTe7/j8rqnp/G+8xnv1jEN06S8oe0tKOOO+e7ealekt+P3Y/s0+TTNQ+VVzM/YwfOfZ7G3sIweSVHcNLIzF/RrTZCffM85PK3TZQzDuzTuLqm+LPtgCdNfXMG23GIeuazfkXWhmTmHuH7OSrbnlvDA2N5MHdrR4aQNR60QTkaXcyCipXtjFR8v7k7rHM9DKvtjOQAAIABJREFUl/Th3n99xUPvbuJ3Y31zHaGIiDSBjW9ARRGkaSOVppRfUsHVLyyntLKa12723sIOICkmlHnXD+Ef//6WJz/dwtqdBTw5ZQBdWh57WlxD2l9Uxqwl23hp2XaKyqo4rVM8fx7fl5HdEh3tOdcYEiJDGNG1ea6tOxHtWoSz6OZh3PTSKu56ZS0780rp1y6G2+evJijAxdzrhjC0Ge09oeLuhwQEQb8rIOMZKD4AEb79F5OJgzvw7f5DPP95Fl1aRnKlH/31QkREGtCq2ZDQHdoPcTpJs1FWWc11s1eyM6+UOdcOptsPrB3yJoEBLn5+XncGpcRx1ytruOjxJTx0SR/GpzXONM2sA8XM+CyT177MprK6htG9k7hxZGdSa+1aKc1XTFgQs68ZzL2vrePRj74BoEdSFM9dld7s1iiquPsxqVPgiyfc20EPvdnpNKfsV2N6kplziAfe2kCnhAiGaYhfRERq278JspfDqD+Cn42CeKvqGsudC1azasdBHp80wOdGGEZ2S3TvprlgNXcvXEtGVm6DTtNcszOfZ/+7lfc37CUowMX4ge244YxOpCRokzj5ruBAF3+7oj9dW0Wx82AJv7mg5wm1SfAXze8Tn4hWvaDNAFj9sl8UdwEuwz8nDWD800u5ae4q3rh1OJ0SvWenKxERcdiXc8AVBP0nOp2kWbDW8vu3N/DBhn3cf2EvLuznmz23kmJCmXfdEB77z7c88ckW1uzM58nJA39w98Ljsdby6Tc5PPvfrSzLzCMqNJCbR3Zm2vDkem15L82XMYabz+zsdAxH+ceK08aUOgX2fQV71jmdpEFEhQbx/NWDCAxwce3slRSUVDodSUREvEFlGaydDz0v9PmlCL7imf9mMueL7dxwRieuOT3F6TinJDDAxd2jujN7+mByD1Uw9oklvLYq+4Teo7K6htdXZ3P+Y/9zb5BxoIRfj+nJF/edzT2je6iwE6kHFXfH02c8BAS7N1bxE+3jwnl2ahq7DpZyy7xVVFbXOB1JRESc9vU7UHoQBmojlabwry+z+b/3v+ai/m24d3QPp+M0mDO6JbL4zhH0axfD3QvX8ouFayn19HD7ISUVVbzweRZn/uVT7nplrbsv2eX9+eyen3D9GZ2I9JGWBiLeQMXd8YTHQY8LYN2rUFXhdJoGMyg5jocv7cuSLbn87q0N+FJLDBERaQRfzobYjpAy0ukkfu9/3+Zwz6J1nNYpnr9e3g+Xy7/WN7aKDuXl64Zw+1ldWPRlNhc/+Tnf7iv63nW5h8p59MPNDPvzxzz4zkbaxIby/NXpfPDTM7gsrZ1f9CMTaWr6U0h9pE6BDa/DN+9Dr7FOp2kwl6W1Y8v+Qzzz3610bRnJtOG+PSVEREROUl4WZH0GZ/0GXPpC3ZjW7yrgppdW0aVlJM9elUZIoH82pT48TXNwShw/XbCGsU8s4Q+X9OGytHbsyC3huf9l8urKnZRX1XBur1bcNLITaR3jnI4t4vNU3NVH57MgqjWsmedXxR3APed1Z2vOIR58ZyPJCRGc2b2l05FERKSpxXaAK1+DVn2dTuLXduaVMH3WCmLCgpg1fTDRoUFOR2p0I7q6p2neMX81P1+4ljlfbGP9rgICXIZxA9pywxmd6dJSm7uJNBT9ea4+XAHQbwJ8+yEU7XM6TYNyuQz/mJBK96Robp+3+pjTJkRExM+5AqDLORDVyukkfutgcQVXv7ic8spqZl8zmKSY5rM5yOFpmnec1YX8kkquH9GJz395Fo9c1l+FnUgDU3FXX6lTwFbDV686naTBRYQEMvPqdEKCArh29kryiv1nbaGIiIjTyiqruXb2CrIPljLz6kEn3SLAlwUGuPjZqO58ds9PuG9MT1pFN5/iVqQpqbirr8Ru0G6Qu+edH24+0jY2jBlXpbG3sIyb5q6ioko7aIqIiJyq6hrL7fNXs3pnPo9NSGVwitaViUjjUXF3IlKnQM4m2L3a6SSNYmCHFvzlsn4sz8rjN298pR00RUREToG1lgfeWs9HG/fxwIW9OL9va6cjiYifU3F3InqPg8BQ98Yqfuri1LbccVYXXl2Zzcz/ZTkdR0RExGc99elW5i7bwY0jO2lHahFpEiruTkRYLPS4EL5aCJVlTqdpND89pxtj+ibx8Hub+M8m/9pARkREpCksWpXNXz7YzCWpbfjlef7TpFxEvJuKuxM1YAqU5cM37zmdpNG4XIa/XZ5KnzYx3DF/NZv2FDodSURExGf895sc7n1tHcO7xPPIZf39rkm5iHgvFXcnKmUkRLd1b6zix8KCA3juqnQiQwO5dtYK9hf670iliIhIQ/kqu4Cb566ia6sonrkyjeBAfdUSkaaj3zgnyhUA/SfB1v9A4R6n0zSqpJhQnr96EPmllVw3ZyUlFVVORxIREfFaO3JLmD5rOS3Cg5k1fRBRzaBJuYh4FxV3JyN1MtgaWLfA6SSNrk/bGP45cQDrdxXw0wVrqK7RDpoiIiJ15XmalFdWW2ZfM0h93ETEESruTkZ8Z+hwmnvXzGbQLuCcXq347YW9+HDjPv783ian44iIiHiV0gp3k/Ld+aU8f3U6XVo2vyblIuIdVNydrNTJcOAbyF7pdJImMX14CtOGJfPc/7J4adl2p+OIiIh4harqGm6fv5o1O/N5bOIA0pPVpFxEnKPi7mT1HgdB4bDGvzdWqe23F/bi7B4teeDN9Xyyeb/TcURERBxlreW3b27g35v28fuxvRndJ8npSCLSzKm4O1khUdBzLKz/F1SWOp2mSQS4DP+cNICeraO57eUv2bhbLRJERKT5euLjLcxfvoObz+zMVaclOx1HRETF3SkZMAXKC+Drd51O0mQiQgJ5/mr3DmDXzl7BPrVIEBGRZqaquoY/Ld7E3z76hksHtOWe87o7HUlEBFBxd2o6ng6xHZrV1Exwt0h4YdogCksruWbWCorL1SJBRESah/2FZUyemcGzn2UyZUgH/jy+H8aoSbmIeAcVd6fC5YL+k2HrJ1CQ7XSaJtWrTTRPTB7Ipj2F3LlgtVokiIiI38vIzOWCxz9nXXY+f5/Qnz+O66sm5SLiVfQb6VT1nwhYWOv/Pe/q+kmPlvx+bG/+vWk/D7270ek4IiIijcJay4zPtjJ5ZgZRIYG8eevpjBvQzulYIiLfU6/izhgz2hiz2RizxRhz7zHOdzTG/McYs84Y86kxpp3neKox5gtjzAbPuQm17plljMkyxqzxPFIb7mM1obgU9/TMNS83i553dU09LZlrhqfw4pJtzF66zek4IiIiDaqwrJKb537Jw4u/ZlSvVrx523C6J6mPnYh4p+MWd8aYAOBJ4HygFzDJGNOrzmV/BeZYa/sBDwJ/8hwvAa6y1vYGRgP/MMbE1rrvF9baVM9jzSl+FucMmAJ5mbAzw+kkjvj1BT05p2crfv/2Bj7+ep/TcURERBrEpj2FjH38cz7atI/fXNCTp6YMJCo0yOlYIiI/qD4jd4OBLdbaTGttBbAAuLjONb2Ajz3PPzl83lr7jbX2W8/z3cB+ILEhgnuVnmMhKAJWz3U6iSPcLRJS6dUmmtvmrWbD7gKnI4mIiJyS11ZlM+6pJZRUVDP/+qFcN6KTNk4REa9Xn+KuLbCz1utsz7Ha1gKXep6PA6KMMfG1LzDGDAaCga21Dv/RM13z78aYkGP948aYG4wxK40xK3NycuoR1wEhke6m5hvegIpip9M4IjzY3SIhNiyIa2atYE9B8+j9JyIi/qWsspr7/vUVdy9cS2r7WN6543QGp8Q5HUtEpF4aakOVnwMjjTGrgZHALqD68EljTGvgJWC6tbbGc/g+oAcwCIgDfnmsN7bWzrDWpltr0xMTvXjQL3UyVBTBpredTuKYVtGhPD9tEMXl1Vw7ayWH1CJBRER8yM68Ei5/5gvmL9/BTSM7M/faIbSMCnU6lohIvdWnuNsFtK/1up3n2BHW2t3W2kuttQOAX3uO5QMYY6KBd4FfW2uX1bpnj3UrB17EPf3Td3UcBi2Sm13Pu7p6to7mySkD2byviNvnfUlVdc3xbxIREXHYJ1/v58LHP2dbbjHPXZXOvef3IDBAm4qLiG+pz2+tFUBXY0yKMSYYmAi8VfsCY0yCMebwe90HvOA5Hgy8jnuzlUV17mnt+WmAS4D1p/JBHGcMpE6BrM/g4Han0zhqZLdEHry4N59szuHBdzZim+EuoiIi4huqayx/+3Az02etoE1sGO/cfjrn9mrldCwRkZNy3OLOWlsF3AZ8AGwCXrXWbjDGPGiMGeu57ExgszHmG6AV8EfP8SuAM4Bpx2h58LIx5ivgKyABeKihPpRj+k8CTLPseVfXlCEdueGMTsz5YjsvLtnmdBwREZHvyT1UztUvLOfxj7dwRXo7Xr9lGB3jI5yOJSJy0gLrc5G1djGwuM6x+2s9XwQsOsZ9c4FjbiFprT3rhJL6gtj2kHKGe2rmGb8AV/OeznHv6B5szy3mD+9upH1cuP4SKiIiXuPLHQe59eUvyS2u4P/G92XCoA5ORxIROWXNu/poDAOuhPztsGOp00kc53IZ/jFhAP3axnDH/NV8la0WCSIi4ixrLbOWZDHh2S8ICnDxr5uHqbATEb+h4q6h9bgQQqJhzTynk3iFsOAAnrs6nbiIYK6dvYLd+WqRICLNgzFmtDFmszFmizHm3mOcn2aMyam1bOG6Wuc6GGM+NMZsMsZsNMYkN2V2f1VcXsUdC9bwu7c3MrJbIm/fdjp92sY4HUtEpMGouGtoweFHe96VH3I6jVdoGRXKC9MGUVpRzTWzVlBUVul0JBGRRmWMCQCeBM4HegGTjDG9jnHpK9baVM9jZq3jc4C/WGt74t5Nen+jh/ZzW/YXcfGTS3h33W7uGd2dGVPTiQkPcjqWiEiDUnHXGFKnQGUxbHzT6SReo3tSFE9dOZBv9x/itnmr1SJBRPzdYGCLtTbTWlsBLAAurs+NniIw0Fr7EYC19pC1tqTxovq/t9buZuwTS8gvqWDudUO45cwuuFzG6VgiIg1OxV1jaD8Y4rs0+553dY3omshDl/Thv9/k8MBbG9QiQUT8WVtgZ63X2Z5jdY03xqwzxiwyxhzuKdsNyDfG/MsYs9oY8xfPSOB3GGNuMMasNMaszMnJafhP4Acqqmr43VsbuGP+anq2juad20cwrHOC07FERBqNirvGYAykTobtSyAv0+k0XmXS4A7cNLIzL2fs4PnPs5yOIyLipLeBZGttP+AjYLbneCAwAvg5MAjoBEyre7O1doa1Nt1am56YmNg0iX3InoJSJsz4gllLt3Ht6SksuGEoSTGhTscSEWlUKu4aS7+JYFywZr7TSbzOPed1Z0zfJP64eBPvr9/rdBwRkcawC2hf63U7z7EjrLW51tpyz8uZQJrneTawxjOlswp4AxjYyHn9yuffHuCCf37ON3uLeHLyQH57YS+CAvSVR0T8n37TNZaYttDpJ7B2PtRofVltLpfh0StS6d8ulp++spq1O/OdjiQi0tBWAF2NMSnGmGBgIvBW7QuMMa1rvRwLbKp1b6wx5vBw3FnAxkbO6xdqaixPfPwtU1/IICEymLduP50L+rU+/o0iIn5CxV1jSp0MBTth22dOJ/E6oUEBzLw6nYTIEK6dvZLsg9orQET8h2fE7TbgA9xF26vW2g3GmAeNMWM9l91hjNlgjFkL3IFn6qW1thr3lMz/GGO+AgzwXFN/Bl+TX1LBdXNW8tcPv+Hi/m1449bhdE6MdDqWiEiTMr60qUV6erpduXKl0zHqr7IM/toNuo+GS2c4ncYrfbuviEufXkqbmDAW3nwa0aHallpEwBizylqb7nQOX+Fz/31sYF9lF3Dzy6vYV1jG/Rf15sohHTBGu2GKiH/6sf9GauSuMQWFQt/xsPEtKCtwOo1X6toqimeuTGNrziFufflLKtUiQURE6slay7yMHYx/einWwsKbhjF1aEcVdiLSbKm4a2ypV0JVqbupuRzT8C4JPDyuL//79gD3v7leLRJEROS4Siuq+fnCdfzq9a8Y2jmet28/ndT2sU7HEhFxVKDTAfxe24GQ0B3WzIO0q51O47WuGNSe7XnFPPnJVpLjI7hxZGenI4mIiJfKOlDMzXNXsXlfET89pyu3n9WVADUlFxFRcdfojIEBU+Cj++HAFkjo4nQir3X3ud3ZnlvCn977mvZx4Yzpqx3ORETku95fv5dfLFxLQIBh1vTBjOymHn8iIodpWmZT6DcBTACsned0Eq/mchn+enl/0jq24K5X1rB6x0GnI4mIiJeoqq7h4cWbuGnuKjq1jOTdO0aosBMRqUPFXVOISoIu57gbmtdUO53Gq4UGBTBjahqtokO5fs5KduapRYKISHO3v7CMyTMzmPFZJlOHduTVG4fSNjbM6VgiIl5HxV1TSZ0MRbsh81Onk3i9+MgQXpw+iMpqy/RZKygorXQ6koiIOGTtznzG/PNzvsou4B8TUvnDJX0ICQxwOpaIiFdScddUup8PYS1gzctOJ/EJnRMjeebKNLbnFnPLy6vUIkFEpJl65IOvCXDBG7cO55IBbZ2OIyLi1VTcNZXAEOh7OWx6B0rznU7jE07rHM+fLu3Hki25/OZ1tUgQEWluKqpqWLX9IGP6tqZ7UpTTcUREvJ6Ku6aUOhmqy2H9a04n8RmXpbXjjrO68MrKnTz9361OxxERkSa0LjufssoahqTEOx1FRMQnqLhrSq1ToWVvd887qbe7zu3GxalteOT9zbyzbrfTcUREpIlkZOUBMDglzuEkIiK+QcVdUzLGPXq3ayXkbHY6jc8wxvB/4/sxKLkFP3t1Lau2q0WCiEhzsCwzl+6tooiLCHY6ioiIT1Bx19T6TQBXoDZWOUGhQQE8OzWdNjHuFgk7ctUiQUTEn1VWu9fbDemkUTsRkfpScdfUIhOh6yhYuwCqq5xO41PiIoJ5cfpgaqxl2qzlFJSoRYKIiL9av6uAkopqrbcTETkBKu6ckDoFDu2DrR87ncTnpCREMGNqOtl5pVwzewULV+5kxbY89heWaTdNERE/ovV2IiInLtDpAM1S11EQHu+emtltlNNpfM7glDj+cnk/frFo3XfW34UHB9AhLpyO8eF0jI+gY3w4yfERdIgLp01sGAEu42BqERE5ERmZuXROjCAxKsTpKCIiPkPFnRMCg91r71bMhJI8CNdfJU/UxaltGdO3NbsOlrItt5gdeSVsO1DCjrxituYU88nmHCqqjjY+DwowtG/x3cLv8PN2LcIICQxw8NOIiEht1TWWldsOclFqG6ejiIj4FBV3TkmdDMuecve8G3y902l8UlCAi+SECJITIr53rqbGsrewzF345ZawLddd+G07UMKKbQc5VH50vaMx0CYmrM6IXzgd4tzPI0L0fxMRkaa0cXchReVVDNGUTBGRE6JvrU5J6gtJ/WD1XBV3jcDlMrSJDaNNbBjDOn/3nLWW3OIKtueWsD23+OjPvBI+2LCXvOKK71yfEBniLvY80zyPFIFx4cSGB2GMpnuKiDSkjKxcAIZ20mYqIiInQsWdk1KnwPu/hH0boFVvp9M0G8YYEiJDSIgMIa1ji++dLyyr9Iz2FX+nAFy6JZd/fbnrO9dGhwZ+b5pnx7hwurSMJD5S60RERE7Gssw8kuPDaRUd6nQUERGfouLOSX0vhw9/A2vmwXl/dDqNeESHBtGnbQx92sZ871xZZTU78kq+U/Rtyy3mq10FvLd+L9U1R3fsTIwKoUdSlOcRTY/WUXRpGan1fSIiP6KmxrJiWx6jeyc5HUVExOeouHNSRDx0Hw3rXoFzfgcBQU4nkuMIDQqgW6sourWK+t65yuqaIxu8bNl/iE17ivh6byGzl26notq9uUuAy9ApIYIeraPpkRRFz9ZRdE+Kpk1MqKZ3iogAX+8toqC0Us3LRUROgoo7p6VOgU1vw7cfQY8xTqeRU1B7g5czu7c8cryquoZtucVHir3Ne4v4cvtB3l67+8g1UaGB9EyKpntSFD1au0f6uidFEanNXESkmTm83m6I1tuJiJwwfXN0WpdzIKKlu+ediju/FBjgokvLKLq0jOKi/ke39S4sq+SbvUVs2lvE13vcRd/rq3dxaNnRnTzbx4XRIymanknuEb4eraNIjo9Qzz4R8VsZmXm0axFG29gwp6OIiPgcFXdOCwiCfldAxjNwKAciE51OJE0kOjSI9OQ40pOPTj2y1pJ9sJTNe92jfJv2FrF5bxH/2bSPw8v5QgJddGvlWcvnmd7ZIylKG7g0M+VV1Vq/KX7HWsvybXn8pNbsBxERqT8Vd95g4NXu4u61a2HKIneTc2mWjDG0jwunfVw45/RqdeR4WWU1W/Yf4mvPKN/Xe4v4ZHMOC1dlH7lGG7j4t4PFFXyRmcuSLQdYujWXrAPFXJ7Wjvsv6kVUqNbrin/Ysv8QecUVWm8nInKSVNx5g8RuMPYJeOMmePMWGDcDXC6nU4kXCQ0KOOYOnjlF5UdG+b72/Jz9xXYqqo69gUuXlpG0jQ2jdUwocRHB2sTFixWXV7F8Wx5LtxxgyZZcNu0txFqIDAlkSEocQzvF8cqKnSzdmstfLu/HsM4JTkcWOWXLsvIAGJqi9XYiIidDxZ23SJ0ERXvgP7+HqCQY9ZDTicQHJEaFkBgVwuldj36xd2/gUuIu+PYU8fXeIlbv+O4GLgDBgS5ax4TSOiaUNjFhJMWE0jo2jNbRobSOdR9Tk/amU1FVw5qd+Z6RuQOs2ZlPZbUlOMDFwI6x/OycbgzrkkD/djEEBrj/+HN5envufnUtk5/LYNqwZH45ugdhwRqpFd+VkZlL65hQ2sdpvZ2IyMlQcedNTr8LCnfD0schqg2cdovTicQHuTdwiaRLy0gu7Hf0eFFZJZk5xewpKGNPQSl7C8rYXVDGnvxSMrLy2FdYRlWtPn0AoUEuWseEkVSr4EuKCaVNbCitY9wjgDFhKgBPRk2NZeOeQpZudY/MrdiWR0lFNS4DfdvGcN2ITgzvnEBaxxY/WLAN7NCCxXeM4P/e/5pZS7fx2bc5PHpFKqntY5v404icOmstGVl5DO8cr98pIiInqV7FnTFmNPAYEADMtNb+uc75jsALQCKQB1xprc32nLsa+I3n0oestbM9x9OAWUAYsBi401r73W+WzY0xcP7/waG98MGv3CN4fS51OpX4iajQIPq3j6V/+2Ofr66x5B4qP1LwHS4C3T/LWLY1l31F5d9p1A4QFhTgHgGsVfC1jgnzvHY/jw4NbPZf1qy1ZB0oZsnWXJZuOcAXmbnkl1QC0KVlJJentWNYlwSGdoonJqz+a+jCggP43djenNurFb9YuJbxTy/lljM7c/tZXQkO1PRu8R1ZB4rJKSpXCwQRkVNw3OLOGBMAPAmcC2QDK4wxb1lrN9a67K/AHGvtbGPMWcCfgKnGmDjgASAdsMAqz70HgaeB64EM3MXdaOC9hvtoPsoVAJc+By+Ng9dvhIhESBnhdCppBgJchpbRobSMDv3BkZ/qGktOUfmRom93vnsE8HAhuGTLAfYVllGn/iM82F0Atok9PAoYdnRKaKx7NDAqxP8KwH2FZSzxrJlbuvUAewrKAGgTE8o5PVsxvEs8wzon0Co69JT/reFdEnj/rjN48O2NPP7xFv6zaT9/n5BK96SoU35vkaaQ4VlvNyRFm6mIiJys+ozcDQa2WGszAYwxC4CLgdrFXS/gZ57nnwBveJ6fB3xkrc3z3PsRMNoY8ykQba1d5jk+B7gEFXduQWEwaT68MBoWTIFr3oNWvZ1OJUKAy5AUE0pSTCgDfuCaquoacg6Vszu/1vTP/DL2FpayO7+Mb/blsL+onLrj9OHBASRFh9IyOoSk6FBa1XokxYTQMsp9zpt3/ywoqeSLzKPF3NacYgBahAcxrHMCw7rEM7xzAh3jwxulkI0ODeKvl/dnVK9W/Or1r7jo8c/52ahuXD+ik3ojitfLyMwlMSqElIQIp6OIiPis+hR3bYGdtV5nA0PqXLMWuBT31M1xQJQxJv4H7m3reWQf4/j3GGNuAG4A6NChQz3i+omwFu62CM+Pgrnj4dqPIPYH5tOJeJHAAJdnemYY0OKY11RW17C/qJy9BaVHisB9heXsLSxjf2EZq3YcZF9h+ZFdP2uLiwimZVQISTGhtIoKpVVMKK3qFITxEcG4mqCYKa2oZsW2PJZsPcDSLbms312Ate5CdXBKHBMHdWBYl3h6JkU3SZ7DRvVOIq1jC379+nr+/N7X/HvjPv56eX+S9aVZvNTh9XZDUuL8bgRfRKQpNdSGKj8HnjDGTAM+A3YB1Q3xxtbaGcAMgPT09Oa1Ji+2PVy5yD2C9/JlMP09CNd0FfF9QQEu2saG0TY2jLSOx77GWktBaSV7C8vYV1jOvoIy9hWWHX1dWMbG3YUcOFT+vWmggS5Dy6gQWkYfLfxaRoceKQCTYtznTnQqaGV1Deuy81myxd1vbvWOfCqqawgKMAzo0II7z+7K8C4J9G8X6/h6t/jIEJ6+ciBvrNnF/W9u4PzH/sevLujJlUM66MuzeJ2dee6p3lpvJyJyaupT3O0Cag8ZtfMcO8Jauxv3yB3GmEhgvLU23xizCzizzr2feu5v92PvKR6tesPEeTD3UlgwGaa+AUGnvj5HxNsZY4gNDyY2PJgeST98XVV1DQcOVRwp/PbXKQCzDhTzxdZcCsuqvndveHAAraJDj44EHhn9OzoSWFRW5dnR8gDLs/IorqjGGOjdJprpw5MZ1iWBQcktCA/2vs2HjTGMG9COoZ3iuWfROn77xno+3LCXRy7r5xlZFfEOy7JyAa23ExE5VfX5NrIC6GqMScFdgE0EJte+wBiTAORZa2uA+3DvnAnwAfCwMebw3KxRwH3W2jxjTKExZijuDVWuAh4/5U/jr1JGwLhnYdF0+Nd1cPls98YrIkJggOvIOsD+P3JdaUU1+wrLahWB5Z4i0P3rxLAJAAAgAElEQVR89Y589haWHXMqKECnhAjGDWzL8M7uHS1bRAQ3zgdqBK1jwphzzWDmZuzg4Xc3Mervn/Hgxb25JLWtRvHEK2Rk5hEXEUzXlpFORxER8WnHLe6stVXGmNtwF2oBwAvW2g3GmAeBldbat3CPzv3JGGNxT8u81XNvnjHmD7gLRIAHD2+uAtzC0VYI76HNVH5cn0uhaC98cB+890sY8xd36wQRqZew4ACSEyJ+dN3Z4amg+2oVfoEuw2md431+pMsYw9ShHRnRJYGfL1zLXa+s5YP1+/jjuD7ER4Y4HU+auYysXAYna72diMipqtc8ImvtYtztCmofu7/W80XAoh+49wWOjuTVPr4S6HMiYZu9026BIk+T8+g2MOJnx79HROqt9lRQf20hkJwQwSs3nsZz/8vk0Q+/4bx/fMbD4/oyqvePzH0VaUS78kvJPljKtaenOB1FRMTnqcOtrznnQeh7Ofzn97BmvtNpRMQHBbgMN43szFu3D6dlVCg3vLSKu19dS2FZpdPRpBnKyDy83k6bqYiInCoVd77G5YKLn4KUkfDWbbDl304nEhEf1SMpmjduHc7tZ3XhjTW7GP33z1iy5YDTsaSZycjMIyYsiB5+OlouItKUVNz5osBgmDAXEnvCK1fB7tVOJxIRHxUc6OLuUd157eZhhAYHMGVmBg+8uZ7SigbpZiNyXBlZuQxKjmvSXpAiIv5KxZ2vCo2GKQshPB5evhzyspxOJCI+LLV9LO/ePoLpw5OZ/cV2Lvjn//hyx0GnY4mf21dYxrbcEoZ2UgsEEZGGoOLOl0W3hitfg5oqmDseijWdSkROXlhwAA9c1Jt51w+hvKqGy55eyl8++PoH20OInKplWm8nItKgVNz5usRuMOkVKNwF866AimKnE4mIjxvWOYH3fzqCy9La8eQnW7n4ySVs2lPodCzxQxlZeUSFBNKrTbTTUURE/IKKO3/QYQhc9oJ77d3C6VBd5XQiEfFxUaFBPHJZf2ZelU5OUTljn/icpz7dQlW1RvGk4WRk5pKe3IIArbcTEWkQKu78RY8L4IK/wbcfwDs/BWudTiQifuCcXq348K4zOLdXKx55fzNXPPsFWQc0Q0BOXU5ROVtzihnSSVMyRUQaioo7f5J+DZzxC1j9Enz6Z6fTiIifiIsI5snJA3lsYipbc4oZ89j/mPPFNmpq9EekH2OMGW2M2WyM2WKMufcY56cZY3KMMWs8j+vqnI82xmQbY55outRNZ3lWHgBDUrSZiohIQ1Fx529+8mtIvRL++2dY+aLTaUTETxhjuDi1LR/edQaDU+K4/80NXPXCcnbnlzodzSsZYwKAJ4HzgV7AJGNMr2Nc+oq1NtXzmFnn3B+Azxo5qmMysnIJDw6gT9sYp6OIiPgNFXf+xhi46B/Q5Vx492ew+T2nE4mIH2kVHcqs6YN4eFxfvtxxkPP+8RmvrcrGaip4XYOBLdbaTGttBbAAuLi+Nxtj0oBWwIeNlM9xGZl5pHVsQVCAvoqIiDQU/Ub1RwFBcMVsaJ3q3mBl5wqnE4mIHzHGMHlIB96/8wx6JkVz98K13PjSKg4cKnc6mjdpC+ys9Trbc6yu8caYdcaYRcaY9gDGGBfwN+DnP/YPGGNuMMasNMaszMnJaajcTSKvuILN+4oYqvV2IiINSsWdvwqOgMmvunvhzbsCDnzrdCIR8TMd4sOZf8NQfjWmB59uzuG8v3/G++v3Oh3Ll7wNJFtr+wEfAbM9x28BFltrs3/sZmvtDGtturU2PTExsZGjNiyttxMRaRwq7vxZZKK7yblxwdxLoUhfukSkYQW4DDec0Zl37jid1rGh3DR3FfcsWqtpmrALaF/rdTvPsSOstbnW2sPDnTOBNM/z04DbjDHbgL8CVxlj/GqXrIysXEKDXPRrF+t0FBERv6Lizt/FdYIpC6E4F16+DMrUiFhEGl63VlG8fstw7jy7K+1bhGNMs+9btgLoaoxJMcYEAxOBt2pfYIxpXevlWGATgLV2irW2g7U2GffUzDnW2u/ttunLMjLzGNihBcGB+hoiItKQ9Fu1OWg7EK6YA/s2wqtToarC6UQi4oeCAlzcdW43bj+7q9NRHGetrQJuAz7AXbS9aq3dYIx50Bgz1nPZHcaYDcaYtcAdwDRn0jatgpJKNu0tZEiK1tuJiDS0QKcDSBPpeg6MfRzevAXevBXGPQsu1fYiIo3FWrsYWFzn2P21nt8H3Hec95gFzGqEeI5ZsS0Pa2FIJ623ExFpaCrumpMBU6BoD3z8B/dGK+c+6HQiERFpZjKycgkOdJHaXuvtREQamoq75mbE3e4Cb8ljENUGht7kdCIREWlGlmflkdo+ltCgAKejiIj4Hc3La26MgfMfgR4Xwvv3wobXnU4kIiLNxKHyKtbvLmSoWiCIiDQKFXfNkSsAxs+E9kPgXzfAts+dTiQiIs3Aym15VNdYhqh5uYhIo1Bx11wFhcGk+dAiGeZPdu+kKSIi0ogysvIICjAM7NDC6SgiIn5JxV1zFh7nbnIeHA5zx0NBttOJRETEj2Vk5tKvXSxhwVpvJyLSGFTcNXexHWDKIqg4BHMvg9KDTicSERE/VFJRxbrsAgZrvZ2ISKNRcSeQ1AcmzIXcLbBgClSWOZ1IRET8zJfb86mqsQxRcSci0mhU3Ilbp5Ew7hnYvgRevwFqqp1OJCIifiQjK5cAlyE9WcWdiEhjUXEnR/W9DEb9ETa+Ce/fB9Y6nUhERPxERmYefdpEExmiFrsiIo1FxZ1817Db4LTbYPmz7kbnIiIip6isspo1O/PVAkFEpJHpz2fyfef+AYr2wL8fgKjW0H+C04lERMSHrd6RT0V1jdbbiYg0MhV38n0uF1zyNBzaD2/eApGJ0Pksp1OJiIiPysjKxRi03k5EpJFpWqYcW2AITHwZEnvAK1epybmIiJy0jMw8erWOJiYsyOkoIiJ+TcWd/LDQGJj8CgRHwLwroGif04lERMTHlFdV8+WOgwxJ0Xo7EZHGpuJOflxMO5i8AEpyYcEkqChxOpGIiPiQddkFlFfVMKSTpmSKiDQ2FXdyfG0GwPiZsOtLeP1GqKlxOpGIiPiIjMxcAAZrvZ2ISKNTcSf10+MCGPUQbHoLPn7Q6TQiIuIjMrLy6JEURYuIYKejiIj4PRV3Un+n3Qpp0+Hzv8OXLzmdRkREvFxldQ2rth9UCwQRkSaiVghSf8bAmL9A/nZ456cQ2x46nel0KhER8VJf7SqgpKJazctFRJqIRu7kxAQEweWzIL6ru0VCzmanE4mIiJfKyMwDYLBG7kREmkS9ijtjzGhjzGZjzBZjzL3HON/BGPOJMWa1MWadMWaM5/gUY8yaWo8aY0yq59ynnvc8fK5lw340aTSHWyQEBsPLl0PxAacTiYiIF8rIyqVLy0gSIkOcjiIi0iwct7gzxgQATwLnA72AScaYXnUu+w3wqrV2ADAReArAWvuytTbVWpsKTAWyrLVrat035fB5a+3+Bvg80lRadIRJC+DQPlgwGSrLnE4kIiJepKq6hpXbtN5ORKQp1WfkbjCwxVqbaa2tABYAF9e5xgLRnucxwO5jvM8kz73iL9qlw7hnYWcGvHkrWOt0IhER8RIb9xRyqLxK6+1ERJpQfYq7tsDOWq+zPcdq+x1wpTEmG1gM3H6M95kAzK9z7EXPlMzfGmPMsf5xY8wNxpiVxpiVOTk59YgrTar3JXD2A7B+EXzysNNpRETESxxebzdUI3ciIk2moTZUmQTMsta2A8YALxljjry3MWYIUGKtXV/rninW2r7ACM9j6rHe2Fo7w1qbbq1NT0xMbKC40qBOvwsGXAmfPQJr6tbvIiLSHGVk5ZKSEEHL6FCno4iINBv1Ke52Ae1rvW7nOVbbtcCrANbaL4BQIKHW+YnUGbWz1u7y/CwC5uGe/im+yBi44O+QPALeuh22LXE6kYiIOKi6xrI8K0/r7UREmlh9irsVQFdjTIoxJhh3ofZWnWt2AGcDGGN64i7ucjyvXcAV1FpvZ4wJNMYkeJ4HARcC6xHfFRgME16CFsnwyhTI3ep0IhERccjXewspLKtiSCcVdyIiTem4xZ21tgq4DfgA2IR7V8wNxpgHjTFjPZfdDVxvjFmLe4RumrVHdtc4A9hprc2s9bYhwAfGmHXAGtwjgc81yCcS54S1gCmvgnG5WySU5DmdSEREHHB4vd2QFG2mIiLSlALrc5G1djHujVJqH7u/1vONwPAfuPdTYGidY8VA2glmFV8Q1wkmzoPZF8ErV8LU1yFQ/Y1ERJqTjKxc2seF0SY2zOkoIiLNSkNtqCJyVIehcMnTsH0JvH2nWiSIiDQjNUfW22nUTkSkqdVr5E7khPW9zL3u7tOHIa4zjPyF04lERKQJfLv/EAdLKrWZioiIA1TcSeMZeQ/kbYVPHoK4FHfBJyIifm15Vi4AQ9W8XESkyWlapjQeY2Ds49BhGLxxC+zIcDqRiIg0smVZebSOCaVdC623ExFpairupHEFhsDElyGmLSyYBHlZTicSEZFGYq0lI9Pd384Y43QcEZFmR8WdNL7wOJi8EGqqYd4VUJrvdCIREWkEmQeKOXConCGakiki4ggVd9I0Erq4R/DysuDVq6C60ulEIiLSwI72t9NmKiIiTlBxJ00n+XT3Grys/8I7d6lFgoiIn8nIyiUxKoSUhAino4iINEvaLVOaVuok9w6an/0F4rvA6T91OpGIiDQArbcTEXGeijtpemf+yt0D798PuFsk9LrY6UQiInKKduSVsLewTOvtREQcpGmZ0vRcLrjkaWg3GP51A2SvcjqRiIicosPr7YZqvZ2IiGNU3IkzgkJh4jyIbAXzJ0L+DqcTiYjIKViWlUt8RDBdWkY6HUVEpNlScSfOiUyEKQuhqhzmTYCyAqcTiYjIScrIzGOw1tuJiDhKxZ04K7E7TJgDB76BhdOhusrpRCIicoKyD5awK79ULRBERBym4k6c1+lMuOBR2PofeO8etUgQEfExR/rbaTMVERFHabdM8Q5pV7tbJCx5zN0i4bRbnE4kIiL1lJGVS2x4EN1bRTkdRUSkWdPInXiPs38HPS+CD34FXy92Oo2IyCkxxow2xmw2xmwxxtx7jPPTjDE5xpg1nsd1nuOpxpgvjDEbjDHrjDETmj79icnIymNQchwul9bbiYg4ScWdeA+XC8bNgDYD4LVrYfcapxOJiJwUY0wA8CRwPtALmGSM6XWMS1+x1qZ6HjM9x0qAq6y1vYHRwD+MMbFNEvwk7C0oY3tuidbbiYh4ARV34l2Cw2HSAgiPd7dIKNjldCIRkZMxGNhirc201lYAC4CL63OjtfYba+23nue7gf1AYqMlPUUZWbkADNV6OxERx6m4E+8T1QomvwLlh2D+BPdPERHf0hbYWet1tudYXeM9Uy8XGWPa1z1pjBkMBANbj3HuBmPMSmPMypycnIbKfcKWZeYRFRpIz9bRjmUQERE3FXfinVr1hstnwb6N7imaNdVOJxIRaWhvA8nW2n7AR8Ds2ieNMa2Bl4Dp1tqaujdba2dYa9OttemJic4N7GVk5TIoOY4ArbcTEXGcijvxXl3PgTGPwDfvwwe/djqNiMiJ2AXUHolr5zl2hLU211pb7nk5E0g7fM4YEw28C/zaWruskbOetP1FZWTmFGu9nYiIl1BxJ95t0HUw9BbIeBqWP+d0GhGR+loBdDXGpBhjgoGJwFu1L/CMzB02FtjkOR4MvA7MsdYuaqK8J2V5lvrbiYh4E/W5E+836iHIy3I3OI/tCN1GOZ1IRORHWWurjDG3AR8AAcAL1toNxpgHgZXW2reAO4wxY4EqIA+Y5rn9CuAMIN4Yc/jYNGut120hnJGZR0RwAH3aaL2diIg3UHEn3s8VAONnwovnw6LpcM0HkNTH6VQiIj/KWrsYWFzn2P21nt8H3HeM++YCcxs9YAPIyMolLTmOwABNBBIR8Qb6bSy+ISTSvYNmSDTMmwBFe51OJCLSrOUVV/DNvkNabyci4kVU3InviG4DkxdA6UF3gVdR7HQiEZFma/mR/nYq7kREvIWKO/EtrfvDZc/D3nXw2vVQVuB0IhGRZmlZZh6hQS76to11OoqIiHiouBPf0/18OO9PsPldeLQ3fPhbKNztdCoRkWYlIyuPtI4tCA7UVwkREW+h38jim4beBDf8171z5hdPwD/6wRu3wP5NTicTEfF7BSWVfL23kCEpaoEgIuJNVNyJ72qTCpe9AHeshvRrYMPr8NRQePkK2PY5WOt0QhERv7R8Wx7WwmBtpiIi4lVU3Inva5EMYx6BuzbAT34Nu1bBrAtg5tmw4Q2oqXY6oYiIX8nIzCU40EVqe623ExHxJiruxH+Ex8HIe+Cu9XDBo1CSBwuvhsfTYMVMqCx1OqGIiF/IyMojtX0soUEBTkcREZFaVNyJ/wkKg0HXwu2r4Io57qLv3bvh733g0/9zF30iInJSisoq2bC7gKGakiki4nVU3In/cgVAr4vhuv/AtMXQNg0+fRge7QWLfwEHtzmdUETE56zcfpAaC0M6aTMVERFvE+h0AJFGZwwkD3c/9m+CpY/DyhfdUzV7XQLD74A2A5xOKSLiEzIy8wgKMAzs0MLpKCIiUodG7qR5adkTLnkKfroOht0OW/4NM86E2Re5n2uHTRGRH5WRlUu/drGEBWu9nYiIt1FxJ81TdBs490H35ivn/gEOfAtzx8Mzp8PaBVBd6XRCERGvU1JRxVfZBQzRejsREa+k4k6at9AY97TMO9fBxU+52ya8fiM8lgpLn4DyIqcTioh4jVXbD1JVY7XeTkTES9WruDPGjDbGbDbGbDHG3HuM8x2MMZ8YY1YbY9YZY8Z4jicbY0qNMWs8j2dq3ZNmjPnK857/NMaYhvtYIicoMBgGTIGbl8LkV9298z78NTzaG/79Oyja63RCERHHZWTmEeAypHXUejsREW903OLOGBMAPAmcD/QCJhljetW57DfAq9baAcBE4Kla57Zaa1M9j5tqHX8auB7o6nmMPvmPIdJAXC7odh5Mfxeu+xg6nwlLHoN/9IU3b4Ocb5xOKCLimIysXPq0jSEyRPuxiYh4o/qM3A0GtlhrM621FcAC4OI611gg2vM8Btj9Y29ojGkNRFtrl1lrLTAHuOSEkos0tnZp7j55t62EAVPhq4Xw5CCYPwl2LHM6nYhIkyqrrGbtTvW3ExHxZvUp7toCO2u9zvYcq+13wJXGmGxgMXB7rXMpnuma/zXGjKj1ntnHeU8AjDE3GGNWGmNW5uTk1COuSAOL7wwXPgp3bYCRv3QXdi+cBzPPhU3vQE2N0wmPr6YGygrcDdy1I6iInIQvdxykorqGIZ1U3ImIeKuGmlcxCZhlrf2bMeY04CVjTB9gD9DBWptrjEkD3jDG9D6RN7bWzgBmAKSnp+tbqTgnIgF+8isYfiesfhm+eAJemQLxXdxtFfpNhKDQhv93rYWqMigrdBdo5Z6f33le+N3nR87Veo7n/z7BUe41hS06en7WesR2gMCQhv8MIuLzMjLzcBlIT1ZxJyLirepT3O0C2td63c5zrLZr8ayZs9Z+YYwJBRKstfuBcs/xVcaYrUA3z/3tjvOeIt4pOAKG3ADp18CmN2HJP+HtO+HjP8KQG2HQtRBWa7OB6qqjhdePFmN1C7Na11VX/Hgm44KQaPfun6HREBoLsR2Pvj58zhjI3wEHt0HuFndvv6qy2m/kbhNRt+g7/IhIdL+HiDQ7GVm59GoTTXRokNNRRETkB9SnuFsBdDXGpOAuwCYCk+tcswM4G5hljOkJhAI5xphEIM9aW22M6YR745RMa22eMabQGDMUyACuAh5vmI8k0kQCAqHPeOh9KWR9Bkv/CR//Af73KMS2P1qoVRYf/72CIjxFWYy7EAtPgLhOR18fORdTq4CrdS448uSKLmvh0D53sfedx3bY+gkU1Vk+GxTuLhqPVfjFdoDg8BPPICJer7yqmtU78rlyaEeno4iIyI84bnFnra0yxtwGfAAEAC9YazcYYx4EVlpr3wLuBp4zxtyFe+7XNGutNcacATxojKkEaoCbrLV5nre+BZgFhAHveR4ivscY6DTS/di7HpbPgNK87xZi3xlVi6n1OgZCoiDAob+EGwNRSe5Hh6HfP19ZCvk7j1H8bXMXtHUL18ikOkVfrUIwMsm9G6mI+Jy1Owsor6pR83IRES9XrzV31trFuDdKqX3s/lrPNwLDj3Hfa8BrP/CeK4E+JxJWxOsl9YGx/3Q6RcMJCoPEbu5HXdZCSW6tgi/r6Kjf9iWw7hWOrPMDCAj5gXV+nmMhkY3+cUTk5GRk5mIMDFZxJyLi1dSoRkROjjHuTWYiEqBd+vfPV1VAwU5P0bf9u6N+O5Z5Nnmppc94GPWQe82fiHiVjKw8ureKIjY82OkoIiLyI1TciUjjCAx2t5GI7/z9c9ZC6cGjxd7uLyFjBnzzAZx5n3tjGqemqorId1RW17Bq+0EmDGp//ItFRMRRWgAjIk3PGAiPg7YDoc+l7hG7W5dBx2Hw4a/h2TNg2xKnU4oIsC67gNLKaq23ExHxASruRMQ7xHWCya/CxHlQfghmjYF/3QiH9judTKRZy8jKBbTeTkTEF6i4ExHvYQz0uABuzYARd8P61+DxNMh41t0vUESaXEZmHl1bRhIfGeJ0FBEROQ4VdyLifYLD4ez74ZZl0DYN3rsHnjsTdi53OplIs1JVXcPKbXkM6aRROxERX6DiTkS8V0IXmPo6XD4binPh+XPhzVuh+IDTyUSahQ27CymuqGZISrzTUUREpB5U3ImIdzMGel8Ct62A4XfC2gXuqZorX4CaaqfTifi1w+vtNHInIuIb1ApBRHxDSCSc+yD0nwyLfw7v3AVfzoEL/uaeuikiDS4jM49OCRG0jAp1OoqInKLKykqys7MpKytzOorUU2hoKO3atSMoqP7toVTciYhvadkDrn4bvlrkbpvw3NmQNs29Ri9cowsiDaW6xrJ8Wx4X9G3tdBQRaQDZ2dlERUWRnJyMMcbpOHIc1lpyc3PJzs4mJSWl3vdpWqaI+B5joN/lcNtKGHqzewTv8TT3z5oap9OJ+IVNewopKqvSlEwRP1FWVkZ8fLwKOx9hjCE+Pv6ER1pV3ImI7wqNhtF/ghs/g8Tu8Nbt8MJ5sGed08lEfF5GVh6ANlMR8SMq7HzLyfzvpeJORHxfUh+Y/h5c8jTkZcKMkbD4HijNdzqZiM/KyMylfVwYbWLDnI4iIiL1pOJORPyDMZA6GW5fBenXworn4IlB7t01rXU6nYhPqfGst9OonYiIb1FxJyL+JSwWLvgrXP8JxHaA12+EF8fAvg1OJxPxGd/sLyK/pJIhKVpvJyINJz8/n6eeeuqE7xszZgz5+ZqNUx/aLVNE/FObVLj2I1j9Evz7d/DMCPfmK2feCyFRTqcT8WoZme71dkM7aeROxB/9/u0NbNxd2KDv2atNNA9c1PtHrzlc3N1yyy3fOV5VVUVg4A+XJYsXL26QjI3lePmbkkbuRMR/uVyQdrV7qubAqfDFk+6pml8t0lRNkR+RkZVLm5hQ2rXQejsRaTj33nsvW7duJTU1lUGDBjFixAjGjh1Lr169ALjkkktIS0ujd+/ezJgx48h9ycnJHDhwgG3bttGzZ0+uv/56evfuzahRoygtLf3Bf++5555j0KBB9O/fn/Hjx1NSUgLAvn37GDduHP3796d///4sXboUgDlz5tCvXz/69+/P1KlTAZg2bRqLFi068p6RkZEAfPrpp/XO//777zNw4ED69+/P2WefTU1NDV27diUnJweAmv9v796jqyrPPI5/Hw6BQEDuNwmYqCgIISIpiVIBQTvYOljthGi90iqtNwQ6naHFZVkFXY6XaaGlWERELEoVi1XrFLEJ1VFBQ0W5WS+AAnIJ4TJEkVue+WOfhBAI5OTCzkl+n7X2Ovvss/d7nvMm5OE57373Li7m7LPPLn1eLe4eN0v//v1dRKTKNua7PzrI/Renuc+5wn37h2FHJBUA8r0O5J14WWoyPxYXF3v/ya/62Pnv1VibIhK+NWvWhB2Cr1+/3nv37u3u7nl5ed68eXNft25d6euFhYXu7v7VV1957969fceOHe7ufsYZZ3hBQYGvX7/eI5GIv/de8PcpOzvbn3rqqQrfr+R4d/eJEyf6tGnT3N195MiR/qtf/crd3Q8dOuS7d+/2VatWeY8ePbygoOCoWG666SZ/7rnnSttJSkqKKf7t27d7cnJy6X4l+0yaNKk0hkWLFvnVV1993M9wvJ/biXKkRu5EpOFI7g+35sJ3HoEt78OMgbD4F3Dgy7AjE6kzPi34kh1FBzTfTkRq3YABA466Qfe0adNIT08nKyuLjRs38vHHHx9zTGpqKueffz4A/fv3Z8OGDRW2v2rVKi6++GLS0tKYN28eq1cH8+9zc3O57bbbAIhEIrRq1Yrc3Fyys7Np3749AG3bnvxvYGXiX7p0KYMGDSrdr6TdH/zgB8ydOxeA2bNnM2rUqJO+X2XUjZNDRUROlUYR+MYtcN53g8LuzV8Hp2kOvx96jQiuulmXFBfDV4VQtBX2bos+boWvd0OX8yF1MLToEHaUUo8sW18IQKbm24lILUtKSipdX7JkCa+99hpvv/02zZs3Z8iQIce9gXfTpk1L1yORyAlPy7z55pt54YUXSE9PZ86cOSxZsiTmGBs3bkxxcTEQnD554MCBasVfolu3bnTq1Inc3Fzeeecd5s2bF3Nsx423RloREYk3Se3hu9ODuXh/+Qk8eyOcNQy+/RC0O6v23//wIfiy4Niibe9WKNp25LFoGxQfOvb4RglQfDBY75QGZw2BMy+BMy6CBM2Tkqpbtm4nHVs2JaVd87BDEZF6pmXLluzdu/e4r+3Zs4c2bdrQvHlzPvzwQ5YuXVrt99u7dy9dunTh4MGDzJs3j65duwIwbBpjYcEAABJcSURBVNgwZsyYwdixYzl8+DBFRUUMHTqUq666ivHjx9OuXTt27txJ27ZtSUlJYfny5YwcOZIXX3yRgwcPxhR/VlYWt99+O+vXryc1NbW0XYBbbrmF66+/nhtuuIFIJFLtzwsq7kSkoeueBaP/Du/Ogrz74HdZMPBu+OZ4aFKF/9weOnCkKNu7FfZuObpYKyngvtoBXnzs8c3bQYvO0LITdOgZPLbsAi06QcvORx4jTWDLCvg0D9YtgaWPwlu/gUhT6J4ZFHpnXQKd04MLy4hUgruzbH0hmWe2w+raKLaIxL127doxcOBA+vTpQ7NmzejUqVPpa8OHD+fRRx+lV69enHvuuWRlZVX7/SZPnkxmZiYdOnQgMzOztLCcOnUqo0eP5vHHHycSiTBjxgwuvPBCJk6cyODBg4lEIvTr1485c+Zw6623cuWVV5Kens7w4cOPGq0rq6L4O3TowMyZM7n66qspLi6mY8eOLF68GIARI0YwatSoGjslE8CCOXnxISMjw/Pz88MOQ0Tqq71bYfG98MEfg3vkXf4gnHt58NrBfWVG1sqOtpUbddu389h2rREkdYgWZl2Cgq2kgGvROSjWWnaGpI7QuEnVYj/wJXz2NqzLCwq+7dH7+jVrC2cOhjOHBAVfmzOq1v4pZmbL3T0j7DjiRU3lxw07vmTIw0uY8t0+XJ8VH78rIlI5a9eupVevXmGHIWXk5+czbtw43njjjQr3Od7P7UQ5UiN3IiIlWnaGq2fCBTfCX/4dnrkmKPL27YH9e47dv1FCtGDrBG1Sg1HAssVayShbUodgrl9tapIEPS4NFgiKzvV/j47s5cHqhcH2tmceGdVLuTi46bvUCjMbDkwFIsAsd3+g3Os3Aw8Bm6Obfuvus6Kv3QTcE90+xd2fPBUxl8y3yzpTF1MREalNDzzwADNmzKixuXYlVNyJiJSX8k348RvwzmOwcdmRAu6o0bYu0KxN3T3lsWUn6DsyWNyh4J9HRvXenw/5jwcjiqdfEBR6Z14Cyd+o+sihHMXMIsB04DJgE/Cumb3o7mvK7fpHd7+z3LFtgV8AGYADy6PH7qrtuJet20n7Fk04q0OL2n4rEZEac8cdd/Dmm28ete3uu++u0dMda9qECROYMGFCjber4k5E5HgiCXDh7cES78ygY89gybotmBe4Of/IqN4bj8DrD0FCEqQMPDKy16Fn3bt6aPwYAHzi7usAzGw+cCVQvrg7nn8BFrv7zuixi4HhwDO1FGupZet3MiC1rebbiUhcmT59etgh1Bkq7kREGprGTYKrap5xEQydCPt2w4b/PTKy9/GrwX4tOgdz9c66JHhs2Tm8mONPV2BjmeebgMzj7Pc9MxsEfASMc/eNFRzbtfyBZjYaGA3QvXv3age8cedXbN69j9GDzqx2WyIiEg4VdyIiDV2z1tDrimAB2P15cAXOkkLvg/nB9o7nHbkwS8rAYJ6fVMdLwDPuvt/MfgQ8CQyt7MHuPhOYCcEFVaobzLL1wcWAMjXfTkQkbqm4ExGRo7XuHlxU5oIbg5uob/0gKPbW5cG7j8PS3wUXk+mWeWRk7/R+tX/RmPiyGehW5nkyRy6cAoC7F5Z5Ogt4sMyxQ8odu6TGIyxn2bpCWjdP4JyOLWv7rUREpJaouBMRkYo1agSnnx8s3xwb3BLi87ePjOzlTQmWxFaQOiha7A0NrsrZsL0L9DCzVIJi7Rrg+2V3MLMu7r4l+nQEsDa6vgi438zaRJ9/C/hZbQe8bP1OBqS0pVEjzbcTEYlXKu5ERKTyEpoFxdtZQ4PrQH65IzqqF13WvgTdsuCHi8KNM2TufsjM7iQo1CLAbHdfbWa/BPLd/UVgjJmNAA4BO4Gbo8fuNLPJBAUiwC9LLq5SW7bs2cfnO7/ipotSavNtRERi0qJFC4qKisIOI66ouBMRkapLag9p/xYs7lD46fHvCdgAufsrwCvltt1bZv1nVDAi5+6zgdm1GmAZ7ZKa8tyPL6Rbm+an6i1FJEz/MwG2rqzZNjunweUPnHy/OHTo0CEaN46PsqmO3qBJRETijhm0Pxu69g87EolRk8aN+EZKWzq3Sgw7FBGpxyZMmHDUbQsmTZrElClTGDZsGBdccAFpaWn8+c9/rlRbRUVFFR43d+5c+vbtS3p6OjfccAMA27Zt46qrriI9PZ309HTeeustNmzYQJ8+fUqPe/jhh5k0aRIAQ4YMYezYsWRkZDB16lReeuklMjMz6devH5deeinbtm0rjWPUqFGkpaXRt29fnn/+eWbPns3YsWNL233ssccYN25clfstFvFRgoqIiIiISM0IaYQtJyeHsWPHcscddwDw7LPPsmjRIsaMGcNpp53Gjh07yMrKYsSIESe932ZiYiILFy485rg1a9YwZcoU3nrrLdq3b8/OncFZ7WPGjGHw4MEsXLiQw4cPU1RUxK5du074HgcOHCA/Px+AXbt2sXTpUsyMWbNm8eCDD/LII48wefJkWrVqxcqVK0v3S0hI4L777uOhhx4iISGBJ554gt///vfV7b5KUXEnIiIiIiK1rl+/fmzfvp0vvviCgoIC2rRpQ+fOnRk3bhyvv/46jRo1YvPmzWzbto3OnU98b1V35+c///kxx+Xm5pKdnU379u0BaNs2uL1Lbm4uc+fOBSASidCqVauTFnc5OTml65s2bSInJ4ctW7Zw4MABUlNTAXjttdeYP39+6X5t2gTXwho6dCgvv/wyvXr14uDBg6SlpcXYW1Wj4k5ERERERE6J7OxsFixYwNatW8nJyWHevHkUFBSwfPlyEhISSElJ4euvvz5pO1U9rqzGjRtTXFxc+rz88UlJR+7netdddzF+/HhGjBjBkiVLSk/frMgtt9zC/fffT8+ePRk1alRMcVWH5tyJiIiIiMgpkZOTw/z581mwYAHZ2dns2bOHjh07kpCQQF5eHp999lml2qnouKFDh/Lcc89RWBjcSrTktMxhw4YxY8YMAA4fPsyePXvo1KkT27dvp7CwkP379/Pyyy+f8P26du0KwJNPPlm6/bLLLjtqHmHJaGBmZiYbN27k6aef5tprr61s91SbijsRERERETklevfuzd69e+natStdunThuuuuIz8/n7S0NObOnUvPnj0r1U5Fx/Xu3ZuJEycyePBg0tPTGT9+PABTp04lLy+PtLQ0+vfvz5o1a0hISODee+9lwIABXHbZZSd870mTJpGdnU3//v1LT/kEuOeee9i1axd9+vQhPT2dvLy80tdGjhzJwIEDS0/VPBXM3U++k9lwYCrBvXpmufsD5V7vDjwJtI7uM8HdXzGzy4AHgCbAAeCn7p4bPWYJ0AXYF23mW+6+/URxZGRkeMmkRhERqb/MbLm7Z4QdR7xQfhSRk1m7di29evUKO4wG5YorrmDcuHEMGzasym0c7+d2ohx50pE7M4sA04HLgfOAa83svHK73QM86+79gGuA30W37wD+1d3TgJuAp8odd527nx9dTljYiYiIiIiI1HW7d+/mnHPOoVmzZtUq7KqiMhdUGQB84u7rAMxsPnAlsKbMPg6cFl1vBXwB4O7vldlnNdDMzJq6+/7qBi4iIiIiIvXbypUrS+9VV6Jp06YsW7YspIhOrnXr1nz00UehvHdliruuwMYyzzcBmeX2mQS8amZ3AUnApcdp53vAP8oVdk+Y2WHgeWCKH+ccUTMbDYwG6N69eyXCFRERERGR8tz9pPePq2vS0tJYsWJF2GGEojLT58qrqQuqXAvMcfdk4NvAU2ZW2raZ9Qb+C/hRmWOui56ueXF0Obokj3L3me6e4e4ZHTp0qKFwRUREREQajsTERAoLC6tUMMip5+4UFhaSmJgY03GVGbnbDHQr8zw5uq2sHwLDo4G8bWaJQHtgu5klAwuBG9390zIBb44+7jWzpwlO/5wbU/QiIiIiInJSycnJbNq0iYKCgrBDkUpKTEwkOTk5pmMqU9y9C/Qws1SCou4a4Pvl9vkcGAbMMbNeQCJQYGatgb8QXD3zzZKdzawx0Nrdd5hZAnAF8FpMkYuIiIiISKUkJCSQmpoadhhSy056Wqa7HwLuBBYBawmuirnazH5pZiOiu/0EuNXM3geeAW6Ozp+7EzgbuNfMVkSXjkBTYJGZfQCsICgaH6vpDyciIiIiItJQVGbkDnd/BXil3LZ7y6yvAQYe57gpwJQKmu1f+TBFRERERETkRGrqgioiIiIiIiISIounK+aYWQHwWTWbaU9wc3WpPPVZ7NRnsVOfxa4+99kZ7q5LJFdSDeVHqN+/U7VFfRYb9Vfs1Gexq+99VmGOjKviriaYWb67Z4QdRzxRn8VOfRY79Vns1GdS0/Q7FTv1WWzUX7FTn8WuIfeZTssUERERERGpB1TciYiIiIiI1AMNsbibGXYAcUh9Fjv1WezUZ7FTn0lN0+9U7NRnsVF/xU59FrsG22cNbs6diIiIiIhIfdQQR+5ERERERETqHRV3IiIiIiIi9UCDKu7MbLiZ/dPMPjGzCWHHU9eZWTczyzOzNWa22szuDjumeGBmETN7z8xeDjuWeGBmrc1sgZl9aGZrzezCsGOq68xsXPTf5Coze8bMEsOOSeKb8mNslB+rTjkyNsqRsWvoObLBFHdmFgGmA5cD5wHXmtl54UZV5x0CfuLu5wFZwB3qs0q5G1gbdhBxZCrwV3fvCaSjvjshM+sKjAEy3L0PEAGuCTcqiWfKj1Wi/Fh1ypGxUY6MgXJkAyrugAHAJ+6+zt0PAPOBK0OOqU5z9y3u/o/o+l6CPyhdw42qbjOzZOA7wKywY4kHZtYKGAQ8DuDuB9x9d7hRxYXGQDMzaww0B74IOR6Jb8qPMVJ+rBrlyNgoR1ZZg86RDam46wpsLPN8E/pDXGlmlgL0A5aFG0md92vgP4DisAOJE6lAAfBE9DSdWWaWFHZQdZm7bwYeBj4HtgB73P3VcKOSOKf8WA3KjzFRjoyNcmSMlCMbVnEnVWRmLYDngbHu/n9hx1NXmdkVwHZ3Xx52LHGkMXABMMPd+wFfAprvcwJm1oZgVCUVOB1IMrPrw41KpGFSfqw85cgqUY6MkXJkwyruNgPdyjxPjm6TEzCzBILENc/d/xR2PHXcQGCEmW0gOK1pqJn9IdyQ6rxNwCZ3L/nGewFBIpOKXQqsd/cCdz8I/Am4KOSYJL4pP1aB8mPMlCNjpxwZuwafIxtScfcu0MPMUs2sCcHkyhdDjqlOMzMjOM97rbv/d9jx1HXu/jN3T3b3FILfr1x3b1DfFsXK3bcCG83s3OimYcCaEEOKB58DWWbWPPpvdBiaYC/Vo/wYI+XH2ClHxk45skoafI5sHHYAp4q7HzKzO4FFBFfOme3uq0MOq64bCNwArDSzFdFtP3f3V0KMSeqfu4B50f9UrgNGhRxPnebuy8xsAfAPgiv2vQfMDDcqiWfKj1Wi/CininJkDJQjwdw97BhERERERESkmhrSaZkiIiIiIiL1loo7ERERERGRekDFnYiIiIiISD2g4k5ERERERKQeUHEnIiIiIiJSD6i4E6lFZnbYzFaUWSbUYNspZraqptoTERE5lZQjRWpeg7nPnUhI9rn7+WEHISIiUgcpR4rUMI3ciYTAzDaY2YNmttLM3jGzs6PbU8ws18w+MLO/mVn36PZOZrbQzN6PLhdFm4qY2WNmttrMXjWzZtH9x5jZmmg780P6mCIiIjFTjhSpOhV3IrWrWblTTnLKvLbH3dOA3wK/jm77DfCku/cF5gHTotunAX9393TgAmB1dHsPYLq79wZ2A9+Lbp8A9Iu28+Pa+nAiIiLVoBwpUsPM3cOOQaTeMrMid29xnO0bgKHuvs7MEoCt7t7OzHYAXdz9YHT7Fndvb2YFQLK77y/TRgqw2N17RJ//J5Dg7lPM7K9AEfAC8IK7F9XyRxUREYmJcqRIzdPInUh4vIL1WOwvs36YI/NovwNMJ/gG810z0/xaERGJJ8qRIlWg4k4kPDllHt+Orr8FXBNdvw54I7r+N+A2ADOLmFmriho1s0ZAN3fPA/4TaAUc882oiIhIHaYcKVIF+qZCpHY1M7MVZZ7/1d1LLvXcxsw+IPhm8drotruAJ8zsp0ABMCq6/W5gppn9kODbx9uALRW8ZwT4QzS5GTDN3XfX2CcSERGpGcqRIjVMc+5EQhCdT5Dh7jvCjkVERKQuUY4UqTqdlikiIiIiIlIPaORORERERESkHtDInYiIiIiISD2g4k5ERERERKQeUHEnIiIiIiJSD6i4ExERERERqQdU3ImIiIiIiNQD/w+PV0AbGOnv7QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1080x504 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Optimize Classifier"
      ],
      "metadata": {
        "id": "8kxaObKrswJQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "  import optuna\n",
        "except:\n",
        "  !pip install optuna\n",
        "  import optuna"
      ],
      "metadata": {
        "id": "2KdU3O0-s80S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "MODEL = ensemble_model\n",
        "EPOCHS = 5\n",
        "DEVICE = device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "train_loader = train_dataloader\n",
        "valid_loader = test_dataloader\n",
        "\n",
        "def objective(trial):\n",
        "\n",
        "    # Generate the model.\n",
        "    model = MODEL\n",
        "\n",
        "    # Generate the optimizers\n",
        "    optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\",\"SGD\"])\n",
        "    lr = trial.suggest_float(\"lr\", 1e-5, 1e-1,log=True)\n",
        "    optimizer = getattr(torch.optim, optimizer_name)(model.parameters(), lr=lr)\n",
        "    #batch_size=trial.suggest_int(\"batch_size\", 16, 128,step=16)\n",
        "\n",
        "    criterion=nn.CrossEntropyLoss()\n",
        "    \n",
        "    # Training of the model.\n",
        "    for epoch in range(EPOCHS):\n",
        "        model.train()\n",
        "       \n",
        "        for batch_idx, (images, labels) in enumerate(train_loader):\n",
        "\n",
        "            images, labels = images.to(DEVICE), labels.to(DEVICE)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            output = model(images)\n",
        "            loss = criterion(output, labels)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "        # Validation of the model.\n",
        "        model.eval()\n",
        "        correct = 0\n",
        "        with torch.no_grad():\n",
        "            for batch_idx, (images, labels) in enumerate(valid_loader):\n",
        "                # Limiting validation images.\n",
        "               # if batch_idx * BATCHSIZE >= N_VALID_EXAMPLES:\n",
        "                #    break\n",
        "                images, labels = images.to(DEVICE), labels.to(DEVICE)\n",
        "                output = model(images)\n",
        "                # Get the index of the max log-probability.\n",
        "                pred = output.argmax(dim=1, keepdim=True)\n",
        "                correct += pred.eq(labels.view_as(pred)).sum().item()\n",
        "\n",
        "        accuracy = correct / len(valid_loader.dataset)\n",
        "\n",
        "        trial.report(accuracy, epoch)\n",
        "\n",
        "        # Handle pruning based on the intermediate value.\n",
        "        if trial.should_prune():\n",
        "            raise optuna.exceptions.TrialPruned()\n",
        "\n",
        "    return accuracy"
      ],
      "metadata": {
        "id": "UIFzCSLyst2C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#create study for optuna \n",
        "torch.manual_seed(42)\n",
        "torch.cuda.manual_seed(42)\n",
        "\n",
        "# Start the timer\n",
        "from timeit import default_timer as timer \n",
        "start_time = timer()\n",
        "\n",
        "study = optuna.create_study(direction='maximize')\n",
        "study.optimize(objective, n_trials=3)\n",
        "\n",
        "trial = study.best_trial\n",
        "\n",
        "print('Accuracy: {}'.format(trial.value))\n",
        "print(\"Best hyperparameters: {}\".format(trial.params))\n",
        "\n",
        "end_time = timer()\n",
        "print(f\"[INFO] Total time to tune hyper parameters: {end_time-start_time:.3f} seconds using {DEVICE}\")\n",
        "\n",
        "df = study.trials_dataframe()\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "QuVuY5Pos_xP",
        "outputId": "084a658c-1287-450d-e202-fe473e31af2f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-12-12 01:51:39,567]\u001b[0m A new study created in memory with name: no-name-695cddb1-5280-456b-8ece-c26790c5559e\u001b[0m\n",
            "\u001b[32m[I 2022-12-12 03:29:00,249]\u001b[0m Trial 0 finished with value: 0.4346415327564895 and parameters: {'optimizer': 'SGD', 'lr': 0.03436145355133335}. Best is trial 0 with value: 0.4346415327564895.\u001b[0m\n",
            "\u001b[32m[I 2022-12-12 05:06:23,367]\u001b[0m Trial 1 finished with value: 0.42629789864029666 and parameters: {'optimizer': 'Adam', 'lr': 3.515118469905006e-05}. Best is trial 0 with value: 0.4346415327564895.\u001b[0m\n",
            "\u001b[32m[I 2022-12-12 06:43:27,693]\u001b[0m Trial 2 finished with value: 0.43170580964153277 and parameters: {'optimizer': 'SGD', 'lr': 5.407498153361748e-05}. Best is trial 0 with value: 0.4346415327564895.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.4346415327564895\n",
            "Best hyperparameters: {'optimizer': 'SGD', 'lr': 0.03436145355133335}\n",
            "[INFO] Total time to tune hyper parameters: 17508.130 seconds using cuda\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-e84943c1-365f-4c38-8048-80952a6e37c8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>number</th>\n",
              "      <th>value</th>\n",
              "      <th>datetime_start</th>\n",
              "      <th>datetime_complete</th>\n",
              "      <th>duration</th>\n",
              "      <th>params_lr</th>\n",
              "      <th>params_optimizer</th>\n",
              "      <th>state</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0.434642</td>\n",
              "      <td>2022-12-12 01:51:39.570247</td>\n",
              "      <td>2022-12-12 03:29:00.248723</td>\n",
              "      <td>0 days 01:37:20.678476</td>\n",
              "      <td>0.034361</td>\n",
              "      <td>SGD</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0.426298</td>\n",
              "      <td>2022-12-12 03:29:00.251812</td>\n",
              "      <td>2022-12-12 05:06:23.367184</td>\n",
              "      <td>0 days 01:37:23.115372</td>\n",
              "      <td>0.000035</td>\n",
              "      <td>Adam</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>0.431706</td>\n",
              "      <td>2022-12-12 05:06:23.370726</td>\n",
              "      <td>2022-12-12 06:43:27.693323</td>\n",
              "      <td>0 days 01:37:04.322597</td>\n",
              "      <td>0.000054</td>\n",
              "      <td>SGD</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e84943c1-365f-4c38-8048-80952a6e37c8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e84943c1-365f-4c38-8048-80952a6e37c8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e84943c1-365f-4c38-8048-80952a6e37c8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   number     value  ... params_optimizer     state\n",
              "0       0  0.434642  ...              SGD  COMPLETE\n",
              "1       1  0.426298  ...             Adam  COMPLETE\n",
              "2       2  0.431706  ...              SGD  COMPLETE\n",
              "\n",
              "[3 rows x 8 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    }
  ]
}