Transformers/src/transformer_gpu.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eReLT1AU7QBa"
      },
      "source": [
        "# TPU Configuration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "QtbP7Ofg7QBi",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# !pip install cloud-tpu-client==0.10 torch==1.9.0 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.9-cp37-cp37m-linux_x86_64.whl -qq"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:40:54.109948Z",
          "iopub.status.busy": "2022-01-07T04:40:54.109654Z",
          "iopub.status.idle": "2022-01-07T04:40:55.684626Z",
          "shell.execute_reply": "2022-01-07T04:40:55.683951Z",
          "shell.execute_reply.started": "2022-01-07T04:40:54.109921Z"
        },
        "id": "-nj5KVNb7QBj",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# import torch_xla\n",
        "# import torch_xla.debug.metrics as met\n",
        "# import torch_xla.distributed.parallel_loader as pl\n",
        "# import torch_xla.utils.utils as xu\n",
        "# import torch_xla.core.xla_model as xm\n",
        "# import torch_xla.distributed.xla_multiprocessing as xmp\n",
        "# import torch_xla.test.test_utils as test_utils\n",
        "\n",
        "# import warnings\n",
        "# warnings.filterwarnings(\"ignore\")\n",
        "# device = xm.xla_device()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1meTcsQISzrq"
      },
      "source": [
        "# Library"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:40:55.686109Z",
          "iopub.status.busy": "2022-01-07T04:40:55.685839Z",
          "iopub.status.idle": "2022-01-07T04:41:05.455032Z",
          "shell.execute_reply": "2022-01-07T04:41:05.453612Z",
          "shell.execute_reply.started": "2022-01-07T04:40:55.686079Z"
        },
        "id": "JkFt8wlvSzrv",
        "outputId": "b7378ac8-d2c2-412c-cb33-6d34ea25c822",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "! pip install Korpora sentencepiece einops wandb torch-summary -qq"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:41:05.458999Z",
          "iopub.status.busy": "2022-01-07T04:41:05.458431Z",
          "iopub.status.idle": "2022-01-07T04:41:07.695876Z",
          "shell.execute_reply": "2022-01-07T04:41:07.695056Z",
          "shell.execute_reply.started": "2022-01-07T04:41:05.458945Z"
        },
        "id": "WxM006iV5StT",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from Korpora import Korpora\n",
        "import pandas as pd\n",
        "from pprint import pprint\n",
        "# from konlpy.tag import Mecab\n",
        "from nltk.tokenize import word_tokenize as en_tokenizer\n",
        "import sentencepiece as spm\n",
        "import urllib.request\n",
        "import csv\n",
        "import numpy as np\n",
        "from einops import rearrange, reduce, repeat\n",
        "from torch.cuda import amp\n",
        "from tqdm import tqdm\n",
        "import wandb\n",
        "import time\n",
        "import copy\n",
        "from collections import defaultdict\n",
        "from sklearn.metrics import mean_squared_error\n",
        "import joblib\n",
        "import gc\n",
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:41:07.697603Z",
          "iopub.status.busy": "2022-01-07T04:41:07.697390Z",
          "iopub.status.idle": "2022-01-07T04:41:07.705323Z",
          "shell.execute_reply": "2022-01-07T04:41:07.704393Z",
          "shell.execute_reply.started": "2022-01-07T04:41:07.697577Z"
        },
        "id": "VuuFjEE8Szry",
        "outputId": "587337a2-eb49-47b1-d814-a3744a2637b2",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Using cpu\n"
          ]
        }
      ],
      "source": [
        "VOCAB_SIZE = 32000 + 7\n",
        "SEQ_LEN = 100\n",
        "PAD_IDX = 0\n",
        "# Trainig Set 모집단의 크기\n",
        "TRAINSET_SIZE = 120000\n",
        "# 실제로 사용할 Training Set의 크기. 이 수만큼 전체 Training Set에서 Random Sampling\n",
        "TRAIN_LEN = 100000\n",
        "VALID_LEN = 10000\n",
        "BATCH_SIZE = 2\n",
        "WANDB_SAVED_PATH = ''\n",
        "if 'device' not in globals():\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(f'Using {device}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 149
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:41:07.706787Z",
          "iopub.status.busy": "2022-01-07T04:41:07.706559Z",
          "iopub.status.idle": "2022-01-07T04:41:15.020744Z",
          "shell.execute_reply": "2022-01-07T04:41:15.020134Z",
          "shell.execute_reply.started": "2022-01-07T04:41:07.706759Z"
        },
        "id": "KWd0k3I0RBD2",
        "outputId": "dcaac222-746e-4b15-b459-c10ff09fae41",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "W&B syncing is set to `offline` in this directory.  <br/>\n",
              "Run `wandb online` or set WANDB_MODE=online to enable cloud syncing."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src=\"?jupyter=true\" style=\"border:none;width:100%;height:420px;display:none;\"></iframe>"
            ],
            "text/plain": [
              "<wandb.sdk.wandb_run.Run at 0x7fee8220a0a0>"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import wandb\n",
        "import os\n",
        "# if want to run in offline mode\n",
        "\n",
        "os.environ[\"WANDB_API_KEY\"] = \"d60a4af56f6cd9cccec7d9da1dbced7960b61310\"\n",
        "os.environ[\"WANDB_MODE\"] = \"dryrun\"\n",
        "wandb.init(project=\"Transformer\", entity=\"jiwon7258\")\n",
        "RUN_PATH = ''\n",
        "\n",
        "\n",
        "# wandb.login(key=\"d60a4af56f6cd9cccec7d9da1dbced7960b61310\")\n",
        "# os.environ[\"WANDB_MODE\"] = \"online\"\n",
        "# wandb.init(project=\"Transformer\", entity=\"jiwon7258\")\n",
        "# wandb.run.name = \"kaggleGpu\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TQrKlhKPSzrz"
      },
      "source": [
        "# Create Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:41:15.022337Z",
          "iopub.status.busy": "2022-01-07T04:41:15.022039Z",
          "iopub.status.idle": "2022-01-07T04:41:31.815226Z",
          "shell.execute_reply": "2022-01-07T04:41:31.814376Z",
          "shell.execute_reply.started": "2022-01-07T04:41:15.022312Z"
        },
        "id": "jZtd6ivuCz_z",
        "outputId": "e1140616-910f-40ac-aa96-fba91dac2142",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "    Korpora 는 다른 분들이 연구 목적으로 공유해주신 말뭉치들을\n",
            "    손쉽게 다운로드, 사용할 수 있는 기능만을 제공합니다.\n",
            "\n",
            "    말뭉치들을 공유해 주신 분들에게 감사드리며, 각 말뭉치 별 설명과 라이센스를 공유 드립니다.\n",
            "    해당 말뭉치에 대해 자세히 알고 싶으신 분은 아래의 description 을 참고,\n",
            "    해당 말뭉치를 연구/상용의 목적으로 이용하실 때에는 아래의 라이센스를 참고해 주시기 바랍니다.\n",
            "\n",
            "    # Description\n",
            "    Author : TRAC (https://trac.edgewall.org/)\n",
            "    Repository : http://opus.nlpl.eu/OpenSubtitles-v2018.php\n",
            "    References :\n",
            "        - P. Lison and J. Tiedemann, 2016, OpenSubtitles2016: Extracting Large Parallel Corpora\n",
            "          from Movie and TV Subtitles. In Proceedings of the 10th International Conference on\n",
            "          Language Resources and Evaluation (LREC 2016)\n",
            "\n",
            "    This is a new collection of translated movie subtitles from http://www.opensubtitles.org/.\n",
            "\n",
            "    [[ IMPORTANT ]]\n",
            "    If you use the OpenSubtitle corpus: Please, add a link to http://www.opensubtitles.org/\n",
            "    to your website and to your reports and publications produced with the data!\n",
            "    I promised this when I got the data from the providers of that website!\n",
            "\n",
            "    This is a slightly cleaner version of the subtitle collection using improved sentence alignment\n",
            "    and better language checking.\n",
            "\n",
            "    62 languages, 1,782 bitexts\n",
            "    total number of files: 3,735,070\n",
            "    total number of tokens: 22.10G\n",
            "    total number of sentence fragments: 3.35G\n",
            "\n",
            "    [[ NOTICE ]]\n",
            "    In original data, the source language is `en` and target language is `ko`. However in Korpora,\n",
            "    we change the language pair so that source language is `ko` and target language is `en`.\n",
            "\n",
            "    # License\n",
            "    Open Data. Details in https://opendefinition.org/od/2.1/en/\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[open_subtitles] download en-ko.tmx.gz: 48.1MB [00:04, 10.2MB/s]                            \n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "decompress /content/open_subtitles/en-ko.tmx.gz\n"
          ]
        }
      ],
      "source": [
        "# dataset = open_subtitles_dataset()\n",
        "corpus = Korpora.load(\"open_subtitles\", root_dir='./')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:41:31.816669Z",
          "iopub.status.busy": "2022-01-07T04:41:31.816438Z",
          "iopub.status.idle": "2022-01-07T04:42:43.891488Z",
          "shell.execute_reply": "2022-01-07T04:42:43.890619Z",
          "shell.execute_reply.started": "2022-01-07T04:41:31.816642Z"
        },
        "id": "CpuQ8uGZOFqQ",
        "outputId": "17911e59-e8ce-4ad4-8167-c3246262394e",
        "trusted": true
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-32e84ea2-c678-45bf-80f0-f76392f9ad61\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>src</th>\n",
              "      <th>trg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Through the snow and sleet and hail, through t...</td>\n",
              "      <td>폭설이 내리고 우박, 진눈깨비가 퍼부어도 눈보라가 몰아쳐도 강풍이 불고 비바람이 휘...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ever faithful, ever true, nothing stops him, h...</td>\n",
              "      <td>우리의 한결같은 심부름꾼 황새 아저씨 가는 길을 그 누가 막으랴!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Look out for Mr Stork That persevering chap</td>\n",
              "      <td>황새 아저씨를 기다리세요</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>He'll come along and drop a bundle in your lap</td>\n",
              "      <td>찾아와 선물을 주실 거예요</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>You may be poor or rich It doesn't matter which</td>\n",
              "      <td>가난하든 부자이든 상관이 없답니다</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-32e84ea2-c678-45bf-80f0-f76392f9ad61')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-32e84ea2-c678-45bf-80f0-f76392f9ad61 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-32e84ea2-c678-45bf-80f0-f76392f9ad61');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                 src                                                trg\n",
              "0  Through the snow and sleet and hail, through t...  폭설이 내리고 우박, 진눈깨비가 퍼부어도 눈보라가 몰아쳐도 강풍이 불고 비바람이 휘...\n",
              "1  ever faithful, ever true, nothing stops him, h...               우리의 한결같은 심부름꾼 황새 아저씨 가는 길을 그 누가 막으랴!\n",
              "2        Look out for Mr Stork That persevering chap                                      황새 아저씨를 기다리세요\n",
              "3     He'll come along and drop a bundle in your lap                                     찾아와 선물을 주실 거예요\n",
              "4    You may be poor or rich It doesn't matter which                                 가난하든 부자이든 상관이 없답니다"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data = pd.DataFrame([corpus.train.pairs, corpus.train.texts], index = ['src', 'trg'])\n",
        "data = data.transpose()\n",
        "data.to_csv('data.txt', index=False)\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:42:43.893752Z",
          "iopub.status.busy": "2022-01-07T04:42:43.893201Z",
          "iopub.status.idle": "2022-01-07T04:42:47.203440Z",
          "shell.execute_reply": "2022-01-07T04:42:47.202045Z",
          "shell.execute_reply.started": "2022-01-07T04:42:43.893708Z"
        },
        "id": "GhEK8vW5Szr1",
        "outputId": "935c3f72-15cb-41e2-a0e4-f58392572758",
        "trusted": true
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-60e56e0e-6a0e-455e-9615-cf59895c3dce\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>src</th>\n",
              "      <th>trg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Through the snow and sleet and hail, through t...</td>\n",
              "      <td>폭설이 내리고 우박, 진눈깨비가 퍼부어도 눈보라가 몰아쳐도 강풍이 불고 비바람이 휘...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ever faithful, ever true, nothing stops him, h...</td>\n",
              "      <td>우리의 한결같은 심부름꾼 황새 아저씨 가는 길을 그 누가 막으랴!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Look out for Mr Stork That persevering chap</td>\n",
              "      <td>황새 아저씨를 기다리세요</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>He'll come along and drop a bundle in your lap</td>\n",
              "      <td>찾아와 선물을 주실 거예요</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>You may be poor or rich It doesn't matter which</td>\n",
              "      <td>가난하든 부자이든 상관이 없답니다</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-60e56e0e-6a0e-455e-9615-cf59895c3dce')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-60e56e0e-6a0e-455e-9615-cf59895c3dce button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-60e56e0e-6a0e-455e-9615-cf59895c3dce');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                 src                                                trg\n",
              "0  Through the snow and sleet and hail, through t...  폭설이 내리고 우박, 진눈깨비가 퍼부어도 눈보라가 몰아쳐도 강풍이 불고 비바람이 휘...\n",
              "1  ever faithful, ever true, nothing stops him, h...               우리의 한결같은 심부름꾼 황새 아저씨 가는 길을 그 누가 막으랴!\n",
              "2        Look out for Mr Stork That persevering chap                                      황새 아저씨를 기다리세요\n",
              "3     He'll come along and drop a bundle in your lap                                     찾아와 선물을 주실 거예요\n",
              "4    You may be poor or rich It doesn't matter which                                 가난하든 부자이든 상관이 없답니다"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data = pd.read_csv('data.txt')\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BYHmUVpWSzr2"
      },
      "source": [
        "## Sentencepiece Train"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "model / vocab 파일을 생성합니다"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:42:47.206548Z",
          "iopub.status.busy": "2022-01-07T04:42:47.206105Z",
          "iopub.status.idle": "2022-01-07T04:42:48.202446Z",
          "shell.execute_reply": "2022-01-07T04:42:48.201657Z",
          "shell.execute_reply.started": "2022-01-07T04:42:47.206510Z"
        },
        "id": "qKGd7SI8OFqR",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "with open('src.txt', mode = 'w', encoding='utf8') as f:\n",
        "    f.write('\\n'.join(data['src']))\n",
        "with open('trg.txt', mode= 'w', encoding='utf8') as f:\n",
        "    f.write('\\n'.join(data['trg']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "7SHuXFE2Szr4",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "corpus = \"src.txt\"\n",
        "prefix = \"src\"\n",
        "vocab_size = VOCAB_SIZE - 7\n",
        "spm.SentencePieceTrainer.train(\n",
        "    f\"--input={corpus} --model_prefix={prefix} --vocab_size={vocab_size + 7}\" +\n",
        "    \" --model_type=bpe\" +\n",
        "    \" --max_sentence_length=999999\" +  # 문장 최대 길이\n",
        "    \" --pad_id=0 --pad_piece=[PAD]\" +  # pad (0)\n",
        "    \" --unk_id=1 --unk_piece=[UNK]\" +  # unknown (1)\n",
        "    \" --bos_id=2 --bos_piece=[BOS]\" +  # begin of sequence (2)\n",
        "    \" --eos_id=3 --eos_piece=[EOS]\" +  # end of sequence (3)\n",
        "    \" --user_defined_symbols=[SEP],[CLS],[MASK]\")  # 사용자 정의 토큰\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "n9O90jzgSzr6",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "corpus = \"trg.txt\"\n",
        "prefix = \"trg\"\n",
        "vocab_size = VOCAB_SIZE - 7\n",
        "spm.SentencePieceTrainer.train(\n",
        "    f\"--input={corpus} --model_prefix={prefix} --vocab_size={vocab_size + 7}\" +\n",
        "    \" --model_type=bpe\" +\n",
        "    \" --max_sentence_length=999999\" +  # 문장 최대 길이\n",
        "    \" --pad_id=0 --pad_piece=[PAD]\" +  # pad (0)\n",
        "    \" --unk_id=1 --unk_piece=[UNK]\" +  # unknown (1)\n",
        "    \" --bos_id=2 --bos_piece=[BOS]\" +  # begin of sequence (2)\n",
        "    \" --eos_id=3 --eos_piece=[EOS]\" +  # end of sequence (3)\n",
        "    \" --user_defined_symbols=[SEP],[CLS],[MASK]\")  # 사용자 정의 토큰\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dceb4asCSzr7"
      },
      "source": [
        "## SRC Data (EN) Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "wandb.restore('src.model', run_path = WANDB_SAVED_PATH, root='./')\n",
        "wandb.restore('src.vocab', run_path = WANDB_SAVED_PATH, root='./')\n",
        "wandb.restore('trg.model', run_path = WANDB_SAVED_PATH, root='./')\n",
        "wandb.restore('trg.vocab', run_path = WANDB_SAVED_PATH, root='./')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "sp_src = spm.SentencePieceProcessor()\n",
        "sp_src.Load('src.model')\n",
        "lines = [\n",
        "    \"I didn't at all think of it this way.\",\n",
        "    \"I have waited a long time for someone to film\",\n",
        "    \"[PAD] [CLS] [BOS] [EOS] [SEP] [UNK] \"\n",
        "]\n",
        "for line in lines:\n",
        "    print(sp_src.EncodeAsPieces(line))\n",
        "    print(sp_src.EncodeAsIds(line))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:44:01.478080Z",
          "iopub.status.busy": "2022-01-07T04:44:01.477785Z",
          "iopub.status.idle": "2022-01-07T04:44:01.619751Z",
          "shell.execute_reply": "2022-01-07T04:44:01.588134Z",
          "shell.execute_reply.started": "2022-01-07T04:44:01.478045Z"
        },
        "id": "m4PVun3kSzr7",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "def en_encode(tmpstr:str) -> np.array :\n",
        "    tmpstr = np.array(sp_src.EncodeAsIds(tmpstr))\n",
        "\n",
        "    # SEQ_LEN보다 길면 짜른다 \n",
        "    if len(tmpstr) > SEQ_LEN :\n",
        "        tmpstr = tmpstr[:SEQ_LEN]\n",
        "\n",
        "    # SEQ_LEN보다 작으면 padding\n",
        "    else :\n",
        "        tmpstr = np.pad(tmpstr, (0, SEQ_LEN - len(tmpstr)), 'constant', constant_values = sp_src.pad_id())\n",
        "    \n",
        "    return tmpstr"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:44:01.640600Z",
          "iopub.status.busy": "2022-01-07T04:44:01.640342Z",
          "iopub.status.idle": "2022-01-07T04:46:07.133828Z",
          "shell.execute_reply": "2022-01-07T04:46:07.132795Z",
          "shell.execute_reply.started": "2022-01-07T04:44:01.640573Z"
        },
        "id": "OeyT2TyGSzr8",
        "outputId": "f7091b15-8203-427c-b64b-531c13dceb46",
        "trusted": true
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([ 6823,    20,  4819,    80,  3222,    51,    80, 16146, 31952,\n",
              "         649,    20, 31760,   301, 31952,   649,    20,  3867,    56,\n",
              "       31952,   649,    20,  1475,    80,   649,    20,  3414, 31952,\n",
              "         372,  3085, 31952,   372,  7886, 31952,   649,    20, 25790,\n",
              "        6930,  6787, 31952,    80,    20, 10726, 10477,  3997, 31952,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0])"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# src_data는 data['src']를 참조한다. (동일 id)\n",
        "src_data = data['src']\n",
        "\n",
        "src_list = []\n",
        "\n",
        "for idx in range(len(src_data)):\n",
        "    src_list.append(en_encode(src_data[idx]))\n",
        "\n",
        "src_list[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NoyUWrCcSzr9"
      },
      "source": [
        "## TRG Data (KO) Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "sp_trg = spm.SentencePieceProcessor()\n",
        "sp_trg.Load('trg.model')\n",
        "lines = [\n",
        "    \"알잘딱깔센 임마 그거 몰라?.\",\n",
        "    \"하.. 존나 싫다...\",\n",
        "    \"가족이 사람을 죽여서 면목이 없다거나 같이 살던 사람들이 살해당해서 책임을 느낀다거나 자신의 이런 저런 일로\",\n",
        "    \"[PAD] [CLS] [BOS] [EOS] [SEP] [UNK] \"\n",
        "]\n",
        "for line in lines:\n",
        "    print(sp_trg.EncodeAsPieces(line))\n",
        "    print(sp_trg.EncodeAsIds(line))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:46:07.135396Z",
          "iopub.status.busy": "2022-01-07T04:46:07.135159Z",
          "iopub.status.idle": "2022-01-07T04:46:07.144610Z",
          "shell.execute_reply": "2022-01-07T04:46:07.143519Z",
          "shell.execute_reply.started": "2022-01-07T04:46:07.135367Z"
        },
        "id": "iLdTNJnESzr-",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "def ko_encode(tmpstr: str) -> np.array:\n",
        "    tmpstr = np.array(sp_trg.EncodeAsIds(tmpstr))\n",
        "    tmpstr = np.insert(tmpstr, 0, sp_trg.bos_id())\n",
        "\n",
        "    if len(tmpstr) >= SEQ_LEN:\n",
        "        # SEQ_LEN -1의 길이로 자른다\n",
        "        tmpstr = tmpstr[:SEQ_LEN-1]\n",
        "        # 마지막에 <eos> 토큰을 넣어줌으로써, 길이를 SEQ_LEN으로 맞춘다\n",
        "        tmpstr = np.pad(tmpstr, (0, 1),\n",
        "                        'constant', constant_values=sp_trg.eos_id())\n",
        "\n",
        "\n",
        "    else:\n",
        "        tmpstr = np.pad(tmpstr, (0, 1),\n",
        "                        'constant', constant_values=sp_trg.eos_id())\n",
        "        tmpstr = np.pad(tmpstr, (0, SEQ_LEN - len(tmpstr)),\n",
        "                        'constant', constant_values=sp_trg.pad_id())\n",
        "\n",
        "    return tmpstr\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:46:07.146070Z",
          "iopub.status.busy": "2022-01-07T04:46:07.145739Z",
          "iopub.status.idle": "2022-01-07T04:49:11.396411Z",
          "shell.execute_reply": "2022-01-07T04:49:11.395621Z",
          "shell.execute_reply.started": "2022-01-07T04:46:07.146028Z"
        },
        "id": "su8a905VSzr-",
        "outputId": "41557adb-f863-4fb6-c269-1249df496ab5",
        "trusted": true
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([    2,   721, 30905, 30546, 11101,    24, 30900, 30558,   130,\n",
              "       30970, 31043,  2579, 28212,  1207,   490, 30600,  2024,  5312,\n",
              "       29426,   548, 15377, 23537,   168, 11300, 30546,  3611, 30841,\n",
              "       30551, 29426,     3,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0])"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# trg_data는 data['trg']를 참조한다. (동일 id)\n",
        "trg_data = data['trg']\n",
        "\n",
        "trg_list = []\n",
        "\n",
        "for idx in range(len(trg_data)):\n",
        "    trg_list.append(ko_encode(trg_data[idx]))   \n",
        "\n",
        "trg_list[0]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "names = [\n",
        "         'src.model',\n",
        "         'src.vocab',\n",
        "         'trg.model',\n",
        "         'trg.vocab',\n",
        "]\n",
        "for name in names:\n",
        "    wandb.save(f'{name}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fOAA4wGI7QBv"
      },
      "source": [
        "## Train / Valid Split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:56:51.675498Z",
          "iopub.status.busy": "2022-01-07T04:56:51.674511Z",
          "iopub.status.idle": "2022-01-07T04:56:56.671758Z",
          "shell.execute_reply": "2022-01-07T04:56:56.670713Z",
          "shell.execute_reply.started": "2022-01-07T04:56:51.675369Z"
        },
        "id": "AIjk8lTL7QBv",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "train_mask = np.random.choice(len(src_list[:TRAINSET_SIZE]), size = TRAIN_LEN, replace = False)\n",
        "valid_mask = np.random.choice(len(trg_list[TRAINSET_SIZE:]), size = VALID_LEN, replace = False)\n",
        "\n",
        "src_train = np.take(src_list, train_mask, axis = 0)\n",
        "trg_train = np.take(trg_list, train_mask, axis = 0)\n",
        "\n",
        "src_valid = np.take(src_list, valid_mask, axis = 0)\n",
        "trg_valid = np.take(trg_list, valid_mask, axis = 0)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T05:05:33.334164Z",
          "iopub.status.busy": "2022-01-07T05:05:33.333803Z",
          "iopub.status.idle": "2022-01-07T05:05:33.351529Z",
          "shell.execute_reply": "2022-01-07T05:05:33.350551Z",
          "shell.execute_reply.started": "2022-01-07T05:05:33.334126Z"
        },
        "id": "m4ckOFtS7QBv",
        "outputId": "54890267-1a8a-4490-e79f-0731a435b713",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(100000, 100)\n",
            "(100000, 100)\n",
            "(10000, 100)\n",
            "(10000, 100)\n"
          ]
        }
      ],
      "source": [
        "print(src_train.shape)\n",
        "print(trg_train.shape)\n",
        "print(src_valid.shape)\n",
        "print(trg_valid.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:49:16.081154Z",
          "iopub.status.busy": "2022-01-07T04:49:16.080915Z",
          "iopub.status.idle": "2022-01-07T04:49:16.091795Z",
          "shell.execute_reply": "2022-01-07T04:49:16.090821Z",
          "shell.execute_reply.started": "2022-01-07T04:49:16.081125Z"
        },
        "id": "ooy_eIWaSzr_",
        "outputId": "4bf90125-f070-4615-e36b-32351a328ecb",
        "trusted": true
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import gc; gc.collect()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T04:51:30.765035Z",
          "iopub.status.busy": "2022-01-07T04:51:30.764596Z",
          "iopub.status.idle": "2022-01-07T04:54:53.129684Z",
          "shell.execute_reply": "2022-01-07T04:54:53.128643Z",
          "shell.execute_reply.started": "2022-01-07T04:51:30.764980Z"
        },
        "id": "9TBYPlt97QBw",
        "outputId": "ab5faae0-f3fa-4187-bc4d-eb09ff7ea4ea",
        "trusted": true
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['trg_list.pkl']"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Save data to local\n",
        "import joblib\n",
        "\n",
        "joblib.dump(src_train, 'src_train.pkl')\n",
        "joblib.dump(trg_train, 'trg_train.pkl')\n",
        "joblib.dump(src_valid, 'src_valid.pkl')\n",
        "joblib.dump(trg_valid, 'trg_valid.pkl')\n",
        "joblib.dump(src_list, 'src_list.pkl')\n",
        "joblib.dump(trg_list, 'trg_list.pkl')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "0ZnsOBMR7QBw"
      },
      "outputs": [],
      "source": [
        "# save data in wandb\n",
        "names = [\n",
        "         'src_train',\n",
        "         'src_valid',\n",
        "         'trg_train',\n",
        "         'trg_valid',\n",
        "         'src_list',\n",
        "         'trg_list',]\n",
        "\n",
        "for name in names:\n",
        "    wandb.save(f'{name}.pkl')\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yu36X8W17QBw"
      },
      "source": [
        "# Load Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0mwMb2G67QBw",
        "outputId": "12671297-76ec-454e-f5f0-810752d86928"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<_io.TextIOWrapper name='./trg_valid.pkl' mode='r' encoding='UTF-8'>"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# wandb.restore('trg_list.pkl', run_path = 'jiwon7258/Transformer/1hdb9bta', root='./')\n",
        "# wandb.restore('src_list.pkl', run_path = 'jiwon7258/Transformer/1hdb9bta', root='./')\n",
        "wandb.restore('src_train.pkl', run_path = WANDB_SAVED_PATH, root='./')\n",
        "wandb.restore('src_valid.pkl', run_path = WANDB_SAVED_PATH, root='./')\n",
        "wandb.restore('trg_train.pkl', run_path = WANDB_SAVED_PATH, root='./')\n",
        "wandb.restore('trg_valid.pkl', run_path = WANDB_SAVED_PATH, root='./')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T04:59:10.683804Z",
          "iopub.status.busy": "2022-01-07T04:59:10.683453Z",
          "iopub.status.idle": "2022-01-07T04:59:10.690379Z",
          "shell.execute_reply": "2022-01-07T04:59:10.689381Z",
          "shell.execute_reply.started": "2022-01-07T04:59:10.683771Z"
        },
        "id": "fhFt9Nxq7QBx",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# # Load Whole Data\n",
        "\n",
        "# src_list_path = 'src_list.pkl'\n",
        "# trg_list_path = 'trg_list.pkl'\n",
        "# src_list = joblib.load(src_list_path)\n",
        "# trg_list = joblib.load(trg_list_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "jsKOLypB7QBx"
      },
      "outputs": [],
      "source": [
        "# or Train / Valid Data\n",
        "src_train_path = 'src_train.pkl'\n",
        "src_valid_path = 'src_valid.pkl'\n",
        "trg_train_path = 'trg_train.pkl'\n",
        "trg_valid_path = 'trg_valid.pkl'\n",
        "\n",
        "src_train = joblib.load(src_train_path)\n",
        "src_valid = joblib.load(src_valid_path)\n",
        "trg_train = joblib.load(trg_train_path)\n",
        "trg_valid = joblib.load(trg_valid_path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nDY5iOWVSzr_"
      },
      "source": [
        "# Create Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:07:12.994490Z",
          "iopub.status.busy": "2022-01-07T05:07:12.994151Z",
          "iopub.status.idle": "2022-01-07T05:07:13.005942Z",
          "shell.execute_reply": "2022-01-07T05:07:13.004991Z",
          "shell.execute_reply.started": "2022-01-07T05:07:12.994456Z"
        },
        "id": "QBtsviGgSzsA",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class TrainDataset(Dataset):\n",
        "    def __init__(self, src_data, trg_data):\n",
        "        super().__init__()\n",
        "\n",
        "        assert len(src_data) == len(trg_data)\n",
        "\n",
        "        self.src_data = src_data\n",
        "        self.trg_data = trg_data\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.src_data)\n",
        "        \n",
        "    def __getitem__ (self, idx):\n",
        "        src = self.src_data[idx]\n",
        "        trg_input = self.trg_data[idx]\n",
        "        trg_output = trg_input[1:SEQ_LEN]\n",
        "        trg_output = np.pad(trg_output, (0,1), 'constant', constant_values =0)\n",
        "        # (seq_len,)\n",
        "        return src, trg_input, trg_output\n",
        "\n",
        "train_dataset = TrainDataset(src_train, trg_train)\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle= True, pin_memory=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:07:14.944389Z",
          "iopub.status.busy": "2022-01-07T05:07:14.943280Z",
          "iopub.status.idle": "2022-01-07T05:07:14.954362Z",
          "shell.execute_reply": "2022-01-07T05:07:14.953365Z",
          "shell.execute_reply.started": "2022-01-07T05:07:14.944337Z"
        },
        "id": "_ps365AhRBEB",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class ValidDataset(Dataset):\n",
        "    def __init__(self, src_data, trg_data):\n",
        "        super().__init__()\n",
        "\n",
        "        assert len(src_data) == len(trg_data)\n",
        "\n",
        "        self.src_data = src_data\n",
        "        self.trg_data = trg_data\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.src_data)\n",
        "        \n",
        "    def __getitem__ (self, idx):\n",
        "        src = self.src_data[idx]\n",
        "        trg_input = self.trg_data[idx]\n",
        "        trg_output = trg_input[1:SEQ_LEN]\n",
        "        trg_output = np.pad(trg_output, (0,1), 'constant',constant_values= 0)\n",
        "\n",
        "        return src, trg_input, trg_output\n",
        "\n",
        "valid_dataset = ValidDataset(src_valid, trg_valid)\n",
        "valid_dataloader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle= False, pin_memory=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T05:07:16.703402Z",
          "iopub.status.busy": "2022-01-07T05:07:16.703099Z",
          "iopub.status.idle": "2022-01-07T05:07:16.711645Z",
          "shell.execute_reply": "2022-01-07T05:07:16.710574Z",
          "shell.execute_reply.started": "2022-01-07T05:07:16.703369Z"
        },
        "id": "bEZqR0Zh7QBy",
        "outputId": "88168c64-dca9-4ef2-ed65-5f6eaf6a3cb8",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([2, 100])\n",
            "torch.Size([2, 100])\n",
            "torch.Size([2, 100])\n"
          ]
        }
      ],
      "source": [
        "for src, trg_input, trg_output in train_dataloader:\n",
        "    print(src.shape)\n",
        "    print(trg_input.shape)\n",
        "    print(trg_output.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HFoLlW9DSzsA"
      },
      "source": [
        "# Transformer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jqaL9w0VSzsA"
      },
      "source": [
        "## Mask Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.641230Z",
          "iopub.status.busy": "2022-01-07T05:13:21.640540Z",
          "iopub.status.idle": "2022-01-07T05:13:21.653161Z",
          "shell.execute_reply": "2022-01-07T05:13:21.652478Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.641180Z"
        },
        "id": "bunvi9rNSzsA",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "'''\n",
        "Mask 행렬을 반환하는 Mask Function\n",
        "\n",
        "Input\n",
        "- Tensor\n",
        "    shape (bs, seq_len)\n",
        "\n",
        "Args\n",
        "- Option\n",
        "    If option is 'padding', function returns padding mask\n",
        "    If option is 'lookahead', function returns lookahead mask\n",
        "\n",
        "Output\n",
        "- Tensor\n",
        "    shpae (bs, seq_len, seq_len)\n",
        "\n",
        "'''\n",
        "\n",
        "\n",
        "def makeMask(tensor, option: str) -> torch.Tensor:\n",
        "\n",
        "    '''\n",
        "    tensor (bs, seq_len)\n",
        "    '''\n",
        "    if option == 'padding':\n",
        "        tmp = torch.full_like(tensor, fill_value=PAD_IDX).to(device)\n",
        "        # tmp : (bs,seq_len)\n",
        "        mask = (tensor != tmp).float()\n",
        "        # mask : (bs, seq_len)\n",
        "        mask = repeat(mask, 'bs seq_len -> bs new_axis seq_len ',\n",
        "                      new_axis=mask.shape[1])\n",
        "        # mask(bs,seq_len,seq_len)\n",
        "\n",
        "        '''\n",
        "        Example of mask\n",
        "        tensor([[\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.]]])\n",
        "        '''\n",
        "\n",
        "    elif option == 'lookahead':\n",
        "        padding_mask = makeMask(tensor, 'padding')\n",
        "\n",
        "        mask = torch.ones_like(padding_mask)\n",
        "        mask = torch.tril(mask)\n",
        "        '''\n",
        "        Example of 'mask'\n",
        "        tensor([[\n",
        "        [1., 0., 0., 0., 0., 0., 0., 0.],\n",
        "        [1., 1., 0., 0., 0., 0., 0., 0.],\n",
        "        [1., 1., 1., 0., 0., 0., 0., 0.],\n",
        "        [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "        [1., 1., 1., 1., 1., 0., 0., 0.],\n",
        "        [1., 1., 1., 1., 1., 1., 0., 0.],\n",
        "        [1., 1., 1., 1., 1., 1., 1., 0.],\n",
        "        [1., 1., 1., 1., 1., 1., 1., 1.]]])\n",
        "        '''\n",
        "\n",
        "        mask = mask * padding_mask\n",
        "\n",
        "        '''\n",
        "        Example\n",
        "        tensor([[\n",
        "         [1., 0., 0., 0., 0., 0., 0., 0.],\n",
        "         [1., 1., 0., 0., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 0., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.],\n",
        "         [1., 1., 1., 1., 0., 0., 0., 0.]]])\n",
        "        '''\n",
        "\n",
        "\n",
        "    return mask\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fb-AM2f2SzsB"
      },
      "source": [
        "## Positional Embedding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.655488Z",
          "iopub.status.busy": "2022-01-07T05:13:21.654875Z",
          "iopub.status.idle": "2022-01-07T05:13:21.672149Z",
          "shell.execute_reply": "2022-01-07T05:13:21.671146Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.655443Z"
        },
        "id": "Xc96DvV_SzsB",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# def pos_embed(input):\n",
        "    # input : (bs, seq_len, hidden_dim)\n",
        "\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "loi-qIjHSzsB"
      },
      "source": [
        "## Multihead Attention"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.675628Z",
          "iopub.status.busy": "2022-01-07T05:13:21.674659Z",
          "iopub.status.idle": "2022-01-07T05:13:21.691412Z",
          "shell.execute_reply": "2022-01-07T05:13:21.690692Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.675573Z"
        },
        "id": "OacFbmDGSzsB",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class Multiheadattention(nn.Module):\n",
        "    def __init__(self, hidden_dim: int, num_head: int):\n",
        "        super().__init__()\n",
        "\n",
        "        # embedding_dim, d_model, 512 in paper\n",
        "        self.hidden_dim = hidden_dim\n",
        "        # 8 in paper\n",
        "        self.num_head = num_head\n",
        "        # head_dim, d_key, d_query, d_value, 64 in paper (= 512 / 8)\n",
        "        self.head_dim = hidden_dim // num_head\n",
        "        self.scale = torch.sqrt(torch.FloatTensor()).to(device)\n",
        "\n",
        "        self.fcQ = nn.Linear(hidden_dim, hidden_dim)\n",
        "        self.fcK = nn.Linear(hidden_dim, hidden_dim)\n",
        "        self.fcV = nn.Linear(hidden_dim, hidden_dim)\n",
        "        self.fcOut = nn.Linear(hidden_dim, hidden_dim)\n",
        "\n",
        "\n",
        "\n",
        "    def forward(self, srcQ, srcK, srcV, mask=None):\n",
        "\n",
        "        ##### SCALED DOT PRODUCT ATTENTION ######\n",
        "\n",
        "        # input : (bs, seq_len, hidden_dim)\n",
        "        Q = self.fcQ(srcQ)\n",
        "        K = self.fcK(srcK)\n",
        "        V = self.fcV(srcV)\n",
        "\n",
        "        Q = rearrange(\n",
        "            Q, 'bs seq_len (num_head head_dim) -> bs num_head seq_len head_dim', num_head=self.num_head)\n",
        "        K_T = rearrange(\n",
        "            K, 'bs seq_len (num_head head_dim) -> bs num_head head_dim seq_len', num_head=self.num_head)\n",
        "        V = rearrange(\n",
        "            V, 'bs seq_len (num_head head_dim) -> bs num_head seq_len head_dim', num_head=self.num_head)\n",
        "\n",
        "        attention_energy = torch.matmul(Q, K_T)\n",
        "        # attention_energy : (bs, num_head, seq_len, seq_len)\n",
        "\n",
        "        if mask is not None :\n",
        "            attention_energy : torch.masked_fill(attention_energy, (mask==0), -1e10)\n",
        "\n",
        "        attention_energy = torch.softmax(attention_energy, dim = -1)\n",
        "        # print(attention_energy[0,0,0,:])\n",
        "\n",
        "        result = torch.matmul(attention_energy,V)\n",
        "        # result (bs, num_head, seq_len, head_dim)\n",
        "\n",
        "        ##### END OF SCALED DOT PRODUCT ATTENTION ######\n",
        "\n",
        "        # CONCAT\n",
        "        result = rearrange(result, 'bs num_head seq_len head_dim -> bs seq_len (num_head head_dim)')\n",
        "        # result : (bs, seq_len, hidden_dim)\n",
        "\n",
        "        # LINEAR\n",
        "\n",
        "        result = self.fcOut(result)\n",
        "\n",
        "        return result\n",
        "        \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.693336Z",
          "iopub.status.busy": "2022-01-07T05:13:21.692829Z",
          "iopub.status.idle": "2022-01-07T05:13:21.710290Z",
          "shell.execute_reply": "2022-01-07T05:13:21.709577Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.693301Z"
        },
        "id": "5JrWZEYOSzsC",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# # TEST CODE #\n",
        "# bs = 32\n",
        "# seq_len = 200\n",
        "# hidden_dim = 128\n",
        "# test_tensor = torch.randn((bs,seq_len,hidden_dim))\n",
        "# print(test_tensor.shape)\n",
        "# test_layer = Multiheadattention(hidden_dim=hidden_dim, num_head =2)\n",
        "# print(test_layer(srcQ = test_tensor, srcK = test_tensor, srcV = test_tensor).shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jTb9C0UTSzsC"
      },
      "source": [
        "## Poistionwise Feedforward Network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.712010Z",
          "iopub.status.busy": "2022-01-07T05:13:21.711741Z",
          "iopub.status.idle": "2022-01-07T05:13:21.729297Z",
          "shell.execute_reply": "2022-01-07T05:13:21.728300Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.711978Z"
        },
        "id": "en_h1fTISzsC",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class FFN(nn.Module):\n",
        "    def __init__ (self, hidden_dim, inner_dim):\n",
        "        super().__init__()\n",
        "\n",
        "        # 512 in paper \n",
        "        self.hidden_dim = hidden_dim\n",
        "        # 2048 in paper\n",
        "        self.inner_dim = inner_dim \n",
        "\n",
        "        self.fc1 = nn.Linear(hidden_dim, inner_dim)\n",
        "        self.fc2 = nn.Linear(inner_dim, hidden_dim)\n",
        "        self.relu = nn.ReLU(inplace=False)\n",
        "\n",
        "        \n",
        "    def forward(self, input):\n",
        "        output = input\n",
        "        output = self.fc1(output)\n",
        "        output2 = self.relu(output)\n",
        "        output3 = self.fc2(output2)\n",
        "\n",
        "        return output3\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "emd_xbYPSzsD"
      },
      "source": [
        "## Encoder Layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.809101Z",
          "iopub.status.busy": "2022-01-07T05:13:21.808739Z",
          "iopub.status.idle": "2022-01-07T05:13:21.820414Z",
          "shell.execute_reply": "2022-01-07T05:13:21.819006Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.809065Z"
        },
        "id": "3_d7aGzFSzsD",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self, hidden_dim, num_head, inner_dim):\n",
        "        super().__init__()\n",
        "\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.num_head = num_head\n",
        "        self.inner_dim = inner_dim\n",
        "        \n",
        "        self.multiheadattention = Multiheadattention(hidden_dim, num_head)\n",
        "        self.ffn = FFN(hidden_dim, inner_dim)\n",
        "        self.layerNorm = nn.LayerNorm(hidden_dim)\n",
        "\n",
        "        self.dropout1 = nn.Dropout(p=0.1)\n",
        "        self.dropout2 = nn.Dropout(p=0.1)\n",
        "\n",
        "\n",
        "    def forward(self, input, mask = None):\n",
        "\n",
        "        # input : (bs, seq_len, hidden_dim)\n",
        "        \n",
        "        # encoder attention\n",
        "        # uses only padding mask\n",
        "        output = self.multiheadattention(srcQ= input, srcK = input, srcV = input, mask = mask)\n",
        "        output = self.dropout1(output)\n",
        "        output = input + output\n",
        "        output = self.layerNorm(output)\n",
        "\n",
        "        output_ = self.ffn(output)\n",
        "        output_ = self.dropout2(output_)\n",
        "        output = output + output_\n",
        "        output = self.layerNorm(output)\n",
        "\n",
        "        # output : (bs, seq_len, hidden_dim)\n",
        "        return output\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z0NInjRrSzsD"
      },
      "source": [
        "## Encoder Architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.824217Z",
          "iopub.status.busy": "2022-01-07T05:13:21.823330Z",
          "iopub.status.idle": "2022-01-07T05:13:21.840593Z",
          "shell.execute_reply": "2022-01-07T05:13:21.839384Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.824168Z"
        },
        "id": "Ln84_dJKSzsD",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__ (self, N, hidden_dim, num_head, inner_dim):\n",
        "        super().__init__()\n",
        "\n",
        "        # N : number of encoder layer repeated \n",
        "        self.N = N\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.num_head = num_head\n",
        "        self.inner_dim = inner_dim\n",
        "\n",
        "        self.embedding = nn.Embedding(num_embeddings=VOCAB_SIZE, embedding_dim=hidden_dim, padding_idx=0)\n",
        "        self.enc_layers = nn.ModuleList(EncoderLayer(hidden_dim, num_head, inner_dim) for _ in range(N))\n",
        "\n",
        "        self.dropout = nn.Dropout(p=0.1)\n",
        "\n",
        "\n",
        "\n",
        "    def forward(self, input):\n",
        "\n",
        "        # input : (bs, seq_len)\n",
        "        print(f'input shape = {input.shape}')\n",
        "        mask = makeMask(input, option='padding')\n",
        "\n",
        "        # embedding layer\n",
        "        output = self.embedding(input)\n",
        "        # output : (bs, seq_len, hidden_dim)\n",
        "\n",
        "        # Positional Embedding\n",
        "        # output = pos_embed(output)\n",
        "\n",
        "        # Dropout\n",
        "        output = self.dropout(output)\n",
        "\n",
        "        # N encoder layer\n",
        "        for layer in self.enc_layers:\n",
        "            output = layer(output, mask)\n",
        "\n",
        "        # output : (bs, seq_len, hidden_dim)\n",
        "\n",
        "        return output\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nRhfhEB-SzsE"
      },
      "source": [
        "## Decoder Layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.842931Z",
          "iopub.status.busy": "2022-01-07T05:13:21.842244Z",
          "iopub.status.idle": "2022-01-07T05:13:21.859676Z",
          "shell.execute_reply": "2022-01-07T05:13:21.858944Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.842879Z"
        },
        "id": "vpUTcWneSzsE",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self, hidden_dim, num_head, inner_dim):\n",
        "        super().__init__()\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.num_head = num_head\n",
        "        self.inner_dim = inner_dim\n",
        "\n",
        "        self.multiheadattention1 = Multiheadattention(hidden_dim, num_head)\n",
        "        self.layerNorm1 = nn.LayerNorm(hidden_dim)\n",
        "        self.multiheadattention2 = Multiheadattention(hidden_dim, num_head)\n",
        "        self.layerNorm2 = nn.LayerNorm(hidden_dim)\n",
        "        self.ffn = FFN(hidden_dim, inner_dim)\n",
        "        self.layerNorm3 = nn.LayerNorm(hidden_dim)\n",
        "\n",
        "        self.dropout1 = nn.Dropout(p=0.1)\n",
        "        self.dropout2 = nn.Dropout(p=0.1)\n",
        "        self.dropout3 = nn.Dropout(p=0.1)\n",
        "\n",
        "    \n",
        "    def forward(self, input, enc_output, paddingMask, lookaheadMask):\n",
        "        # input : (bs, seq_len, hidden_dim)\n",
        "\n",
        "        # first multiheadattention\n",
        "        output = self.multiheadattention1(input, input, input, lookaheadMask)\n",
        "        output = self.dropout1(output)\n",
        "        output = output + input\n",
        "        output = self.layerNorm1(output)\n",
        "\n",
        "        # second multiheadattention\n",
        "        output_ = self.multiheadattention2(output, enc_output, enc_output, paddingMask)\n",
        "        output_ = self.dropout2(output_)\n",
        "        output = output_ + output\n",
        "        output = self.layerNorm2(output)\n",
        "\n",
        "        # Feedforward Network\n",
        "        output_ = self.ffn(output)\n",
        "        output_ = self.dropout3(output_)\n",
        "        output = output + output_\n",
        "        output = self.layerNorm3(output)\n",
        "\n",
        "        return output\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ACUVck9eSzsE"
      },
      "source": [
        "## Decoder Architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.861790Z",
          "iopub.status.busy": "2022-01-07T05:13:21.861343Z",
          "iopub.status.idle": "2022-01-07T05:13:21.881968Z",
          "shell.execute_reply": "2022-01-07T05:13:21.881032Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.861748Z"
        },
        "id": "dXokLRkjSzsE",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__ (self, N, hidden_dim, num_head, inner_dim):\n",
        "        super().__init__()\n",
        "\n",
        "        # N : number of encoder layer repeated \n",
        "        self.N = N\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.num_head = num_head\n",
        "        self.inner_dim = inner_dim\n",
        "\n",
        "        self.embedding = nn.Embedding(num_embeddings=VOCAB_SIZE, embedding_dim=hidden_dim, padding_idx=0)\n",
        "\n",
        "        self.dec_layers = nn.ModuleList(DecoderLayer(hidden_dim, num_head, inner_dim) for _ in range(N))\n",
        "\n",
        "        self.dropout = nn.Dropout(p=0.1)\n",
        "        \n",
        "        self.finalFc = nn.Linear(hidden_dim, VOCAB_SIZE)\n",
        "\n",
        "\n",
        "    def forward(self, input, enc_src, enc_output):\n",
        "\n",
        "        # input : (bs, seq_len)\n",
        "        # enc_src : (bs, seq_len)\n",
        "        # enc_output : (bs, seq_len,hidden_dim)\n",
        "\n",
        "        lookaheadMask = makeMask(input, option='lookahead')\n",
        "        paddingMask = makeMask(enc_src, option = 'padding')\n",
        "\n",
        "        # embedding layer\n",
        "        output = self.embedding(input)\n",
        "\n",
        "        # Positional Embedding\n",
        "        # output = pos_embed(output)\n",
        "\n",
        "        # Dropout\n",
        "        output = self.dropout(output)\n",
        "\n",
        "        # N decoder layer\n",
        "        for layer in self.dec_layers:\n",
        "            output = layer(output, enc_output, paddingMask, lookaheadMask)\n",
        "        # output : (bs, seq_len, hidden_dim)\n",
        "\n",
        "        logits = self.finalFc(output)\n",
        "        # logits : (bs, seq_len, VOCAB_SIZE)\n",
        "        output = torch.softmax(logits, dim = -1)\n",
        "\n",
        "        output = torch.argmax(output, dim = -1)\n",
        "        # output : (bs, seq_len)\n",
        "\n",
        "\n",
        "\n",
        "        return output, logits\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tgOKqiCdSzsF"
      },
      "source": [
        "## Transformer Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.884701Z",
          "iopub.status.busy": "2022-01-07T05:13:21.884075Z",
          "iopub.status.idle": "2022-01-07T05:13:21.895575Z",
          "shell.execute_reply": "2022-01-07T05:13:21.894905Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.884663Z"
        },
        "id": "DhoHNUHPSzsF",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "class Transformer(nn.Module):\n",
        "    def __init__(self, N = 6, hidden_dim = 512, num_head = 8, inner_dim = 2048):\n",
        "        super().__init__()\n",
        "        self.encoder = Encoder(N, hidden_dim, num_head, inner_dim)\n",
        "        self.decoder = Decoder(N, hidden_dim, num_head, inner_dim)\n",
        "\n",
        "    def forward(self, enc_src, dec_src):\n",
        "        # enc_src : (bs, seq_len)\n",
        "        # dec_src : (bs, seq_len)\n",
        "\n",
        "        # print(f'enc_src : {enc_src.shape}')\n",
        "        # print(f'dec_src : {dec_src.shape}')\n",
        "\n",
        "        enc_output = self.encoder(enc_src)\n",
        "        output, logits = self.decoder(dec_src, enc_src, enc_output.detach())\n",
        "        # logits = (bs, seq_len, VOCAB_SIZE) \n",
        "\n",
        "        return output, logits\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gJhmIN-QSzsF"
      },
      "source": [
        "# Model Train"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fywH11aFSzsF"
      },
      "source": [
        "# Create Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:21.897517Z",
          "iopub.status.busy": "2022-01-07T05:13:21.897003Z",
          "iopub.status.idle": "2022-01-07T05:13:22.772060Z",
          "shell.execute_reply": "2022-01-07T05:13:22.770986Z",
          "shell.execute_reply.started": "2022-01-07T05:13:21.897471Z"
        },
        "id": "UkQQSxnhSzsF",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "model = Transformer().to(device)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Check Model Structure"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "input shape = torch.Size([2, 100])\n",
            "===============================================================================================\n",
            "Layer (type:depth-idx)                        Output Shape              Param #\n",
            "===============================================================================================\n",
            "├─Encoder: 1-1                                [-1, 100, 512]            --\n",
            "|    └─Embedding: 2-1                         [-1, 100, 512]            16,387,584\n",
            "|    └─Dropout: 2-2                           [-1, 100, 512]            --\n",
            "|    └─ModuleList: 2                          []                        --\n",
            "|    |    └─EncoderLayer: 3-1                 [-1, 100, 512]            3,151,360\n",
            "|    |    └─EncoderLayer: 3-2                 [-1, 100, 512]            3,151,360\n",
            "|    |    └─EncoderLayer: 3-3                 [-1, 100, 512]            3,151,360\n",
            "|    |    └─EncoderLayer: 3-4                 [-1, 100, 512]            3,151,360\n",
            "|    |    └─EncoderLayer: 3-5                 [-1, 100, 512]            3,151,360\n",
            "|    |    └─EncoderLayer: 3-6                 [-1, 100, 512]            3,151,360\n",
            "├─Decoder: 1-2                                [-1, 100]                 --\n",
            "|    └─Embedding: 2-3                         [-1, 100, 512]            16,387,584\n",
            "|    └─Dropout: 2-4                           [-1, 100, 512]            --\n",
            "|    └─ModuleList: 2                          []                        --\n",
            "|    |    └─DecoderLayer: 3-7                 [-1, 100, 512]            4,204,032\n",
            "|    |    └─DecoderLayer: 3-8                 [-1, 100, 512]            4,204,032\n",
            "|    |    └─DecoderLayer: 3-9                 [-1, 100, 512]            4,204,032\n",
            "|    |    └─DecoderLayer: 3-10                [-1, 100, 512]            4,204,032\n",
            "|    |    └─DecoderLayer: 3-11                [-1, 100, 512]            4,204,032\n",
            "|    |    └─DecoderLayer: 3-12                [-1, 100, 512]            4,204,032\n",
            "|    └─Linear: 2-5                            [-1, 100, 32007]          16,419,591\n",
            "===============================================================================================\n",
            "Total params: 93,327,111\n",
            "Trainable params: 93,327,111\n",
            "Non-trainable params: 0\n",
            "Total mult-adds (M): 230.49\n",
            "===============================================================================================\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 34.58\n",
            "Params size (MB): 356.01\n",
            "Estimated Total Size (MB): 390.59\n",
            "===============================================================================================\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "===============================================================================================\n",
              "Layer (type:depth-idx)                        Output Shape              Param #\n",
              "===============================================================================================\n",
              "├─Encoder: 1-1                                [-1, 100, 512]            --\n",
              "|    └─Embedding: 2-1                         [-1, 100, 512]            16,387,584\n",
              "|    └─Dropout: 2-2                           [-1, 100, 512]            --\n",
              "|    └─ModuleList: 2                          []                        --\n",
              "|    |    └─EncoderLayer: 3-1                 [-1, 100, 512]            3,151,360\n",
              "|    |    └─EncoderLayer: 3-2                 [-1, 100, 512]            3,151,360\n",
              "|    |    └─EncoderLayer: 3-3                 [-1, 100, 512]            3,151,360\n",
              "|    |    └─EncoderLayer: 3-4                 [-1, 100, 512]            3,151,360\n",
              "|    |    └─EncoderLayer: 3-5                 [-1, 100, 512]            3,151,360\n",
              "|    |    └─EncoderLayer: 3-6                 [-1, 100, 512]            3,151,360\n",
              "├─Decoder: 1-2                                [-1, 100]                 --\n",
              "|    └─Embedding: 2-3                         [-1, 100, 512]            16,387,584\n",
              "|    └─Dropout: 2-4                           [-1, 100, 512]            --\n",
              "|    └─ModuleList: 2                          []                        --\n",
              "|    |    └─DecoderLayer: 3-7                 [-1, 100, 512]            4,204,032\n",
              "|    |    └─DecoderLayer: 3-8                 [-1, 100, 512]            4,204,032\n",
              "|    |    └─DecoderLayer: 3-9                 [-1, 100, 512]            4,204,032\n",
              "|    |    └─DecoderLayer: 3-10                [-1, 100, 512]            4,204,032\n",
              "|    |    └─DecoderLayer: 3-11                [-1, 100, 512]            4,204,032\n",
              "|    |    └─DecoderLayer: 3-12                [-1, 100, 512]            4,204,032\n",
              "|    └─Linear: 2-5                            [-1, 100, 32007]          16,419,591\n",
              "===============================================================================================\n",
              "Total params: 93,327,111\n",
              "Trainable params: 93,327,111\n",
              "Non-trainable params: 0\n",
              "Total mult-adds (M): 230.49\n",
              "===============================================================================================\n",
              "Input size (MB): 0.00\n",
              "Forward/backward pass size (MB): 34.58\n",
              "Params size (MB): 356.01\n",
              "Estimated Total Size (MB): 390.59\n",
              "==============================================================================================="
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from torchsummary import summary\n",
        "test1 = torch.randint(low = 0, high = 1000, size = (SEQ_LEN,))\n",
        "test2 = torch.randint(low = 0, high = 1000, size = (SEQ_LEN,))\n",
        "summary(model, [(SEQ_LEN,), (SEQ_LEN,)], dtypes = [torch.int, torch.int])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H35nu2mHSzsF"
      },
      "source": [
        "# Weight Init"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:22.774317Z",
          "iopub.status.busy": "2022-01-07T05:13:22.773978Z",
          "iopub.status.idle": "2022-01-07T05:13:23.188383Z",
          "shell.execute_reply": "2022-01-07T05:13:23.187236Z",
          "shell.execute_reply.started": "2022-01-07T05:13:22.774272Z"
        },
        "id": "rzoP52AVRBEH",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "for param in model.named_parameters():\n",
        "    if 'weight' in param[0] and 'layerNorm' not in param[0] :\n",
        "        torch.nn.init.xavier_uniform_(param[1])\n",
        "        "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GBFeBnwHSzsG"
      },
      "source": [
        "# Optimizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:23.190429Z",
          "iopub.status.busy": "2022-01-07T05:13:23.190075Z",
          "iopub.status.idle": "2022-01-07T05:13:23.197817Z",
          "shell.execute_reply": "2022-01-07T05:13:23.197017Z",
          "shell.execute_reply.started": "2022-01-07T05:13:23.190390Z"
        },
        "id": "Q8L264UYSzsG",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "optimizer = torch.optim.Adam(params = model.parameters())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OI124S8rRBEI"
      },
      "source": [
        "# Loss Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:23.200297Z",
          "iopub.status.busy": "2022-01-07T05:13:23.199248Z",
          "iopub.status.idle": "2022-01-07T05:13:23.212529Z",
          "shell.execute_reply": "2022-01-07T05:13:23.211783Z",
          "shell.execute_reply.started": "2022-01-07T05:13:23.200247Z"
        },
        "id": "b1LfUu55RBEI",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "def criterion(logits: torch.tensor, targets: torch.tensor):\n",
        "    return nn.CrossEntropyLoss(ignore_index=PAD_IDX)(logits.view(-1,VOCAB_SIZE), targets.view(-1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-XEBeN4NRBEI"
      },
      "source": [
        "# Training Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:15:25.576698Z",
          "iopub.status.busy": "2022-01-07T05:15:25.576359Z",
          "iopub.status.idle": "2022-01-07T05:15:25.591098Z",
          "shell.execute_reply": "2022-01-07T05:15:25.590051Z",
          "shell.execute_reply.started": "2022-01-07T05:15:25.576665Z"
        },
        "id": "xHMJ5lROSzsG",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n",
        "    # train 모드로 변경\n",
        "    model.train()\n",
        "\n",
        "    # for the Mixed Precision\n",
        "    # Pytorch 예제 : https://pytorch.org/docs/stable/notes/amp_examples.html#amp-examples\n",
        "    scaler = amp.GradScaler()\n",
        "\n",
        "    dataset_size = 0\n",
        "    running_loss = 0\n",
        "\n",
        "    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n",
        "    \n",
        "    for step, (src, trg_input, trg_output) in bar:\n",
        "        src = src.to(device)\n",
        "        trg_input = trg_input.to(device)\n",
        "        trg_output = trg_output.to(device)\n",
        "\n",
        "        batch_size = src.shape[0]\n",
        "\n",
        "        with amp.autocast(enabled=True):\n",
        "            output, logits = model(enc_src = src, dec_src = trg_input)\n",
        "            loss = criterion(logits, trg_output)\n",
        "            \n",
        "        # logits (bs, seq_len, VOCAB_SIZE)\n",
        "        # trg_output (bs, seq_len)\n",
        "\n",
        "        # loss를 Scale\n",
        "        # Scaled Grdients를 계산(call)하기 위해 scaled loss를 backward()\n",
        "        scaler.scale(loss).backward()\n",
        "        # loss.backward()\n",
        "\n",
        "        # scaler.step() first unscales the gradients of the optimizer's assigned params.\n",
        "        # If these gradients do not contain infs or NaNs, optimizer.step() is then called,\n",
        "        # otherwise, optimizer.step() is skipped.\n",
        "        scaler.step(optimizer)\n",
        "        # optimizer.step()\n",
        "        \n",
        "        # Updates the scale for next iteration.\n",
        "        scaler.update()\n",
        "\n",
        "        # zero the parameter gradients\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # change learning rate by Scheduler\n",
        "        if scheduler is not None:\n",
        "            scheduler.step()\n",
        "\n",
        "        # loss.item()은 loss를 Python Float으로 반환\n",
        "        # loss.item()은 batch data의 average loss이므로, sum of loss를 구하기 위해 batch_size를 곱해준다\n",
        "        running_loss += loss.item() * batch_size\n",
        "        dataset_size += batch_size\n",
        "\n",
        "        epoch_loss = running_loss / dataset_size\n",
        "\n",
        "        bar.set_postfix(\n",
        "            Epoch=epoch, Train_Loss=epoch_loss, LR=optimizer.param_groups[0][\"lr\"]\n",
        "        )\n",
        "\n",
        "    # Garbage Collector\n",
        "    gc.collect()\n",
        "\n",
        "    return epoch_loss"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jqW-6Gzz7QB5"
      },
      "source": [
        "# Validation Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:13:23.232205Z",
          "iopub.status.busy": "2022-01-07T05:13:23.231513Z",
          "iopub.status.idle": "2022-01-07T05:13:23.247180Z",
          "shell.execute_reply": "2022-01-07T05:13:23.246252Z",
          "shell.execute_reply.started": "2022-01-07T05:13:23.232156Z"
        },
        "id": "jbwnuTo7RBEJ",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "@torch.no_grad()\n",
        "def valid_one_epoch(model, dataloader, device, epoch):\n",
        "    model.eval()\n",
        "\n",
        "    dataset_size = 0\n",
        "    running_loss = 0\n",
        "\n",
        "\n",
        "    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n",
        "\n",
        "    for step, (src, trg_input, trg_output) in bar:\n",
        "        src = src.to(device)\n",
        "        trg_input = trg_input.to(device)\n",
        "        trg_output = trg_output.to(device)\n",
        "\n",
        "        batch_size = src.shape[0]\n",
        "\n",
        "        output, logits = model(enc_src = src, dec_src = trg_input)\n",
        "        loss = criterion(logits, trg_output)\n",
        "\n",
        "        running_loss += loss.item() * batch_size\n",
        "        dataset_size += batch_size\n",
        "\n",
        "        # 실시간으로 정보를 표시하기 위한 epoch loss\n",
        "        val_loss = running_loss / dataset_size\n",
        "\n",
        "        bar.set_postfix(\n",
        "            Epoch=epoch, Valid_Loss=val_loss, LR=optimizer.param_groups[0][\"lr\"]\n",
        "        )\n",
        "\n",
        "    gc.collect()\n",
        "\n",
        "    return val_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T05:15:55.917148Z",
          "iopub.status.busy": "2022-01-07T05:15:55.916760Z",
          "iopub.status.idle": "2022-01-07T05:15:55.935166Z",
          "shell.execute_reply": "2022-01-07T05:15:55.933889Z",
          "shell.execute_reply.started": "2022-01-07T05:15:55.917112Z"
        },
        "id": "fIT6cHpsRBEJ",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "def run_training(\n",
        "    model,\n",
        "    optimizer,\n",
        "    scheduler,\n",
        "    device,\n",
        "    num_epochs,\n",
        "    metric_prefix=\"\",\n",
        "    file_prefix=\"\",\n",
        "    early_stopping=True,\n",
        "    early_stopping_step=10,\n",
        "):\n",
        "    # To automatically log graidents\n",
        "    wandb.watch(model, log_freq=100)\n",
        "\n",
        "    if torch.cuda.is_available():\n",
        "        print(\"[INFO] Using GPU:{}\\n\".format(torch.cuda.get_device_name()))\n",
        "\n",
        "    start = time.time()\n",
        "    best_model_wts = copy.deepcopy(model.state_dict())\n",
        "    best_loss = np.inf\n",
        "    history = defaultdict(list)\n",
        "    early_stop_counter = 0\n",
        "\n",
        "    # num_epochs만큼, train과 val을 실행한다\n",
        "    for epoch in range(1, num_epochs + 1):\n",
        "        gc.collect()\n",
        "\n",
        "        train_epoch_loss = train_one_epoch(\n",
        "            model,\n",
        "            optimizer,\n",
        "            scheduler,\n",
        "            dataloader= train_dataloader,\n",
        "            device=device,\n",
        "            epoch=epoch,\n",
        "        )\n",
        "\n",
        "        val_loss = valid_one_epoch(\n",
        "            model, valid_dataloader, device=device, epoch=epoch\n",
        "        )\n",
        "\n",
        "        history[f\"{metric_prefix}Train Loss\"].append(train_epoch_loss)\n",
        "        history[f\"{metric_prefix}Valid Loss\"].append(val_loss)\n",
        "\n",
        "        # Log the metrics\n",
        "        wandb.log(\n",
        "            {\n",
        "                f\"{metric_prefix}Train Loss\": train_epoch_loss,\n",
        "                f\"{metric_prefix}Valid Loss\": val_loss,\n",
        "            }\n",
        "        )\n",
        "\n",
        "        print(f\"Valid Loss : {val_loss}\")\n",
        "\n",
        "        # deep copy the model\n",
        "        if val_loss <= best_loss:\n",
        "            early_stop_counter = 0\n",
        "\n",
        "            print(\n",
        "                f\"Validation Loss improved( {best_loss} ---> {val_loss}  )\"\n",
        "            )\n",
        "\n",
        "            # Update Best Loss\n",
        "            best_loss = val_loss\n",
        "            \n",
        "            # Update Best Model Weight\n",
        "            # run.summary['Best RMSE'] = best_loss\n",
        "            best_model_wts = copy.deepcopy(model.state_dict())\n",
        "\n",
        "            PATH = \"{}epoch{:.0f}_Loss{:.4f}.bin\".format(file_prefix, epoch, best_loss)\n",
        "            torch.save(model.state_dict(), PATH)\n",
        "            torch.save(model.state_dict(), f\"{file_prefix}best_{epoch}epoch.bin\")\n",
        "            # Save a model file from the current directory\n",
        "            wandb.save(PATH)\n",
        "\n",
        "            print(f\"Model Saved\")\n",
        "\n",
        "        elif early_stopping:\n",
        "            early_stop_counter += 1\n",
        "            if early_stop_counter > early_stopping_step:\n",
        "                break\n",
        "\n",
        "        print()\n",
        "\n",
        "    end = time.time()\n",
        "    time_elapsed = end - start\n",
        "    print(\n",
        "        \"Training complete in {:.0f}h {:.0f}m {:.0f}s\".format(\n",
        "            time_elapsed // 3600,\n",
        "            (time_elapsed % 3600) // 60,\n",
        "            (time_elapsed % 3600) % 60,\n",
        "        )\n",
        "    )\n",
        "    print(\"Best Loss: {:.4f}\".format(best_loss))\n",
        "\n",
        "    # load best model weights\n",
        "    model.load_state_dict(best_model_wts)\n",
        "\n",
        "    return model, history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 526
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T05:16:00.174682Z",
          "iopub.status.busy": "2022-01-07T05:16:00.174344Z",
          "iopub.status.idle": "2022-01-07T05:16:00.195317Z",
          "shell.execute_reply": "2022-01-07T05:16:00.193944Z",
          "shell.execute_reply.started": "2022-01-07T05:16:00.174650Z"
        },
        "id": "mCqLo1s4RBEJ",
        "outputId": "dda2fc71-a9e3-4240-a4b0-2ea6f696395b",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/home/jiwon/anaconda3/lib/python3.9/site-packages/torch/cuda/amp/grad_scaler.py:115: UserWarning: torch.cuda.amp.GradScaler is enabled, but CUDA is not available.  Disabling.\n",
            "  warnings.warn(\"torch.cuda.amp.GradScaler is enabled, but CUDA is not available.  Disabling.\")\n",
            "  0%|          | 0/782 [00:00<?, ?it/s]/home/jiwon/anaconda3/lib/python3.9/site-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        }
      ],
      "source": [
        "run_training(\n",
        "    model = model,\n",
        "    optimizer = optimizer,\n",
        "    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer=optimizer, T_max=100, eta_min=1e-6),\n",
        "    device = device,\n",
        "    num_epochs = 20,\n",
        "    metric_prefix=\"\",\n",
        "    file_prefix=\"\",\n",
        "    early_stopping=True,\n",
        "    early_stopping_step=10,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dVufCm3-SzsG"
      },
      "source": [
        "# Inference"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'en_encode' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m/tmp/ipykernel_3200/2077280996.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0msrc_sentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'i don\\'t love you, pleas go away from me. Do you understand?'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msrc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0men_encode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc_sentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'en_encode' is not defined"
          ]
        }
      ],
      "source": [
        "src_sentence = 'i don\\'t love you, pleas go away from me. Do you understand?'\n",
        "enc_src = en_encode(src_sentence)\n",
        "last = None\n",
        "dec_src = torch.tensor(SOS_TOKEN)\n",
        "enc_output = model.encoder(enc_src)\n",
        "\n",
        "while last is not EOS_TOKEN :\n",
        "    \n",
        "\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "jqaL9w0VSzsA",
        "fb-AM2f2SzsB",
        "loi-qIjHSzsB",
        "jTb9C0UTSzsC",
        "emd_xbYPSzsD",
        "z0NInjRrSzsD",
        "ACUVck9eSzsE"
      ],
      "name": "transformer_gpu.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}