bart_YOR_ENG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "t5_YOR_ENG.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/fajemila/yorubanlptoen/blob/main/bart_YOR_ENG.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KF7jd0UWcier",
        "outputId": "46632d25-1c88-4678-dc1c-5b6a0016fd89"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-Pk18Ccerz3D",
        "outputId": "be3160c9-7eff-4b1d-bcec-2e1ae271a0ec"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers.git\r\n",
        "!pip install -U ./transformers\r\n",
        "!pip install git+https://github.com/huggingface/nlp.git"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 57703, done.\u001b[K\n",
            "remote: Total 57703 (delta 0), reused 0 (delta 0), pack-reused 57703\u001b[K\n",
            "Receiving objects: 100% (57703/57703), 43.00 MiB | 29.75 MiB/s, done.\n",
            "Resolving deltas: 100% (40498/40498), done.\n",
            "Processing ./transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (1.19.4)\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (20.8)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 20.7MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (2019.12.20)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 43.4MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (3.0.12)\n",
            "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (4.41.1)\n",
            "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (0.8)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==4.2.0.dev0) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.2.0.dev0) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.2.0.dev0) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.2.0.dev0) (1.0.0)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (2020.12.5)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (2.10)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.2.0.dev0-cp36-none-any.whl size=1527222 sha256=6beb9d823dbea84d1d0520e210f5ecf09f780bb33d684d3c76501e34a80b288a\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-bda_qv8g/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "Successfully built transformers\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=11904e0ccbbfe27505ffe5d73bfbf853ee628213e65bb9c1c671839abc587e82\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.2.0.dev0\n",
            "Collecting git+https://github.com/huggingface/nlp.git\n",
            "  Cloning https://github.com/huggingface/nlp.git to /tmp/pip-req-build-gcrwhaol\n",
            "  Running command git clone -q https://github.com/huggingface/nlp.git /tmp/pip-req-build-gcrwhaol\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.6/dist-packages (from datasets==1.1.3) (1.19.4)\n",
            "Collecting pyarrow>=0.17.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/e1/27958a70848f8f7089bff8d6ebe42519daf01f976d28b481e1bfd52c8097/pyarrow-2.0.0-cp36-cp36m-manylinux2014_x86_64.whl (17.7MB)\n",
            "\u001b[K     |████████████████████████████████| 17.7MB 150kB/s \n",
            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from datasets==1.1.3) (0.3.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from datasets==1.1.3) (1.1.5)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.6/dist-packages (from datasets==1.1.3) (2.23.0)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.6/dist-packages (from datasets==1.1.3) (4.41.1)\n",
            "Collecting xxhash\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f7/73/826b19f3594756cb1c6c23d2fbd8ca6a77a9cd3b650c9dec5acc85004c38/xxhash-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl (242kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 58.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.6/dist-packages (from datasets==1.1.3) (0.70.11.1)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from datasets==1.1.3) (0.8)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->datasets==1.1.3) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas->datasets==1.1.3) (2.8.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->datasets==1.1.3) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->datasets==1.1.3) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->datasets==1.1.3) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->datasets==1.1.3) (1.24.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.7.3->pandas->datasets==1.1.3) (1.15.0)\n",
            "Building wheels for collected packages: datasets\n",
            "  Building wheel for datasets (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for datasets: filename=datasets-1.1.3-cp36-none-any.whl size=159186 sha256=5326f11cb8ecf275083ca009b124c7fe1397d6b85ab193d5bbd362944647c661\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-cdeu64k1/wheels/a2/98/bd/15f115cb85f1a049a96b915aa14466d3b51a023cd597e08e0d\n",
            "Successfully built datasets\n",
            "Installing collected packages: pyarrow, xxhash, datasets\n",
            "  Found existing installation: pyarrow 0.14.1\n",
            "    Uninstalling pyarrow-0.14.1:\n",
            "      Successfully uninstalled pyarrow-0.14.1\n",
            "Successfully installed datasets-1.1.3 pyarrow-2.0.0 xxhash-2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "ezt9CF56k9MO",
        "outputId": "146cc0a9-d364-4e81-d577-a37fc214a664"
      },
      "source": [
        "import transformers\n",
        "transformers.__version__"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'4.2.0dev0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XpJG25txC-Q8"
      },
      "source": [
        "# Importing stock libraries\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# Importing the T5 modules from huggingface/transformers\n",
        "from transformers import T5Tokenizer, T5ForConditionalGeneration"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "spf4aj-4DCxj",
        "outputId": "1706732d-4ac6-4d42-875e-4aad75ea82fb"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thu Dec 31 17:29:03 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.27.04    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   47C    P8     9W /  70W |      0MiB / 15079MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zqoUoCznDFga"
      },
      "source": [
        "# # Setting up the device for GPU usage\n",
        "from torch import cuda\n",
        "device = 'cuda' if cuda.is_available() else 'cpu'"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rweQ4bQzDubT"
      },
      "source": [
        "class CustomDataset(Dataset):\n",
        "\n",
        "    def __init__(self, dataframe, tokenizer, source_len, target_len):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.data = dataframe\n",
        "        self.source_len = source_len\n",
        "        self.target_len = target_len\n",
        "        self.English = self.data.English\n",
        "        self.Yoruba = self.data.Yoruba\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.English)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        Yoruba = str(self.Yoruba[index])\n",
        "        Yoruba = ' '.join(Yoruba.split())\n",
        "\n",
        "        English = str(self.English[index])\n",
        "        English = ' '.join(English.split())\n",
        "\n",
        "        source = self.tokenizer.batch_encode_plus([Yoruba], max_length= self.source_len, pad_to_max_length=True,return_tensors='pt')\n",
        "        target = self.tokenizer.batch_encode_plus([English], max_length= self.target_len, pad_to_max_length=True,return_tensors='pt')\n",
        "\n",
        "        source_ids = source['input_ids'].squeeze()\n",
        "        source_mask = source['attention_mask'].squeeze()\n",
        "        target_ids = target['input_ids'].squeeze()\n",
        "        target_mask = target['attention_mask'].squeeze()\n",
        "\n",
        "        return {\n",
        "            'source_ids': source_ids.to(dtype=torch.long), \n",
        "            'source_mask': source_mask.to(dtype=torch.long), \n",
        "            'target_ids': target_ids.to(dtype=torch.long),\n",
        "            'target_mask': target_mask.to(dtype=torch.long)\n",
        "        }"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mGCVKu3ecBQ8"
      },
      "source": [
        "class Inference(Dataset):\n",
        "\n",
        "    def __init__(self, dataframe, tokenizer, source_len):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.data = dataframe\n",
        "        self.source_len = source_len\n",
        "        self.Yoruba = self.data.Yoruba\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.Yoruba)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        Yoruba = str(self.Yoruba[index])\n",
        "        Yoruba = ' '.join(Yoruba.split())\n",
        "\n",
        "        source = self.tokenizer.batch_encode_plus([Yoruba], max_length= self.source_len, pad_to_max_length=True,return_tensors='pt',truncation=True)\n",
        "\n",
        "        source_ids = source['input_ids'].squeeze()\n",
        "        source_mask = source['attention_mask'].squeeze()\n",
        "        return {\n",
        "            'source_ids': source_ids.to(dtype=torch.long), \n",
        "            'source_mask': source_mask.to(dtype=torch.long),\n",
        "        }"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xwDoenm_D34Z"
      },
      "source": [
        "def train(epoch, tokenizer, model, device, loader, optimizer):\n",
        "    model.train()\n",
        "    epoch_loss = 0\n",
        "    for _,data in enumerate(loader, 0):\n",
        "        y = data['target_ids'].to(device, dtype = torch.long)\n",
        "        y_ids = y[:, :-1].contiguous()\n",
        "        lm_labels = y[:, 1:].clone().detach()\n",
        "        lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100\n",
        "        ids = data['source_ids'].to(device, dtype = torch.long)\n",
        "        mask = data['source_mask'].to(device, dtype = torch.long)\n",
        "        dmask = data['target_mask'].to(device, dtype = torch.long)\n",
        "        outputs = model(input_ids = ids, attention_mask = mask, decoder_input_ids=y_ids, labels=lm_labels)\n",
        "        loss = outputs[0]\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        epoch_loss += loss.item()\n",
        "    return epoch_loss / len(loader)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ksW9Z7SwP3SQ"
      },
      "source": [
        "def evaluate(epoch, tokenizer, model, device, loader):\n",
        "    model.eval()\n",
        "    predictions = []\n",
        "    actuals = []\n",
        "    epoch_loss = 0\n",
        "    with torch.no_grad():\n",
        "        for _, data in enumerate(loader, 0):\n",
        "            y = data['target_ids'].to(device, dtype = torch.long)\n",
        "            yt_ids = y[:, :-1].contiguous()\n",
        "            lm_labels = y[:, 1:].clone().detach()\n",
        "            lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100\n",
        "            ids = data['source_ids'].to(device, dtype = torch.long)\n",
        "            mask = data['source_mask'].to(device, dtype = torch.long)\n",
        "            dmask = data['target_mask'].to(device, dtype = torch.long)\n",
        "            preds = model(input_ids = ids,attention_mask = mask,decoder_input_ids=yt_ids, labels=lm_labels,dec)\n",
        "            # generated_ids = model.generate(\n",
        "            #     input_ids = ids,\n",
        "            #     attention_mask = mask)\n",
        "            val_loss = preds[0]\n",
        "            epoch_loss += val_loss.item()\n",
        "            # preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "            # target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True)for t in y]\n",
        "            # if _%100==0:\n",
        "            #     print(f'Completed {_}')\n",
        "\n",
        "            # predictions.extend(preds)\n",
        "            # actuals.extend(target)\n",
        "    return epoch_loss / len(loader)#,predictions, actuals"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hn0aZB3ETmk"
      },
      "source": [
        "def validate(epoch, tokenizer, model, device, loader):\n",
        "    model.eval()\n",
        "    predictions = []\n",
        "    actuals = []\n",
        "    with torch.no_grad():\n",
        "        for _, data in enumerate(loader, 0):\n",
        "            y = data['target_ids'].to(device, dtype = torch.long)\n",
        "            ids = data['source_ids'].to(device, dtype = torch.long)\n",
        "            mask = data['source_mask'].to(device, dtype = torch.long)\n",
        "\n",
        "            generated_ids = model.generate(\n",
        "                input_ids = ids,\n",
        "                attention_mask = mask, \n",
        "                max_length=150, \n",
        "                num_beams=2,\n",
        "                repetition_penalty=2.5, \n",
        "                length_penalty=1.0, \n",
        "                early_stopping=True\n",
        "                )\n",
        "            preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "            target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True)for t in y]\n",
        "            if _%100==0:\n",
        "                print(f'Completed {_}')\n",
        "\n",
        "            predictions.extend(preds)\n",
        "            actuals.extend(target)\n",
        "    return predictions, actuals"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HBSYvff7k3kc"
      },
      "source": [
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o8Pp8Sd3vlN_"
      },
      "source": [
        "import time\r\n",
        "import math\r\n",
        "from transformers import BartTokenizer,BartForConditionalGeneration\r\n",
        "TRAIN_BATCH_SIZE = 1    # input batch size for training (default: 64)\r\n",
        "VALID_BATCH_SIZE = 2    # input batch size for testing (default: 1000)\r\n",
        "TRAIN_EPOCHS = 2        # number of epochs to train (default: 10)\r\n",
        "VAL_EPOCHS = 1 \r\n",
        "LEARNING_RATE = 1e-4    # learning rate (default: 0.01)\r\n",
        "SEED = 2020               # random seed (default: 2020)\r\n",
        "MAX_LEN = 150 \r\n",
        "import random\r\n",
        "# Set random seeds and deterministic pytorch for reproducibility\r\n",
        "random.seed(SEED)\r\n",
        "np.random.seed(SEED)\r\n",
        "torch.manual_seed(SEED)\r\n",
        "torch.backends.cudnn.deterministic = True\r\n",
        "torch.backends.cudnn.benchmark = False\r\n",
        "torch.cuda.manual_seed_all(SEED)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Te6t7DLEf3H",
        "scrolled": true,
        "outputId": "7fcfa7f9-697b-4516-8728-9daf0d69c07f"
      },
      "source": [
        "# tokenzier for encoding the text\n",
        "tokenizer = BartTokenizer.from_pretrained(\"facebook/bart-large\")\n",
        "\n",
        "# Importing and Pre-Processing the domain data\n",
        "# Selecting the needed columns only. \n",
        "# Adding the summarzie text in front of the text. This is to format the dataset similar to how T5 model was trained for summarization task. \n",
        "df = pd.read_csv('drive/MyDrive/yorubatoEN/Train.csv')\n",
        "inf_data = pd.read_csv('drive/MyDrive/yorubatoEN/Test.csv')\n",
        "df = df[['Yoruba','English']]\n",
        "df.Yoruba = 'translate: ' + df.Yoruba\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "# Creation of Dataset and Dataloader\n",
        "# Defining the train size. So 80% of the data will be used for training and the rest will be used for validation. \n",
        "train_size = 0.95\n",
        "train_dataset=df.sample(frac=train_size, random_state = SEED).reset_index(drop=True)\n",
        "val_dataset=df.drop(train_dataset.index).reset_index(drop=True)\n",
        "\n",
        "\n",
        "print(\"FULL Dataset: {}\".format(df.shape))\n",
        "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
        "print(\"TEST Dataset: {}\".format(val_dataset.shape))\n",
        "\n",
        "\n",
        "# Creating the Training and Validation dataset for further creation of Dataloader\n",
        "training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN, MAX_LEN)\n",
        "val_set = CustomDataset(val_dataset, tokenizer, MAX_LEN, MAX_LEN)\n",
        "inf_set = Inference(inf_data,tokenizer,MAX_LEN)\n",
        "\n",
        "# Defining the parameters for creation of dataloaders\n",
        "train_params = {\n",
        "    'batch_size': TRAIN_BATCH_SIZE,\n",
        "    'shuffle': True,\n",
        "    'num_workers': 0\n",
        "    }\n",
        "\n",
        "val_params = {\n",
        "    'batch_size': VALID_BATCH_SIZE,\n",
        "    'shuffle': False,\n",
        "    'num_workers': 0\n",
        "    }\n",
        "\n",
        "# Creation of Dataloaders for testing and validation. This will be used down for training and validation stage for the model.\n",
        "training_loader = DataLoader(training_set, **train_params)\n",
        "val_loader = DataLoader(val_set, **val_params)\n",
        "inf_loader = DataLoader(inf_set)\n",
        "\n",
        "\n",
        "# Defining the model. We are using t5-base model and added a Language model layer on top for generation of Summary. \n",
        "# Further this model is sent to device (GPU/TPU) for using the hardware.\n",
        "model = BartForConditionalGeneration.from_pretrained(\"facebook/bart-large\")\n",
        "model = model.to(device)\n",
        "\n",
        "# Defining the optimizer that will be used to tune the weights of the network in the training session. \n",
        "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "# Training loop\n",
        "print('Initiating Fine-Tuning for the model on our dataset')\n",
        "best_valid_score = float('-inf')\n",
        "start_time = time.time()\n",
        "for epoch in range(TRAIN_EPOCHS):\n",
        "    train_loss= train(epoch, tokenizer, model, device, training_loader, optimizer)\n",
        "    valid_loss = evaluate(epoch, tokenizer, model, device, val_loader)\n",
        "    # valid_score = bleu_score(predictions, actuals)\n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    torch.save(model.state_dict(), 'bart.pt_{}'.format(epoch))\n",
        "    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
        "    print(f'\\tVal Loss: {valid_loss:.3f}' )#| Val Bleu: {Bleu_score:7.3f}')\n",
        "    # print(f'Valid Bleu: {valid_scorre}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                                              Yoruba                                            English\n",
            "0  translate: A ṣètò Ìgbìmọ̀ Tó Ń Ṣètò Ìrànwọ́ Ní...  A Disaster Relief Committee was formed to orga...\n",
            "1  translate: Ìrọ̀lẹ́ May 22, 2018 ni wọ́n fàṣẹ ọ...  Brother Solovyev was arrested on the evening o...\n",
            "2            translate: Iléeṣẹ́ Creative Commons náà                  Creative Commons the Organization\n",
            "3  translate: Pè̩lú Egypt, Morocco àti Tunisia tí...  With Egypt, Morocco and Tunisia out of the Wor...\n",
            "4  translate: Adájọ́ àgbà lórílẹ̀ èdè Náíjíríà (A...  The Attorney General of the Federation, Justic...\n",
            "FULL Dataset: (10054, 2)\n",
            "TRAIN Dataset: (9551, 2)\n",
            "TEST Dataset: (503, 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Initiating Fine-Tuning for the model on our dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01 | Time: 49m 53s\n",
            "\tTrain Loss: 7.240 | Train PPL: 1394.400\n",
            "\tVal Loss: 10.324\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HpnpS0Bu0336"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j-MCiyfs03zt"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u7owEtaH03av"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "76iBqxzR03XT"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DwBc4fB503T7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_hyYDwQ03Qf"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q_ybWsvP03M-"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mPkDsbOb03JL"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FMIDccRa03Fm"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TF-2a_Jk03Ab"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0z49EGkuP3Si"
      },
      "source": [
        "train_loss= train(epoch, tokenizer, model, device, training_loader, optimizer)\n",
        "        valid_loss,predictions, actuals = validate(epoch, tokenizer, model, device, val_loader)\n",
        "        Bleu_score = bleu_score(predictions, actuals)\n",
        "        end_time = time.time()\n",
        "    \n",
        "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "        if valid_scorre > best_valid_score:\n",
        "            best_valid_loss = valid_loss\n",
        "            torch.save(model.state_dict(), 'tut6-model.pt')\n",
        "        print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
        "        print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
        "        print(f'\\tVal Loss: {train_loss:.3f} | Val Bleu: {Bleu_score:7.3f}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BO2YFvl4TdTc"
      },
      "source": [
        "predictions, actuals = validate(epoch, tokenizer, model, device, val_loader)\n",
        "valid_scorre = bleu_score(predictions, actuals)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3W1X9oRgwO0q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "465a9b03-22e8-4ebd-f46f-7904ab2aa0ff"
      },
      "source": [
        "inf_data = pd.read_csv('drive/MyDrive/yorubatoEN/Test.csv')\r\n",
        "inf_data.Yoruba = 'translate: ' + inf_data.Yoruba\r\n",
        "print(inf_data.head())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "            ID                                             Yoruba\n",
            "0  ID_AAAitMaH  translate: Nínú ìpè kan lẹ́yìn ìgbà náà, wọ́n ...\n",
            "1  ID_AAKKdQwr  translate: Nítorí kò sí nǹkan tí ọkùnrin ò lè ...\n",
            "2  ID_ABgAyEOp       translate: Bí i kó pariwo. Kí ó kígbe mọ́ ẹ?\n",
            "3  ID_ACFgfKQs  translate: Tí ó ń lé e lọ sọ́nà etí odò Akókur...\n",
            "4  ID_ACNPmlhf  translate: Èṣúńiyì mọ̀ iṣẹ́ rẹ̀ dunjú. Màmá ti...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eNnaG1TVuvAn"
      },
      "source": [
        "inf_set = Inference(inf_data,tokenizer,MAX_LEN)\r\n",
        "inf_loader = DataLoader(inf_set,shuffle=False,batch_size=32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "soHn6ohwxUaC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7ca826f4-e042-4d12-891e-ee2890a2218b"
      },
      "source": [
        "model.eval()\r\n",
        "predictions = []\r\n",
        "with torch.no_grad():\r\n",
        "  for _, data in enumerate(inf_loader, 0):\r\n",
        "    ids = data['source_ids'].to(device, dtype = torch.long)\r\n",
        "    mask = data['source_mask'].to(device, dtype = torch.long)\r\n",
        "\r\n",
        "    generated_ids = model.generate(\r\n",
        "        input_ids = ids,\r\n",
        "        do_sample=True, \r\n",
        "        top_p = 0.92,\r\n",
        "        top_k=50,\r\n",
        "        early_stopping=True\r\n",
        "        )\r\n",
        "    preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\r\n",
        "    if _%100==0:\r\n",
        "        print(f'Completed {_}')\r\n",
        "\r\n",
        "    predictions.extend(preds)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Completed 0\n",
            "Completed 100\n",
            "Completed 200\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9eUNmm6gygmb"
      },
      "source": [
        "inf_data['English'] = predictions\r\n",
        "inf_data[['ID','English']].to_csv('t5yor.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 430
        },
        "id": "fGdsz0SOvo5G",
        "outputId": "8f3a41d3-c9d9-404f-d648-e7799ef3a028"
      },
      "source": [
        "for _,data in enumerate(training_loader, 0):\r\n",
        "    y = data['target_ids'].to(device, dtype = torch.long)\r\n",
        "    y_ids = y[:, 1:].contiguous()\r\n",
        "    lm_labels = y[:, 1:].clone().detach()\r\n",
        "    lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100\r\n",
        "    ids = data['source_ids'].to(device, dtype = torch.long)\r\n",
        "    mask = data['source_mask'].to(device, dtype = torch.long)\r\n",
        "    d_mask = data['target_mask'].to(device, dtype = torch.long)\r\n",
        "    outputs = model(input_ids = ids,attention_mask = mask,decoder_input_ids=y,decoder_attention_mask=d_mask)\r\n",
        "    # print(torch.nn.CrossEntropyLoss()(outputs[1],y))\r\n",
        "    print(y.shape)\r\n",
        "    print(outputs[0].shape)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-33-922161310dfb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0md_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'target_mask'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mattention_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdecoder_input_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdecoder_attention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0md_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCrossEntropyLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/loss.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, target)\u001b[0m\n\u001b[1;32m    960\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    961\u001b[0m         return F.cross_entropy(input, target, weight=self.weight,\n\u001b[0;32m--> 962\u001b[0;31m                                ignore_index=self.ignore_index, reduction=self.reduction)\n\u001b[0m\u001b[1;32m    963\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mcross_entropy\u001b[0;34m(input, target, weight, size_average, ignore_index, reduce, reduction)\u001b[0m\n\u001b[1;32m   2466\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msize_average\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mreduce\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2467\u001b[0m         \u001b[0mreduction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_Reduction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_get_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize_average\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2468\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mnll_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2469\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2470\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlog_softmax\u001b[0;34m(input, dim, _stacklevel, dtype)\u001b[0m\n\u001b[1;32m   1603\u001b[0m         \u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_softmax_dim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'log_softmax'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_stacklevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1604\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1605\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1606\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1607\u001b[0m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'tuple' object has no attribute 'log_softmax'"
          ]
        }
      ]
    }
  ]
}