grade.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7sKtpZx-TI_e",
        "outputId": "54678890-b383-467a-f0be-2cee04452bda"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 7.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 54.2 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 7.5 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
            "\u001b[K     |████████████████████████████████| 880 kB 42.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 58.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895260 sha256=8da10d9292cf37669d5824f662132f0552bf595692f65e99d7a98e5f07002344\n",
            "  Stored in directory: /root/.cache/pip/wheels/87/39/dd/a83eeef36d0bf98e7a4d1933a4ad2d660295a40613079bafc9\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.5.1 pyyaml-6.0 sacremoses-0.0.53 tokenizers-0.12.1 transformers-4.18.0\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "adGqsGHTTZwx"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import torch\n",
        "import re\n",
        "import sklearn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score\n",
        "from transformers import XLMRobertaTokenizerFast, XLMRobertaForSequenceClassification, Trainer, TrainingArguments, AutoModelForSequenceClassification, AutoTokenizer, RobertaTokenizerFast\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "import numpy as np\n",
        "import random\n",
        "from transformers.file_utils import is_tf_available, is_torch_available, is_torch_tpu_available"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AKYuuEwHTcd8"
      },
      "outputs": [],
      "source": [
        "model_path = \"/content/drive/MyDrive/clef_data/models/1A_en\"\n",
        "test_path = '/content/drive/MyDrive/clef_data/tests/CT22_english_1A_checkworthy_test.tsv'\n",
        "result_path = '/content/drive/MyDrive/clef_data/tests/subtask1A_checkworthy_english.tsv'\n",
        "max_length = 64\n",
        "saved_model = AutoModelForSequenceClassification.from_pretrained(model_path, num_labels=2)\n",
        "saved_tokenizer = AutoTokenizer.from_pretrained(model_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-30dwzsKTw_G",
        "outputId": "2d0a1145-bb0b-4f42-cc17-aac1904906d6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "XLMRobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(250002, 1024, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 1024)\n",
              "      (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (12): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (13): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (14): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (15): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (16): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (17): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (18): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (19): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (20): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (21): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (22): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (23): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "if torch.cuda.is_available():    \n",
        "    device = torch.device(\"cuda\")\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "\n",
        "# Push model to device\n",
        "saved_model.to(device)\n",
        "\n",
        "# Model to evaluation mode\n",
        "saved_model.eval()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uyZGgXMGUBRs"
      },
      "outputs": [],
      "source": [
        "def get_prediction(text):\n",
        "    text = nltk_preprocess(text)\n",
        "    # prepare our text into tokenized sequence\n",
        "    inputs = saved_tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors=\"pt\")\n",
        "    # perform inference to our model\n",
        "    outputs = saved_model(**inputs)\n",
        "    # get output probabilities by doing softmax\n",
        "    probs = outputs[0].softmax(1)\n",
        "\n",
        "    return int(probs.argmax())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "epB_fK8ZVID_"
      },
      "outputs": [],
      "source": [
        "def clean_text(text):\n",
        "    text = str(text).replace(r'http[\\w:\\/\\.]+', '@link')\n",
        "    text = str(text).replace(r'\\s\\s+', ' ')\n",
        "    return text\n",
        "\n",
        "def nltk_preprocess(text):\n",
        "    text = clean_text(text)\n",
        "    return  text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "mUT4Q_jyUjyU"
      },
      "outputs": [],
      "source": [
        "test_df = pd.read_csv(test_path, sep='\\t')\n",
        "grade = []\n",
        "modelId = []\n",
        "for index, row in test_df.iterrows():\n",
        "    cur = get_prediction(row['tweet_text'])\n",
        "    grade.append(cur)\n",
        "    modelId.append(\"RoBERTa-large\")\n",
        "    print(row['tweet_text'], cur)\n",
        "test_df = test_df.drop(['tweet_url', 'tweet_text'],axis=1)\n",
        "test_df['class_label'] = grade\n",
        "test_df['run_id'] = modelId\n",
        "print(test_df.head())\n",
        "test_df.to_csv(result_path, index=False, sep=\"\\t\")"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "grade.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}