albert-act.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a6f9f5cd",
   "metadata": {},
   "source": [
    "# Instalacion de dependencias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab594585",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pip install happytransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ea762b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn, optim #Pytorch\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b44ea733",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "RANDOM_SEED = 42\n",
    "BATCH_SIZE = 16\n",
    "np.random.seed(RANDOM_SEED)\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07ad0cd9",
   "metadata": {},
   "source": [
    "## Cargar datos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3f0f82a8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset csv (C:/Users/anabe/.cache/huggingface/datasets/csv/default-093a8e447c963488/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317)\n",
      "Loading cached split indices for dataset at C:\\Users\\anabe\\.cache\\huggingface\\datasets\\csv\\default-093a8e447c963488\\0.0.0\\6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317\\cache-d995bbfd5f578a3e.arrow and C:\\Users\\anabe\\.cache\\huggingface\\datasets\\csv\\default-093a8e447c963488\\0.0.0\\6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317\\cache-974f951674d0b4c2.arrow\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset('csv', data_files=['C:/Users/anabe/Desktop/PRUEBAS/APPSTORE1 - ACTUALIZADO/corpus/book4.csv'], split='train')\n",
    "datasets = dataset.train_test_split(test_size=0.20) # Test del 10%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "09798832",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['name', 'general', 'descrip'],\n",
       "        num_rows: 2496\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['name', 'general', 'descrip'],\n",
       "        num_rows: 625\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fa47d820",
   "metadata": {},
   "outputs": [],
   "source": [
    "def etiquetas(examples):\n",
    "    if examples[\"general\"] == 'Linguistics':\n",
    "        label = 0\n",
    "    else: \n",
    "        label = 1\n",
    "    return {\"labels\": label}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e2ce22d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def etiquetas(examples):\n",
    "    if examples [\"general\"] == 'applied science':\n",
    "        label = 0\n",
    "    elif examples [\"general\"] == 'natural sciences':\n",
    "        label = 1\n",
    "    elif examples [\"general\"] == 'math':\n",
    "        label = 2\n",
    "    elif examples [\"general\"] == 'social sciences ':\n",
    "        label = 3\n",
    "    elif examples [\"general\"] == 'art':\n",
    "        label = 4\n",
    "    elif examples [\"general\"] == 'linguistics':\n",
    "        label = 5\n",
    "    else:\n",
    "        label = 6\n",
    "    return {\"labels\": label}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d580f6fb",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\anabe\\.cache\\huggingface\\datasets\\csv\\default-093a8e447c963488\\0.0.0\\6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317\\cache-8805b999bb461f46.arrow\n",
      "Loading cached processed dataset at C:\\Users\\anabe\\.cache\\huggingface\\datasets\\csv\\default-093a8e447c963488\\0.0.0\\6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317\\cache-0d6d123db84ac67f.arrow\n"
     ]
    }
   ],
   "source": [
    "dataset = datasets.map(etiquetas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2b8e9746",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['name', 'general', 'descrip', 'labels'],\n",
       "        num_rows: 2496\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['name', 'general', 'descrip', 'labels'],\n",
       "        num_rows: 625\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ab50f772",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'Ananas Language Exchange',\n",
       " 'general': 'linguistics',\n",
       " 'descrip': 'The language app for today. Learn and exchange languages. Think global, be local.',\n",
       " 'labels': 5}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train'][5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8744d57",
   "metadata": {},
   "source": [
    "## Tokenizacion DataSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5f88922c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_checkpoint = \"albert-base-v2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1d380522",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "413f1d191dbb40a3beb984efd1248a4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/684 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anabe\\anaconda3\\lib\\site-packages\\huggingface_hub\\file_download.py:133: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\anabe\\.cache\\huggingface\\hub. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fa69cdd00484cf3b1a4cf49c8e6459a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)ve/main/spiece.model:   0%|          | 0.00/760k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0ebfe3b297648c2b3c0bf258d947eac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.31M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
    "#, use_fast=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c402fead",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Vamos a pasar el tokenizador en cada fila del dataset\n",
    "def tokenize_reviews(examples):\n",
    "    return tokenizer(examples[\"descrip\"], padding=True, truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2ab4ad6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(dataset['train']['descrip'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7f7245ea",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2496 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/625 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "columns = dataset[\"train\"].column_names\n",
    "columns.remove(\"labels\")\n",
    "#encoded_dataset = dset.map(tokenize_reviews, batched=True, remove_columns=columns)\n",
    "encoded_dataset = dataset.map(tokenize_reviews, batched=True, remove_columns=columns )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f67fc13f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30000"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.vocab_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c2afe39",
   "metadata": {},
   "source": [
    "## Cargar el modelo preentrenado"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e2c5fe62",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2b3efac7b69486ab5145183718458bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/47.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at albert-base-v2 were not used when initializing AlbertForSequenceClassification: ['predictions.LayerNorm.bias', 'predictions.bias', 'predictions.LayerNorm.weight', 'predictions.decoder.weight', 'predictions.dense.weight', 'predictions.dense.bias', 'predictions.decoder.bias']\n",
      "- This IS expected if you are initializing AlbertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing AlbertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of AlbertForSequenceClassification were not initialized from the model checkpoint at albert-base-v2 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "num_labels = 7\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ca70fa19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'labels': Value(dtype='int64', id=None),\n",
       " 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None),\n",
       " 'token_type_ids': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None),\n",
       " 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_dataset['train'].features\n",
    "#encoded_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8ad13345",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'labels': Value(dtype='int64', id=None),\n",
       " 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None),\n",
       " 'token_type_ids': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None),\n",
       " 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_dataset.set_format(\"torch\", columns=['input_ids','attention_mask','labels'])\n",
    "encoded_dataset['train'].features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6afd7c22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['labels', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 2496\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['labels', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 625\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "734b25ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    labels = eval_pred.label_ids\n",
    "    preds = eval_pred.predictions.argmax(-1)\n",
    "    f1 = f1_score(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\"accuracy\": acc, \"f1\": f1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "24316dc9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached shuffled indices for dataset at C:\\Users\\anabe\\.cache\\huggingface\\datasets\\csv\\default-093a8e447c963488\\0.0.0\\6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317\\cache-29f6f055abb65cea.arrow\n"
     ]
    }
   ],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "model_name = model_checkpoint.split(\"/\")[-1]\n",
    "\n",
    "batch_size = 64\n",
    "num_train_epochs=4\n",
    "num_train_samples = 2_000\n",
    "train_dataset = encoded_dataset[\"train\"].shuffle(seed=42).select(range(num_train_samples))\n",
    "#train_dataset = encoded_dataset[\"train\"]\n",
    "\n",
    "#logging_steps = len(encoded_dataset['train'])//batch_size\n",
    "logging_steps = len(train_dataset) // (2 * batch_size * num_train_epochs)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"albert-act\",\n",
    "    num_train_epochs=num_train_epochs,     \n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"f1\",\n",
    "    weight_decay=0.01,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\", \n",
    "    logging_steps=logging_steps,\n",
    "    #push_to_hub=True,\n",
    "    #push_to_hub_model_id=f\"{model_name}-finetuned-wiki_multi\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0adff1f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model, \n",
    "    args=training_args, \n",
    "    compute_metrics=compute_metrics,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=encoded_dataset[\"test\"],\n",
    "    tokenizer=tokenizer\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e0e8c369",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anabe\\anaconda3\\lib\\site-packages\\transformers\\optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "You're using a AlbertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='128' max='128' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [128/128 4:26:31, Epoch 4/4]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.207000</td>\n",
       "      <td>1.169852</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.500723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.749700</td>\n",
       "      <td>0.885140</td>\n",
       "      <td>0.724800</td>\n",
       "      <td>0.670785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.685600</td>\n",
       "      <td>0.763425</td>\n",
       "      <td>0.769600</td>\n",
       "      <td>0.726567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.593500</td>\n",
       "      <td>0.710291</td>\n",
       "      <td>0.780800</td>\n",
       "      <td>0.739451</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=128, training_loss=0.9258596515282989, metrics={'train_runtime': 16163.9994, 'train_samples_per_second': 0.495, 'train_steps_per_second': 0.008, 'total_flos': 107968203600000.0, 'train_loss': 0.9258596515282989, 'epoch': 4.0})"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23f35347",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}