finetuning-lora-gemma-2b-with-medium-articles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "11f90076",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-03-10T13:05:35.763136Z",
     "iopub.status.busy": "2024-03-10T13:05:35.762348Z",
     "iopub.status.idle": "2024-03-10T13:05:36.523918Z",
     "shell.execute_reply": "2024-03-10T13:05:36.522942Z"
    },
    "papermill": {
     "duration": 0.770349,
     "end_time": "2024-03-10T13:05:36.526191",
     "exception": false,
     "start_time": "2024-03-10T13:05:35.755842",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/medium-data-ft/Finetune_data.csv\n",
      "/kaggle/input/gemma/keras/gemma_instruct_2b_en/2/config.json\n",
      "/kaggle/input/gemma/keras/gemma_instruct_2b_en/2/tokenizer.json\n",
      "/kaggle/input/gemma/keras/gemma_instruct_2b_en/2/metadata.json\n",
      "/kaggle/input/gemma/keras/gemma_instruct_2b_en/2/model.weights.h5\n",
      "/kaggle/input/gemma/keras/gemma_instruct_2b_en/2/assets/tokenizer/vocabulary.spm\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f6b74bc9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:05:36.537118Z",
     "iopub.status.busy": "2024-03-10T13:05:36.536666Z",
     "iopub.status.idle": "2024-03-10T13:05:40.326401Z",
     "shell.execute_reply": "2024-03-10T13:05:40.325365Z"
    },
    "papermill": {
     "duration": 3.797595,
     "end_time": "2024-03-10T13:05:40.328696",
     "exception": false,
     "start_time": "2024-03-10T13:05:36.531101",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>Prefixed_Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mental Note Vol. 24</td>\n",
       "      <td>Photo by Josh Riemer on Unsplash\\n\\nMerry Chri...</td>\n",
       "      <td>Tell me more about Mental Note Vol. 24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Your Brain On Coronavirus</td>\n",
       "      <td>Your Brain On Coronavirus\\n\\nA guide to the cu...</td>\n",
       "      <td>Show me Your Brain On Coronavirus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mind Your Nose</td>\n",
       "      <td>Mind Your Nose\\n\\nHow smell training can chang...</td>\n",
       "      <td>I'm curious about Mind Your Nose</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The 4 Purposes of Dreams</td>\n",
       "      <td>Passionate about the synergy between science a...</td>\n",
       "      <td>I want to learn about The 4 Purposes of Dreams</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Surviving a Rod Through the Head</td>\n",
       "      <td>You’ve heard of him, haven’t you? Phineas Gage...</td>\n",
       "      <td>Tell me more about Surviving a Rod Through the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>Some Best Practices in Using Android Notification</td>\n",
       "      <td>The picture is provided by Unsplash\\n\\nBasical...</td>\n",
       "      <td>Explain Some Best Practices in Using Android N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>A Message to Sam Pancake From a Gay Elder: Tha...</td>\n",
       "      <td>O n behalf of geezers everywhere, thank you. T...</td>\n",
       "      <td>Share with me A Message to Sam Pancake From a ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>DAGS. D’ya Like DAGS?</td>\n",
       "      <td>In the age of connectivity, the new breed of i...</td>\n",
       "      <td>I want to learn about DAGS. D’ya Like DAGS?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>WordPress Security Tips to make your website h...</td>\n",
       "      <td>According to Norton, the cost of a single data...</td>\n",
       "      <td>I need information on WordPress Security Tips ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>Off the Rails</td>\n",
       "      <td>Spiralbound\\n\\nComics for life, brought to lif...</td>\n",
       "      <td>Help me understand Off the Rails</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title  \\\n",
       "0                                    Mental Note Vol. 24   \n",
       "1                              Your Brain On Coronavirus   \n",
       "2                                         Mind Your Nose   \n",
       "3                               The 4 Purposes of Dreams   \n",
       "4                       Surviving a Rod Through the Head   \n",
       "...                                                  ...   \n",
       "19995  Some Best Practices in Using Android Notification   \n",
       "19996  A Message to Sam Pancake From a Gay Elder: Tha...   \n",
       "19997                              DAGS. D’ya Like DAGS?   \n",
       "19998  WordPress Security Tips to make your website h...   \n",
       "19999                                      Off the Rails   \n",
       "\n",
       "                                                    text  \\\n",
       "0      Photo by Josh Riemer on Unsplash\\n\\nMerry Chri...   \n",
       "1      Your Brain On Coronavirus\\n\\nA guide to the cu...   \n",
       "2      Mind Your Nose\\n\\nHow smell training can chang...   \n",
       "3      Passionate about the synergy between science a...   \n",
       "4      You’ve heard of him, haven’t you? Phineas Gage...   \n",
       "...                                                  ...   \n",
       "19995  The picture is provided by Unsplash\\n\\nBasical...   \n",
       "19996  O n behalf of geezers everywhere, thank you. T...   \n",
       "19997  In the age of connectivity, the new breed of i...   \n",
       "19998  According to Norton, the cost of a single data...   \n",
       "19999  Spiralbound\\n\\nComics for life, brought to lif...   \n",
       "\n",
       "                                          Prefixed_Title  \n",
       "0                 Tell me more about Mental Note Vol. 24  \n",
       "1                      Show me Your Brain On Coronavirus  \n",
       "2                       I'm curious about Mind Your Nose  \n",
       "3         I want to learn about The 4 Purposes of Dreams  \n",
       "4      Tell me more about Surviving a Rod Through the...  \n",
       "...                                                  ...  \n",
       "19995  Explain Some Best Practices in Using Android N...  \n",
       "19996  Share with me A Message to Sam Pancake From a ...  \n",
       "19997        I want to learn about DAGS. D’ya Like DAGS?  \n",
       "19998  I need information on WordPress Security Tips ...  \n",
       "19999                   Help me understand Off the Rails  \n",
       "\n",
       "[20000 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"/kaggle/input/medium-data-ft/Finetune_data.csv\", nrows=20000)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "82da4d20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:05:40.339920Z",
     "iopub.status.busy": "2024-03-10T13:05:40.339629Z",
     "iopub.status.idle": "2024-03-10T13:05:40.343382Z",
     "shell.execute_reply": "2024-03-10T13:05:40.342607Z"
    },
    "papermill": {
     "duration": 0.011537,
     "end_time": "2024-03-10T13:05:40.345425",
     "exception": false,
     "start_time": "2024-03-10T13:05:40.333888",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !pip install wurlitzer\n",
    "# !pip install -q -U keras-nlp\n",
    "# !pip install -q -U keras>=3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ea783a3a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:05:40.356478Z",
     "iopub.status.busy": "2024-03-10T13:05:40.356212Z",
     "iopub.status.idle": "2024-03-10T13:05:40.360282Z",
     "shell.execute_reply": "2024-03-10T13:05:40.359492Z"
    },
    "papermill": {
     "duration": 0.011771,
     "end_time": "2024-03-10T13:05:40.362065",
     "exception": false,
     "start_time": "2024-03-10T13:05:40.350294",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"KERAS_BACKEND\"] = \"jax\"  # Or \"torch\" or \"tensorflow\".\n",
    "# Avoid memory fragmentation on JAX backend.\n",
    "os.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"]=\"1.00\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "26cf8377",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:05:40.372581Z",
     "iopub.status.busy": "2024-03-10T13:05:40.372322Z",
     "iopub.status.idle": "2024-03-10T13:06:51.809501Z",
     "shell.execute_reply": "2024-03-10T13:06:51.808649Z"
    },
    "papermill": {
     "duration": 71.445231,
     "end_time": "2024-03-10T13:06:51.811970",
     "exception": false,
     "start_time": "2024-03-10T13:05:40.366739",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-10 13:05:44.364830: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-03-10 13:05:44.364979: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-03-10 13:05:44.516968: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "Attaching 'config.json' from model 'keras/gemma/keras/gemma_instruct_2b_en/2' to your Kaggle notebook...\n",
      "Attaching 'config.json' from model 'keras/gemma/keras/gemma_instruct_2b_en/2' to your Kaggle notebook...\n",
      "Attaching 'model.weights.h5' from model 'keras/gemma/keras/gemma_instruct_2b_en/2' to your Kaggle notebook...\n",
      "Attaching 'tokenizer.json' from model 'keras/gemma/keras/gemma_instruct_2b_en/2' to your Kaggle notebook...\n",
      "Attaching 'assets/tokenizer/vocabulary.spm' from model 'keras/gemma/keras/gemma_instruct_2b_en/2' to your Kaggle notebook...\n",
      "normalizer.cc(51) LOG(INFO) precompiled_charsmap is empty. use identity normalization.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import keras_nlp\n",
    "gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset(\"gemma_instruct_2b_en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6fe05426",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:06:51.824791Z",
     "iopub.status.busy": "2024-03-10T13:06:51.824458Z",
     "iopub.status.idle": "2024-03-10T13:07:20.041230Z",
     "shell.execute_reply": "2024-03-10T13:07:20.040140Z"
    },
    "papermill": {
     "duration": 28.225739,
     "end_time": "2024-03-10T13:07:20.043739",
     "exception": false,
     "start_time": "2024-03-10T13:06:51.818000",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is Keras API and how to use it ?\n",
      "\n",
      "**Keras API**\n",
      "\n",
      "The Keras API is a high-level interface that allows you to build and train deep learning models in a consistent and efficient manner. It provides a wide range of functions for loading data, building models, training, evaluating, and saving models.\n",
      "\n",
      "**How to use the Keras API**\n",
      "\n",
      "1. **Import the Keras library:**\n",
      "```python\n",
      "import keras\n",
      "```\n",
      "\n",
      "2. **Load the data:**\n",
      "```python\n",
      "# Load the MNIST dataset\n",
      "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
      "```\n",
      "\n",
      "3. **Build the model:**\n",
      "```python\n",
      "# Create a model with 2D input, 10 neurons in the hidden layer, and 10 neurons in the output layer\n",
      "model = keras.Sequential([\n",
      "    keras.layers.InputLayer(input_shape=(28, 28, 1)),\n",
      "    keras.layers.Conv2D(32, (3, 3), activation='relu'),\n",
      "    keras.layers.MaxPooling2D((2, 2)),\n",
      "    keras.layers.Conv2D(64, (3, 3), activation='relu'),\n",
      "    keras.layers.MaxPooling2D((2, 2)),\n",
      "    keras.layers.Conv2D(128, (3, 3), activation='relu'),\n",
      "    keras.layers.MaxPooling2D((2, 2)),\n",
      "    keras.layers.Conv2D(10, (3, 3), activation='softmax')\n",
      "])\n",
      "```\n",
      "\n",
      "4. **Compile the model:**\n",
      "```python\n",
      "# Compile the model with Adam optimizer and 100 epochs\n",
      "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
      "```\n",
      "\n",
      "5. **Train the model:**\n",
      "```python\n",
      "# Train the model with the training data\n",
      "model.fit(x_train, y_train, epochs=100)\n",
      "```\n",
      "\n",
      "6. **Evaluate the model:**\n",
      "```python\n",
      "# Evaluate the model on the test data\n",
      "loss, accuracy = model.evaluate(x_test, y_test)\n",
      "print('Loss:', loss)\n",
      "print('Accuracy:', accuracy)\n",
      "```\n",
      "\n",
      "7. **Save the model:**\n",
      "```python\n",
      "# Save the model to a file\n",
      "model.save('mnist_model.h5')\n",
      "```\n",
      "\n",
      "**Additional notes:**\n",
      "\n",
      "* The Keras API is designed to be modular and extensible. You can create your own layers and models by extending the base classes provided by the library.\n",
      "* The API also provides a wide range of options for hyperparameter tuning, allowing you to optimize your models for specific tasks.\n",
      "* The Keras API is a powerful tool for building and training deep learning models. It can be used to build a wide variety of models for different tasks, including image classification, natural language processing, and reinforcement learning.\n"
     ]
    }
   ],
   "source": [
    "print(gemma_lm.generate(\"What is Keras API and how to use it ?\", max_length=1024))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "06c63806",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:07:20.058594Z",
     "iopub.status.busy": "2024-03-10T13:07:20.058186Z",
     "iopub.status.idle": "2024-03-10T13:07:20.075021Z",
     "shell.execute_reply": "2024-03-10T13:07:20.073623Z"
    },
    "papermill": {
     "duration": 0.027745,
     "end_time": "2024-03-10T13:07:20.077574",
     "exception": false,
     "start_time": "2024-03-10T13:07:20.049829",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>Prefixed_Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19990</th>\n",
       "      <td>Quit Google Analytics and use self-hosted Gats...</td>\n",
       "      <td>Photo by Hal Gatewood on Unsplash\\n\\nThere are...</td>\n",
       "      <td>Tell me about Quit Google Analytics and use se...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19991</th>\n",
       "      <td>Game Design with Singleton Pattern</td>\n",
       "      <td>Game Design with Singleton Pattern\\n\\nThere ca...</td>\n",
       "      <td>I want to learn about Game Design with Singlet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19992</th>\n",
       "      <td>Dendrons</td>\n",
       "      <td>Unwitnessed, I swim dreams\\n\\nand disappear in...</td>\n",
       "      <td>What are your thoughts on Dendrons</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19993</th>\n",
       "      <td>7 Sequential Operators You Should Know From Sw...</td>\n",
       "      <td>7 Sequential Operators You Should Know From Sw...</td>\n",
       "      <td>What's the deal with 7 Sequential Operators Yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19994</th>\n",
       "      <td>Why strong leadership is crucial for cybersecu...</td>\n",
       "      <td>In the movies, cyber-crime is often portrayed ...</td>\n",
       "      <td>Can you explain Why strong leadership is cruci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>Some Best Practices in Using Android Notification</td>\n",
       "      <td>The picture is provided by Unsplash\\n\\nBasical...</td>\n",
       "      <td>Explain Some Best Practices in Using Android N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>A Message to Sam Pancake From a Gay Elder: Tha...</td>\n",
       "      <td>O n behalf of geezers everywhere, thank you. T...</td>\n",
       "      <td>Share with me A Message to Sam Pancake From a ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>DAGS. D’ya Like DAGS?</td>\n",
       "      <td>In the age of connectivity, the new breed of i...</td>\n",
       "      <td>I want to learn about DAGS. D’ya Like DAGS?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>WordPress Security Tips to make your website h...</td>\n",
       "      <td>According to Norton, the cost of a single data...</td>\n",
       "      <td>I need information on WordPress Security Tips ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>Off the Rails</td>\n",
       "      <td>Spiralbound\\n\\nComics for life, brought to lif...</td>\n",
       "      <td>Help me understand Off the Rails</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title  \\\n",
       "19990  Quit Google Analytics and use self-hosted Gats...   \n",
       "19991                 Game Design with Singleton Pattern   \n",
       "19992                                           Dendrons   \n",
       "19993  7 Sequential Operators You Should Know From Sw...   \n",
       "19994  Why strong leadership is crucial for cybersecu...   \n",
       "19995  Some Best Practices in Using Android Notification   \n",
       "19996  A Message to Sam Pancake From a Gay Elder: Tha...   \n",
       "19997                              DAGS. D’ya Like DAGS?   \n",
       "19998  WordPress Security Tips to make your website h...   \n",
       "19999                                      Off the Rails   \n",
       "\n",
       "                                                    text  \\\n",
       "19990  Photo by Hal Gatewood on Unsplash\\n\\nThere are...   \n",
       "19991  Game Design with Singleton Pattern\\n\\nThere ca...   \n",
       "19992  Unwitnessed, I swim dreams\\n\\nand disappear in...   \n",
       "19993  7 Sequential Operators You Should Know From Sw...   \n",
       "19994  In the movies, cyber-crime is often portrayed ...   \n",
       "19995  The picture is provided by Unsplash\\n\\nBasical...   \n",
       "19996  O n behalf of geezers everywhere, thank you. T...   \n",
       "19997  In the age of connectivity, the new breed of i...   \n",
       "19998  According to Norton, the cost of a single data...   \n",
       "19999  Spiralbound\\n\\nComics for life, brought to lif...   \n",
       "\n",
       "                                          Prefixed_Title  \n",
       "19990  Tell me about Quit Google Analytics and use se...  \n",
       "19991  I want to learn about Game Design with Singlet...  \n",
       "19992                 What are your thoughts on Dendrons  \n",
       "19993  What's the deal with 7 Sequential Operators Yo...  \n",
       "19994  Can you explain Why strong leadership is cruci...  \n",
       "19995  Explain Some Best Practices in Using Android N...  \n",
       "19996  Share with me A Message to Sam Pancake From a ...  \n",
       "19997        I want to learn about DAGS. D’ya Like DAGS?  \n",
       "19998  I need information on WordPress Security Tips ...  \n",
       "19999                   Help me understand Off the Rails  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b6efa869",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:07:20.098755Z",
     "iopub.status.busy": "2024-03-10T13:07:20.098252Z",
     "iopub.status.idle": "2024-03-10T13:07:21.489170Z",
     "shell.execute_reply": "2024-03-10T13:07:21.488297Z"
    },
    "papermill": {
     "duration": 1.406596,
     "end_time": "2024-03-10T13:07:21.491767",
     "exception": false,
     "start_time": "2024-03-10T13:07:20.085171",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "dataset = []\n",
    "for index, row in df.iterrows():\n",
    "    instruction, response = row['Prefixed_Title'], row['text']\n",
    "    template = (f\"Instruction:\\n{instruction}\\n\\nResponse:\\n{response}\")\n",
    "    dataset.append(template)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a2cf7638",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:07:21.506658Z",
     "iopub.status.busy": "2024-03-10T13:07:21.505859Z",
     "iopub.status.idle": "2024-03-10T13:07:21.940777Z",
     "shell.execute_reply": "2024-03-10T13:07:21.939938Z"
    },
    "papermill": {
     "duration": 0.444926,
     "end_time": "2024-03-10T13:07:21.943392",
     "exception": false,
     "start_time": "2024-03-10T13:07:21.498466",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Enable LoRA for the model and set the LoRA rank to 64.\n",
    "gemma_lm.backbone.enable_lora(rank=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "feff9e6b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:07:21.957451Z",
     "iopub.status.busy": "2024-03-10T13:07:21.957120Z",
     "iopub.status.idle": "2024-03-10T13:07:22.043863Z",
     "shell.execute_reply": "2024-03-10T13:07:22.043050Z"
    },
    "papermill": {
     "duration": 0.096541,
     "end_time": "2024-03-10T13:07:22.046165",
     "exception": false,
     "start_time": "2024-03-10T13:07:21.949624",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "gemma_lm.preprocessor.sequence_length = 314\n",
    "# Use AdamW (a common optimizer for transformer models).\n",
    "optimizer = keras.optimizers.AdamW(\n",
    "    learning_rate=5e-5,\n",
    "    weight_decay=0.01,\n",
    "    beta_1=0.9,          # Adjust beta_1 parameter\n",
    "    beta_2=0.999         # Adjust beta_2 parameter\n",
    ")\n",
    "# Exclude layernorm and bias terms from decay.\n",
    "optimizer.exclude_from_weight_decay(var_names=[\"bias\", \"scale\"])\n",
    "\n",
    "gemma_lm.compile(\n",
    "    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "    optimizer=optimizer,\n",
    "    weighted_metrics=[keras.metrics.SparseCategoricalAccuracy()],)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fe7181bb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T13:07:22.060791Z",
     "iopub.status.busy": "2024-03-10T13:07:22.060425Z",
     "iopub.status.idle": "2024-03-10T18:03:19.497495Z",
     "shell.execute_reply": "2024-03-10T18:03:19.496661Z"
    },
    "papermill": {
     "duration": 17757.447005,
     "end_time": "2024-03-10T18:03:19.499411",
     "exception": false,
     "start_time": "2024-03-10T13:07:22.052406",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "\u001b[1m10000/10000\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8892s\u001b[0m 887ms/step - loss: 2.5929 - sparse_categorical_accuracy: 0.4488\n",
      "Epoch 2/2\n",
      "\u001b[1m10000/10000\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8861s\u001b[0m 886ms/step - loss: 2.4588 - sparse_categorical_accuracy: 0.4651\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x7d97946af310>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gemma_lm.fit(dataset, epochs=2, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7df89944",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T18:03:22.879552Z",
     "iopub.status.busy": "2024-03-10T18:03:22.878863Z",
     "iopub.status.idle": "2024-03-10T18:04:38.097408Z",
     "shell.execute_reply": "2024-03-10T18:04:38.087136Z"
    },
    "papermill": {
     "duration": 76.850649,
     "end_time": "2024-03-10T18:04:38.103735",
     "exception": false,
     "start_time": "2024-03-10T18:03:21.253086",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "gemma_lm.save(\"version_finetuned.keras\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4788993a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-10T18:04:41.562706Z",
     "iopub.status.busy": "2024-03-10T18:04:41.561775Z",
     "iopub.status.idle": "2024-03-10T18:05:03.369206Z",
     "shell.execute_reply": "2024-03-10T18:05:03.368328Z"
    },
    "papermill": {
     "duration": 23.546093,
     "end_time": "2024-03-10T18:05:03.375215",
     "exception": false,
     "start_time": "2024-03-10T18:04:39.829122",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instruction:\n",
      "How to code in python and c#\n",
      "\n",
      "Response:\n",
      "How to code in python and c#\n",
      "\n",
      "How to use Python for your web development tasks and C# for your mobile development tasks\n",
      "\n",
      "Photo by David Castillo on Unsplash\n",
      "\n",
      "As a developer, you can work on both backend and front-end. If you want to work in web development, you need the right tools and libraries to do it.\n",
      "\n",
      "Python is the most popular Python framework for web development. It is easy to learn and it offers a lot of tools to build a web application.\n",
      "\n",
      "If you want to work with a mobile device, you can work on it with C# as the language. It is a powerful language that can be used for both mobile and web development.\n",
      "\n",
      "In this article, I will show you how to code in python and c#.\n",
      "\n",
      "What is Python?\n",
      "\n",
      "Python is a programming language that is used for web development. It is easy to learn and it can be used for both front-end and back-end tasks.\n",
      "\n",
      "Python can be used to create websites. It is also used to write scripts that can be used in the command prompt to automate tasks.\n",
      "\n",
      "Python also has libraries that can be used by other programming languages.\n",
      "\n",
      "Why Python?\n",
      "\n",
      "Python is a popular programming language for web development. It has lots of libraries and tools that can be used for web development.\n",
      "\n",
      "It is also easy to learn and it is easy to use.\n",
      "\n",
      "Why C#?\n",
      "\n",
      "C# is a programming language that is used for mobile development\n"
     ]
    }
   ],
   "source": [
    "instruction=\"How to code in python and c#\"\n",
    "response=\"\"\n",
    "prompt = f\"Instruction:\\n{instruction}\\n\\nResponse:\\n{response}\"\n",
    "print(gemma_lm.generate(prompt, max_length=1024))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4be5ab8e",
   "metadata": {
    "papermill": {
     "duration": 1.722016,
     "end_time": "2024-03-10T18:05:06.720456",
     "exception": false,
     "start_time": "2024-03-10T18:05:04.998440",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "datasetId": 4567716,
     "sourceId": 7800873,
     "sourceType": "datasetVersion"
    },
    {
     "modelInstanceId": 5388,
     "sourceId": 11372,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30664,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 17979.874229,
   "end_time": "2024-03-10T18:05:12.817382",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-03-10T13:05:32.943153",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}