main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You are all resolved rather to die than to famish?\n",
      "\n",
      "All:\n",
      "Resolved. resolved.\n",
      "\n",
      "First Citizen:\n",
      "First, you know Caius Marcius is chief enemy to the people.\n",
      "\n",
      "All:\n",
      "We know't, we know't.\n",
      "\n",
      "First Citizen:\n",
      "Let us kill him, and we'll have corn at our own price.\n",
      "Is't a verdict?\n",
      "\n",
      "All:\n",
      "No more talking on't; let it be done: away, away!\n",
      "\n",
      "Second Citizen:\n",
      "One word, good citizens.\n",
      "\n",
      "First Citizen:\n",
      "We are accounted poor citizens, the patricians good.\n",
      "What authority surfeits on would relieve us: if they\n",
      "would yield us but the superfluity, while it were\n",
      "wholesome, we might guess they relieved us humanely;\n",
      "but they think we are too dear: the leanness that\n",
      "afflicts us, the object of our misery, is as an\n",
      "inventory to particularise their abundance; our\n",
      "sufferance is a gain to them Let us revenge this with\n",
      "our pikes, ere we become rakes: for the gods know I\n",
      "speak this in hunger for bread, not in thirst for revenge.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "#load dataset\n",
    "with open('input.txt', 'r') as f:\n",
    "    text = f.read()\n",
    "\n",
    "print(text[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1115394\n"
     ]
    }
   ],
   "source": [
    "print(len(text)) #dataset length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
      "65\n"
     ]
    }
   ],
   "source": [
    "#identify vocabulary\n",
    "chars = sorted(list(set(text)))\n",
    "vocab_size = len(chars)\n",
    "print(''.join(chars))\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create mapping of characters to integers and vice versa\n",
    "stoi = dict((c, i) for i, c in enumerate(chars)) #string to int\n",
    "itos = dict((i, c) for i, c in enumerate(chars)) #int to string\n",
    "encode = lambda s: [stoi[c] for c in s] #encode string as list of ints\n",
    "decode = lambda e: ''.join([itos[i] for i in e]) #decode list of ints to string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1115394]) torch.int64\n",
      "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 14, 43, 44,\n",
      "        53, 56, 43,  1, 61, 43,  1, 54, 56, 53, 41, 43, 43, 42,  1, 39, 52, 63,\n",
      "         1, 44, 59, 56, 58, 46, 43, 56,  6,  1, 46, 43, 39, 56,  1, 51, 43,  1,\n",
      "        57, 54, 43, 39, 49,  8,  0,  0, 13, 50, 50, 10,  0, 31, 54, 43, 39, 49,\n",
      "         6,  1, 57, 54, 43, 39, 49,  8,  0,  0, 18, 47, 56, 57, 58,  1, 15, 47,\n",
      "        58, 47, 64, 43, 52, 10,  0, 37, 53, 59])\n"
     ]
    }
   ],
   "source": [
    "#torch initialization\n",
    "import torch\n",
    "data = torch.tensor(encode(text), dtype=torch.long)\n",
    "print(data.shape, data.dtype)\n",
    "print(data[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1003854 111540\n"
     ]
    }
   ],
   "source": [
    "train_data, test_data = data[:int(0.9*len(data))], data[int(0.9*len(data)):] #90% train, 10% test\n",
    "print(len(train_data), len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block_size = 8\n",
    "train_data[:block_size+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for input tensor([18]) target is 47\n",
      "for input tensor([18, 47]) target is 56\n",
      "for input tensor([18, 47, 56]) target is 57\n",
      "for input tensor([18, 47, 56, 57]) target is 58\n",
      "for input tensor([18, 47, 56, 57, 58]) target is 1\n",
      "for input tensor([18, 47, 56, 57, 58,  1]) target is 15\n",
      "for input tensor([18, 47, 56, 57, 58,  1, 15]) target is 47\n",
      "for input tensor([18, 47, 56, 57, 58,  1, 15, 47]) target is 58\n"
     ]
    }
   ],
   "source": [
    "x = train_data[:block_size]\n",
    "y = train_data[1:block_size+1] #shifted by 1\n",
    "for t in range(block_size):\n",
    "    context = x[:t+1] #models goal is to predict next character given previous characters\n",
    "    target = y[t] #target is next character\n",
    "    print(f'for input {context} target is {target}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1, 51, 63,  1, 51, 39, 57, 58],\n",
      "        [42,  1, 58, 59, 56, 52,  1, 58],\n",
      "        [60, 47, 52, 43, 63, 39, 56, 42],\n",
      "        [ 0,  0, 25, 13, 30, 21, 13, 26]])\n",
      "tensor([[51, 63,  1, 51, 39, 57, 58, 43],\n",
      "        [ 1, 58, 59, 56, 52,  1, 58, 46],\n",
      "        [47, 52, 43, 63, 39, 56, 42,  1],\n",
      "        [ 0, 25, 13, 30, 21, 13, 26, 13]])\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(1)\n",
    "batch_size = 4 #number of sequences in a mini-batch\n",
    "block_size = 8 #numbers of characters in a sequence\n",
    "\n",
    "def get_batch(split):\n",
    "    '''create a small batch of data from the text corpus'''\n",
    "    data = train_data if split == 'train' else test_data\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,)) #get random starting indices for sequences\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix]) #convert to 2D tensor\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix]) #shifted by 1\n",
    "    return x,y\n",
    "\n",
    "x,y = get_batch('train')\n",
    "print(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "torch.manual_seed(1)\n",
    "\n",
    "class BigramLanguageModel(nn.Module):\n",
    "    \n",
    "\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        #each token reads off logits for next token from lookup table\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n",
    "\n",
    "\n",
    "    def forward(self, idx, targets=None):\n",
    "        '''\n",
    "        idx: (B,T) tensor of ints\n",
    "        targets: (B,T) tensor of ints\n",
    "        '''\n",
    "        logits = self.token_embedding_table(idx) #(B,T,C)\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C)\n",
    "            targets = targets.view(B*T) \n",
    "            loss = F.cross_entropy(logits, targets) #classification loss\n",
    "        return logits, loss\n",
    "    \n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        '''\n",
    "        idx: (B,T) array of indices in the current context\n",
    "        '''\n",
    "        for _ in range(max_new_tokens):\n",
    "            logits, loss = self(idx) #get predictions\n",
    "            logits = logits[:, -1, :] #focus on last timestep only. result is (B,C) tensor.\n",
    "            probs = F.softmax(logits, dim=-1) #get probabilities for each class in vocabulary\n",
    "            idx_next = torch.multinomial(probs, num_samples=1) #sample from distribution. result (B,1) tensor.\n",
    "            idx = torch.cat([idx, idx_next], dim=-1) #append to context\n",
    "        return idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 65]) tensor(4.7781, grad_fn=<NllLossBackward>)\n",
      "\n",
      "-e$PyhE&YW$Hd.Z;mA:\n",
      "3ANF!DG:'gIX,bHZ'yadMLRffLKKDQry3MvpSEcrHEkHapAD&nHNXlNYNcZNJVQ:AdIjZ;jJFz$hSY&C\n"
     ]
    }
   ],
   "source": [
    "m = BigramLanguageModel(vocab_size)\n",
    "logits, loss = m(x, y)\n",
    "print(logits.shape, loss)\n",
    "generated_text = m.generate(idx=torch.zeros((1,1), dtype=torch.long), max_new_tokens=100)[0].tolist()\n",
    "print(decode(generated_text))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create optimizer\n",
    "optimizer = torch.optim.AdamW(m.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.6214325428009033\n"
     ]
    }
   ],
   "source": [
    "batch_size=32\n",
    "for step in range(5000):\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    x,y = get_batch('train')\n",
    "    logits, loss = m(x,y)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Til tXZ? r st s thiky,\n",
      "\n",
      "Hean w:\n",
      "FYor he bd P, sDour w'HERue?\n",
      "DHaf YORorgritrepyowandu bquranglerexco t llDUColi3waswhenordk jor, bj! Sisht.\n",
      "G me:\n",
      "YXll;-Vaselp Lapo shvelorXbusengaryK:\n",
      "Kil c,\n",
      "Heyoutrdoure dxtstxVithuWAn rud:\n",
      "ThBur-\n",
      "rousoouerd S: 'dren be. '\n",
      "Ourin s m,d tant.T?\n",
      "TUzitinger'Whal:\n",
      "Me havUMThe patRO:z, heardpudUD$XLAn herotWhe oo; Uve indXbey jutheayroremyopfouimaipis s tyeIAnth?\n",
      "STURULowe. juCHUKA be we fStle mousods IFritnor hithomer dPSYont Par beste sis:\n",
      "TreBew--thaIwo f bucatheFo\n"
     ]
    }
   ],
   "source": [
    "print(decode(m.generate(idx = torch.zeros((1, 1), dtype=torch.long), max_new_tokens=500)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "torch.manual_seed(1)\n",
    "\n",
    "class LSTMLanguageModel(nn.Module):\n",
    "    '''simple LSTM language model that predicts the next character.'''\n",
    "\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n",
    "        self.lstm = nn.LSTM(vocab_size, 128, batch_first=True, num_layers=2)\n",
    "        self.linear = nn.Linear(128, vocab_size)\n",
    "    \n",
    "    def forward(self, idx, targets=None):\n",
    "        '''\n",
    "        idx: (B,T) tensor of ints\n",
    "        targets: (B,T) tensor of ints\n",
    "        '''\n",
    "        x = self.token_embedding_table(idx)\n",
    "        x, _ = self.lstm(x)\n",
    "        logits = self.linear(x)\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C)\n",
    "            targets = targets.view(B*T) \n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "        return logits, loss\n",
    "    \n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        '''\n",
    "        idx: (B,T) array of indices in the current context\n",
    "        '''\n",
    "        for _ in range(max_new_tokens):\n",
    "            logits, _ = self(idx) #get predictions\n",
    "            logits = logits[:, -1, :] #focus on last timestep only. result is (B,C) tensor.\n",
    "            probs = F.softmax(logits, dim=-1) #get probabilities for each class in vocabulary\n",
    "            idx_next = torch.multinomial(probs, num_samples=1) #sample from distribution. result (B,1) tensor.\n",
    "            idx = torch.cat([idx, idx_next], dim=-1) #append to context\n",
    "        return idx\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm = LSTMLanguageModel(vocab_size)\n",
    "\n",
    "#create optimizer\n",
    "lstm_optimizer = torch.optim.AdamW(lstm.parameters(), lr=1e-3)\n",
    "batch_size=32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7586300373077393\n"
     ]
    }
   ],
   "source": [
    "#train\n",
    "for step in range(5000):\n",
    "    lstm_optimizer.zero_grad(set_to_none=True)\n",
    "    x,y = get_batch('train')\n",
    "    logits, loss = lstm(x,y)\n",
    "    loss.backward()\n",
    "    lstm_optimizer.step()\n",
    "print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "LUCIO:\n",
      "I fay Honse: your may, prowsly.\n",
      "\n",
      "MENENIUS:\n",
      "In than yours deputing this not me; Tork.\n",
      "Vorselder years, and gley,\n",
      "Cricks, for your only: hear\n",
      "Marrined, the did, are the kisnates and friends.\n",
      "Doir fames. The grove a lie on your delisnemand, your chird\n",
      "By othere, though art prespermaties in my more no friervatience, for scratter'd not trumble of him! well art trunked.\n",
      "\n",
      "LUCIO:\n",
      "Did I senful there chast an Like.\n",
      "\n",
      "MENENIUS:\n",
      "Whithie Those croves, you knot for flem trutchumently Trusiname?\n",
      "\n",
      "QUEEN'\n"
     ]
    }
   ],
   "source": [
    "print(decode(lstm.generate(idx = torch.zeros((1, 1), dtype=torch.long), max_new_tokens=500)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "av",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}