Initial One Step Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import common\n",
    "versePairs = common.loadTrainingData()\n",
    "X, Y = common.cleanAndSplitVerses(versePairs)\n",
    "Xenc, Yenc = common.encXandY(X, Y)\n",
    "# Use this to check that regexClean and regexUnclean are perfect inverses on the training data.\n",
    "# If the result is non empty you probably need to clean up the corresponding lines of your training set.\n",
    "assert len([v for v in versePairs if common.regexUnclean(common.regexClean(v)) != v]) == 0\n",
    "maxlen = 100\n",
    "Xnp, Ynp = common.padXandY(Xenc, Yenc, maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make all the training sequences the same length.\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "Xnp = pad_sequences(Xenc)\n",
    "Ynp = pad_sequences(Yenc, value=[1,0,0,0])\n",
    "\n",
    "# Define the model. Uncomment this if you don't have the saved model available.\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import LSTM, GRU, Dense, TimeDistributed, Bidirectional, Input, Embedding\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.layers.core import Dropout\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Bidirectional(GRU(256, return_sequences=True, dropout=0.25), input_shape=(191, 31)))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(TimeDistributed(Dense(4, activation='sigmoid', use_bias=False)))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', sample_weight_mode='temporal', metrics=['categorical_accuracy'])\n",
    "\n",
    "#from keras.models import load_model\n",
    "#model = load_model(\"saved_1step_model\")\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Generate a weighting for the different characters. We want to penalise the model according to how rare a symbol is\n",
    "# so that rare symbols are more important to place correctly than common ones.\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "Yclass = np.argmax(Ynp, axis=2)\n",
    "chars = Counter(\"\".join(Y))\n",
    "freq = chars.values()\n",
    "total = np.sum(list(freq))\n",
    "toReplace = {}\n",
    "toReplace[0] = total/chars[\"0\"]\n",
    "toReplace[1] = total/chars[\"|\"]\n",
    "toReplace[2] = total/chars[\"·\"]\n",
    "toReplace[3] = total/chars[\"*\"]\n",
    "def replace(clas):\n",
    "    return toReplace[clas]\n",
    "sample_weight = np.vectorize(replace)(Yclass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A callback to display a particular verse after each epoch.\n",
    "from keras.callbacks import Callback\n",
    "class ShowVerse(Callback):\n",
    "    def __init__(self, verse=0):\n",
    "        self.verse = verse\n",
    "    def on_epoch_end(self, batch, logs={}):\n",
    "        pred = self.model.predict_classes(Xnp[self.verse:self.verse+1])\n",
    "        toComb = common.decClasses(pred[0])\n",
    "        print(common.regexUnclean(common.mergeStrings(X[self.verse], toComb)))\n",
    "        pred = self.model.predict(Xnp[self.verse:self.verse+1], batch_size=256)\n",
    "        toComb = common.getToComb(pred[0])\n",
    "        print(common.regexUnclean(common.mergeStrings(X[self.verse], toComb)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train the model for an amount of time. You can interrupt this process with Kernel-> Interrupt and when\n",
    "# you run this again training will continue from when you left off (unless you ran the define the model code again.)\n",
    "model.fit(Xnp, Ynp, epochs=10, batch_size=256, sample_weight=sample_weight, validation_split=0.1, callbacks=[])\n",
    "# Save the model\n",
    "model.save(\"saved_1step_model_gru\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = model.predict_classes(Xnp, batch_size=256)\n",
    "for i in range(pred.shape[0]):\n",
    "    toComb = common.decClasses(pred[i])\n",
    "    print(common.regexUnclean(common.mergeStrings(X[i], toComb)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = model.predict(Xnp, batch_size=256)\n",
    "for i in range(pred.shape[0]):\n",
    "    toComb = common.getToComb(pred[i])\n",
    "    print(common.regexUnclean(common.mergeStrings(X[i], toComb)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load and predict on some test data.\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "test = []\n",
    "with open(\"testCleaned.txt\", 'r', encoding=\"utf-8\") as file:\n",
    "    for line in file:\n",
    "        test.append(line[:-1])\n",
    "testEnc = [common.encString(l) for l in test]\n",
    "testNp = pad_sequences(testEnc, maxlen=191)\n",
    "testPred = model.predict_classes(testNp, batch_size=256)\n",
    "for i in range(len(test)):\n",
    "    toComb = common.decClasses(testPred[i])\n",
    "    print(common.regexUnclean(common.mergeStrings(test[i], toComb)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}