Model_Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "$('.nbp-app-bar').toggle()"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%javascript\n",
    "$('.nbp-app-bar').toggle()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/evankranzler/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import json\n",
    "import random\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import itertools as it\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Reshape, Masking\n",
    "from keras.layers import Convolution2D, Convolution1D, MaxPooling2D, MaxPooling1D, LSTM, TimeDistributed\n",
    "from keras.losses import categorical_crossentropy\n",
    "from keras.utils import np_utils\n",
    "from keras.datasets import mnist\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subsets(iterable, minimum=0):\n",
    "    for i in range(minimum, len(iterable) + 1):\n",
    "        for x in it.combinations(iterable, i):\n",
    "            yield x\n",
    "\n",
    "\n",
    "def one_in_another(s1, s2):\n",
    "    if s1 == s2:\n",
    "        return True\n",
    "    if len(s1) > len(s2):\n",
    "        return one_in_another(s2, s1)\n",
    "    for i in range(len(s2) + 1):\n",
    "        for j in range(i):\n",
    "            if s2.replace(s2[j:i], '') == s1:\n",
    "                return True\n",
    "    return False\n",
    "\n",
    "\n",
    "class Chord(object):\n",
    "    def __init__(self, root=None, intervals=set()):\n",
    "        self.root = root\n",
    "        self.intervals = intervals\n",
    "        if root is not None:\n",
    "            self.intervals.add(root)\n",
    "\n",
    "    def __str__(self):\n",
    "        return '(root: ' + str(self.root) + ' intervals: ' + ', '.join([str(x) for x in self.intervals]) + ')'\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.__str__()\n",
    "\n",
    "    def __eq__(self, other):\n",
    "        return self.root == other.root and self.intervals == other.intervals\n",
    "\n",
    "    def transpose(self, half_steps=1):\n",
    "        if self.root == None:\n",
    "            return self\n",
    "        return Chord((self.root + half_steps) % 12, set((i + half_steps) % 12 for i in self.intervals))\n",
    "\n",
    "    def normalize(self):\n",
    "        if self.root is None:\n",
    "            return self\n",
    "        return self.transpose(-1 * self.root)\n",
    "\n",
    "    def add_note(self, note):\n",
    "        return Chord(self.root, self.intervals.union([note % 12]))\n",
    "\n",
    "    def signature(self):\n",
    "        return [1 * (i == self.root) for i in range(12)] + [1 * (i in self.intervals) for i in range(12)]\n",
    "\n",
    "    @staticmethod\n",
    "    def all_chords():\n",
    "        for x in subsets(range(12), minimum=3):\n",
    "            temp = set(x)\n",
    "            for i in temp:\n",
    "                yield Chord(i, temp)\n",
    "\n",
    "\n",
    "notes = [s + q for q in 'b#' for s in 'ABCDEFG'] + [s for s in 'ABCDEFG']\n",
    "\n",
    "\n",
    "def isolate_stem(chord_word):\n",
    "    if 'Bass' in chord_word:\n",
    "        return 'Bass'\n",
    "    if '\\\\' in chord_word:\n",
    "        return\n",
    "    elif '/' in chord_word:\n",
    "        temp = chord_word.split('/')[0]\n",
    "        for n in notes:\n",
    "            if n in temp:\n",
    "                return temp.replace(n, '')\n",
    "    else:\n",
    "        for n in notes:\n",
    "            if n in chord_word:\n",
    "                return chord_word.replace(n, '')\n",
    "\n",
    "\n",
    "def stematize(chord):\n",
    "    for note in notes:\n",
    "        if note in chord:\n",
    "            return stematize(chord.replace(note, str(note_to_num(note))))\n",
    "    return chord\n",
    "\n",
    "\n",
    "def note_to_num(note):\n",
    "    notedict = {'A': 9, 'B': 11, 'C': 0, 'D': 2, 'E': 4, 'F': 5, 'G': 7}\n",
    "    try:\n",
    "        number = notedict[note[0]]\n",
    "    except:\n",
    "        number = 0\n",
    "    if '#' in note:\n",
    "        number = (number + 1) % 12\n",
    "    if 'b' in note:\n",
    "        number = (number - 1) % 12\n",
    "    return number\n",
    "\n",
    "\n",
    "note_cycle = dict()\n",
    "for note in notes:\n",
    "    temp = []\n",
    "    n = note_to_num(note)\n",
    "    for note2 in notes:\n",
    "        if (note_to_num(note2) - 1) % 12 == n:\n",
    "            temp += [note2]\n",
    "    note_cycle[note] = temp\n",
    "\n",
    "\n",
    "def stretch_chords(chords, length):\n",
    "    c_len = len(chords)\n",
    "    if c_len == 0:\n",
    "        return []\n",
    "    if c_len == length:\n",
    "        return chords\n",
    "    if length % c_len == 0:\n",
    "        m = length // c_len\n",
    "        out = []\n",
    "        for chord in chords:\n",
    "            out += [chord] * m\n",
    "        return out\n",
    "    return stretch_chords(chords + ['/'], length)\n",
    "\n",
    "\n",
    "def process_leadsheets():\n",
    "    sheet_dict = dict()\n",
    "    for path in ['./the-imaginary-book-part-1-A-M/', './the-imaginary-book-part-2-N-Z/']:\n",
    "        for fn in os.listdir(path=path):\n",
    "            if '.ls' in fn:\n",
    "                temp = []\n",
    "                with open(path + fn) as file:\n",
    "                    for line in file.readlines():\n",
    "                        if 'meter' in line:\n",
    "                            meter = line\n",
    "                            meter = meter.replace(')', '').replace('(meter', '')\n",
    "                            meter = meter.split()\n",
    "                            meter = int(meter[0])\n",
    "                        if '|' in line:\n",
    "                            temp += re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\", line.replace('\\n', '')).replace(')', '').split('|')\n",
    "                temp = [x.split() for x in temp]\n",
    "                temp = [x for x in temp if x != []]\n",
    "                while True:\n",
    "                    loop = False\n",
    "                    for t in temp:\n",
    "                        if len(t) > meter:\n",
    "                            meter *= 2\n",
    "                            loop = True\n",
    "                            break\n",
    "                    if not loop:\n",
    "                        break\n",
    "                temp = [stretch_chords(x, meter) for x in temp]\n",
    "                temp = sum(temp, [])\n",
    "                temp2 = []\n",
    "                for i, t in enumerate(temp):\n",
    "                    if t == '/':\n",
    "                        temp2 += [temp2[i - 1]]\n",
    "                    else:\n",
    "                        temp2 += [t]\n",
    "                temp = ' '.join(temp2)\n",
    "                sheet_dict[fn.replace('.ls', '')] = temp\n",
    "    with open('sheets.json', 'w') as file:\n",
    "        json.dump(sheet_dict, file)\n",
    "\n",
    "\n",
    "new_chord_defs = (\n",
    "    'C.............. same as CM\\nCM............. C major (c e g)\\nC2............. same as CMadd9\\nC5............. C five (c g)\\nC6............. same as CM6\\nC69............ same as CM69\\nC6#11.......... same as CM6#11\\nC69#11......... same as CM69#11\\nC6b5........... same as CM6#11\\nCM13........... C major thirteen (c e g b d a)\\nCM13#11........ C major thirteen sharp eleven (c e g b d f# a)\\nCmaj13......... same as CM13\\nCMaj13......... same as CM13\\nCmaj13#11...... same as CM13#11\\nCMaj13#11...... same as CM13#11\\nCM6............ C major six (c e g a)\\nCM6#11......... C major six sharp eleven (c e g a f#)\\nCM6b5.......... same as CM6#11\\nCM69#11........ C major six nine sharp eleven (c e g a d f#)\\nCM69........... C major six nine (c e g a d)\\nCM7#11......... C major seven sharp eleven (c e g b f#)\\nCM7............ C major seven (c e g b)\\nCmaj7.......... same as CM7\\nCMaj7.......... same as CM7\\nCmaj7#11....... same as CM7#11\\nCMaj7#11....... same as CM7#11\\nCM7add13....... C major seven add 13 (c e g a b d)\\nCM7b5.......... C major seven flat five (c e gb b)\\nCM7b6.......... C major seven flat six (c e g ab b)\\nCM7b9.......... C major seven flat nine (c e g b db)\\nCM9............ C major nine (c e g b d)\\nCM9#11......... C major nine sharp eleven (c e g b d f#)\\nCmaj9.......... same as CM9\\nCMaj9.......... same as CM9\\nCmaj9#11....... same as CM9#11\\nCMaj9#11....... same as CM9#11\\nCM9b5.......... C major nine flat five (c e gb b d)\\nCMadd9......... C major add nine (c e g d)\\nCMb5........... C major flat five (c e gb)\\nCMb6........... C major flat six (c e ab)\\nCadd2.......... same as CMadd9\\nCadd9.......... same as CMadd9\\nCadd9no3....... same as CMsus2\\n\\nMinor Chords\\n\\nCm#5........... C minor sharp five (c eb g#)\\nCm+............ same as Cm#5\\nCm............. C minor (c eb g)\\nCm11#5......... C minor eleven sharp five (c eb ab bb d f)\\nCm11........... C minor eleven (c eb g bb d f)\\nCm11b5......... C minor eleven flat five (c eb bb gb d f)\\nCm13........... C minor thirteen (c eb g bb d f a)\\nCm6............ C minor six (c eb g a)\\nCm69........... C minor six nine (c eb g a d)\\nCm7#5.......... C minor seven sharp five (c eb ab bb)\\nCm7............ C minor seven (c eb g bb)\\nCm7b5.......... C minor seven flat five (c eb gb bb)\\nCh............. same as Cm7b5 (h for \\\"half-diminished\\\")\\nCm9#5.......... C minor nine sharp five (c eb ab bb d)\\nCm9............ C minor nine (c eb g bb d)\\nCm9b5.......... C minor nine flat five (c eb bb gb d)\\nCmM7........... C minor major seven (c eb g b)\\nCmM7b6......... C minor major seven flat six (c eb g ab b)\\nCmM9........... C minor major nine (c eb g b d)\\nCmadd9......... C minor add nine (c eb g d)\\nCmb6........... C minor flat six (c eb ab)\\nCmb6M7......... C minor flat six major 7 (c eb ab b)\\nCmb6b9......... C minor flat six flat nine (c eb ab db)\\n\\nDiminished Chords\\n\\nCdim........... C diminished triad (c eb gb)\\nCo............. same as Cdim\\nCdim7.......... C diminished seventh (c eb gb a)\\nCo7............ same as Cdim7\\nCoM7........... C diminished major seventh (c eb gb b)\\nCo7M7.......... C diminished seventh major seventh (c eb gb a b)\\n\\nAugmented Chords\\n\\nCM#5........... C major sharp five (c e g#)\\nC+............. same as CM#5\\nCaug........... same as CM#5\\nC+7............ same as C7#5\\nCM#5add9....... C major sharp five add 9 (c e g# d)\\nCM7#5.......... C major seven sharp five (c e g# b)\\nCM7+........... same as CM7#5\\nCM9#5.......... C major nine sharp five (c e g# b d)\\nC+add9......... same as CM#5add9\\n\\nDominant Chords\\n\\nC7............. C seven (c e g bb)\\nC7#5........... C seven sharp five (c e g# bb)\\nC7+............ same as C7#5\\nCaug7.......... same as C7#5\\nC7aug.......... same as C7#5\\nC7#5#9......... C seven sharp five sharp nine (c e g# bb d#)\\nC7alt.......... same as C7#5#9\\nC7b13.......... C seven flat thirteen (c e g bb ab)\\nC7b5#9......... same as C7#9#11\\nC7b5........... C seven flat five (c e gb bb)\\nC7b5b13........ same as C7#11b13\\nC7b5b9......... same as C7b9#11\\nC7b5b9b13...... same as C7b9#11b13\\nC7b6........... C seven flat six (c e g ab bb)\\nC7b9#11........ C seven flat nine sharp eleven (c e g bb db f#)\\nC7b9#11b13..... C seven flat nine sharp eleven flat thirteen (c e g bb db f# ab)\\nC7b9........... C seven flat nine (c e g bb db)\\nC7b9b13#11..... C seven flat nine flat thirteen sharp eleven (c e g bb db f# ab)\\nC7b9b13........ C seven flat nine flat thirteen (c e g bb db ab)\\nC7no5.......... C seven no five (c e bb)\\nC7#11.......... C seven sharp eleven (c e g bb f#)\\nC7#11b13....... C seven sharp eleven flat thirteen (c e g bb f# ab)\\nC7#5b9#11...... C seven sharp five flat nine sharp 11 (c e g# bb db f#)\\nC7#5b9......... C seven sharp five flat nine (c e g# bb db)\\nC7#9#11........ C seven sharp nine sharp eleven (c e g bb d# f#)\\nC7#9#11b13..... C seven sharp nine sharp eleven flat thirteen (c e g bb d# f# ab)\\nC7#9........... C seven sharp nine (c e g bb d#)\\nC7#9b13........ C seven sharp nine flat thirteen (c e g bb d# ab)\\n\\nC9............. C nine (c e g bb d)\\nC9#5........... C nine sharp five (c e g# bb d)\\nC9+............ same as C9#5\\nC9#11.......... C nine sharp eleven (c e g bb d f#)\\nC9#11b13....... C nine sharp eleven flat thirteen (c e g bb d f# ab)\\nC9#5#11........ C nine sharp five sharp eleven (c e g# bb d f#)\\nC9b13.......... C nine flat thirteen (c e g bb d ab)\\nC9b5........... C nine flat five (c e gb bb d)\\nC9b5b13........ same as C9#11b13\\nC9no5.......... C nine no five (c e bb d)\\n\\nC13#11......... C thirteen sharp eleven (c e g bb d f# a)\\nC13#9#11....... C thirteen sharp nine sharp eleven (c e g bb d# f# a)\\nC13#9.......... C thirteen sharp nine (c e g bb d# a)\\nC13............ C thirteen (c e g bb d a)\\nC13b5.......... C thirteen flat five (c e gb a bb)\\nC13b9#11....... C thirteen flat nine sharp eleven (c e g bb db f# a)\\nC13b9.......... C thirteen flat nine (c e g bb db a)\\n\\nSuspensions\\n\\nCMsus2......... C major sus two (c d g)\\nCMsus4......... C major sus four (c f g)\\nCsus2.......... same as CMsus2\\nCsus24......... C sus two four (c d f g)\\nCsus4.......... same as CMsus4\\nCsus4add9...... same as Csus24\\nCsusb9......... C sus flat nine (c db f g)\\nC4............. C four (c f bb eb)\\nCquartal....... same as C4\\nC7b9b13sus4.... same as C7sus4b9b13\\nC7b9sus........ same as C7susb9\\nC7b9sus4....... same as C7sus4b9\\nC7b9sus4....... same as C7susb9\\nC7sus.......... same as C7sus4\\nC7sus4......... C seven sus four (c f g bb)\\nC7sus4b9....... C seven sus four flat nine (c f g bb db)\\nC7sus4b9b13.... C seven sus four flat nine flat thirteen (c f g bb db ab)\\nC7susb9........ C seven sus flat nine (c db f g bb)\\nC9sus4......... C nine sus four (c f g bb d)\\nC9sus.......... same as C9sus4\\nC11............ C eleven (c e g bb d f)\\nC13sus......... same as C13sus4\\nC13sus4........ C thirteen sus four (c f g bb d a)\\n\\nMiscellaneous\\n\\nCBlues......... C Blues (c eb f gb g bb) (Use upper case to avoid confusion with Cb = C flat)\\nCBass.......... C Bass (c) (Use upper case to avoid confusion with Cb = C flat)\\n')\n",
    "chord_defs = [[y for y in x.split('.') if y != ''] for x in new_chord_defs.split('\\n') if '.' in x]\n",
    "equivs = [[x[0][1:], x[1].replace(' same as ', '').replace(' (h for \"half-diminished\")', '')[1:]] for x in chord_defs if\n",
    "          ' same as ' in x[1]]\n",
    "equivs += [['mMaj7', 'mM7'], ['h7', 'm7b5'], ['mb5', 'dim']]\n",
    "\n",
    "\n",
    "def make_into_intervals(chord_def):\n",
    "    temp = re.findall('\\(([^\\)]+)\\)', chord_def[1])[0]\n",
    "    temp = [chord_def[0][1:], [note_to_num(x.capitalize()) for x in temp.split()]]\n",
    "    return [temp[0], [1 * (i in temp[1]) for i in range(12)]]\n",
    "\n",
    "\n",
    "equiv_dict = dict([make_into_intervals(x) for x in chord_defs if ' same as ' not in x[1]])\n",
    "equiv_dict = {**equiv_dict, **dict([[x[0], equiv_dict[x[1]]] for x in equivs])}\n",
    "equiv_dict['NC'] = [0] * 12\n",
    "\n",
    "\n",
    "def process_chord(chord):\n",
    "    if 'NC' in chord:\n",
    "        return Chord()\n",
    "    if 'Bass' in chord:\n",
    "        for note in notes:\n",
    "            if chord[:len(note)] == note:\n",
    "                break\n",
    "        return Chord(note_to_num(note), {note_to_num(note)})\n",
    "    if '\\\\' in chord:\n",
    "        [poly, rest] = chord.split('\\\\')\n",
    "        poly = process_chord(poly)\n",
    "    else:\n",
    "        poly = None\n",
    "        rest = chord\n",
    "    if '/' in chord:\n",
    "        [shape, root] = rest.split('/')\n",
    "    else:\n",
    "        shape = rest\n",
    "        root = None\n",
    "    for note in notes:\n",
    "        if shape[:len(note)] == note:\n",
    "            break\n",
    "    stem = shape.replace(note, '')\n",
    "    chord_obj = Chord(0, set([i for i in range(12) if equiv_dict[stem][i]]))\n",
    "    chord_obj = chord_obj.transpose(note_to_num(note))\n",
    "    if root is not None:\n",
    "        chord_obj.root = note_to_num(root)\n",
    "    if poly is not None:\n",
    "        for n in poly.intervals:\n",
    "            chord_obj = chord_obj.add_note(n)\n",
    "    return chord_obj\n",
    "\n",
    "\n",
    "def process_leadsheet(sheet):\n",
    "    return [process_chord(x) for x in sheet.split()]\n",
    "\n",
    "\n",
    "def all_transpositions(sheet):\n",
    "    temp = process_leadsheet(sheet)\n",
    "    return [[chord.transpose(i) for chord in temp] for i in range(12)]\n",
    "\n",
    "\n",
    "def transpose_note(note):\n",
    "    return note_cycle[note][-1]\n",
    "\n",
    "\n",
    "def transpose_chord_symbol(chord, interval=1):\n",
    "    if interval == 0:\n",
    "        return chord\n",
    "    elif interval == 1:\n",
    "        if 'NC' in chord:\n",
    "            return 'NC'\n",
    "        if 'Bass' in chord:\n",
    "            for note in notes:\n",
    "                if chord[:len(note)] == note:\n",
    "                    break\n",
    "            return transpose_note(note) + 'Bass'\n",
    "        if '\\\\' in chord:\n",
    "            return '\\\\'.join([transpose_chord_symbol(x) for x in chord.split('\\\\')])\n",
    "        if '/' in chord:\n",
    "            return '/'.join([transpose_chord_symbol(x) for x in chord.split('/')])\n",
    "        for note in notes:\n",
    "            if chord[:len(note)] == note:\n",
    "                break\n",
    "        stem = chord.replace(note, '')\n",
    "        return transpose_note(note) + stem\n",
    "    elif 1 < interval < 12:\n",
    "        return transpose_chord_symbol(transpose_chord_symbol(chord, interval=interval - 1))\n",
    "    else:\n",
    "        return transpose_chord_symbol(chord, interval=(interval % 12))\n",
    "\n",
    "\n",
    "def seq_generator(training_data, seq_len):\n",
    "    i = 0\n",
    "    l = len(training_data)\n",
    "    while True:\n",
    "        if i + seq_len + 1 > l:\n",
    "            i = 0\n",
    "        yield training_data[i:i + seq_len]  # , training_data[i + 1:i + seq_len + 1]\n",
    "        i += 1\n",
    "\n",
    "\n",
    "def batch_generator(seq_gen, batch_size):\n",
    "    while True:\n",
    "        X_out = []\n",
    "        y_out = []\n",
    "        for i in range(batch_size + 1):\n",
    "            temp = next(seq_gen)\n",
    "            if i != batch_size:\n",
    "                X_out += [temp]\n",
    "            if i != 0:\n",
    "                y_out += [temp]\n",
    "        yield np.array(X_out), np.array(y_out)\n",
    "\n",
    "\n",
    "def strat_gen(data, batch_size, seq_len):\n",
    "    data_len = len(data)\n",
    "    data_slices = data_len // batch_size\n",
    "    staggers = [data[i * data_slices:] + data[:i * data_slices] for i in range(batch_size)]\n",
    "    staggers = [it.cycle(x) for x in staggers]\n",
    "    X = [[next(x) for _ in range(seq_len)] for x in staggers]\n",
    "    while True:\n",
    "        y = [x[1:] + [next(staggers[i])] for i, x in enumerate(X)]\n",
    "        yield np.array(X), np.array(y)\n",
    "        X = [x for x in y]\n",
    "\n",
    "def filename_string(dt):\n",
    "    return '_{}_{}_{}_{}'.format(dt.toordinal(), dt.hour, dt.minute, dt.second)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(seq_len, batches, dropout_value=0.5, is_stateful=False, LSTM_size=512, LSTM_count=3):\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(Masking(\n",
    "        batch_input_shape=(batches, seq_len, 24),\n",
    "        input_shape=(seq_len, 24)\n",
    "    ))\n",
    "\n",
    "    model.add(TimeDistributed(Dense(96, activation='relu')))\n",
    "    model.add(Dropout(dropout_value))\n",
    "\n",
    "    model.add(TimeDistributed(Dense(96, activation='relu')))\n",
    "    model.add(Dropout(dropout_value))\n",
    "\n",
    "    for _ in range(LSTM_count):\n",
    "        model.add(LSTM(\n",
    "            LSTM_size,\n",
    "            return_sequences=True,\n",
    "            recurrent_dropout=dropout_value,\n",
    "            dropout=dropout_value,\n",
    "            stateful=is_stateful,\n",
    "            implementation=2,\n",
    "#             unroll=True,\n",
    "#             batch_input_shape=(batches, seq_len, LSTM_size)\n",
    "        ))\n",
    "\n",
    "    model.add(TimeDistributed(Dense(24, activation='sigmoid')))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sheets loaded\n"
     ]
    }
   ],
   "source": [
    "with open('sheets.json') as file:\n",
    "    sheet_dict = json.load(file)\n",
    "print('Sheets loaded')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transpositions applied\n"
     ]
    }
   ],
   "source": [
    "every_sheet = []\n",
    "for sheet in sheet_dict.values():\n",
    "    every_sheet += all_transpositions(sheet)\n",
    "print('Transpositions applied')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data processed\n"
     ]
    }
   ],
   "source": [
    "training_data = [[x.signature() for x in sheet] for sheet in every_sheet]\n",
    "print('Training data processed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(training_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data ready for usage\n"
     ]
    }
   ],
   "source": [
    "out = []\n",
    "for x in training_data:\n",
    "    out += x\n",
    "    out += [24 * [0]] * 8\n",
    "print('Training data ready for usage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_not_stateful_2_epochs_736788_6_26_16\n",
      "model_not_stateful_1_epochs_736788_1_14_33\n",
      "model_not_stateful_3_epochs_736788_11_36_55\n"
     ]
    }
   ],
   "source": [
    "for name in os.listdir(path='./'):\n",
    "    if '736788' in name:\n",
    "        print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "masking_1 (Masking)          (128, 32, 24)             0         \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (128, 32, 48)             1200      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (128, 32, 48)             0         \n",
      "_________________________________________________________________\n",
      "time_distributed_2 (TimeDist (128, 32, 48)             2352      \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (128, 32, 48)             0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (128, 32, 512)            1148928   \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (128, 32, 512)            2099200   \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (128, 32, 512)            2099200   \n",
      "_________________________________________________________________\n",
      "time_distributed_3 (TimeDist (128, 32, 24)             12312     \n",
      "=================================================================\n",
      "Total params: 5,363,192\n",
      "Trainable params: 5,363,192\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = load_model('./model_not_stateful_1_epochs_736786_21_47_54')\n",
    "# model = load_model('./model_not_stateful_1_epochs_736788_1_14_33')\n",
    "# model = load_model('./model_not_stateful_2_epochs_736788_6_26_16')\n",
    "# model = load_model('./model_not_stateful_3_epochs_736788_11_36_55')\n",
    "# last_epoch = 5\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_epoch = 0\n",
    "epoch_stride = 1\n",
    "\n",
    "seq_len = 32\n",
    "is_stateful = False\n",
    "dropout_value = 0.5\n",
    "LSTM_size = 1024\n",
    "LSTM_count = 3\n",
    "if is_stateful:\n",
    "    batches = 1\n",
    "else:\n",
    "    batches = 4 * seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "masking_28 (Masking)         (128, 32, 24)             0         \n",
      "_________________________________________________________________\n",
      "time_distributed_70 (TimeDis (128, 32, 96)             2400      \n",
      "_________________________________________________________________\n",
      "dropout_57 (Dropout)         (128, 32, 96)             0         \n",
      "_________________________________________________________________\n",
      "time_distributed_71 (TimeDis (128, 32, 96)             9312      \n",
      "_________________________________________________________________\n",
      "dropout_58 (Dropout)         (128, 32, 96)             0         \n",
      "_________________________________________________________________\n",
      "lstm_44 (LSTM)               (128, 32, 1024)           4591616   \n",
      "_________________________________________________________________\n",
      "lstm_45 (LSTM)               (128, 32, 1024)           8392704   \n",
      "_________________________________________________________________\n",
      "lstm_46 (LSTM)               (128, 32, 1024)           8392704   \n",
      "_________________________________________________________________\n",
      "time_distributed_72 (TimeDis (128, 32, 24)             24600     \n",
      "=================================================================\n",
      "Total params: 21,413,336\n",
      "Trainable params: 21,413,336\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = build_model(\n",
    "    seq_len=seq_len,\n",
    "    batches=batches,\n",
    "    dropout_value=dropout_value,\n",
    "    is_stateful=is_stateful,\n",
    "    LSTM_size=LSTM_size,\n",
    "    LSTM_count=LSTM_count\n",
    ")\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if is_stateful:\n",
    "#     gen = strat_gen(out, batches, seq_len)\n",
    "# else:\n",
    "out = np.asarray(out)\n",
    "gen = batch_generator(seq_generator(out, seq_len), batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "  305/35159 [..............................] - ETA: 5:25:55 - loss: 0.4732 - acc: 0.7895"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-103-d8d7318077aa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_epoch\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mepoch_stride\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlast_epoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m )\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1274\u001b[0m                                         \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1275\u001b[0m                                         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1276\u001b[0;31m                                         initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1277\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1278\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   2222\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   2223\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2224\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   2225\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2226\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1881\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1882\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1883\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1884\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1885\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2476\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2478\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2479\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1137\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1355\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1356\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1359\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1361\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1362\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1338\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[0;32m-> 1340\u001b[0;31m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(\n",
    "    gen,\n",
    "    steps_per_epoch=(len(out) // batches),\n",
    "    epochs=(last_epoch + epoch_stride),\n",
    "    verbose=1,\n",
    "    initial_epoch=last_epoch\n",
    ")\n",
    "\n",
    "last_epoch += epoch_stride"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'model_'\n",
    "if is_stateful:\n",
    "    model_name += 'is_stateful_'\n",
    "else:\n",
    "    model_name += 'not_stateful_'\n",
    "\n",
    "right_now = datetime.datetime.now()\n",
    "\n",
    "model_name += str(last_epoch)\n",
    "model_name += '_epochs'\n",
    "model_name += filename_string(right_now)\n",
    "\n",
    "model.save(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [],
   "source": [
    "chords = set()\n",
    "for key in sheet_dict:\n",
    "    temp = sheet_dict[key].split()\n",
    "    for chord in temp:\n",
    "        chords.add(chord)\n",
    "\n",
    "chord_formats = set()\n",
    "for x in chords:\n",
    "    chord_formats.add(isolate_stem(x))\n",
    "\n",
    "chord_objs = dict()\n",
    "for chord in chords:\n",
    "    temp = process_chord(chord)\n",
    "    name = chord\n",
    "    for _ in range(12):\n",
    "        name = transpose_chord_symbol(name)\n",
    "        temp = temp.transpose()\n",
    "        chord_objs[name] = temp\n",
    "\n",
    "canon_sigs = [np.array(chord_objs[x].signature()) for x in canonical_chords]\n",
    "\n",
    "def remove_if_possible(list_in, element):\n",
    "    out = []\n",
    "    for d in list_in:\n",
    "        cont = False\n",
    "        for x in d:\n",
    "            if element in x:\n",
    "                cont = True\n",
    "                break\n",
    "        if cont:\n",
    "            temp = [x for x in d if element not in x]\n",
    "            if len(temp) > 0:\n",
    "                out += [temp]\n",
    "            else:\n",
    "                out += [d]\n",
    "        else:\n",
    "            out += [d]\n",
    "    return out\n",
    "\n",
    "eclasses = []\n",
    "for value in chord_objs.values():\n",
    "    if value.root == 0:\n",
    "        temp = []\n",
    "        for key in chord_objs.keys():\n",
    "            if chord_objs[key] == value:\n",
    "                temp += [key]\n",
    "        if temp not in eclasses:\n",
    "            eclasses += [temp]\n",
    "\n",
    "            \n",
    "decisions = [x for x in eclasses if len(x) > 1]\n",
    "\n",
    "stuff = remove_if_possible(decisions, 'maj')\n",
    "\n",
    "out2 = []\n",
    "for d in stuff:\n",
    "    cont = False\n",
    "    for x in d:\n",
    "        if 'sus' in x:\n",
    "            cont = True\n",
    "            break\n",
    "    if cont:\n",
    "        temp = [x for x in d if 'sus' not in x or 'sus4' in x or 'sus2' in x]\n",
    "        if len(temp) > 0:\n",
    "            out2 += [temp]\n",
    "        else:\n",
    "            out2 += [d]\n",
    "    else:\n",
    "        out2 += [d]\n",
    "\n",
    "out3 = remove_if_possible(out2, '\\\\')\n",
    "out4 = remove_if_possible(out3, '6')\n",
    "out5 = remove_if_possible(out4, '/')\n",
    "\n",
    "out6 = []\n",
    "for x in out5:\n",
    "    if len(x) == 2 and one_in_another(*x):\n",
    "        out6 += [[min(x,key=len)]]\n",
    "    else:\n",
    "        out6 += [x]\n",
    "        \n",
    "decided = [x[0] for x in out6 + [y for y in eclasses if len(y) > 0]] + ['NC']\n",
    "\n",
    "canonical_chords = []\n",
    "for d in decided:\n",
    "    for i in range(12):\n",
    "        temp = transpose_chord_symbol(d, i)\n",
    "        if temp not in canonical_chords:\n",
    "            canonical_chords += [temp]\n",
    "\n",
    "reverse_dict = dict()\n",
    "for chord in canonical_chords:\n",
    "    temp = chord_objs[chord].signature()\n",
    "    reverse_dict[''.join([str(x) for x in temp])] = chord\n",
    "\n",
    "def binary_crossentropy(y_true, y_predict):\n",
    "    return -sum([(y1 * np.log(y2)) + ((1 - y1) * np.log(1 - y2)) for y1, y2 in zip(y_true, y_predict)])\n",
    "\n",
    "def hamming(s1, s2):\n",
    "    assert(len(s1) == len(s2))\n",
    "    return sum([x != y for x, y in zip(s1, s2)])\n",
    "\n",
    "def sample_chord(chord):\n",
    "    prob_array = np.array([1 / binary_crossentropy(x, c) for x in canon_sigs])\n",
    "    prob_array = prob_array / prob_array.sum()\n",
    "    return canon_sigs[np.random.choice(np.arange(len(canon_sigs)),p=prob_array)]\n",
    "\n",
    "def guess_chord(chord):\n",
    "    root = np.random.choice(np.arange(12), p=(chord[:12] / sum(chord[:12])))\n",
    "    temp = 1 * (np.array(chord[12:]) > np.random.random(12))\n",
    "    return np.array([1 * (i == root) for i in range(12)] + [1 if temp[i] == 1 or i == root else 0 for i in range(12)])\n",
    "\n",
    "def process_prediction(predix):\n",
    "    return [1 * (i == predix[:12].argmax() and max(predix) > .5) for i in range(12)] + [1 * (x > 0.5) for x in predix[12:]]\n",
    "\n",
    "def turn_to_word(chord):\n",
    "    temp = [str(x) for x in (chord)]\n",
    "    for i in range(12):\n",
    "        if temp[i] == '1':\n",
    "            temp[i + 12] = '1'\n",
    "    temp = ''.join(temp)\n",
    "    try:\n",
    "        symbol = reverse_dict[temp]\n",
    "    except:\n",
    "        symbol = min([[np.linalg.norm(np.array(chord_objs[x].signature()) - chord), x] for x in canonical_chords])\n",
    "#         symbol2 = reverse_dict[min([[hamming(temp, key), key] for key in reverse_dict.keys()])[1]]\n",
    "        print('mismatch:', temp, symbol[1], symbol[0])\n",
    "        symbol = symbol[1]\n",
    "    return symbol\n",
    "\n",
    "def eightify(stuff):\n",
    "#     assert(len(stuff) == 32)\n",
    "    out = ''\n",
    "    for i in range(len(stuff)):\n",
    "        if i % 8 == 0 and i != 0:\n",
    "            out += '\\n'\n",
    "        out += stuff[i]\n",
    "        out += ' | '\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen1 = seq_generator(out, seq_len)\n",
    "# [next(gen1) for _ in range(128)]\n",
    "t = next(gen1)\n",
    "T = np.stack([t] * batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100.0%           \r"
     ]
    }
   ],
   "source": [
    "chords_to_make = 64\n",
    "for i in range(chords_to_make):\n",
    "    predix = model.predict(T[:,-32:,:], batch_size=batches)\n",
    "    predix = np.expand_dims(predix[:,-1], axis=1)\n",
    "    predix = np.array([[sample_chord(predix[0][0])]] * batches)\n",
    "    T = np.concatenate([T, predix], axis=1)\n",
    "    t = next(gen1)\n",
    "    print(round((100 * i + 100) / chords_to_make, 2), end='%           \\r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [],
   "source": [
    "thing = eightify([turn_to_word(y) for y in T[0][32:]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Co7M7 | G7sus | G#7/D | C/F# | B7#5b9/E | Bm6 | G#M7#11/D# | EM7/G# | \n",
      "EMb5 | Amaj7 | E6/G# | G#7+ | G#M7#5/G | A#7b9/D# | G#7b9/C# | D7sus | \n",
      "Am9/G | B/A | BM9#11 | C#7sus4/G# | F#m7/A# | D#7sus | C#9 | A69/C# | \n",
      "D#7/A# | C#o/E | C#o/G# | G#m9/C# | F#7alt | G9b5 | C#7#5#9/F | A11 | \n",
      "C#/F# | Em9/A | A | A#5 | G13b9#11 | Adim | BM9#11 | Fm7/A | \n",
      "Am69 | G#13b5 | DM7/F | E7#9/B | D#7b6 | F7/C | F#o/A | Dm11b5 | \n",
      "D#m/D | CM6 | FM7/B | C#7#11 | D#7b9#11 | D#7b6/A# | G#7b9 | G9/F# | \n",
      "AM7b5/G# | A7sus4/E | A\\F | AM7#5/E | EM9/B | F#6/A# | EM7#5/C | Bm/E | \n",
      "\n",
      "A7 | A7 | NC | NC | NC | NC | NC | NC | \n",
      "NC | NC | A#m | A#m | A#m | A#m | D#m | D#m | \n",
      "D#m | D#m | A#m | A#m | A#m | A#m | D#7 | D#7 | \n",
      "D#7 | D#7 | C# | C# | Bo7/D | Bo7/D | D#m7 | D#m7 | \n"
     ]
    }
   ],
   "source": [
    "print(thing)\n",
    "print()\n",
    "print(eightify([turn_to_word(y) for y in np.array(t)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dupe_counter(pred):\n",
    "#     dupes = []\n",
    "    return len(set([turn_to_word(x) for x in pred]))\n",
    "#         if x not in dupes:\n",
    "#             dupes += [x]\n",
    "#     return (dupes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100.0                     \r"
     ]
    }
   ],
   "source": [
    "gen1 = seq_generator(out, seq_len)\n",
    "dupes = []\n",
    "l = len(out) // 32\n",
    "for i in range(l):\n",
    "    p = round((100 * i) / l, 2)\n",
    "    dupes += [dupe_counter(next(gen1))]\n",
    "    [next(gen1) for _ in range(31)]\n",
    "    print(p, end='       \\r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mismatch: 000000000001000000100101 B7 1.1018787761197362\n",
      "mismatch: 000000000001000000100101 B5 1.0568486048282173\n",
      "mismatch: 000000100000010010100000 F#7 0.980127709639786\n",
      "mismatch: 000000100000000000100001 B5 1.2353488807056856\n",
      "mismatch: 000000100000010000100101 B9/F# 1.1228914932501608\n",
      "mismatch: 000000000001000000100101 B7sus4 1.3497807502163373\n",
      "mismatch: 000000100000010010100000 F#7 0.847338682131144\n",
      "mismatch: 000000100000010010100000 F#7 0.9294703115272414\n",
      "mismatch: 000000100000010010100101 F#m7 0.7716156006973176\n",
      "mismatch: 000000100000010000100101 F#m 1.034712632712423\n",
      "mismatch: 000000000001000000100101 F#m/B 1.2150483420933096\n",
      "mismatch: 000000100000000010100100 F#m7 1.0214412184902102\n",
      "mismatch: 000000100000000010100100 F#m7 1.0016301368839198\n",
      "mismatch: 000001000000001001001100 F13#9 1.0017180631207045\n",
      "mismatch: 000000010000101001010001 G7 0.9416414875987901\n",
      "mismatch: 000000000100100000010100 Am7b5 1.2433754737778897\n",
      "mismatch: 000000100000000000100101 Esus4/F# 1.3369455231974399\n",
      "mismatch: 000000000001000000100101 B7sus4 1.233172448023035\n",
      "mismatch: 000000100000010010100000 F#m7 0.9890987583097527\n",
      "mismatch: 000000000001000000100101 F#m/B 1.218918269538519\n",
      "mismatch: 000000100000010010100000 F#7 0.67936570618394\n",
      "mismatch: 000000100000010010100000 F#7sus4 0.8315718552618647\n",
      "mismatch: 000000000001000110100001 B 1.2165040809483412\n",
      "mismatch: 000000100000010010100101 F#m7 1.0500347737724982\n",
      "mismatch: 000000100000000000100001 F#sus4 1.2807858225909021\n",
      "mismatch: 000000100000000010100011 Bsus4/F# 1.267825833470407\n",
      "mismatch: 000000100000010010100011 F#7 1.0557046130101515\n",
      "mismatch: 000000010000001001010000 G7 1.0642285018275617\n",
      "mismatch: 000010000000001010010000 Em7 0.7864974598097177\n",
      "mismatch: 000010000000001010000010 Em7b5 0.8171224132796049\n",
      "mismatch: 000000000100000001000100 Dm/A 1.2142267674504161\n",
      "mismatch: 000010000000001010001000 E7 1.0349643891574234\n",
      "mismatch: 000010000000010010101000 C#m/E 0.8113067719868228\n",
      "mismatch: 000010000000010010000000 NC 1.2878143475842652\n",
      "mismatch: 000010000000010010000001 E5 1.3036565525456278\n",
      "mismatch: 000010000000010010100000 F#7sus4/E 1.208581420476288\n",
      "mismatch: 000010000000000010100100 A6/E 0.9330497786693781\n",
      "mismatch: 000000000001000000100101 B7sus4 0.9673784139574964\n",
      "mismatch: 000000100000010010100101 F#7sus4 0.7492185154487253\n",
      "mismatch: 000000100000010010100101 F#7sus4 0.9295951185389886\n",
      "mismatch: 000000100000010010100101 F#m7 0.7244527082472532\n",
      "mismatch: 000000100000010010100101 F#7sus4 0.8126428523931721\n",
      "mismatch: 000000100000010010100101 F#m7 0.9754150611836253\n",
      "mismatch: 000000100000010010100101 F#m7 0.9854814991907845\n",
      "mismatch: 000000000010000001010010 C7sus4/A# 0.9042058445576782\n",
      "mismatch: 000000000010100000010010 NC 1.3269855944247038\n",
      "mismatch: 000010000000010010101101 E69 0.908339643619576\n",
      "mismatch: 000000100000010010100000 F#m7 0.8001144016465317\n",
      "mismatch: 000000100000000000100101 Esus4/F# 1.1259179088784235\n",
      "mismatch: 000000100000000000100001 F#sus4 1.136545332278187\n",
      "mismatch: 000000100000010010100101 F#7sus4 0.7199462909068424\n",
      "mismatch: 000000000001000000100101 B7 1.0812318634492122\n",
      "mismatch: 000000100000000000100101 F#sus4 1.1698804420478521\n",
      "mismatch: 000000100000010010100101 F#7sus4 0.7655489593443763\n",
      "mismatch: 000000100000010010100101 F#7sus4 0.6794329898293371\n",
      "mismatch: 000000100000000010100100 F#m7 0.9025470241774853\n",
      "mismatch: 000000000001000000100101 B5 1.2542078559248842\n",
      "mismatch: 000010000000000110000001 B/E 1.2106170173164041\n",
      "mismatch: 000000100000000010100100 F#m7 1.0800253780941076\n",
      "mismatch: 000100000000010100000010 NC 1.3200785436672338\n",
      "mismatch: 000000001000100000101000 G#7 0.9349319616351597\n",
      "mismatch: 000100000000000100001000 NC 1.2916581015475157\n",
      "mismatch: 010000000000010011001000 C# 0.8805623283459131\n",
      "mismatch: 010000000000010011001000 C#7#9 0.8718825400008784\n",
      "mismatch: 000000100000010010100000 NC 1.3648970155608098\n",
      "mismatch: 000000100000000010100010 F#7 0.9196937757040167\n",
      "mismatch: 000000100000010010100000 F#7 0.9257115611675321\n",
      "mismatch: 000000100000010010100011 F#7 0.8824251309503868\n",
      "mismatch: 000000100000010010100011 F#7sus4 0.9150289744911454\n",
      "mismatch: 000000100000010010100101 F#m7 0.8385699069051247\n",
      "mismatch: 000000100000010010100101 Bm11/F# 0.816811356902958\n",
      "mismatch: 010000000000010011001000 C# 0.8802096197163224\n",
      "mismatch: 000000000001000000100101 B7 1.2124974518215992\n",
      "2.8 7.55 4.75       \r"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-167-3e2ee0cfbda9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mpredix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mpredix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredix\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mT\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredix\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/models.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1023\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1024\u001b[0m         return self.model.predict(x, batch_size=batch_size, verbose=verbose,\n\u001b[0;32m-> 1025\u001b[0;31m                                   steps=steps)\n\u001b[0m\u001b[1;32m   1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1027\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_on_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1833\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1834\u001b[0m         return self._predict_loop(f, ins, batch_size=batch_size,\n\u001b[0;32m-> 1835\u001b[0;31m                                   verbose=verbose, steps=steps)\n\u001b[0m\u001b[1;32m   1836\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1837\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_predict_loop\u001b[0;34m(self, f, ins, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1328\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1330\u001b[0;31m                 \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1331\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1332\u001b[0m                     \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2476\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2478\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2479\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1137\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1355\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1356\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1359\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1361\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1362\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1338\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[0;32m-> 1340\u001b[0;31m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "gen1 = seq_generator(out, seq_len)\n",
    "dupes2 = []\n",
    "# l = len(out) // 32\n",
    "l = 1000\n",
    "t = next(gen1)\n",
    "for i in range(l):\n",
    "    T = np.stack([t] * batches)\n",
    "    p = round((100 * i) / l, 2)\n",
    "    for _ in range(64):\n",
    "        predix = model.predict(T[:,-32:,:], batch_size=batches)\n",
    "        predix = np.expand_dims(predix[:,-1], axis=1)\n",
    "        T = np.concatenate([T, predix], axis=1)\n",
    "    [next(gen1) for _ in range(31)]\n",
    "    t = next(gen1)\n",
    "    dupes2 += [dupe_counter(T[0][64:])]\n",
    "#     print(([turn_to_word(x) for x in T[0][64:]]))\n",
    "    print(p, round(np.mean(dupes2), 2), round(np.std(dupes2), 2), end='       \\r')\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33.02319203812074"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binary_crossentropy(canon_sigs[0], c)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}