Tutorial_5 - An understandable example to implement Multi-LSTM for MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**tensorflow 版本： 1.2.1**\n",
    "\n",
    "通过本例，你可以了解到单层 LSTM 的实现，多层 LSTM 的实现。输入输出数据的格式。 RNN 的 dropout layer 的实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n"
     ]
    }
   ],
   "source": [
    "# -*- coding:utf-8 -*-\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.contrib import rnn\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "# 设置 GPU 按需增长\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "\n",
    "# 首先导入数据，看一下数据的形式\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
    "print mnist.train.images.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ** 一、首先设置好模型用到的各个超参数 **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "input_size = 28      # 每个时刻的输入特征是28维的，就是每个时刻输入一行，一行有 28 个像素\n",
    "timestep_size = 28   # 时序持续长度为28，即每做一次预测，需要先输入28行\n",
    "hidden_size = 256    # 隐含层的数量\n",
    "layer_num = 2        # LSTM layer 的层数\n",
    "class_num = 10       # 最后输出分类类别数量，如果是回归预测的话应该是 1\n",
    "\n",
    "_X = tf.placeholder(tf.float32, [None, 784])\n",
    "y = tf.placeholder(tf.float32, [None, class_num])\n",
    "# 在训练和测试的时候，我们想用不同的 batch_size.所以采用占位符的方式\n",
    "batch_size = tf.placeholder(tf.int32, [])  # 注意类型必须为 tf.int32, batch_size = 128\n",
    "keep_prob = tf.placeholder(tf.float32, [])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ** 二、开始搭建 LSTM 模型，其实普通 RNNs 模型也一样 **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 把784个点的字符信息还原成 28 * 28 的图片\n",
    "# 下面几个步骤是实现 RNN / LSTM 的关键\n",
    "####################################################################\n",
    "# **步骤1：RNN 的输入shape = (batch_size, timestep_size, input_size) \n",
    "X = tf.reshape(_X, [-1, 28, 28])\n",
    "\n",
    "# # **步骤2：定义一层 LSTM_cell，只需要说明 hidden_size, 它会自动匹配输入的 X 的维度\n",
    "# lstm_cell = rnn.BasicLSTMCell(num_units=hidden_size, forget_bias=1.0, state_is_tuple=True)\n",
    "\n",
    "# # **步骤3：添加 dropout layer, 一般只设置 output_keep_prob\n",
    "# lstm_cell = rnn.DropoutWrapper(cell=lstm_cell, input_keep_prob=1.0, output_keep_prob=keep_prob)\n",
    "\n",
    "# # **步骤4：调用 MultiRNNCell 来实现多层 LSTM\n",
    "# mlstm_cell = rnn.MultiRNNCell([lstm_cell] * layer_num, state_is_tuple=True)\n",
    "# mlstm_cell = rnn.MultiRNNCell([lstm_cell for _ in range(layer_num)] , state_is_tuple=True)\n",
    "\n",
    "# 在 tf 1.0.0 版本中，可以使用上面的 三个步骤创建多层 lstm， 但是在 tf 1.2.1 版本中，可以通过下面方式来创建\n",
    "def lstm_cell():\n",
    "    cell = rnn.LSTMCell(hidden_size, reuse=tf.get_variable_scope().reuse)\n",
    "    return rnn.DropoutWrapper(cell, output_keep_prob=keep_prob)\n",
    "\n",
    "mlstm_cell = tf.contrib.rnn.MultiRNNCell([lstm_cell() for _ in range(layer_num)], state_is_tuple = True)\n",
    "\n",
    "# **步骤5：用全零来初始化state\n",
    "init_state = mlstm_cell.zero_state(batch_size, dtype=tf.float32)\n",
    "\n",
    "# **步骤6：方法一，调用 dynamic_rnn() 来让我们构建好的网络运行起来\n",
    "# ** 当 time_major==False 时， outputs.shape = [batch_size, timestep_size, hidden_size] \n",
    "# ** 所以，可以取 h_state = outputs[:, -1, :] 作为最后输出\n",
    "# ** state.shape = [layer_num, 2, batch_size, hidden_size], \n",
    "# ** 或者，可以取 h_state = state[-1][1] 作为最后输出\n",
    "# ** 最后输出维度是 [batch_size, hidden_size]\n",
    "# outputs, state = tf.nn.dynamic_rnn(mlstm_cell, inputs=X, initial_state=init_state, time_major=False)\n",
    "# h_state = state[-1][1]\n",
    "\n",
    "# *************** 为了更好的理解 LSTM 工作原理，我们把上面 步骤6 中的函数自己来实现 ***************\n",
    "# 通过查看文档你会发现， RNNCell 都提供了一个 __call__()函数，我们可以用它来展开实现LSTM按时间步迭代。\n",
    "# **步骤6：方法二，按时间步展开计算\n",
    "outputs = list()\n",
    "state = init_state\n",
    "with tf.variable_scope('RNN'):\n",
    "    for timestep in range(timestep_size):\n",
    "        if timestep > 0:\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "        # 这里的state保存了每一层 LSTM 的状态\n",
    "        (cell_output, state) = mlstm_cell(X[:, timestep, :],state)\n",
    "        outputs.append(cell_output)\n",
    "h_state = outputs[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ** 三、最后设置 loss function 和 优化器，展开训练并完成测试 **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter0, step 200, training accuracy 0.9375\n",
      "Iter0, step 400, training accuracy 0.945312\n",
      "Iter1, step 600, training accuracy 0.9375\n",
      "Iter1, step 800, training accuracy 0.976562\n",
      "Iter2, step 1000, training accuracy 0.96875\n",
      "Iter2, step 1200, training accuracy 1\n",
      "Iter3, step 1400, training accuracy 0.976562\n",
      "Iter3, step 1600, training accuracy 0.992188\n",
      "Iter4, step 1800, training accuracy 0.976562\n",
      "Iter4, step 2000, training accuracy 0.984375\n",
      "test accuracy 0.9839\n"
     ]
    }
   ],
   "source": [
    "############################################################################\n",
    "# 以下部分其实和之前写的多层 CNNs 来实现 MNIST 分类是一样的。\n",
    "# 只是在测试的时候也要设置一样的 batch_size.\n",
    "\n",
    "# 上面 LSTM 部分的输出会是一个 [hidden_size] 的tensor，我们要分类的话，还需要接一个 softmax 层\n",
    "# 首先定义 softmax 的连接权重矩阵和偏置\n",
    "# out_W = tf.placeholder(tf.float32, [hidden_size, class_num], name='out_Weights')\n",
    "# out_bias = tf.placeholder(tf.float32, [class_num], name='out_bias')\n",
    "# 开始训练和测试\n",
    "W = tf.Variable(tf.truncated_normal([hidden_size, class_num], stddev=0.1), dtype=tf.float32)\n",
    "bias = tf.Variable(tf.constant(0.1,shape=[class_num]), dtype=tf.float32)\n",
    "y_pre = tf.nn.softmax(tf.matmul(h_state, W) + bias)\n",
    "\n",
    "\n",
    "# 损失和评估函数\n",
    "cross_entropy = -tf.reduce_mean(y * tf.log(y_pre))\n",
    "train_op = tf.train.AdamOptimizer(lr).minimize(cross_entropy)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "for i in range(2000):\n",
    "    _batch_size = 128\n",
    "    batch = mnist.train.next_batch(_batch_size)\n",
    "    if (i+1)%200 == 0:\n",
    "        train_accuracy = sess.run(accuracy, feed_dict={\n",
    "            _X:batch[0], y: batch[1], keep_prob: 1.0, batch_size: _batch_size})\n",
    "        # 已经迭代完成的 epoch 数: mnist.train.epochs_completed\n",
    "        print \"Iter%d, step %d, training accuracy %g\" % ( mnist.train.epochs_completed, (i+1), train_accuracy)\n",
    "    sess.run(train_op, feed_dict={_X: batch[0], y: batch[1], keep_prob: 0.5, batch_size: _batch_size})\n",
    "\n",
    "# 计算测试数据的准确率\n",
    "print \"test accuracy %g\"% sess.run(accuracy, feed_dict={\n",
    "    _X: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0, batch_size:mnist.test.images.shape[0]})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们一共只迭代不到5个epoch，在测试集上就已经达到了0.98的准确率，可以看出来 LSTM 在做这个字符分类的任务上还是比较有效的，而且我们最后一次性对 10000 张测试图片进行预测，才占了 725 MiB 的显存。而我们在之前的两层 CNNs 网络中，预测 10000 张图片一共用了 8721 MiB 的显存，差了整整 12 倍呀！！ 这主要是因为 RNN/LSTM 网络中，每个时间步所用的权值矩阵都是共享的，可以通过前面介绍的 LSTM 的网络结构分析一下，整个网络的参数非常少。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 四、可视化看看 LSTM 的是怎么做分类的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毕竟 LSTM 更多的是用来做时序相关的问题，要么是文本，要么是序列预测之类的，所以很难像 CNNs 一样非常直观地看到每一层中特征的变化。在这里，我想通过可视化的方式来帮助大家理解 LSTM 是怎么样一步一步地把图片正确的给分类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 784) (5, 10)\n",
      "_outputs.shape = (28, 5, 256)\n",
      "arr_state.shape = (2, 2, 5, 256)\n"
     ]
    }
   ],
   "source": [
    "# 手写的结果 shape\n",
    "_batch_size = 5\n",
    "X_batch, y_batch = mnist.test.next_batch(_batch_size)\n",
    "print X_batch.shape, y_batch.shape\n",
    "_outputs, _state = np.array(sess.run([outputs, state],feed_dict={\n",
    "            _X: X_batch, y: y_batch, keep_prob: 1.0, batch_size: _batch_size}))\n",
    "print '_outputs.shape =', np.asarray(_outputs).shape\n",
    "print 'arr_state.shape =', np.asarray(_state).shape\n",
    "# 可见 outputs.shape = [ batch_size, timestep_size, hidden_size]\n",
    "# state.shape = [layer_num, 2, batch_size, hidden_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 784) (5, 10)\n",
      "_outputs.shape = (28, 5, 256)\n",
      "arr_state.shape = (2, 2, 5, 256)\n"
     ]
    }
   ],
   "source": [
    "_batch_size = 5\n",
    "X_batch, y_batch = mnist.test.next_batch(_batch_size)\n",
    "print X_batch.shape, y_batch.shape\n",
    "_outputs, _state = sess.run([outputs, state],feed_dict={\n",
    "            _X: X_batch, y: y_batch, keep_prob: 1.0, batch_size: _batch_size})\n",
    "print '_outputs.shape =', np.asarray(_outputs).shape\n",
    "print 'arr_state.shape =', np.asarray(_state).shape\n",
    "# 可见 outputs.shape = [ batch_size, timestep_size, hidden_size]\n",
    "# state.shape = [layer_num, 2, batch_size, hidden_size]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "看下面我找了一个字符 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "print mnist.train.labels[4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们先来看看这个字符样子,上半部分还挺像 2 来的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADhJJREFUeJzt3X+MFPUZx/HPIz/+EImitScBKmjUBA2h5kKaiE1NS0Vt\nAk3UlBhB23BV26YaY2qsxt+J1tZaTWxyKAFMq63Ri4ikWNEgatOIxqqAItXD3olQgxH6h6FwT//Y\nwZ5w+5293dmdPZ73KyG3O8/O7pPlPjcz+92Zr7m7AMRzRNkNACgH4QeCIvxAUIQfCIrwA0ERfiAo\nwg8ERfiBoAg/ENToVr6YmfF1QqDJ3N1qeVxDW34zm2tm75rZVjO7vpHnAtBaVu93+81slKQtkuZI\n6pP0qqQF7r4psQ5bfqDJWrHlnyVpq7u/7+57JT0maV4DzweghRoJ/yRJ/xp0vy9b9iVm1mVmG8xs\nQwOvBaBgTf/Az927JXVL7PYD7aSRLX+/pCmD7k/OlgEYARoJ/6uSTjGzaWY2VtIPJK0spi0AzVb3\nbr+77zOzn0paI2mUpKXuvrGwzgA0Vd1DfXW9GMf8QNO15Es+AEYuwg8ERfiBoAg/EBThB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxA\nUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Kqe4puSTKzXkl7JO2XtM/dO4toCsMzffr0qrUrrrgi\nue5pp52WrM+ZMydZN6tpQtghvfLKK8n6ypUrk/Vly5Yl6zt27BhuS6E0FP7MOe7+SQHPA6CF2O0H\ngmo0/C7pOTN7zcy6imgIQGs0uts/2937zeyrkv5qZu+4+4uDH5D9UeAPA9BmGtryu3t/9nOnpB5J\ns4Z4TLe7d/JhINBe6g6/mY0zs/EHbkv6rqS3i2oMQHM1stvfIaknG+oZLemP7v6XQroC0HTm7q17\nMbPWvdgIcvTRRyfr9913X7K+YMGCqrUxY8bU1dNI0NfXl6yffPLJVWv79u0rup224e41ffmCoT4g\nKMIPBEX4gaAIPxAU4QeCIvxAUEWc1Ycceae9XnTRRcn6woULi2znsDF58uRkvZHTjSNgyw8ERfiB\noAg/EBThB4Ii/EBQhB8IivADQXFKbwuMGzcuWd+9e3dDzz8wMFC1tnnz5uS6jz/+eLKed/nrWbMO\nuXjTl1x++eXJeiM+/fTTZP2EE06oWuOUXrb8QFiEHwiK8ANBEX4gKMIPBEX4gaAIPxAU5/O3wP79\n+5P1bdu2Jesnnnhisp76nsCMGTOS6+aZMGFCsn7bbbc19PyNuPvuu5P1w3ksvwhs+YGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gqNxxfjNbKul7kna6+xnZsmMl/UnSVEm9ki529/TJ1YF9/vnnyfrcuXOT\n9csuuyxZP/XUU4fb0hdGj07/Cpx99tnJ+vHHH1/3a+dZt25dsp43dTnSatnyL5N08G/n9ZLWuvsp\nktZm9wGMILnhd/cXJe06aPE8Scuz28slzS+4LwBNVu8xf4e7b89ufyypo6B+ALRIw9/td3dPXZvP\nzLokdTX6OgCKVe+Wf4eZTZSk7OfOag90925373T3zjpfC0AT1Bv+lZIWZbcXSXqqmHYAtEpu+M3s\nUUl/k3SamfWZ2Y8k3SVpjpm9J+k72X0AIwjX7T/MTZ06NVnPOyf+wgsvLLCb4Xn++eeT9bvuSm9z\nUuu38ve+1bhuP4Akwg8ERfiBoAg/EBThB4Ii/EBQDPUdBs4888yqtdWrVyfXbeYpuWW75557qtbu\nv//+5LofffRR0e20DEN9AJIIPxAU4QeCIvxAUIQfCIrwA0ERfiAoxvnbQN7ls1Pj+JK0atWqqrXj\njjuurp4Od3mnKvf09LSok+Ixzg8gifADQRF+ICjCDwRF+IGgCD8QFOEHgmKcvw2cd955yXpqHL9s\nW7duTdZ37Tp4jtf/e+edd5LrLly4sK6eavHss88m63n/J+2McX4ASYQfCIrwA0ERfiAowg8ERfiB\noAg/EFT6RHJJZrZU0vck7XT3M7Jlt0haLOnf2cNucPf0BeJRVTOnwX755ZeT9Q8++CBZX7JkSbK+\nadOmZD01zn/NNdck122mvO8YRFDLln+ZpLlDLP+tu8/M/hF8YITJDb+7vyip+p9vACNSI8f8PzOz\nN81sqZlNKKwjAC1Rb/h/L+kkSTMlbZf0m2oPNLMuM9tgZhvqfC0ATVBX+N19h7vvd/cBSUskzUo8\nttvdO929s94mARSvrvCb2cRBd78v6e1i2gHQKrUM9T0q6VuSvmJmfZJulvQtM5spySX1SvpxE3sE\n0AS54Xf3BUMsfrgJvYT10ksvJesXXHBBsr5+/fqqtUsuuSS57t69e5P1Rl155ZVVa7feemtTX3tg\nYKBqbfVqRqf5hh8QFOEHgiL8QFCEHwiK8ANBEX4gKC7djaSxY8cm6zfddFOyftVVV1WtHXPMMXX1\nVKubb765au2OO+5o6muXiUt3A0gi/EBQhB8IivADQRF+ICjCDwRF+IGgck/pxeHtuuuuS9YvvfTS\nZP30008vsp1huf3225P1O++8s0WdjExs+YGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gqDDj/Hnj1Y88\n8kiLOhm+I488Mlk/99xzq9ZuvPHG5LozZsxI1o84onnbh927dyfr9957b7KeN47fymtVjERs+YGg\nCD8QFOEHgiL8QFCEHwiK8ANBEX4gqNzr9pvZFEkrJHVIcknd7v47MztW0p8kTZXUK+lid/8057lC\nDryeddZZyfqkSZOS9WuvvTZZ7+zsHHZPRdm4cWOy/sADD1StrVu3Lrnuli1b6uopuiKv279P0rXu\nPl3SNyT9xMymS7pe0lp3P0XS2uw+gBEiN/zuvt3dX89u75G0WdIkSfMkLc8etlzS/GY1CaB4wzrm\nN7Opkr4u6e+SOtx9e1b6WJXDAgAjRM3f7TezoyQ9Ielqd99t9v/DCnf3asfzZtYlqavRRgEUq6Yt\nv5mNUSX4f3D3J7PFO8xsYlafKGnnUOu6e7e7d7p7eZ9KAThEbvitsol/WNJmdx98mtVKSYuy24sk\nPVV8ewCapZahvtmS1kt6S9JAtvgGVY77/yzpa5K2qTLUtyvnudp2qG/UqFHJ+uLFi6vW5s9Pf9Z5\nzjnnJOujR7fvmdUPPvhgsp53yvBnn31WZDuoQa1Dfbm/de7+kqRqT/bt4TQFoH3wDT8gKMIPBEX4\ngaAIPxAU4QeCIvxAUO07wFywvKmk804vnTBhQpHtFGpgYKBq7aGHHkqum3f56/7+/mSdy2OPXGz5\ngaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCoMOP806ZNS9bLHMffs2dPsr5ixYpk/ZlnnqlaW7NmTV09\n4fDHlh8IivADQRF+ICjCDwRF+IGgCD8QFOEHggozzt/b25us5421jx8/vmqtp6cnue7TTz+drL/w\nwgvJ+ocffpisA/Vgyw8ERfiBoAg/EBThB4Ii/EBQhB8IivADQVneddfNbIqkFZI6JLmkbnf/nZnd\nImmxpH9nD73B3VfnPBcXeQeazN2tlsfVEv6Jkia6++tmNl7Sa5LmS7pY0n/c/de1NkX4gearNfy5\n3/Bz9+2Stme395jZZkmTGmsPQNmGdcxvZlMlfV3S37NFPzOzN81sqZkNeR0sM+sysw1mtqGhTgEU\nKne3/4sHmh0laZ2kO939STPrkPSJKp8D3K7KocEPc56D3X6gyQo75pckMxsjaZWkNe5+7xD1qZJW\nufsZOc9D+IEmqzX8ubv9ZmaSHpa0eXDwsw8CD/i+pLeH2ySA8tTyaf9sSeslvSXpwFzQN0haIGmm\nKrv9vZJ+nH04mHoutvxAkxW6218Uwg80X2G7/QAOT4QfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwRF+IGgWj1F9yeStg26/5VsWTtq197atS+J3upVZG8n1vrAlp7Pf8iL\nm21w987SGkho197atS+J3upVVm/s9gNBEX4gqLLD313y66e0a2/t2pdEb/UqpbdSj/kBlKfsLT+A\nkpQSfjOba2bvmtlWM7u+jB6qMbNeM3vLzN4oe4qxbBq0nWb29qBlx5rZX83sveznkNOkldTbLWbW\nn713b5jZ+SX1NsXMXjCzTWa20cx+ni0v9b1L9FXK+9by3X4zGyVpi6Q5kvokvSppgbtvamkjVZhZ\nr6ROdy99TNjMvinpP5JWHJgNycx+JWmXu9+V/eGc4O6/aJPebtEwZ25uUm/VZpa+TCW+d0XOeF2E\nMrb8syRtdff33X2vpMckzSuhj7bn7i9K2nXQ4nmSlme3l6vyy9NyVXprC+6+3d1fz27vkXRgZulS\n37tEX6UoI/yTJP1r0P0+tdeU3y7pOTN7zcy6ym5mCB2DZkb6WFJHmc0MIXfm5lY6aGbptnnv6pnx\numh84Heo2e4+U9J5kn6S7d62Ja8cs7XTcM3vJZ2kyjRu2yX9psxmspmln5B0tbvvHlwr870boq9S\n3rcywt8vacqg+5OzZW3B3fuznzsl9ahymNJOdhyYJDX7ubPkfr7g7jvcfb+7D0haohLfu2xm6Sck\n/cHdn8wWl/7eDdVXWe9bGeF/VdIpZjbNzMZK+oGklSX0cQgzG5d9ECMzGyfpu2q/2YdXSlqU3V4k\n6akSe/mSdpm5udrM0ir5vWu7Ga/dveX/JJ2vyif+/5T0yzJ6qNLXSZL+kf3bWHZvkh5VZTfwv6p8\nNvIjScdJWivpPUnPSTq2jXp7RJXZnN9UJWgTS+pttiq79G9KeiP7d37Z712ir1LeN77hBwTFB35A\nUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4L6H8FXlvHOhDl0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1c10e5ffd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X3 = mnist.train.images[4]\n",
    "img3 = X3.reshape([28, 28])\n",
    "plt.imshow(img3, cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们看看在分类的时候，一行一行地输入，分为各个类别的概率会是什么样子的。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(28, 1, 256)\n",
      "(28, 256)\n"
     ]
    }
   ],
   "source": [
    "X3.shape = [-1, 784]\n",
    "y_batch = mnist.train.labels[0]\n",
    "y_batch.shape = [-1, class_num]\n",
    "\n",
    "X3_outputs = np.array(sess.run(outputs, feed_dict={\n",
    "            _X: X3, y: y_batch, keep_prob: 1.0, batch_size: 1}))\n",
    "print X3_outputs.shape\n",
    "X3_outputs.shape = [28, hidden_size]\n",
    "print X3_outputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAABftJREFUeJzt3dF14jgAQFHYSqbMaWzq837ka/ZkA4kRlvTuLYAY2X6S\njSH34zhuAHT8c/UGAPBewg8QI/wAMcIPECP8ADHCDxAj/AAxwg8QI/wAMcIPECP8ADHbhv/X7z/H\nr99/Xv5DRCNec0XGYRxjO86oLqxm6fD/dCdeMSl89TcdjMA7TR/+K4I4W4Rn2x5gbdOHH243V0Xw\nSsLPp2a7HXbmdYG/CX9Y5QPwR5PNO7dlJFdFPEv4ga0mwFF2GiPhB760U/D4IPwwSOUW0/9x6+mx\nq8ZI+GEhPgBfz4zjLvzAj1nVX+fMuAs/QIzwA8sZ9X2QCuEHuM33+cnICUz4AWKEHyBG+AFihB8g\nRvgBYoQfIEb4AWKEHyBG+AFihB8gRvgBYu7H4beMAEqs+AFihB8gRvgBYoQfIEb4AWK2Db9/v9Zi\nX49jbB9bbYy2DT8AnxN+gFvrLoHwA8QIPyyktCplHOEHiBH+T1hRATsTft5qtkl1tu0Z6d3v1W2p\neQn/mzgJgFkIP0CM8PMjrl643VzJrkr4AWKEfwFWVMArCf8LCfQH4wBzE36mMep+sYkI/jZ9+L86\naX2wdI7xG8vYcsbI83P68I9wZkBHrUhni8SM2wS8RjL8rMdE9Jgx4lnCDxcQ6HN2Gb+rJmvhB4gR\nfoAY4YfJXHHpv8utk5Iz+0z4ga34kPsx4QcYaMZJSPiBjNm+w3MV4QeIuR/HNpMYAE+w4geIEX6A\nGOEHiBF+gBjhB4gR/m/a6Vne2fjG5VjG9hozjrvwA8QIP0CM8APECD9AjPADxAg/MOWTJ4wj/AAx\nwg8QI/wAMcIPEJMMv58GYGaOTUZLhh+gTPgBYoQfIEb4AU5a7XMZ4eflVjsJ6PBgxwfhB4gRfliI\nFSuvIPwwiEAzq+nDXzh5rlrFWT1C0/Thh9vNJDUr+2VNws+nHp3QK53sM27rjNtEx9Lht9qAJuf9\nOUuH/ytXTAqPVsgOVpjXbOfoyO3ZNvxnzLTzb7f5tueMUe/lp6+729iOeD9nxnanhxZGveYVx+79\nOLY57gF4ghU/QIzwA8QIP0CM8APECD9AjPC/0E6PBl5htueod2Ns93Jmfwo/QIzwA8QIPzCEW3fz\nEn6AGOEHiBF+3sqlP1xP+AFihB824cNUniX8ADHCDxAj/CzBbQx4HeEHOGm1RYnwA8QI/zetNrMD\n/JfwA8QIP0CM8JPm1h1Fwg8QI/wAMcIP8ISdbgsKP0DMtuE/8xX/nWZ2eIZjvmXb8APwuenDP2Il\nstoPfo3a1lFXRSuN7WpWO3aZ0/04HEMAJdOv+AF4LeEHiBF+gBjhB4gRfoAY4QeIEf4FeG57nBnH\ndsZt2oXvQXwQfoAY4QeIEX6AGOEHhnA/fV7CDxAj/AAxwg8QI/wAMcIPECP8ADHCDxAj/AAxwg/8\nmC9prUn4ASY0clIVfoAY4QeIEX7gS+7h70f4ARZ0ZkIW/m+y+oE9lZ5QEn6Ai1w12Qg/QIzwM40z\nq5/KJXrpdgTjCP8nnFiPrRToGffniG0yKfAs4X+T1U5Kq+/9VPbLqHNtp/G7H8c27wWAJ1jxA8QI\nP0CM8APECD9AjPADxAg/QEwy/Ks9U78aYzuOY3ec0tgmww9QJvwAMcIPECP8ADHCDxAj/AAxwg8Q\nI/wAMcIPECP8ADHCDxAj/AAxwg8QI/wAMcIPECP8ADHCDxAj/AAxwg8QI/wAMcI/gdI/eQaut234\nxRTgc9uGfycmMOCVhP9NXIEAs7gfhxYBlFjxA8QIP0CM8APECD9AjPADxAg/QEwy/J6pH8vYjuPY\nHac0tsnwA5QJP0CM8APECD9AjPADxAg/QIzwA8QIP0CM8APECD9AjPADxAg/QIzwA8QIP0CM8APE\nCD9AjPADxAg/QIzwA8QIP0CM8APECD9AjPADxAg/QMz9OI6rtwGAN7LiB4gRfoAY4QeIEX6AGOEH\niBF+gBjhB4gRfoAY4QeIEX6AGOEHiBF+gBjhB4gRfoAY4QeIEX6AGOEHiBF+gBjhB4gRfoAY4QeI\nEX6AGOEHiPkXmKN4V4tf4jQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1c10bba9d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h_W = sess.run(W, feed_dict={\n",
    "            _X:X3, y: y_batch, keep_prob: 1.0, batch_size: 1})\n",
    "h_bias = sess.run(bias, feed_dict={\n",
    "            _X:X3, y: y_batch, keep_prob: 1.0, batch_size: 1})\n",
    "h_bias.shape = [-1, 10]\n",
    "\n",
    "bar_index = range(class_num)\n",
    "for i in xrange(X3_outputs.shape[0]):\n",
    "    plt.subplot(7, 4, i+1)\n",
    "    X3_h_shate = X3_outputs[i, :].reshape([-1, hidden_size])\n",
    "    pro = sess.run(tf.nn.softmax(tf.matmul(X3_h_shate, h_W) + h_bias))\n",
    "    plt.bar(bar_index, pro[0], width=0.2 , align='center')\n",
    "    plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在上面的图中，为了更清楚地看到线条的变化，我把坐标都去了，每一行显示了 4 个图，共有 7 行，表示了一行一行读取过程中，模型对字符的识别。可以看到，在只看到前面的几行像素时，模型根本认不出来是什么字符，随着看到的像素越来越多，最后就基本确定了它是字符 3."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}