weight_evolution.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "weight_evolution.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNp/+Ntk4nFc/noHzBmnDee",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/joaogui1/Recreational/blob/master/weight_evolution.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iWOI9baNqVgT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install tensorflow==2.1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5grc8klUK1e0",
        "colab_type": "code",
        "outputId": "c795af6f-d7d0-4f85-c4f8-d76f54b3ddd3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        " import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow.keras import layers, Sequential\n",
        "from keras.datasets import mnist\n",
        "from keras.utils import  to_categorical\n",
        "import numpy as np\n",
        "from scipy.stats import entropy as kl_div\n",
        "from numpy.linalg import svd\n",
        "print(tf.__version__)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f9H9r08GMDcH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "c354c51c-ed2d-4123-c7c1-b9049b31e5f7"
      },
      "source": [
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "x_train = (x_train.reshape(-1, 784) - 177.5)/177.5\n",
        "x_test = (x_test.reshape(-1, 784) - 177.5)/177.5\n",
        "y_train = to_categorical(y_train)\n",
        "y_test = to_categorical(y_test)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J9_zY5a9MkSE",
        "colab_type": "code",
        "outputId": "1d02954b-b44e-47a2-c2ef-55d661452a37",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "x_train.shape"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60000, 784)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRw-qCyHMevK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Sequential([layers.Dense(1024, activation='relu', input_dim=(28*28)),\n",
        "                    layers.Dense(512, activation='relu'),\n",
        "                    layers.Dense(10, activation='softmax')])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "czY_rf5vOirv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_layers(model, epoch=0):\n",
        "  for idx, layer in enumerate(model.layers):\n",
        "    W, b = layer.weights[0].numpy(), layer.weights[1].numpy()\n",
        "    plt.hist(W.reshape(np.prod(W.shape)))\n",
        "    plt.savefig(f'W_{idx}_{epoch}.png')\n",
        "    plt.close()\n",
        "    plt.hist(b)\n",
        "    plt.savefig(f'b_{idx}_{epoch}.png')\n",
        "    plt.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MB6Ij6hVSu7M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PlotCallback(tf.keras.callbacks.Callback):\n",
        "  def on_epoch_end(self, epoch, logs=None):\n",
        "    plot_layers(self.model, epoch + 1)\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c2qiGBFerRkC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PrintSingularValues(tf.keras.callbacks.Callback):\n",
        "  def on_epoch_end(self, epoch, logs=[]):\n",
        "    for idx, layer in enumerate(self.model.layers):\n",
        "      W = layer.weights[0].numpy()\n",
        "      __, s, _ = svd(W)\n",
        "      s = np.sort(s)\n",
        "      print(f\"Largest Singular Value for {idx}th layer:\", s[-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9P6TyFjGWWsn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#WIP\n",
        "class PrintKL(tf.keras.callbacks.Callback):\n",
        "  def on_epoch_end(self, epoch, logs=[]):\n",
        "    if epoch > 0:\n",
        "      for idx, layer in enumerate(self.model.layers):\n",
        "        W_hist, b_hist = np.histogram(layer.weights[0].numpy()), np.histogram(layer.weights[1].numpy())\n",
        "        kl_W = kl_div(W_hist, self.W_hist[idx])\n",
        "        kl_b = kl_div(b_hist, self.b_hist[idx])\n",
        "        print(f\"Weights KL Divergence: {kl_W}, bias KL Divergence {kl_b}\")\n",
        "        self.W_hist[idx], self.b_hist[idx] = W_hist, b_hist\n",
        "    else: \n",
        "      self.W_hist = []\n",
        "      self.b_hist = []\n",
        "      print(logs)\n",
        "      for idx, layer in enumerate(self.model.layers):\n",
        "        W_hist, b_hist = np.histogram(layer.weights[0].numpy()), np.histogram(layer.weights[1].numpy())\n",
        "        self.W_hist.append(W_hist)\n",
        "        self.b_hist.append(b_hist)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XtZciZALTfYf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot_layers(model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4BF_PpQdNKaz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile('adam', 'categorical_crossentropy', metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H3ZyVwULQYHF",
        "colab_type": "code",
        "outputId": "939895f1-c6b1-4617-9574-654ec9be3b07",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 953
        }
      },
      "source": [
        "model.fit(x_train, y_train, batch_size=128, epochs=10, callbacks=[PlotCallback(), PrintSingularValues()])"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples\n",
            "Epoch 1/10\n",
            "58752/60000 [============================>.] - ETA: 0s - loss: 0.0212 - acc: 0.9932Largest Singular Value for 0th layer: 11.651933\n",
            "Largest Singular Value for 1th layer: 9.572143\n",
            "Largest Singular Value for 2th layer: 3.529023\n",
            "60000/60000 [==============================] - 3s 55us/sample - loss: 0.0209 - acc: 0.9932\n",
            "Epoch 2/10\n",
            "58624/60000 [============================>.] - ETA: 0s - loss: 0.0165 - acc: 0.9942Largest Singular Value for 0th layer: 11.837514\n",
            "Largest Singular Value for 1th layer: 9.697352\n",
            "Largest Singular Value for 2th layer: 3.5807707\n",
            "60000/60000 [==============================] - 3s 49us/sample - loss: 0.0164 - acc: 0.9942\n",
            "Epoch 3/10\n",
            "59008/60000 [============================>.] - ETA: 0s - loss: 0.0182 - acc: 0.9941Largest Singular Value for 0th layer: 11.961409\n",
            "Largest Singular Value for 1th layer: 9.853334\n",
            "Largest Singular Value for 2th layer: 3.6637583\n",
            "60000/60000 [==============================] - 3s 51us/sample - loss: 0.0179 - acc: 0.9942\n",
            "Epoch 4/10\n",
            "59520/60000 [============================>.] - ETA: 0s - loss: 0.0176 - acc: 0.9940Largest Singular Value for 0th layer: 11.98366\n",
            "Largest Singular Value for 1th layer: 10.077137\n",
            "Largest Singular Value for 2th layer: 3.7294214\n",
            "60000/60000 [==============================] - 3s 56us/sample - loss: 0.0175 - acc: 0.9940\n",
            "Epoch 5/10\n",
            "58752/60000 [============================>.] - ETA: 0s - loss: 0.0151 - acc: 0.9950Largest Singular Value for 0th layer: 12.144734\n",
            "Largest Singular Value for 1th layer: 10.193441\n",
            "Largest Singular Value for 2th layer: 3.766798\n",
            "60000/60000 [==============================] - 3s 50us/sample - loss: 0.0156 - acc: 0.9948\n",
            "Epoch 6/10\n",
            "58368/60000 [============================>.] - ETA: 0s - loss: 0.0206 - acc: 0.9935Largest Singular Value for 0th layer: 12.226597\n",
            "Largest Singular Value for 1th layer: 10.374943\n",
            "Largest Singular Value for 2th layer: 3.8121274\n",
            "60000/60000 [==============================] - 3s 51us/sample - loss: 0.0206 - acc: 0.9934\n",
            "Epoch 7/10\n",
            "59648/60000 [============================>.] - ETA: 0s - loss: 0.0153 - acc: 0.9947Largest Singular Value for 0th layer: 12.334215\n",
            "Largest Singular Value for 1th layer: 10.5168295\n",
            "Largest Singular Value for 2th layer: 3.8493502\n",
            "60000/60000 [==============================] - 3s 51us/sample - loss: 0.0153 - acc: 0.9947\n",
            "Epoch 8/10\n",
            "58240/60000 [============================>.] - ETA: 0s - loss: 0.0127 - acc: 0.9956Largest Singular Value for 0th layer: 12.47388\n",
            "Largest Singular Value for 1th layer: 10.6215725\n",
            "Largest Singular Value for 2th layer: 3.8751643\n",
            "60000/60000 [==============================] - 3s 51us/sample - loss: 0.0134 - acc: 0.9954\n",
            "Epoch 9/10\n",
            "58624/60000 [============================>.] - ETA: 0s - loss: 0.0200 - acc: 0.9936Largest Singular Value for 0th layer: 12.578565\n",
            "Largest Singular Value for 1th layer: 10.877382\n",
            "Largest Singular Value for 2th layer: 3.8904655\n",
            "60000/60000 [==============================] - 3s 52us/sample - loss: 0.0198 - acc: 0.9937\n",
            "Epoch 10/10\n",
            "59392/60000 [============================>.] - ETA: 0s - loss: 0.0082 - acc: 0.9970Largest Singular Value for 0th layer: 12.587915\n",
            "Largest Singular Value for 1th layer: 10.953655\n",
            "Largest Singular Value for 2th layer: 3.9079876\n",
            "60000/60000 [==============================] - 3s 50us/sample - loss: 0.0084 - acc: 0.9969\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fbd013ac320>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TqTmM-19UVgu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "for epoch in range(11):\n",
        "  for idx in range(3):\n",
        "    files.download(f'W_{idx}_{epoch}.png')\n",
        "    files.download(f'b_{idx}_{epoch}.png')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}