examples/colab/tf_hub_delf_module.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RUymE2l9GZfO"
      },
      "source": [
        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "cellView": "code",
        "colab": {},
        "colab_type": "code",
        "id": "JMyTNwSJGGWg"
      },
      "outputs": [],
      "source": [
        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License.\n",
        "# =============================================================================="
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "0DmDwGPOGfaQ"
      },
      "source": [
        "# How to match images using DELF and TensorFlow Hub\n",
        "\n",
        "\u003ctable align=\"left\"\u003e\u003ctd\u003e\n",
        "  \u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/hub/blob/master/examples/colab/tf_hub_delf_module.ipynb\"\u003e\n",
        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\n",
        "  \u003c/a\u003e\n",
        "\u003c/td\u003e\u003ctd\u003e\n",
        "  \u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/hub/blob/master/examples/colab/tf_hub_delf_module.ipynb\"\u003e\n",
        "    \u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
        "\u003c/td\u003e\u003c/table\u003e\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "f3nk38tIKytQ"
      },
      "source": [
        "TensorFlow Hub (TF-Hub) is a platform to share machine learning expertise packaged in reusable resources, notably pre-trained **modules**.\n",
        "\n",
        "In this colab, we will use a module that packages the [DELF](https://github.com/tensorflow/models/tree/master/research/delf) neural network and logic for processing images to identify keypoints and their descriptors. The weights of the neural network were trained on images of landmarks as described in [this paper](https://arxiv.org/abs/1612.06321)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "SI7eVflHHxvi"
      },
      "outputs": [],
      "source": [
        "import matplotlib.image as mpimg\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from PIL import Image, ImageOps\n",
        "from scipy.spatial import cKDTree\n",
        "from skimage.feature import plot_matches\n",
        "from skimage.measure import ransac\n",
        "from skimage.transform import AffineTransform\n",
        "from six import BytesIO\n",
        "%tensorflow_version 1.x\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "from six.moves.urllib.request import urlopen"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "qquo2HiONiDK"
      },
      "source": [
        "## The data\n",
        "\n",
        "In the next cell, we specify the URLs of two images we would like to process with DELF in order to match and compare them."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "l93ye4WFIqIV"
      },
      "outputs": [],
      "source": [
        "IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/2/28/Bridge_of_Sighs%2C_Oxford.jpg'\n",
        "IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/c/c3/The_Bridge_of_Sighs_and_Sheldonian_Theatre%2C_Oxford.jpg'\n",
        "\n",
        "IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/1/1e/Golden_gate2.jpg'\n",
        "IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/3/3e/GoldenGateBridge.jpg'\n",
        "\n",
        "IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/c/ce/2006_01_21_Ath%C3%A8nes_Parth%C3%A9non.JPG'\n",
        "IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/5/5c/ACROPOLIS_1969_-_panoramio_-_jean_melis.jpg'\n",
        "\n",
        "# IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/d/d8/Eiffel_Tower%2C_November_15%2C_2011.jpg'\n",
        "# IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/a/a8/Eiffel_Tower_from_immediately_beside_it%2C_Paris_May_2008.jpg'\n",
        "\n",
        "# IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/2/2d/Parthenon-Restoration-Nov-2005-a.jpg'\n",
        "# IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/5/57/White_House_06.02.08.jpg'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ttlHtcmiN6QF"
      },
      "source": [
        "Download, resize, save and display the images."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "AANtNUgKJ_Yl"
      },
      "outputs": [],
      "source": [
        "# The names that will be used for the resized local images.\n",
        "\n",
        "IMAGE_1_JPG = 'image_1.jpg'\n",
        "IMAGE_2_JPG = 'image_2.jpg'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "cellView": "form",
        "colab": {},
        "colab_type": "code",
        "id": "JDs4t8rPdyNJ"
      },
      "outputs": [],
      "source": [
        "#@title The images that will be processed by DELF\n",
        "def download_and_resize_image(url, filename, new_width=256, new_height=256):\n",
        "  response = urlopen(url)\n",
        "  image_data = response.read()\n",
        "  image_data = BytesIO(image_data)\n",
        "  pil_image = Image.open(image_data)\n",
        "  pil_image = ImageOps.fit(pil_image, (new_width, new_height), Image.ANTIALIAS)\n",
        "  pil_image_rgb = pil_image.convert('RGB')\n",
        "  pil_image_rgb.save(filename, format='JPEG', quality=90)\n",
        "\n",
        "download_and_resize_image(IMAGE_1_URL, IMAGE_1_JPG)\n",
        "download_and_resize_image(IMAGE_2_URL, IMAGE_2_JPG)\n",
        "\n",
        "def show_images(image_path_list):\n",
        "  plt.figure()\n",
        "  for i, image_path in enumerate(image_path_list):\n",
        "    plt.subplot(1, len(image_path_list), i+1)\n",
        "    plt.imshow(np.asarray(Image.open(image_path)))\n",
        "    plt.title(image_path)\n",
        "    plt.grid(False)\n",
        "    plt.yticks([])\n",
        "    plt.xticks([])\n",
        "  plt.show()\n",
        "\n",
        "show_images([IMAGE_1_JPG, IMAGE_2_JPG])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "CXvEAgIyQr9L"
      },
      "source": [
        "Next, we define a function to read the images into a tensor so that they can be processed by TensorFlow."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "a-e98I4mQqp6"
      },
      "outputs": [],
      "source": [
        "def image_input_fn():\n",
        "  filename_queue = tf.train.string_input_producer(\n",
        "      [IMAGE_1_JPG, IMAGE_2_JPG], shuffle=False)\n",
        "  reader = tf.WholeFileReader()\n",
        "  _, value = reader.read(filename_queue)\n",
        "  image_tf = tf.image.decode_jpeg(value, channels=3)\n",
        "  return tf.image.convert_image_dtype(image_tf, tf.float32)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "leKqkoT9OP7r"
      },
      "source": [
        "## Apply the DELF module to the data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "A3WoT1-SPoTI"
      },
      "source": [
        "The DELF module takes an image as input and will describe noteworthy points with vectors. The following cell contains the core of this colab's logic."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "bECFwKHnKW14"
      },
      "outputs": [],
      "source": [
        "tf.reset_default_graph()\n",
        "tf.logging.set_verbosity(tf.logging.FATAL)\n",
        "\n",
        "m = hub.Module('https://tfhub.dev/google/delf/1')\n",
        "\n",
        "# The module operates on a single image at a time, so define a placeholder to\n",
        "# feed an arbitrary image in.\n",
        "image_placeholder = tf.placeholder(\n",
        "    tf.float32, shape=(None, None, 3), name='input_image')\n",
        "\n",
        "module_inputs = {\n",
        "    'image': image_placeholder,\n",
        "    'score_threshold': 100.0,\n",
        "    'image_scales': [0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0],\n",
        "    'max_feature_num': 1000,\n",
        "}\n",
        "\n",
        "module_outputs = m(module_inputs, as_dict=True)\n",
        "\n",
        "image_tf = image_input_fn()\n",
        "\n",
        "with tf.train.MonitoredSession() as sess:\n",
        "  results_dict = {}  # Stores the locations and their descriptors for each image\n",
        "  for image_path in [IMAGE_1_JPG, IMAGE_2_JPG]:\n",
        "    image = sess.run(image_tf)\n",
        "    print('Extracting locations and descriptors from %s' % image_path)\n",
        "    results_dict[image_path] = sess.run(\n",
        "        [module_outputs['locations'], module_outputs['descriptors']],\n",
        "        feed_dict={image_placeholder: image})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NByyBA5yOL2b"
      },
      "source": [
        "## Use the locations and description vectors to match the images"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "cellView": "form",
        "colab": {},
        "colab_type": "code",
        "id": "mVaKXT3cMSib"
      },
      "outputs": [],
      "source": [
        "#@title TensorFlow is not needed for this post-processing and visualization\n",
        "def match_images(results_dict, image_1_path, image_2_path):\n",
        "  distance_threshold = 0.8\n",
        "\n",
        "  # Read features.\n",
        "  locations_1, descriptors_1 = results_dict[image_1_path]\n",
        "  num_features_1 = locations_1.shape[0]\n",
        "  print(\"Loaded image 1's %d features\" % num_features_1)\n",
        "  locations_2, descriptors_2 = results_dict[image_2_path]\n",
        "  num_features_2 = locations_2.shape[0]\n",
        "  print(\"Loaded image 2's %d features\" % num_features_2)\n",
        "\n",
        "  # Find nearest-neighbor matches using a KD tree.\n",
        "  d1_tree = cKDTree(descriptors_1)\n",
        "  _, indices = d1_tree.query(\n",
        "      descriptors_2, distance_upper_bound=distance_threshold)\n",
        "\n",
        "  # Select feature locations for putative matches.\n",
        "  locations_2_to_use = np.array([\n",
        "      locations_2[i,]\n",
        "      for i in range(num_features_2)\n",
        "      if indices[i] != num_features_1\n",
        "  ])\n",
        "  locations_1_to_use = np.array([\n",
        "      locations_1[indices[i],]\n",
        "      for i in range(num_features_2)\n",
        "      if indices[i] != num_features_1\n",
        "  ])\n",
        "\n",
        "  # Perform geometric verification using RANSAC.\n",
        "  _, inliers = ransac(\n",
        "      (locations_1_to_use, locations_2_to_use),\n",
        "      AffineTransform,\n",
        "      min_samples=3,\n",
        "      residual_threshold=20,\n",
        "      max_trials=1000)\n",
        "\n",
        "  print('Found %d inliers' % sum(inliers))\n",
        "\n",
        "  # Visualize correspondences.\n",
        "  _, ax = plt.subplots()\n",
        "  img_1 = mpimg.imread(image_1_path)\n",
        "  img_2 = mpimg.imread(image_2_path)\n",
        "  inlier_idxs = np.nonzero(inliers)[0]\n",
        "  plot_matches(\n",
        "      ax,\n",
        "      img_1,\n",
        "      img_2,\n",
        "      locations_1_to_use,\n",
        "      locations_2_to_use,\n",
        "      np.column_stack((inlier_idxs, inlier_idxs)),\n",
        "      matches_color='b')\n",
        "  ax.axis('off')\n",
        "  ax.set_title('DELF correspondences')\n",
        "\n",
        "match_images(results_dict, IMAGE_1_JPG, IMAGE_2_JPG)\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "RUymE2l9GZfO"
      ],
      "default_view": {},
      "name": "TF-Hub Delf module",
      "provenance": [],
      "version": "0.3.2",
      "views": {}
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}