TensorFlow 1.3 update follow-up and cleanups (#123)

- adds more tests for different versions of TF
- adds dependencies to the official proto interfaces instead of a local copy

* update to tensorflow 1.3.0

* proper fix

* cleanups

* update to tf v1.3

1. Removing the tensorflow framework java sources in favor of the
   tensorflow.proto project
2. Made changes to unit tests to w.r.t. the new API

* travis update

* travis update

Author: phi-dbq

.travis.yml

@@ -1,30 +1,67 @@
-language: scala
-
-scala:
-  - 2.10.6
+dist: trusty
 
-jdk:
-  - oraclejdk8
+jdk: oraclejdk8
 
 sudo: required
 
-dist: trusty
-
 cache:
   directories:
-    - $HOME/.ivy2/
-    - $HOME/.sbt/launchers/
-    - $HOME/.cache/spark-versions/
-    - $HOME/.sbt/boot/scala-2.10.6/
-    - $HOME/.sbt/boot/scala-2.11.8/
+  - $HOME/.ivy2/
+  - $HOME/.sbt/launchers/
+  - $HOME/.cache/spark-versions/
+  - $HOME/.sbt/boot/scala-2.10.6/
+  - $HOME/.sbt/boot/scala-2.11.8/
 
 env:
   matrix:
-    - PIP_BIN=pip PYSPARK_PYTHON=python2 TF_BUILD=tensorflow-1.0.1-cp27-none-linux_x86_64.whl SCALA_BINARY_VERSION="2.11.8" SPARK_VERSION="2.0.2" SPARK_BUILD="spark-2.0.2-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz"
-    - PIP_BIN=pip PYSPARK_PYTHON=python2 TF_BUILD=tensorflow-1.0.1-cp27-none-linux_x86_64.whl SCALA_BINARY_VERSION="2.11.8" SPARK_VERSION="2.1.0" SPARK_BUILD="spark-2.1.0-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz"
-    - PIP_BIN=pip PYSPARK_PYTHON=python2 TF_BUILD=tensorflow-1.0.1-cp27-none-linux_x86_64.whl SCALA_BINARY_VERSION="2.10.6" SPARK_VERSION="2.1.0" SPARK_BUILD="spark-2.1.0-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz"
-    - PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_BUILD=tensorflow-1.0.1-cp35-cp35m-linux_x86_64.whl SCALA_BINARY_VERSION="2.11.8" SPARK_VERSION="2.1.0" SPARK_BUILD="spark-2.1.0-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz"
-    - PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_BUILD=tensorflow-1.0.1-cp35-cp35m-linux_x86_64.whl SCALA_BINARY_VERSION="2.10.6" SPARK_VERSION="2.1.0" SPARK_BUILD="spark-2.1.0-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz"
+  - SCALA_BINARY_VERSION=2.10.6 SPARK_VERSION=2.0.2 SPARK_BUILD="spark-2.0.2-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz"
+    PIP_BIN=pip2 PYSPARK_PYTHON=python2 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.10.6 SPARK_VERSION=2.0.2 SPARK_BUILD="spark-2.0.2-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz"
+    PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.10.6 SPARK_VERSION=2.1.1 SPARK_BUILD="spark-2.1.1-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz"
+    PIP_BIN=pip2 PYSPARK_PYTHON=python2 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.10.6 SPARK_VERSION=2.1.1 SPARK_BUILD="spark-2.1.1-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz"
+    PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.10.6 SPARK_VERSION=2.2.0 SPARK_BUILD="spark-2.2.0-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz"
+    PIP_BIN=pip2 PYSPARK_PYTHON=python2 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.10.6 SPARK_VERSION=2.2.0 SPARK_BUILD="spark-2.2.0-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz"
+    PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.0.2 SPARK_BUILD="spark-2.0.2-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz"
+    PIP_BIN=pip2 PYSPARK_PYTHON=python2 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.0.2 SPARK_BUILD="spark-2.0.2-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz"
+    PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.1.1 SPARK_BUILD="spark-2.1.1-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz"
+    PIP_BIN=pip2 PYSPARK_PYTHON=python2 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.1.1 SPARK_BUILD="spark-2.1.1-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz"
+    PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.2.0 SPARK_BUILD="spark-2.2.0-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz"
+    PIP_BIN=pip2 PYSPARK_PYTHON=python2 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.2.0 SPARK_BUILD="spark-2.2.0-bin-hadoop2.7"
+    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz"
+    PIP_BIN=pip3 PYSPARK_PYTHON=python3 TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl
+    TF_PY3_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl TF_BUILD=tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl
 
 before_install:
  - ./bin/download_travis_dependencies.sh
@@ -34,7 +71,7 @@ install:
  - $PIP_BIN install --user --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/$TF_BUILD
  # We also install a python 2 version of Tensorflow. It is used in the Scala tests to make sure that
  # the protos produced by scala and python are binary identical.
- - pip install --user --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp27-none-linux_x86_64.whl
+ - pip install --user --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/${TF_PY2_BUILD}
 
 script:
  - rm -rf /home/travis/.javacpp
@@ -45,4 +82,3 @@ script:
  # Run the python unit tests.
  - sbt -Dspark.version=$SPARK_VERSION -Dscala.version=$SCALA_BINARY_VERSION tfs_testing/assembly
  - SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD ./python/run-tests.sh
-

project/Build.scala

@@ -51,7 +51,9 @@ object Shading extends Build {
     "com.typesafe.scala-logging" %% "scala-logging-api" % "2.1.2",
     "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2",
     // TensorFlow dependencies
-    "org.tensorflow" % "tensorflow" % targetTensorFlowVersion
+    "org.tensorflow" % "tensorflow" % targetTensorFlowVersion,
+    "org.tensorflow" % "proto" % targetTensorFlowVersion,
+    "org.tensorflow" % "libtensorflow" % targetTensorFlowVersion
   )
 
   lazy val testDependencies = Seq(