Dockerized tests in order to run in Ubuntu 16.04 (required by tf >= 1.5) (#143)

* Dockerized tests in order to run in Ubuntu 16.04 (required by tf >= 1.5)

* Udpated Readme.md to recommend using conda environment.

Author: tomasatdatabricks

.travis.yml

@@ -1,11 +1,14 @@
-dist: xenial
+dist: trusty
 
 language: java
 
 jdk: oraclejdk8
 
 sudo: required
 
+services:
+  - docker
+
 cache:
   directories:
   - $HOME/.ivy2/
@@ -28,9 +31,6 @@ env:
   - SCALA_BINARY_VERSION=2.10.6 SPARK_VERSION=2.2.0 SPARK_BUILD="spark-2.2.0-bin-hadoop2.7"
     SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz"
     TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl PYTHON_VERSION=3.6.2
-  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.0.2 SPARK_BUILD="spark-2.0.2-bin-hadoop2.7"
-    SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz"
-    TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl PYTHON_VERSION=2.7.13
   - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.1.1 SPARK_BUILD="spark-2.1.1-bin-hadoop2.7"
     SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz"
     TF_PY2_BUILD=tensorflow-1.3.0-cp27-none-linux_x86_64.whl PYTHON_VERSION=2.7.13
@@ -52,45 +52,68 @@ env:
       PYTHON_VERSION=3.6.2
 
 before_install:
- - ./bin/download_travis_dependencies.sh
+  - ./bin/download_travis_dependencies.sh
+  - if [[ "$PYTHON_VERSION" == 2.* ]]; then
+        export CONDA_URL="repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
+        export PYSPARK_PYTHON=python2;
+      else
+        export CONDA_URL="repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh";
+        export PYSPARK_PYTHON=python3;
+      fi
+  - docker run -e "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
+               -e SPARK_VERSION
+               -e SPARK_BUILD
+               -e SCALA_BINARY_VERSION
+               -e PYTHON_VERSION
+               -e PYSPARK_PYTHON
+               -e CONDA_URL
+               -d --name ubuntu-test -v $HOME ubuntu:16.04 tail -f /dev/null
+  - docker cp `pwd` ubuntu-test:$HOME/
+  - docker cp $HOME/.cache ubuntu-test:$HOME/
+  - docker ps
 
 # See this page: http://conda.pydata.org/docs/travis.html
 install:
-  - sudo apt-get update
-  # We do this conditionally because it saves us some downloading if the
-  # version is the same.
-  - if [[ "$PYTHON_VERSION" == 2.* ]]; then
-      wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
-      export PYSPARK_PYTHON=python2;
-    else
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-      export PYSPARK_PYTHON=python3;
-    fi
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - hash -r
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
-  # Useful for debugging any issues with conda
-  - conda info -a
+  # install needed ubuntu packages
+  - docker exec -t ubuntu-test bash -c "apt-get update && apt-get upgrade -y"
+  - docker exec -t ubuntu-test bash -c "apt-get install -y curl bzip2 openjdk-8-jdk"
+  # download and set up miniconda
+  - docker exec -t ubuntu-test bash -c "
+      curl https://$CONDA_URL >> $HOME/miniconda.sh;
+      bash $HOME/miniconda.sh -b -p $HOME/miniconda;
+      bash $HOME/miniconda.sh -b -p $HOME/miniconda;
+      $HOME/miniconda/bin/conda config --set always_yes yes --set changeps1 no;
+      $HOME/miniconda/bin/conda update -q conda;
+      $HOME/miniconda/bin/conda info -a;
+      $HOME/miniconda/bin/conda create -q -n test-environment python=$PYTHON_VERSION"
 
-  # Create and activate conda environment
-  - conda create -q -n test-environment python=$PYTHON_VERSION
-  - source activate test-environment
-  # Log python & pip versions
-  - python --version
-  - pip --version
-  # Install dependencies from requirements.txt
-  - pip install --user -r ./python/requirements.txt
-  # Install tensorflow
-  - pip install --user tensorflow==1.6
+  # Activate conda environment ad install required packages
+  - docker exec -t ubuntu-test bash -c "
+      source $HOME/miniconda/bin/activate test-environment;
+      python --version;
+      pip --version;
+      pip install --user -r $HOME/tensorframes/python/requirements.txt;"
 
 script:
- - rm -rf /home/travis/.javacpp
- # Manually remove previous versions, otherwise the snaphsot will not get updated.
- - rm -rf /home/travis/.ivy2/cache/org.bytedeco.javacpp-presets/*
  # Run the scala unit tests first
- - sbt -Dspark.version=$SPARK_VERSION -Dpython.version=$PYSPARK_PYTHON -Dscala.version=$SCALA_BINARY_VERSION tfs_testing/test
- # Run the python unit tests.
- - sbt -Dspark.version=$SPARK_VERSION -Dscala.version=$SCALA_BINARY_VERSION tfs_testing/assembly
- - SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD ./python/run-tests.sh
+ - docker exec -t ubuntu-test bash -c "
+     source $HOME/miniconda/bin/activate test-environment;
+     cd $HOME/tensorframes;
+     ./build/sbt -Dspark.version=$SPARK_VERSION
+                 -Dpython.version=$PYSPARK_PYTHON
+                 -Dscala.version=$SCALA_BINARY_VERSION
+                 tfs_testing/test"
+
+ # Build the assembly
+ - docker exec -t ubuntu-test bash -c "
+     source $HOME/miniconda/bin/activate test-environment;
+     cd $HOME/tensorframes;
+     ./build/sbt -Dspark.version=$SPARK_VERSION
+                 -Dscala.version=$SCALA_BINARY_VERSION
+                 tfs_testing/assembly"
+
+ # Run python tests
+ - docker exec -t ubuntu-test bash -c "
+     source $HOME/miniconda/bin/activate test-environment;
+     cd $HOME/tensorframes;
+     SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD ./python/run-tests.sh"

README.md

@@ -178,20 +178,23 @@ df2.collect()
 ```
 
 ## How to compile and install for developers
-
-It is recommended you use [Nix](http://nixos.org/nix/) to guarantee that the build environment
-can be reproduced. Once you have installed Nix, you can set the environment from
+It is recommended you use [Conda Environment](https://conda.io/docs/user-guide/tasks/manage-environments.html) to guarantee that the build environment
+can be reproduced. Once you have installed Conda, you can set the environment from
 the root of project:
 
 ```bash
-nix-shell --pure default.nix
+conda create -q -n tensorframes-environment python=$PYTHON_VERSION
 ```
 
-This will create a python 2.7 environment with all the dependencies. If you
-want to work with Python 3.5, use `default-3.5.nix` instead.
+This will create an environment for your project. We recommend using Python version 3.6.2 or 2.7.13.
+After the environemnt is created, you can activate it and install all dependencies as follows:
+
+```bash
+conda activate tensorframes-environment
+pip install --user -r python/requirements.txt
+```
 
- The C++ bindings are already compiled though, so you should only have to deal with compiling
- the scala code. The recommended procedure is to use the assembly:
+You also need to compile the scala code. The recommended procedure is to use the assembly:
 
 ```bash
 build/sbt tfs_testing/assembly

python/requirements.txt

@@ -1,3 +1,4 @@
 # This file should list any python package dependencies.
 nose>=1.3.3
 pandas>=0.19.1
+tensorflow==1.6

python/run-tests.sh

@@ -93,7 +93,7 @@ export TF_CPP_MIN_LOG_LEVEL=2  # Warning level
 if [[ "$python_major" == "2" ]]; then
 
   # Horrible hack for spark 1.x: we manually remove some log lines to stay below the 4MB log limit on Travis.
-  $PYSPARK_DRIVER_PYTHON `which nosetests` -v --all-modules -w "$PROJECT_HOME/src/main/python" 2>&1 | grep -vE "INFO (ParquetOutputFormat|SparkContext|ContextCleaner|ShuffleBlockFetcherIterator|MapOutputTrackerMaster|TaskSetManager|Executor|MemoryStore|CacheManager|BlockManager|DAGScheduler|PythonRDD|TaskSchedulerImpl|ZippedPartitionsRDD2)";
+  $PYSPARK_DRIVER_PYTHON -m "nose" -v --all-modules -w "$PROJECT_HOME/src/main/python" 2>&1 | grep -vE "INFO (ParquetOutputFormat|SparkContext|ContextCleaner|ShuffleBlockFetcherIterator|MapOutputTrackerMaster|TaskSetManager|Executor|MemoryStore|CacheManager|BlockManager|DAGScheduler|PythonRDD|TaskSchedulerImpl|ZippedPartitionsRDD2)";
 
 else