From 0df074cc09411c663e4b3c3efebf3611ab480fc3 Mon Sep 17 00:00:00 2001 From: Morten Dahl Date: Fri, 17 Apr 2020 20:35:40 +0200 Subject: [PATCH] Upgrade to TF 2.1 (#61) * upgrade to TF 2.1 * update Circle CI to only build for TF 2.1 * update tests to not (directly) use TF sessions * [should fix] disable CC tests * parameterize tests * remove outdated CC test * copy test util from TFE to avoid dependency * ship without testing util * bring back skipped tests --- .circleci/config.yml | 178 ++--------- configure.sh | 110 ++++--- requirements-dev.txt | 1 + setup.py | 6 +- tf_big/BUILD | 20 +- tf_big/__init__.py | 2 + tf_big/cc/big_tensor_test.cc | 30 -- tf_big/python/ops/big_ops_test.py | 185 ++++++----- tf_big/python/tensor_test.py | 316 ++++++++++--------- tf_big/python/test/BUILD | 21 ++ tf_big/python/test/__init__.py | 5 + tf_big/python/test/execution_context.py | 42 +++ tf_big/python/test/execution_context_test.py | 28 ++ third_party/tf/tf_configure.bzl | 48 +-- 14 files changed, 474 insertions(+), 518 deletions(-) delete mode 100644 tf_big/cc/big_tensor_test.cc create mode 100644 tf_big/python/test/BUILD create mode 100644 tf_big/python/test/__init__.py create mode 100644 tf_big/python/test/execution_context.py create mode 100644 tf_big/python/test/execution_context_test.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 8876660..ba6c860 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -303,9 +303,9 @@ workflows: quicktest: jobs: - build-linux: - name: build-linux-py3.6-tf1.14.0 + name: build-linux-py3.6-tf2.1.0 python-version: "3.6" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" filters: branches: ignore: master @@ -316,7 +316,7 @@ workflows: name: bundle-linux-py3.6 python-version: "3.6" requires: - - build-linux-py3.6-tf1.14.0 + - build-linux-py3.6-tf2.1.0 filters: branches: ignore: master @@ -324,9 +324,9 @@ workflows: ignore: /.*/ - whltest-linux: - name: whltest-linux-py3.6-tf1.14.0 + name: whltest-linux-py3.6-tf2.1.0 python-version: "3.6" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" requires: - bundle-linux-py3.6 filters: @@ -338,27 +338,9 @@ workflows: linux-py3.5: jobs: - build-linux: - name: build-linux-py3.5-tf1.13.1 + name: build-linux-py3.5-tf2.1.0 python-version: "3.5" - tensorflow-version: "1.13.1" - filters: - branches: - only: master - tags: - only: /.*/ - - build-linux: - name: build-linux-py3.5-tf1.13.2 - python-version: "3.5" - tensorflow-version: "1.13.2" - filters: - branches: - only: master - tags: - only: /.*/ - - build-linux: - name: build-linux-py3.5-tf1.14.0 - python-version: "3.5" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" filters: branches: only: master @@ -369,9 +351,7 @@ workflows: name: bundle-linux-py3.5 python-version: "3.5" requires: - - build-linux-py3.5-tf1.13.1 - - build-linux-py3.5-tf1.13.2 - - build-linux-py3.5-tf1.14.0 + - build-linux-py3.5-tf2.1.0 filters: branches: only: master @@ -379,31 +359,9 @@ workflows: only: /.*/ - whltest-linux: - name: whltest-linux-py3.5-tf1.13.1 - python-version: "3.5" - tensorflow-version: "1.13.1" - requires: - - bundle-linux-py3.5 - filters: - branches: - only: master - tags: - only: /.*/ - - whltest-linux: - name: whltest-linux-py3.5-tf1.13.2 - python-version: "3.5" - tensorflow-version: "1.13.2" - requires: - - bundle-linux-py3.5 - filters: - branches: - only: master - tags: - only: /.*/ - - whltest-linux: - name: whltest-linux-py3.5-tf1.14.0 + name: whltest-linux-py3.5-tf2.1.0 python-version: "3.5" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" requires: - bundle-linux-py3.5 filters: @@ -415,9 +373,7 @@ workflows: - store: name: store-linux-py3.5 requires: - - whltest-linux-py3.5-tf1.13.1 - - whltest-linux-py3.5-tf1.13.2 - - whltest-linux-py3.5-tf1.14.0 + - whltest-linux-py3.5-tf2.1.0 filters: branches: only: master @@ -448,27 +404,9 @@ workflows: linux-py3.6: jobs: - build-linux: - name: build-linux-py3.6-tf1.13.1 - python-version: "3.6" - tensorflow-version: "1.13.1" - filters: - branches: - only: master - tags: - only: /.*/ - - build-linux: - name: build-linux-py3.6-tf1.13.2 - python-version: "3.6" - tensorflow-version: "1.13.2" - filters: - branches: - only: master - tags: - only: /.*/ - - build-linux: - name: build-linux-py3.6-tf1.14.0 + name: build-linux-py3.6-tf2.1.0 python-version: "3.6" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" filters: branches: only: master @@ -479,9 +417,7 @@ workflows: name: bundle-linux-py3.6 python-version: "3.6" requires: - - build-linux-py3.6-tf1.13.1 - - build-linux-py3.6-tf1.13.2 - - build-linux-py3.6-tf1.14.0 + - build-linux-py3.6-tf2.1.0 filters: branches: only: master @@ -489,31 +425,9 @@ workflows: only: /.*/ - whltest-linux: - name: whltest-linux-py3.6-tf1.13.1 - python-version: "3.6" - tensorflow-version: "1.13.1" - requires: - - bundle-linux-py3.6 - filters: - branches: - only: master - tags: - only: /.*/ - - whltest-linux: - name: whltest-linux-py3.6-tf1.13.2 - python-version: "3.6" - tensorflow-version: "1.13.2" - requires: - - bundle-linux-py3.6 - filters: - branches: - only: master - tags: - only: /.*/ - - whltest-linux: - name: whltest-linux-py3.6-tf1.14.0 + name: whltest-linux-py3.6-tf2.1.0 python-version: "3.6" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" requires: - bundle-linux-py3.6 filters: @@ -525,9 +439,7 @@ workflows: - store: name: store-linux-py3.6 requires: - - whltest-linux-py3.6-tf1.13.1 - - whltest-linux-py3.6-tf1.13.2 - - whltest-linux-py3.6-tf1.14.0 + - whltest-linux-py3.6-tf2.1.0 filters: branches: only: master @@ -558,27 +470,9 @@ workflows: macos-py3.6: jobs: - build-macos: - name: build-macos-py3.6-tf1.13.1 - python-version: "3.6" - tensorflow-version: "1.13.1" - filters: - branches: - only: master - tags: - only: /.*/ - - build-macos: - name: build-macos-py3.6-tf1.13.2 - python-version: "3.6" - tensorflow-version: "1.13.2" - filters: - branches: - only: master - tags: - only: /.*/ - - build-macos: - name: build-macos-py3.6-tf1.14.0 + name: build-macos-py3.6-tf2.1.0 python-version: "3.6" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" filters: branches: only: master @@ -589,9 +483,7 @@ workflows: name: bundle-macos-py3.6 python-version: "3.6" requires: - - build-macos-py3.6-tf1.13.1 - - build-macos-py3.6-tf1.13.2 - - build-macos-py3.6-tf1.14.0 + - build-macos-py3.6-tf2.1.0 filters: branches: only: master @@ -599,31 +491,9 @@ workflows: only: /.*/ - whltest-macos: - name: whltest-macos-py3.6-tf1.13.1 - python-version: "3.6" - tensorflow-version: "1.13.1" - requires: - - bundle-macos-py3.6 - filters: - branches: - only: master - tags: - only: /.*/ - - whltest-macos: - name: whltest-macos-py3.6-tf1.13.2 - python-version: "3.6" - tensorflow-version: "1.13.2" - requires: - - bundle-macos-py3.6 - filters: - branches: - only: master - tags: - only: /.*/ - - whltest-macos: - name: whltest-macos-py3.6-tf1.14.0 + name: whltest-macos-py3.6-tf2.1.0 python-version: "3.6" - tensorflow-version: "1.14.0" + tensorflow-version: "2.1.0" requires: - bundle-macos-py3.6 filters: @@ -635,9 +505,7 @@ workflows: - store: name: store-macos-py3.6 requires: - - whltest-macos-py3.6-tf1.13.1 - - whltest-macos-py3.6-tf1.13.2 - - whltest-macos-py3.6-tf1.14.0 + - whltest-macos-py3.6-tf2.1.0 filters: branches: only: master diff --git a/configure.sh b/configure.sh index dffa143..afc0532 100755 --- a/configure.sh +++ b/configure.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== +PLATFORM="$(uname -s | tr 'A-Z' 'a-z')" + function write_to_bazelrc() { echo "$1" >> .bazelrc } @@ -21,42 +23,55 @@ function write_action_env_to_bazelrc() { write_to_bazelrc "build --action_env $1=\"$2\"" } +function is_linux() { + [[ "${PLATFORM}" == "linux" ]] +} + +function is_macos() { + [[ "${PLATFORM}" == "darwin" ]] +} + +function is_windows() { + # On windows, the shell script is actually running in msys + [[ "${PLATFORM}" =~ msys_nt*|mingw*|cygwin*|uwin* ]] +} + +function is_ppc64le() { + [[ "$(uname -m)" == "ppc64le" ]] +} + + # Remove .bazelrc if it already exist [ -e .bazelrc ] && rm .bazelrc -# Check if we are building GPU or CPU ops, default CPU -while [[ "$TF_NEED_CUDA" == "" ]]; do - read -p "Do you want to build ops again TensorFlow CPU pip package?"\ -" Y or enter for CPU (tensorflow), N for GPU (tensorflow-gpu). [Y/n] " INPUT - case $INPUT in - [Yy]* ) echo "Build with CPU pip package."; TF_NEED_CUDA=0;; - [Nn]* ) echo "Build with GPU pip package."; TF_NEED_CUDA=1;; - "" ) echo "Build with CPU pip package."; TF_NEED_CUDA=0;; - * ) echo "Invalid selection: " $INPUT;; - esac -done +TF_NEED_CUDA=0 + +if is_windows; then + PIP_MANYLINUX2010=0 +else + PIP_MANYLINUX2010=1 +fi +TF_CUDA_VERSION=10.1 # CPU if [[ "$TF_NEED_CUDA" == "0" ]]; then - # Check if it's installed if [[ $(pip show tensorflow) == *tensorflow* ]] || [[ $(pip show tf-nightly) == *tf-nightly* ]] ; then echo 'Using installed tensorflow' else - echo 'Please install tensorflow' - exit 1 - fi - -else - - # Check if it's installed - if [[ $(pip show tensorflow-gpu) == *tensorflow-gpu* ]] || [[ $(pip show tf-nightly-gpu) == *tf-nightly-gpu* ]]; then - echo 'Using installed tensorflow-gpu' - else - echo 'Please install tensorflow-gpu' - exit 1 + # Uninstall GPU version if it is installed. + if [[ $(pip show tensorflow-gpu) == *tensorflow-gpu* ]]; then + echo 'Already have gpu version of tensorflow installed. Uninstalling......\n' + pip uninstall tensorflow-gpu + elif [[ $(pip show tf-nightly-gpu) == *tf-nightly-gpu* ]]; then + echo 'Already have gpu version of tensorflow installed. Uninstalling......\n' + pip uninstall tf-nightly-gpu + fi + # Install CPU version + echo 'Installing tensorflow......\n' + pip install tensorflow fi fi @@ -64,33 +79,52 @@ fi TF_CFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') ) TF_LFLAGS="$(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))')" -write_to_bazelrc "build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true" -write_to_bazelrc "build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain" +# write_to_bazelrc "build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true" +# if [[ "$PIP_MANYLINUX2010" == "0" ]]; then +# write_to_bazelrc "build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain" +# fi +# # Add Ubuntu toolchain flags +# if is_linux; then +# write_to_bazelrc "build:manylinux2010cuda100 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain" +# write_to_bazelrc "build:manylinux2010cuda101 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain" +# fi write_to_bazelrc "build --spawn_strategy=standalone" write_to_bazelrc "build --strategy=Genrule=standalone" write_to_bazelrc "build -c opt" -write_action_env_to_bazelrc "TF_HEADER_DIR" ${TF_CFLAGS:2} -SHARED_LIBRARY_DIR=${TF_LFLAGS:2} +if is_windows; then + # Use pywrap_tensorflow instead of tensorflow_framework on Windows + SHARED_LIBRARY_DIR=${TF_CFLAGS:2:-7}"python" +else + SHARED_LIBRARY_DIR=${TF_LFLAGS:2} +fi SHARED_LIBRARY_NAME=$(echo $TF_LFLAGS | rev | cut -d":" -f1 | rev) if ! [[ $TF_LFLAGS =~ .*:.* ]]; then - if [[ "$(uname)" == "Darwin" ]]; then - SHARED_LIBRARY_NAME="libtensorflow_framework.so" + if is_macos; then + SHARED_LIBRARY_NAME="libtensorflow_framework.dylib" + elif is_windows; then + # Use pywrap_tensorflow's import library on Windows. It is in the same dir as the dll/pyd. + SHARED_LIBRARY_NAME="_pywrap_tensorflow_internal.lib" else SHARED_LIBRARY_NAME="libtensorflow_framework.so" fi fi + +HEADER_DIR=${TF_CFLAGS:2} +if is_windows; then + SHARED_LIBRARY_DIR=${SHARED_LIBRARY_DIR//\\//} + SHARED_LIBRARY_NAME=${SHARED_LIBRARY_NAME//\\//} + HEADER_DIR=${HEADER_DIR//\\//} +fi +write_action_env_to_bazelrc "TF_HEADER_DIR" ${HEADER_DIR} write_action_env_to_bazelrc "TF_SHARED_LIBRARY_DIR" ${SHARED_LIBRARY_DIR} write_action_env_to_bazelrc "TF_SHARED_LIBRARY_NAME" ${SHARED_LIBRARY_NAME} write_action_env_to_bazelrc "TF_NEED_CUDA" ${TF_NEED_CUDA} -# TODO(yifeif): do not hardcode path -if [[ "$TF_NEED_CUDA" == "1" ]]; then - write_action_env_to_bazelrc "CUDNN_INSTALL_PATH" "/usr/lib/x86_64-linux-gnu" - write_action_env_to_bazelrc "TF_CUDA_VERSION" "10.0" - write_action_env_to_bazelrc "TF_CUDNN_VERSION" "7" - write_action_env_to_bazelrc "CUDA_TOOLKIT_PATH" "/usr/local/cuda" - write_to_bazelrc "build --config=cuda" - write_to_bazelrc "test --config=cuda" +if is_linux; then + if [[ "$PIP_MANYLINUX2010" == "1" ]]; then + write_to_bazelrc "build --config=manylinux2010cuda101" + write_to_bazelrc "test --config=manylinux2010cuda101" + fi fi diff --git a/requirements-dev.txt b/requirements-dev.txt index bf5dd1b..1226dee 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +absl-py==0.9.0 cpplint==1.4.4 numpy==1.16.4 pip==19.2.3 diff --git a/setup.py b/setup.py index ce60a4a..8f71500 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def has_ext_modules(self): setuptools.setup( name="tf-big", - version="0.1.3", + version="0.2.0", packages=setuptools.find_packages(), package_data={ '': ['*.so'], @@ -25,10 +25,10 @@ def has_ext_modules(self): python_requires=">=3.5", install_requires=[ "numpy >=1.14", - "tensorflow >=1.13.1, <2", + "tensorflow ==2.1.0", ], extras_require={ - "tf": ["tensorflow >=1.13.1, <2"], + "tf": ["tensorflow ==2.1.0"], }, license="Apache License 2.0", url="https://github.com/tf-encrypted/tf-big", diff --git a/tf_big/BUILD b/tf_big/BUILD index 8c5916c..8b6aeed 100644 --- a/tf_big/BUILD +++ b/tf_big/BUILD @@ -19,22 +19,6 @@ cc_binary( copts = ["-pthread", "-std=c++11", "-D_GLIBCXX_USE_CXX11_ABI=0", "-fPIC"], ) -cc_test( - name = "big_ops_test", - srcs = [ - "cc/big_tensor_test.cc", - "cc/big_tensor.h", - "cc/big_tensor.cc", - ], - deps = [ - "@com_google_googletest//:gtest", - "@local_config_tf//:libtensorflow_framework", - "@local_config_tf//:tf_header_lib", - "@libgmp//:lib" - ], - copts = ["-pthread", "-std=c++11", "-D_GLIBCXX_USE_CXX11_ABI=0", "-fPIC"], -) - py_library( name = "big_ops_py", srcs = ([ @@ -55,6 +39,7 @@ py_test( main = "python/ops/big_ops_test.py", deps = [ ":big_ops_py", + "//tf_big/python/test:test_py", ], srcs_version = "PY2AND3", ) @@ -67,6 +52,7 @@ py_test( main = "python/tensor_test.py", deps = [ ":big_ops_py", + "//tf_big/python/test:test_py", ], srcs_version = "PY2AND3", ) @@ -79,7 +65,7 @@ py_library( "python/ops/__init__.py", ]), deps = [ - ":big_ops_py" + ":big_ops_py", ], srcs_version = "PY2AND3", ) diff --git a/tf_big/__init__.py b/tf_big/__init__.py index 67204de..b9e43a8 100644 --- a/tf_big/__init__.py +++ b/tf_big/__init__.py @@ -1,6 +1,7 @@ from tf_big.python.tensor import set_secure_default from tf_big.python.tensor import get_secure_default +from tf_big.python import test from tf_big.python.tensor import Tensor from tf_big.python.tensor import constant @@ -21,6 +22,7 @@ 'set_secure_default', 'get_secure_default', + 'test', 'Tensor', 'constant', diff --git a/tf_big/cc/big_tensor_test.cc b/tf_big/cc/big_tensor_test.cc deleted file mode 100644 index 5255924..0000000 --- a/tf_big/cc/big_tensor_test.cc +++ /dev/null @@ -1,30 +0,0 @@ -#include - -#include "tensorflow/core/framework/variant_tensor_data.h" -#include "gtest/gtest.h" - -#include "tf_big/cc/big_tensor.h" - -using tf_big::BigTensor; - -TEST(BigTensorTest, EncodeDecode) { - std::string input("2344134134"); - BigTensor b(mpz_class(input, 10)); - - tensorflow::VariantTensorData d; - - b.Encode(&d); - - BigTensor b2; - - b2.Decode(d); - - EXPECT_EQ(b2(0, 0).get_str(10), input); -} - -// TODO(justin1121) I don't think we need a main function but I couldn't make it -// work without! -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tf_big/python/ops/big_ops_test.py b/tf_big/python/ops/big_ops_test.py index 9495dd3..835451e 100644 --- a/tf_big/python/ops/big_ops_test.py +++ b/tf_big/python/ops/big_ops_test.py @@ -1,6 +1,8 @@ +import unittest +from absl.testing import parameterized + import numpy as np import tensorflow as tf -from tensorflow.python.platform import test from tf_big.python.ops.big_ops import big_import from tf_big.python.ops.big_ops import big_export @@ -9,150 +11,139 @@ from tf_big.python.ops.big_ops import big_mul from tf_big.python.ops.big_ops import big_mod from tf_big.python.ops.big_ops import big_pow +from tf_big.python.test import tf_execution_context -class BigTest(test.TestCase): - """BigTest test""" - - def test_import_export(self): - with tf.Session() as sess: - inp = [[b"43424"]] - variant = big_import(inp) - output = big_export(variant, tf.string) - - assert sess.run(output) == inp - def test_import_export_int32(self): - with tf.Session() as sess: - inp = [[43424]] - variant = big_import(inp) - output = big_export(variant, tf.int32) - - expected = [[43424]] - assert sess.run(output) == expected - - def test_add(self): - with tf.Session() as sess: - a = "5453452435245245245242534" - b = "1424132412341234123412341234134" +class BigTest(parameterized.TestCase): + """BigTest test""" - expected = int(a) + int(b) + @parameterized.parameters( + {"run_eagerly": run_eagerly, "raw": raw, "dtype": dtype} + for run_eagerly in (True, False) + for raw, dtype in ( + ([[43424]], tf.int32), + ([[b"43424"]], tf.string), + ) + ) + def test_import_export(self, run_eagerly, raw, dtype): + context = tf_execution_context(run_eagerly) + with context.scope(): + variant = big_import(raw) + output = big_export(variant, dtype) + assert context.evaluate(output) == raw + + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_add(self, run_eagerly): + a = "5453452435245245245242534" + b = "1424132412341234123412341234134" + expected = int(a) + int(b) + + context = tf_execution_context(run_eagerly) + with context.scope(): a_var = big_import([[a]]) b_var = big_import([[b]]) - c_var = big_add(a_var, b_var) - c_str = big_export(c_var, tf.string) - output = sess.run(c_str) - - assert int(output) == expected - - def test_pow(self): - with tf.Session() as sess: - base = "54" - exp = "3434" - modulus = "34" - - base_var = big_import([[base]]) - exp_var = big_import([[exp]]) - mod_var = big_import([[modulus]]) - - out = big_pow(base_var, exp_var, mod_var, secure=False) + np.testing.assert_equal(int(context.evaluate(c_str)), expected) - out_str = big_export(out, tf.string) - - output = sess.run(out_str) + @parameterized.parameters( + {"run_eagerly": run_eagerly, "secure": secure} + for run_eagerly in (True, False) + for secure in (True, False) + ) + def test_pow(self, run_eagerly, secure): + base = "54" + exp = "3434" + modulus = "35" + expected = pow(54, 3434, 35) - assert int(output) == 8 - - def test_pow_secure(self): - with tf.Session() as sess: - base = "54" - exp = "3434" - modulus = "35" + context = tf_execution_context(run_eagerly) + with context.scope(): base_var = big_import([[base]]) exp_var = big_import([[exp]]) mod_var = big_import([[modulus]]) - - out = big_pow(base_var, exp_var, mod_var, secure=True) - + out = big_pow(base_var, exp_var, mod_var, secure=secure) out_str = big_export(out, tf.string) - output = sess.run(out_str) - - assert int(output) == 11 + np.testing.assert_equal(int(context.evaluate(out_str)), expected) - def test_2d_matrix_add(self): - with tf.Session() as sess: - a = np.array([[5, 5], [5, 5]]).astype(np.int32) - b = np.array([[6, 6], [6, 6]]).astype(np.int32) + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_2d_matrix_add(self, run_eagerly): + a = np.array([[5, 5], [5, 5]]).astype(np.int32) + b = np.array([[6, 6], [6, 6]]).astype(np.int32) + expected = a + b - expected = a + b + context = tf_execution_context(run_eagerly) + with context.scope(): a_var = big_import(a) b_var = big_import(b) - c_var = big_add(a_var, b_var) - c_str = big_export(c_var, tf.int32) - output = sess.run(c_str) + np.testing.assert_equal(context.evaluate(c_str), expected) - np.testing.assert_equal(output, expected) + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_matmul(self, run_eagerly): + a = np.array([[5, 5], [5, 5]]).astype(np.int32) + b = np.array([[6, 6], [6, 6]]).astype(np.int32) + expected = a.dot(b) - def test_matmul(self): - with tf.Session() as sess: - a = np.array([[5, 5], [5, 5]]).astype(np.int32) - b = np.array([[6, 6], [6, 6]]).astype(np.int32) - - expected = a.dot(b) + context = tf_execution_context(run_eagerly) + with context.scope(): a_var = big_import(a) b_var = big_import(b) - c_var = big_matmul(a_var, b_var) - c_str = big_export(c_var, tf.int32) - output = sess.run(c_str) - - np.testing.assert_equal(output, expected) + np.testing.assert_equal(context.evaluate(c_str), expected) - def test_mul(self): - with tf.Session() as sess: - a = np.array([[5, 5], [5, 5]]).astype(np.int32) - b = np.array([[6, 6], [6, 6]]).astype(np.int32) + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_mul(self, run_eagerly): + a = np.array([[5, 5], [5, 5]]).astype(np.int32) + b = np.array([[6, 6], [6, 6]]).astype(np.int32) + expected = a * b - expected = a * b + context = tf_execution_context(run_eagerly) + with context.scope(): a_var = big_import(a) b_var = big_import(b) - c_var = big_mul(a_var, b_var) - c_str = big_export(c_var, tf.int32) - output = sess.run(c_str) + np.testing.assert_equal(context.evaluate(c_str), expected) - np.testing.assert_equal(output, expected) + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_mod(self, run_eagerly): + x = np.array([[123, 234], [345, 456]]).astype(np.int32) + n = np.array([[37]]).astype(np.int32) + expected = x % n - def test_mod(self): - with tf.Session() as sess: - x = np.array([[123, 234], [345, 456]]).astype(np.int32) - n = np.array([[37]]).astype(np.int32) - - expected = x % n + context = tf_execution_context(run_eagerly) + with context.scope(): x_big = big_import(x) n_big = big_import(n) y_big = big_mod(x_big, n_big) + y_str = big_export(y_big, tf.int32) - actual = sess.run(big_export(y_big, tf.int32)) - - np.testing.assert_equal(actual, expected) + np.testing.assert_equal(context.evaluate(y_str), expected) if __name__ == '__main__': - test.main() \ No newline at end of file + unittest.main() diff --git a/tf_big/python/tensor_test.py b/tf_big/python/tensor_test.py index ea3110b..9f39021 100644 --- a/tf_big/python/tensor_test.py +++ b/tf_big/python/tensor_test.py @@ -1,86 +1,101 @@ +import unittest +from absl.testing import parameterized + import numpy as np import tensorflow as tf -from tensorflow.python.keras.utils import tf_utils -from tensorflow.python.platform import test +from tf_big.python.tensor import convert_from_tensor from tf_big.python.tensor import convert_to_tensor from tf_big.python.tensor import random_uniform +from tf_big.python.test import tf_execution_context -class EvaluationTest(test.TestCase): - - def test_session_run(self): - x_raw = np.array([[123456789123456089123456089, 123406789123456789123406789]]) - x = convert_to_tensor(x_raw) +class EvaluationTest(parameterized.TestCase): - with tf.Session() as sess: - res = sess.run(x) - np.testing.assert_array_equal(res, x_raw.astype(str)) - - def test_eval(self): + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_eval(self, run_eagerly): x_raw = np.array([[123456789123456789123456789, 123456789123456789123456789]]) - x = convert_to_tensor(x_raw) - with tf.Session() as sess: - res = x.eval(session=sess) - np.testing.assert_array_equal(res, x_raw.astype(str)) + context = tf_execution_context(run_eagerly) + with context.scope(): + x = convert_to_tensor(x_raw) + x = convert_from_tensor(x) + + np.testing.assert_array_equal(context.evaluate(x).astype(str), x_raw.astype(str)) -class RandomTest(test.TestCase): +class RandomTest(parameterized.TestCase): - def test_uniform_random(self): + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_uniform_random(self, run_eagerly): shape = (2, 2) maxval = 2**100 - x = random_uniform(shape=shape, maxval=maxval) + context = tf_execution_context(run_eagerly) + with context.scope(): + x = random_uniform(shape=shape, maxval=maxval) + x = convert_from_tensor(x) + assert x.shape == shape - - with tf.Session() as sess: - res = sess.run(x) - assert res.shape == shape + assert context.evaluate(x).shape == shape -class ArithmeticTest(test.TestCase): +class ArithmeticTest(parameterized.TestCase): - def _core_test(self, op): + @parameterized.parameters( + {"run_eagerly": run_eagerly, "op_name": op_name, "op": op} + for run_eagerly in (True, False) + for op_name, op in ( + ("add", lambda x, y: x + y), + ("sub", lambda x, y: x - y), + ("mul", lambda x, y: x * y), + ) + ) + def test_op(self, run_eagerly, op_name, op): x_raw = np.array([[123456789123456789687293389, 123456789125927572056789]]) y_raw = np.array([[123456785629362289123456789, 123456789123456723456789]]) z_raw = op(x_raw, y_raw) - x = convert_to_tensor(x_raw) - y = convert_to_tensor(y_raw) - z = op(x, y) + context = tf_execution_context(run_eagerly) + with context.scope(): - with tf.Session() as sess: - res = sess.run(z) - np.testing.assert_array_equal(res, z_raw.astype(str)) + x = convert_to_tensor(x_raw) + y = convert_to_tensor(y_raw) + z = op(x, y) + z = convert_from_tensor(z) - def test_add(self): - self._core_test(lambda x, y: x + y) - - def test_sub(self): - self._core_test(lambda x, y: x - y) - - def test_mul(self): - self._core_test(lambda x, y: x * y) + np.testing.assert_array_equal(context.evaluate(z).astype(str), z_raw.astype(str)) -class NumberTheoryTest(test.TestCase): +class NumberTheoryTest(parameterized.TestCase): - def test_mod(self): + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_mod(self, run_eagerly): x_raw = np.array([[123456789123456789123456789, 123456789123456789123456789]]) n_raw = np.array([[10000]]) y_raw = x_raw % n_raw - x = convert_to_tensor(x_raw) - n = convert_to_tensor(n_raw) - y = x % n + context = tf_execution_context(run_eagerly) + with context.scope(): - with tf.Session() as sess: - res = sess.run(y) - np.testing.assert_array_equal(res, y_raw.astype(str)) + x = convert_to_tensor(x_raw) + n = convert_to_tensor(n_raw) + y = x % n + y = convert_from_tensor(y) - def test_inv(self): + np.testing.assert_array_equal(context.evaluate(y).astype(str), y_raw.astype(str)) + + + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_inv(self, run_eagerly): def egcd(a, b): if a == 0: @@ -96,120 +111,109 @@ def inv(a, m): n_raw = np.array([[10000000]]) y_raw = np.array([[inv(123456789123456789123456789, 10000000)]]) - x = convert_to_tensor(x_raw) - n = convert_to_tensor(n_raw) - y = x.inv(n) - - with tf.Session() as sess: - res = sess.run(y) - np.testing.assert_array_equal(res, y_raw.astype(str)) - - -class ConvertTest(test.TestCase): - - def _core_test(self, in_np, out_np, convert_to_tf_tensor): - if convert_to_tf_tensor: - in_tf = tf.convert_to_tensor(in_np) - x = convert_to_tensor(in_tf) - else: - x = convert_to_tensor(in_np) - - with tf.Session() as sess: - res = sess.run(x) - np.testing.assert_array_equal(res, out_np) - - def test_constant_int32(self): - x = np.array([[1,2,3,4]]).astype(np.int32) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=False, - ) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=True, - ) - - def test_constant_int64(self): - x = np.array([[1,2,3,4]]).astype(np.int64) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=False, - ) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=True, - ) - - def test_constant_string(self): - x = np.array([["123456789123456789123456789", "123456789123456789123456789"]]) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=False, - ) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=True, - ) - - def test_constant_bytes(self): - x = np.array([[b"123456789123456789123456789", b"123456789123456789123456789"]]) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=False, - ) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=True, - ) - - def test_constant_numpy_object(self): - x = np.array([[123456789123456789123456789]]) - self._core_test( - in_np=x, - out_np=x.astype(str), - convert_to_tf_tensor=False, - ) - - def test_is_tensor(self): - x = convert_to_tensor(np.array([[10, 20]])) - #assert tf.is_tensor(x) # for TensorFlow >=1.14 - assert tf.contrib.framework.is_tensor(x) + context = tf_execution_context(run_eagerly) + with context.scope(): + + x = convert_to_tensor(x_raw) + n = convert_to_tensor(n_raw) + y = x.inv(n) + y = convert_from_tensor(y) + + np.testing.assert_array_equal(context.evaluate(y).astype(str), y_raw.astype(str)) + + +class ConvertTest(parameterized.TestCase): + + @parameterized.parameters( + { + "x": x, + "tf_cast": tf_cast, + "np_cast": np_cast, + "expected": expected, + "run_eagerly": run_eagerly, + "convert_to_tf_tensor": convert_to_tf_tensor, + } + for x, tf_cast, np_cast, expected in ( + ( + np.array([[1,2,3,4]]).astype(np.int32), + tf.int32, + None, + np.array([[1,2,3,4]]).astype(np.int32), + ), + ( + np.array([[1,2,3,4]]).astype(np.int64), + tf.int32, + None, + np.array([[1,2,3,4]]).astype(np.int32), + ), + ( + np.array([["123456789123456789123456789", "123456789123456789123456789"]]), + tf.string, + str, + np.array([["123456789123456789123456789", "123456789123456789123456789"]]).astype(str), + ), + ( + np.array([[b"123456789123456789123456789", b"123456789123456789123456789"]]), + tf.string, + str, + np.array([[b"123456789123456789123456789", b"123456789123456789123456789"]]).astype(str), + ) + ) + for run_eagerly in (True, False) + for convert_to_tf_tensor in (True, False) + ) + def test_foo( + self, + x, + tf_cast, + np_cast, + expected, + convert_to_tf_tensor, + run_eagerly, + ): + + context = tf_execution_context(run_eagerly) + with context.scope(): + + y = tf.convert_to_tensor(x) if convert_to_tf_tensor else x + y = convert_to_tensor(y) + z = convert_from_tensor(y, dtype=tf_cast) + + actual = context.evaluate(z) + actual = actual.astype(np_cast) if np_cast else actual + assert actual.dtype == expected.dtype, "'{}' did not match expected '{}'".format(actual.dtype, expected.dtype) + np.testing.assert_array_equal(actual, expected) + + @parameterized.parameters( + {"run_eagerly": run_eagerly} for run_eagerly in (True, False) + ) + def test_is_tensor(self, run_eagerly): + context = tf_execution_context(run_eagerly) + + with context.scope(): + x = convert_to_tensor(np.array([[10, 20]])) + + assert tf.is_tensor(x) def test_register_tensor_conversion_function(self): - x = convert_to_tensor(np.array([[10, 20]])) - y = tf.convert_to_tensor(np.array([[30, 40]])) - z = x + y - with tf.Session() as sess: - res = sess.run(z) - np.testing.assert_array_equal(res, np.array([["40", "60"]])) - - def test_convert_to_tensor(self): - x = convert_to_tensor(np.array([[10, 20]])) - y = tf.convert_to_tensor(x) - assert y.dtype is tf.string + context = tf_execution_context(False) + + with context.scope(): + x = convert_to_tensor(np.array([[10, 20]])) + y = tf.convert_to_tensor(np.array([[30, 40]])) + z = x + y + np.testing.assert_array_equal(context.evaluate(z), np.array([["40", "60"]])) -class IntegrationTest(test.TestCase): + def test_convert_to_tensor(self): + context = tf_execution_context(False) - def test_register_symbolic(self): - x = convert_to_tensor(np.array(10)) - assert tf_utils.is_symbolic_tensor(x) + with context.scope(): + x = convert_to_tensor(np.array([[10, 20]])) + y = tf.convert_to_tensor(x) - # def test_use_in_model(self): - # x = convert_to_tensor(np.array(10)) - # model = tf.keras.models.Sequential([ - # tf.keras.layers.Dense(10) - # ]) - # model(x) + assert y.dtype is tf.string if __name__ == '__main__': - test.main() + unittest.main() diff --git a/tf_big/python/test/BUILD b/tf_big/python/test/BUILD new file mode 100644 index 0000000..1562023 --- /dev/null +++ b/tf_big/python/test/BUILD @@ -0,0 +1,21 @@ +package(default_visibility = ["//visibility:public"]) + + +py_library( + name = "test_py", + srcs = ([ + "__init__.py", + "execution_context.py", + ]), +) + +py_test( + name = "execution_context_py_test", + srcs = [ + "execution_context_test.py", + ], + main = "execution_context_test.py", + deps = [ + ":test_py", + ], +) diff --git a/tf_big/python/test/__init__.py b/tf_big/python/test/__init__.py new file mode 100644 index 0000000..4ac4e90 --- /dev/null +++ b/tf_big/python/test/__init__.py @@ -0,0 +1,5 @@ +from .execution_context import tf_execution_context + +__all__ = [ + "tf_execution_context", +] diff --git a/tf_big/python/test/execution_context.py b/tf_big/python/test/execution_context.py new file mode 100644 index 0000000..18582ee --- /dev/null +++ b/tf_big/python/test/execution_context.py @@ -0,0 +1,42 @@ +import contextlib + +import tensorflow as tf + + +class EagerExecutionContext: + def scope(self): + return contextlib.suppress() + + def evaluate(self, value): + return value.numpy() + + +class GraphExecutionContext: + def __init__(self): + self._graph = None + self._session = None + + @property + def graph(self): + if self._graph is None: + self._graph = tf.Graph() + return self._graph + + @property + def session(self): + if self._session is None: + with self._graph.as_default(): + self._session = tf.compat.v1.Session() + return self._session + + def scope(self): + return self.graph.as_default() + + def evaluate(self, value): + return self.session.run(value) + + +def tf_execution_context(run_eagerly): + if run_eagerly: + return EagerExecutionContext() + return GraphExecutionContext() diff --git a/tf_big/python/test/execution_context_test.py b/tf_big/python/test/execution_context_test.py new file mode 100644 index 0000000..45423e4 --- /dev/null +++ b/tf_big/python/test/execution_context_test.py @@ -0,0 +1,28 @@ +# pylint: disable=missing-docstring +import unittest + +import numpy as np +import tensorflow as tf +from absl.testing import parameterized + +from tf_big.python.test import tf_execution_context + + +class TestExecutionContext(parameterized.TestCase): + @parameterized.parameters({"run_eagerly": True}, {"run_eagerly": False}) + def test_tf_execution_mode(self, run_eagerly): + context = tf_execution_context(run_eagerly) + with context.scope(): + x = tf.fill(dims=(2, 2), value=5.0) + assert tf.executing_eagerly() == run_eagerly + + assert isinstance(x, tf.Tensor) + actual_result = context.evaluate(x) + assert isinstance(actual_result, np.ndarray) + + expected_result = np.array([[5.0, 5.0], [5.0, 5.0]]) + np.testing.assert_equal(actual_result, expected_result) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/tf/tf_configure.bzl b/third_party/tf/tf_configure.bzl index da25da7..a67bf93 100644 --- a/third_party/tf/tf_configure.bzl +++ b/third_party/tf/tf_configure.bzl @@ -33,19 +33,15 @@ def _execute( error_details = None, empty_stdout_fine = False): """Executes an arbitrary shell command. - - Helper for executes an arbitrary shell command. - Args: - repository_ctx: the repository_ctx object. - cmdline: list of strings, the command to execute. - error_msg: string, a summary of the error if the command fails. - error_details: string, details about the error or steps to fix it. + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise - it's an error. - - Returns: - The result of repository_ctx.execute(cmdline). + it's an error + Return: + the result of repository_ctx.execute(cmdline) """ result = repository_ctx.execute(cmdline) if result.stderr or not (empty_stdout_fine or result.stdout): @@ -58,17 +54,9 @@ def _execute( def _read_dir(repository_ctx, src_dir): """Returns a string with all files in a directory. - Finds all files inside a directory, traversing subfolders and following symlinks. The returned string contains the full path of all files separated by line breaks. - - Args: - repository_ctx: the repository_ctx object. - src_dir: directory to find files from. - - Returns: - A string of all files inside the given dir. """ if _is_windows(repository_ctx): src_dir = src_dir.replace("/", "\\") @@ -129,7 +117,8 @@ def _symlink_genrule_for_dir( dest_dir, genrule_name, src_files = [], - dest_files = []): + dest_files = [], + tf_pip_dir_rename_pair = []): """Returns a genrule to symlink(or copy if on Windows) a set of files. If src_dir is passed, files will be read from the given directory; otherwise @@ -142,20 +131,33 @@ def _symlink_genrule_for_dir( genrule_name: genrule name. src_files: list of source files instead of src_dir. dest_files: list of corresonding destination files. + tf_pip_dir_rename_pair: list of the pair of tf pip parent directory to + replace. For example, in TF pip package, the source code is under + "tensorflow_core", and we might want to replace it with + "tensorflow" to match the header includes. Returns: genrule target that creates the symlinks. """ + # Check that tf_pip_dir_rename_pair has the right length + tf_pip_dir_rename_pair_len = len(tf_pip_dir_rename_pair) + if tf_pip_dir_rename_pair_len != 0 and tf_pip_dir_rename_pair_len !=2: + _fail("The size of argument tf_pip_dir_rename_pair should be either 0 or 2, but %d is given." % tf_pip_dir_rename_pair_len) + if src_dir != None: src_dir = _norm_path(src_dir) dest_dir = _norm_path(dest_dir) files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines())) # Create a list with the src_dir stripped to use for outputs. - dest_files = files.replace(src_dir, "").splitlines() + if tf_pip_dir_rename_pair_len: + dest_files = files.replace(src_dir, "").replace(tf_pip_dir_rename_pair[0], tf_pip_dir_rename_pair[1]).splitlines() + else: + dest_files = files.replace(src_dir, "").splitlines() src_files = files.splitlines() command = [] outs = [] + for i in range(len(dest_files)): if dest_files[i] != "": # If we have only one file to link we do not want to use the dest_dir, as @@ -166,6 +168,7 @@ def _symlink_genrule_for_dir( cmd = "cp -f" command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) outs.append(' "' + dest_dir + dest_files[i] + '",') + dest_dir = "abc" genrule = _genrule( genrule_name, " && ".join(command), @@ -180,6 +183,7 @@ def _tf_pip_impl(repository_ctx): tf_header_dir, "include", "tf_header_include", + tf_pip_dir_rename_pair = ["tensorflow_core", "tensorflow"] ) tf_shared_library_dir = repository_ctx.os.environ[_TF_SHARED_LIBRARY_DIR] @@ -192,7 +196,7 @@ def _tf_pip_impl(repository_ctx): "", "libtensorflow_framework.so", [tf_shared_library_path], - [tf_shared_library_name], + ["_pywrap_tensorflow_internal.lib" if _is_windows(repository_ctx) else "libtensorflow_framework.so"], ) _tpl(repository_ctx, "BUILD", {