diff --git a/WORKSPACE b/WORKSPACE index b5b90f39b..afab6bb30 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -4,11 +4,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "icu", - strip_prefix = "icu-release-64-2", - sha256 = "dfc62618aa4bd3ca14a3df548cd65fe393155edd213e49c39f3a30ccd618fc27", + strip_prefix = "icu-release-77-1", + sha256 = "e424ba5282d95ad38b52639a08fb82164f0b0cbd7f17b53ae16bf14f8541855f", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-64-2.zip", - "https://github.com/unicode-org/icu/archive/release-64-2.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-77-1.zip", + "https://github.com/unicode-org/icu/archive/release-77-1.zip", ], build_file = "//third_party/icu:BUILD.bzl", patches = ["//third_party/icu:udata.patch"], @@ -56,10 +56,10 @@ http_archive( http_archive( name = "org_tensorflow", - strip_prefix = "tensorflow-40998f44c0c500ce0f6e3b1658dfbc54f838a82a", - sha256 = "5a5bc4599964c71277dcac0d687435291e5810d2ac2f6283cc96736febf73aaf", + strip_prefix = "tensorflow-735467e89ccfd7ace190363412bb5698164628b5", + sha256 = "c75a6fde64ba60d435f795a1a63f5caec0db8b352f544e222b8f51a1f6fec63e", urls = [ - "https://github.com/tensorflow/tensorflow/archive/40998f44c0c500ce0f6e3b1658dfbc54f838a82a.zip" + "https://github.com/tensorflow/tensorflow/archive/735467e89ccfd7ace190363412bb5698164628b5.zip" ], ) @@ -95,7 +95,7 @@ load("@org_tensorflow//third_party/py:python_init_rules.bzl", "python_init_rules python_init_rules() -load("//tensorflow_text:tftext.bzl", "py_deps_profile") +load("//tensorflow_text:py_deps_profile.bzl", "py_deps_profile") py_deps_profile( name = "release_or_nightly", @@ -158,7 +158,22 @@ load( python_wheel_version_suffix_repository(name = "tf_wheel_version_suffix") load( - "@local_xla//third_party/gpus/cuda/hermetic:cuda_json_init_repository.bzl", + "@rules_ml_toolchain//cc/deps:cc_toolchain_deps.bzl", + "cc_toolchain_deps", +) + +cc_toolchain_deps() + +register_toolchains("@rules_ml_toolchain//cc:linux_x86_64_linux_x86_64") + +register_toolchains("@rules_ml_toolchain//cc:linux_x86_64_linux_x86_64_cuda") + +register_toolchains("@rules_ml_toolchain//cc:linux_aarch64_linux_aarch64") + +register_toolchains("@rules_ml_toolchain//cc:linux_aarch64_linux_aarch64_cuda") + +load( + "@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_json_init_repository.bzl", "cuda_json_init_repository", ) @@ -170,7 +185,7 @@ load( "CUDNN_REDISTRIBUTIONS", ) load( - "@local_xla//third_party/gpus/cuda/hermetic:cuda_redist_init_repositories.bzl", + "@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_redist_init_repositories.bzl", "cuda_redist_init_repositories", "cudnn_redist_init_repository", ) @@ -184,21 +199,21 @@ cudnn_redist_init_repository( ) load( - "@local_xla//third_party/gpus/cuda/hermetic:cuda_configure.bzl", + "@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_configure.bzl", "cuda_configure", ) cuda_configure(name = "local_config_cuda") load( - "@local_xla//third_party/nccl/hermetic:nccl_redist_init_repository.bzl", + "@rules_ml_toolchain//third_party/nccl/hermetic:nccl_redist_init_repository.bzl", "nccl_redist_init_repository", ) nccl_redist_init_repository() load( - "@local_xla//third_party/nccl/hermetic:nccl_configure.bzl", + "@rules_ml_toolchain//third_party/nccl/hermetic:nccl_configure.bzl", "nccl_configure", ) diff --git a/oss_scripts/pip_package/BUILD b/oss_scripts/pip_package/BUILD index ff9c51d42..359636abd 100644 --- a/oss_scripts/pip_package/BUILD +++ b/oss_scripts/pip_package/BUILD @@ -1,6 +1,7 @@ # Tools for building the TF.Text pip package. -load("@python//:defs.bzl", "compile_pip_requirements") + load("@python_version_repo//:py_version.bzl", "REQUIREMENTS") +load("@rules_python//python:pip.bzl", "compile_pip_requirements") package(default_visibility = ["//visibility:private"]) diff --git a/oss_scripts/pip_package/requirements.in b/oss_scripts/pip_package/requirements.in index 9cc9f75d0..196788b2f 100644 --- a/oss_scripts/pip_package/requirements.in +++ b/oss_scripts/pip_package/requirements.in @@ -1,8 +1,6 @@ setuptools==70.0.0 dm-tree==0.1.8 # Limit for macos support. numpy -protobuf==4.25.3 # b/397977335 - Fix crash on python 3.9, 3.10. tensorflow tf-keras tensorflow-datasets -tensorflow-metadata diff --git a/tensorflow_text/BUILD b/tensorflow_text/BUILD index 8078fd391..c2cfcd20b 100644 --- a/tensorflow_text/BUILD +++ b/tensorflow_text/BUILD @@ -1,7 +1,8 @@ -# Placeholder: load py_library -# Placeholder: load py_test +load("@org_tensorflow//tensorflow:tensorflow.bzl", "if_oss") load("@org_tensorflow//tensorflow/lite:build_def.bzl", "tflite_cc_shared_object") load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python:py_test.bzl", "py_test") load("//tensorflow_text:tftext.bzl", "extra_py_deps", "py_tf_text_library") # [internal] load build_test.bzl @@ -123,9 +124,11 @@ py_library( ":wordpiece_tokenizer", ":wordshape_ops", # python/util:all_util tensorflow dep, - "//tensorflow_text/core/pybinds:tflite_registrar", "//tensorflow_text/tools/wordpiece_vocab", - ] + extra_py_deps(), + ] + if_oss( + ["//tensorflow_text/core/pybinds:tflite_registrar_library"], + ["//tensorflow_text/core/pybinds:tflite_registrar"], + ) + extra_py_deps(), ) # build_test diff --git a/tensorflow_text/core/kernels/BUILD b/tensorflow_text/core/kernels/BUILD index 8e6fc14d2..b77af8388 100644 --- a/tensorflow_text/core/kernels/BUILD +++ b/tensorflow_text/core/kernels/BUILD @@ -337,6 +337,7 @@ cc_library( deps = [ "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@darts_clone", ], ) diff --git a/tensorflow_text/core/kernels/darts_clone_trie_builder.cc b/tensorflow_text/core/kernels/darts_clone_trie_builder.cc index b7fafefaa..4eb492b14 100644 --- a/tensorflow_text/core/kernels/darts_clone_trie_builder.cc +++ b/tensorflow_text/core/kernels/darts_clone_trie_builder.cc @@ -19,6 +19,7 @@ #include #include "absl/container/flat_hash_set.h" +#include "absl/strings/str_cat.h" #include "include/darts.h" namespace tensorflow { diff --git a/tensorflow_text/core/pybinds/BUILD b/tensorflow_text/core/pybinds/BUILD index 68b03161b..a8334cc84 100644 --- a/tensorflow_text/core/pybinds/BUILD +++ b/tensorflow_text/core/pybinds/BUILD @@ -1,7 +1,10 @@ # Code that exposes C++ libraries to Python via pybind11. -# Placeholder: load py_test -load("@org_tensorflow//tensorflow:tensorflow.bzl", "pybind_extension") +load("@rules_python//python:py_test.bzl", "py_test") + +# This comment separates the load statements, so copybara reverse transformation works. + +load("@org_tensorflow//tensorflow:tensorflow.default.bzl", "pybind_extension", "pywrap_library") licenses(["notice"]) @@ -11,6 +14,11 @@ package(default_visibility = [ "//tensorflow_text:__subpackages__", ]) +pywrap_library( + name = "tflite_registrar_library", + deps = [":tflite_registrar"], +) + pybind_extension( name = "tflite_registrar", srcs = [ diff --git a/tensorflow_text/py_deps_profile.bzl b/tensorflow_text/py_deps_profile.bzl new file mode 100644 index 000000000..b13937f11 --- /dev/null +++ b/tensorflow_text/py_deps_profile.bzl @@ -0,0 +1,49 @@ +""" +Repository rule for open source dependencies. +""" + +def _py_deps_profile_impl(ctx): + is_switch = False + for var_name, var_val in ctx.attr.switch.items(): + is_switch = is_switch or ctx.os.environ.get(var_name, "") == var_val + + prefix = ctx.attr.pip_repo_name + reqirements_name = ctx.attr.requirements_in.name + requirements_in_substitutions = {} + build_content = ['exports_files(["{}"])'.format(reqirements_name)] + for k, v in ctx.attr.deps_map.items(): + repo_name = v[0] if is_switch else k + requirements_in_substitutions[k + "\n"] = repo_name + "\n" + requirements_in_substitutions[k + "\r\n"] = repo_name + "\r\n" + aliased_targets = ["pkg"] + v[1:] + norm_repo_name = repo_name.replace("-", "_") + norm_alas_name = k.replace("-", "_") + for target in aliased_targets: + alias_name = "{}_{}".format(norm_alas_name, target) + alias_value = "@{}_{}//:{}".format(prefix, norm_repo_name, target) + build_content.append(""" +alias( + name = "{}", + actual = "{}", + visibility = ["//visibility:public"] +) +""".format(alias_name, alias_value)) + + ctx.file("BUILD", "".join(build_content)) + ctx.template( + reqirements_name, + ctx.attr.requirements_in, + executable = False, + substitutions = requirements_in_substitutions, + ) + +py_deps_profile = repository_rule( + implementation = _py_deps_profile_impl, + attrs = { + "requirements_in": attr.label(mandatory = True), + "deps_map": attr.string_list_dict(mandatory = True), + "pip_repo_name": attr.string(mandatory = True), + "switch": attr.string_dict(mandatory = True), + }, + local = True, +) diff --git a/tensorflow_text/tftext.bzl b/tensorflow_text/tftext.bzl index 8343de92e..bc6e7cd11 100644 --- a/tensorflow_text/tftext.bzl +++ b/tensorflow_text/tftext.bzl @@ -1,6 +1,8 @@ """ Build rule for open source tf.text libraries. """ +load("@rules_python//python:py_library.bzl", "py_library") + def py_tf_text_library( name, @@ -85,7 +87,7 @@ def py_tf_text_library( }), ) - native.py_library( + py_library( name = name, srcs = srcs, srcs_version = "PY2AND3", @@ -139,6 +141,12 @@ def tf_cc_library( oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/time") oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/types:variant") oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/functional:any_invocable") + oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/log:check") + oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/log:log") + oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/log:absl_check") + oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/log:absl_log") + oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/container:btree") + oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/container:flat_hash_set") deps += select({ "@org_tensorflow//tensorflow:mobile": [ "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", @@ -227,49 +235,3 @@ def extra_py_deps(): "@pypi_tensorflow_datasets//:pkg", "@pypi_tensorflow_metadata//:pkg", ] - -def _py_deps_profile_impl(ctx): - is_switch = False - for var_name, var_val in ctx.attr.switch.items(): - is_switch = is_switch or ctx.os.environ.get(var_name, "") == var_val - - prefix = ctx.attr.pip_repo_name - reqirements_name = ctx.attr.requirements_in.name - requirements_in_substitutions = {} - build_content = ['exports_files(["{}"])'.format(reqirements_name)] - for k, v in ctx.attr.deps_map.items(): - repo_name = v[0] if is_switch else k - requirements_in_substitutions[k + "\n"] = repo_name + "\n" - requirements_in_substitutions[k + "\r\n"] = repo_name + "\r\n" - aliased_targets = ["pkg"] + v[1:] - norm_repo_name = repo_name.replace("-", "_") - norm_alas_name = k.replace("-", "_") - for target in aliased_targets: - alias_name = "{}_{}".format(norm_alas_name, target) - alias_value = "@{}_{}//:{}".format(prefix, norm_repo_name, target) - build_content.append(""" -alias( - name = "{}", - actual = "{}", - visibility = ["//visibility:public"] -) -""".format(alias_name, alias_value)) - - ctx.file("BUILD", "".join(build_content)) - ctx.template( - reqirements_name, - ctx.attr.requirements_in, - executable = False, - substitutions = requirements_in_substitutions, - ) - -py_deps_profile = repository_rule( - implementation = _py_deps_profile_impl, - attrs = { - "requirements_in": attr.label(mandatory = True), - "deps_map": attr.string_list_dict(mandatory = True), - "pip_repo_name": attr.string(mandatory = True), - "switch": attr.string_dict(mandatory = True), - }, - local = True, -) diff --git a/tensorflow_text/tools/wordpiece_vocab/BUILD b/tensorflow_text/tools/wordpiece_vocab/BUILD index 608f3c276..62093ff9d 100644 --- a/tensorflow_text/tools/wordpiece_vocab/BUILD +++ b/tensorflow_text/tools/wordpiece_vocab/BUILD @@ -1,5 +1,5 @@ -# Placeholder: load py_library -# Placeholder: load py_test +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python:py_test.bzl", "py_test") licenses(["notice"]) diff --git a/third_party/icu/udata.patch b/third_party/icu/udata.patch index eac684427..fdef60e1f 100644 --- a/third_party/icu/udata.patch +++ b/third_party/icu/udata.patch @@ -1,5 +1,5 @@ diff --git a/icu4c/source/common/udata.cpp b/icu4c/source/common/udata.cpp -index efcd2a2f97..80f275c415 100644 +index 68b62272901..5a107e7126a 100644 --- a/icu4c/source/common/udata.cpp +++ b/icu4c/source/common/udata.cpp @@ -18,11 +18,10 @@ @@ -18,38 +18,33 @@ index efcd2a2f97..80f275c415 100644 #endif #include "unicode/putil.h" -@@ -649,10 +648,11 @@ extern "C" const DataHeader U_DATA_API U_ICUDATA_ENTRY_POINT; +@@ -651,10 +650,9 @@ extern "C" const DataHeader U_DATA_API U_ICUDATA_ENTRY_POINT; * partial-data-library access functions where each returns a pointer * to its data package, if it is linked in. */ -/* --extern const void *uprv_getICUData_collation(void) ATTRIBUTE_WEAK; --extern const void *uprv_getICUData_conversion(void) ATTRIBUTE_WEAK; +-extern const void *uprv_getICUData_collation() ATTRIBUTE_WEAK; +-extern const void *uprv_getICUData_conversion() ATTRIBUTE_WEAK; -*/ -+ +U_CDECL_BEGIN -+const void *uprv_getICUData_other(void) ATTRIBUTE_WEAK; ++const void *uprv_getICUData_conversion(void) ATTRIBUTE_WEAK; +U_CDECL_END -+ /*----------------------------------------------------------------------* * * -@@ -709,11 +709,11 @@ openCommonData(const char *path, /* Path from OpenChoice? */ - /* +@@ -713,10 +711,11 @@ openCommonData(const char *path, /* Path from OpenChoice? */ if (uprv_getICUData_collation) { - setCommonICUDataPointer(uprv_getICUData_collation(), FALSE, pErrorCode); -+ }*/ -+ if (uprv_getICUData_other) { -+ setCommonICUDataPointer(uprv_getICUData_other(), FALSE, pErrorCode); + setCommonICUDataPointer(uprv_getICUData_collation(), false, pErrorCode); + } ++ */ + if (uprv_getICUData_conversion) { + setCommonICUDataPointer(uprv_getICUData_conversion(), false, pErrorCode); } -- if (uprv_getICUData_conversion) { -- setCommonICUDataPointer(uprv_getICUData_conversion(), FALSE, pErrorCode); -- } - */ + - #if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP Platform does not support dll icu data at this time - setCommonICUDataPointer(&U_ICUDATA_ENTRY_POINT, FALSE, pErrorCode); - { + #if !defined(ICU_DATA_DIR_WINDOWS) + // When using the Windows system data, we expect only a single data file. + setCommonICUDataPointer(&U_ICUDATA_ENTRY_POINT, false, pErrorCode); diff --git a/icu4c/source/common/unicode/uconfig.h b/icu4c/source/common/unicode/uconfig.h index e61eb5795c..95c43f2334 100644 --- a/icu4c/source/common/unicode/uconfig.h diff --git a/third_party/tensorflow/tf_configure.bzl b/third_party/tensorflow/tf_configure.bzl index e224a2e7a..36c4f5ecb 100644 --- a/third_party/tensorflow/tf_configure.bzl +++ b/third_party/tensorflow/tf_configure.bzl @@ -4,7 +4,10 @@ This is used for the generation of the dynamic libraries used for custom ops. See: http://github.com/tensorflow/custom-op """ -load("@python//:defs.bzl", "interpreter") +load( + "@org_tensorflow//third_party/py:python_init_toolchains.bzl", + "get_toolchain_name_per_python_version", +) load("@python_version_repo//:py_version.bzl", "REQUIREMENTS_WITH_LOCAL_WHEELS") load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse") @@ -55,6 +58,11 @@ cc_library( additive_build_content = tensorflow_annotation, ), }, - python_interpreter_target = interpreter, + python_interpreter_target = _get_python_interpreter(), requirements_lock = REQUIREMENTS_WITH_LOCAL_WHEELS, ) + +def _get_python_interpreter(): + return "@{}_host//:python".format( + get_toolchain_name_per_python_version("python"), + )