Fix TF-text for TF 2.20.

This is a combination of a collection of changes to fix the TF-text build.  All are required
for a successful build.

1. Upgrade ICU to v77.1
   - Requires changes to ICU build file, updating ICU data, updating test goldens
   - Added `custom_http_archive` rule to allow excluding files.
2. Replace `tensorflow.bzl:pybind_extension` with `pywrap.bzl:pybind_extension`, `pywrap_libraries`, `pywrap_binaries`
   - Forces creation of pybind bundled .so file to include in the wheel.
3. Patch TF build file to exclude `libtensorflow_framework` source files.
   - Avoids ODR issues causing double-registrations.
4. Update requirements files
   - TF 3.9, 3.10 are somewhat fundamentally broken because of tensorflow-metadata,
     which restricts the protobuf version below what TF requires.  Though this turns out
     not to be a hard requirement, by skipping the requirements.update in `configure` and manually
     setting package versions, we can create a successful build.
5. Replace the `build_pip_package.sh` with a python version
   -  Allows using the hermetic python toolchain to build the wheel.  Avoids the need for docker.
6. Fix random missing deps.

PiperOrigin-RevId: 854285564

Author: cantonios

WORKSPACE

@@ -1,37 +1,42 @@
 workspace(name = "org_tensorflow_text")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("//third_party/bazel:http.bzl", "custom_http_archive")
 
-http_archive(
+custom_http_archive(
     name = "icu",
-    strip_prefix = "icu-release-64-2",
-    sha256 = "dfc62618aa4bd3ca14a3df548cd65fe393155edd213e49c39f3a30ccd618fc27",
+    exclude = [
+        # Build file does not contain all required targets.
+        "icu4c/source/common/BUILD.bazel",
+    ],
+    files = {
+        "BUILD.bazel": "//third_party/icu:BUILD.bzl",
+    },
+    sha256 = "e424ba5282d95ad38b52639a08fb82164f0b0cbd7f17b53ae16bf14f8541855f",
+    strip_prefix = "icu-release-77-1",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-64-2.zip",
-        "https://github.com/unicode-org/icu/archive/release-64-2.zip",
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-77-1.zip",
+        "https://github.com/unicode-org/icu/archive/release-77-1.zip",
     ],
-    build_file = "//third_party/icu:BUILD.bzl",
-    patches = ["//third_party/icu:udata.patch"],
-    patch_args = ["-p1"],
 )
 
 http_archive(
     name = "com_google_sentencepiece",
-    strip_prefix = "sentencepiece-0.1.96",
+    build_file = "//third_party/sentencepiece:BUILD",
+    patch_args = ["-p1"],
+    patches = ["//third_party/sentencepiece:sp.patch"],
     sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754",
+    strip_prefix = "sentencepiece-0.1.96",
     urls = [
-        "https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip"
+        "https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip",
     ],
-    build_file = "//third_party/sentencepiece:BUILD",
-    patches = ["//third_party/sentencepiece:sp.patch"],
-    patch_args = ["-p1"],
 )
 
 http_archive(
     name = "cppitertools",
-    urls = ["https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.0.zip"],
     sha256 = "e56741b108d6baced98c4ccd83fd0d5a545937f2845978799c28d0312c0dee3d",
     strip_prefix = "cppitertools-2.0",
+    urls = ["https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.0.zip"],
 )
 
 http_archive(
@@ -56,10 +61,12 @@ http_archive(
 
 http_archive(
     name = "org_tensorflow",
-    strip_prefix = "tensorflow-40998f44c0c500ce0f6e3b1658dfbc54f838a82a",
-    sha256 = "5a5bc4599964c71277dcac0d687435291e5810d2ac2f6283cc96736febf73aaf",
+    sha256 = "213edf03ac7c4e74d8eb2074216ae8c8ae4f325c6bc22efd16cfdeed2073bd66",
+    strip_prefix = "tensorflow-2.20.0",
+    patch_args = ["-p1"],
+    patches = ["//third_party/tensorflow:tensorflow.core.BUILD.patch"],
     urls = [
-        "https://github.com/tensorflow/tensorflow/archive/40998f44c0c500ce0f6e3b1658dfbc54f838a82a.zip"
+        "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.20.0.zip",
     ],
 )
 
@@ -74,13 +81,13 @@ http_archive(
 
 http_archive(
     name = "pybind11",
+    build_file = "//third_party/pybind11:BUILD.bzl",
+    sha256 = "efc901aa0aab439a3fea6efeaf930b5a349fb06394bf845c64ce15a9cf8f0240",
+    strip_prefix = "pybind11-2.13.4",
     urls = [
         "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.13.4.tar.gz",
         "https://github.com/pybind/pybind11/archive/v2.13.4.tar.gz",
     ],
-    sha256 = "efc901aa0aab439a3fea6efeaf930b5a349fb06394bf845c64ce15a9cf8f0240",
-    strip_prefix = "pybind11-2.13.4",
-    build_file = "//third_party/pybind11:BUILD.bzl",
 )
 
 http_archive(
@@ -95,31 +102,37 @@ load("@org_tensorflow//third_party/py:python_init_rules.bzl", "python_init_rules
 
 python_init_rules()
 
-load("//tensorflow_text:tftext.bzl", "py_deps_profile")
+load("//third_party/bazel:py_deps_profile.bzl", "py_deps_profile")
 
 py_deps_profile(
     name = "release_or_nightly",
-    requirements_in = "//oss_scripts/pip_package:requirements.in",
-    pip_repo_name = "pypi",
     deps_map = {
-        "tensorflow": ["tf-nightly", "tf_header_lib", "libtensorflow_framework"],
-        "tf-keras": ["tf-keras-nightly"]
+        "tensorflow": [
+            "tf-nightly",
+            "tf_headers",
+            "tf_header_lib",
+            "libtensorflow_framework",
+        ],
+        "tf-keras": ["tf-keras-nightly"],
     },
+    pip_repo_name = "pypi",
+    requirements_in = "//oss_scripts/pip_package:requirements.in",
     switch = {
-        "IS_NIGHTLY": "nightly"
-    }
+        "IS_NIGHTLY": "nightly",
+    },
 )
 
 load("@org_tensorflow//third_party/py:python_init_repositories.bzl", "python_init_repositories")
 
 python_init_repositories(
+    default_python_version = "system",
     requirements = {
         "3.9": "//oss_scripts/pip_package:requirements_lock_3_9.txt",
         "3.10": "//oss_scripts/pip_package:requirements_lock_3_10.txt",
         "3.11": "//oss_scripts/pip_package:requirements_lock_3_11.txt",
         "3.12": "//oss_scripts/pip_package:requirements_lock_3_12.txt",
+        "3.13": "//oss_scripts/pip_package:requirements_lock_3_13.txt",
     },
-    default_python_version = "system",
 )
 
 load("@org_tensorflow//third_party/py:python_init_toolchains.bzl", "python_init_toolchains")
@@ -136,18 +149,28 @@ install_deps()
 
 # Initialize TensorFlow dependencies.
 load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
+
 tf_workspace3()
+
 load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
+
 tf_workspace2()
+
 load("@org_tensorflow//tensorflow:workspace1.bzl", "tf_workspace1")
+
 tf_workspace1()
+
 load("@org_tensorflow//tensorflow:workspace0.bzl", "tf_workspace0")
+
 tf_workspace0()
 
 # Set up Android.
 load("@org_tensorflow//third_party/android:android_configure.bzl", "android_configure")
-android_configure(name="local_config_android")
+
+android_configure(name = "local_config_android")
+
 load("@local_config_android//:android.bzl", "android_workspace")
+
 android_workspace()
 
 load(

oss_scripts/configure.sh

@@ -32,19 +32,26 @@ function is_macos() {
 # Remove .bazelrc if it already exist
 [ -e .bazelrc ] && rm .bazelrc
 
-if [[ $(pip show tensorflow) == *tensorflow* ]] ||
-   [[ $(pip show tensorflow-macos) == *tensorflow-macos* ]] ||
-   [[ $(pip show tf-nightly) == *tf-nightly* ]]; then
-  echo 'Using installed tensorflow.'
-else
-  echo 'Installing tensorflow.'
-  if [[ "$IS_NIGHTLY" == "nightly" ]]; then
-    pip install tf-nightly
-  else
-    pip install tensorflow==2.18.0
-  fi
+# Don't install tensorflow locally.
+if (which python3) | grep -q "python3"; then
+  installed_python="python3"
+elif (which python) | grep -q "python"; then
+  installed_python="python"
 fi
 
+# if [[ $($installed_python -m pip show tensorflow) == *tensorflow* ]] ||
+#    [[ $($installed_python -m pip show tensorflow-macos) == *tensorflow-macos* ]] ||
+#    [[ $($installed_python -m pip show tf-nightly) == *tf-nightly* ]]; then
+#   echo 'Using installed tensorflow.'
+# else
+#   echo 'Installing tensorflow.'
+#   if [[ "$IS_NIGHTLY" == "nightly" ]]; then
+#     $installed_python -m pip install --no-deps tf-nightly
+#   else
+#     $installed_python -m pip install --no-deps tensorflow==2.20
+#   fi
+# fi
+
 # Copy the current bazelversion of TF.
 curl https://raw.githubusercontent.com/tensorflow/tensorflow/master/.bazelversion -o .bazelversion
 
@@ -54,12 +61,7 @@ curl https://raw.githubusercontent.com/tensorflow/tensorflow/master/.bazelrc -o
 sed -i -e 's/build --noincompatible_remove_legacy_whole_archive//' .bazelrc
 
 write_to_bazelrc "build:manylinux2014 --config=release_cpu_linux"
-
-if (which python3) | grep -q "python3"; then
-  installed_python="python3"
-elif (which python) | grep -q "python"; then
-  installed_python="python"
-fi
+write_to_bazelrc "test --test_env=TF_USE_LEGACY_KERAS=1"
 
 if [ -z "$HERMETIC_PYTHON_VERSION" ]; then
   if [ -n "$PY_VERSION" ]; then
@@ -70,13 +72,16 @@ if [ -z "$HERMETIC_PYTHON_VERSION" ]; then
 fi
 export HERMETIC_PYTHON_VERSION
 
-echo "TF_VERSION=$TF_VERSION"
-REQUIREMENTS_EXTRA_FLAGS="--upgrade"
-if [[ "$TF_VERSION" == *"rc"* ]]; then
-  REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre"
-fi
-
-bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS
+# NIGHTLY broken, don't try to upgrade deps.
+# Only auto-upgrade requirements for nightly and if >= 3.11.
+# HERMETIC_PYTHON_VERSION_MINOR="${HERMETIC_PYTHON_VERSION#*.}"
+# if [[ "$IS_NIGHTLY" == "nightly" ]] && [[ 11 -le $HERMETIC_PYTHON_VERSION_MINOR  ]]; then
+#   REQUIREMENTS_EXTRA_FLAGS="--upgrade"
+#   if [[ "$TF_VERSION" == *"rc"* ]]; then
+#     REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre"
+#   fi
+#   bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS
+# fi
 
 TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi)
 SHARED_LIBRARY_NAME="libtensorflow_framework.so.2"

oss_scripts/pip_package/BUILD

@@ -27,14 +27,18 @@ py_binary(
     ],
 )
 
-sh_binary(
+py_binary(
     name = "build_pip_package",
-    srcs = ["build_pip_package.sh"],
+    srcs = ["build_pip_package.py"],
     data = [
         "LICENSE",
         "MANIFEST.in",
         "setup.nightly.py",
         "setup.py",
         "//tensorflow_text",
+        "//tensorflow_text/core/pybinds:pybinds_binaries",
+    ],
+    deps = [
+        "@pypi_setuptools//:pkg",
     ],
 )