Update to TF 2.9 and also update build scripts to use TF SIG Build container image to make the binary compatible to TF 2.9+.

tfx-copybara · tfx-copybara · commit 61f0b2b7b465 · 2022-05-19T16:36:26.000-07:00
TF 2.9+ uses cxx11 ABI[1] and s2t should be compiled with cxx11 ABI enabled. To make things simpler, I changed the base image for the build from plain manylinux to TF SIG Build image. The image contains all required toolset already. One caveat is that the `gold` linker included in the image fails to link with following error: ``` /dt9/usr/bin/../lib/gcc/x86_64-pc-linux-gnu/9/crtend.o(.gnu.build.attributes+0x14): error: relocation refers to local symbol "" [2], which is defined in a discarded section section group signature: "(null)" ``` So I upgraded binutils(which includes gold) to 2.35.1 and it worked fine. Also deleted redundant flatbuffer dependency definition. We can use the flatbuffer definition in tensorflow and avoid the version mismatch. [1] https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html PiperOrigin-RevId: 449859520
diff --git a/.bazelrc b/.bazelrc
@@ -24,6 +24,9 @@ build --apple_platform_type=macos
 # Enable using platform specific build settings
 build --enable_platform_specific_config
 
+# Enable using 'cc_shared_library' rule.
+build --experimental_cc_shared_library
+
 # Flag to enable remote config
 common --experimental_repo_remote_exec
 
@@ -33,8 +36,10 @@ build:macos --copt=-w
 
 # By default, build TF in C++ 14 mode.
 build:linux --cxxopt=-std=c++14
+build:linux --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1
 build:linux --host_cxxopt=-std=c++14
 build:macos --cxxopt=-std=c++14
+build:macos --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1
 build:macos --host_cxxopt=-std=c++14
 
 # Suppress all warning messages.
diff --git a/.bazelversion b/.bazelversion
@@ -0,0 +1 @@
+5.1.1
diff --git a/README.md b/README.md
@@ -47,8 +47,8 @@ command:
 pip install --extra-index-url https://pypi-nightly.tensorflow.org/simple struct2tensor
 ```
 
-This will install the nightly packages for the major dependencies of Fairness
-Indicators such as TensorFlow Metadata (TFMD).
+This will install the nightly packages for the major dependencies of
+struct2tensor such as TensorFlow Metadata (TFMD).
 
 ## Creating a PIP package.
 
@@ -73,26 +73,16 @@ cd ~/struct2tensor
 ### Use docker-compose
 Install [docker-compose](https://docs.docker.com/compose/).
 
-Use it to build a pip wheel for Python 3.6 with tensorflow version 2:
+Use it to build a pip wheel for Python 3.7 with tensorflow version 2:
 
 ```bash
-docker-compose build manylinux2010
-docker-compose run -e PYTHON_VERSION=36 -e TF_VERSION=RELEASED_TF_2 manylinux2010
-```
-
-Or build a pip wheel for Python 3.7 with tensorflow version 2 (note that if you
-run one of these docker-compose commands after the other, the second will erase
-the result from the first):
-
-```bash
-docker-compose build manylinux2010
-docker-compose run -e PYTHON_VERSION=37 -e TF_VERSION=RELEASED_TF_2 manylinux2010
+docker-compose build --build-arg PYTHON_VERSION=3.7 manylinux2014
+docker-compose run -e TF_VERSION=RELEASED_TF_2 manylinux2014
 ```
 
 This will create a manylinux package in the ~/struct2tensor/dist directory.
 
 
-
 ## Creating a static library
 
 In order to construct a static library for tensorflow-serving, we run:
diff --git a/RELEASE.md b/RELEASE.md
@@ -4,8 +4,14 @@
 
 ## Major Features and Improvements
 
+*   Linux wheels now specifically conform to
+    [manylinux2014](https://peps.python.org/pep-0599/), an upgrade from
+    manylinux2010. This is aligned with TensorFlow 2.9 requirement.
+
 ## Bug Fixes and Other Changes
 
+*   Depends on `tensorflow>=2.9.0,<2.10`.
+
 ## Breaking Changes
 
 ## Deprecations
diff --git a/WORKSPACE b/WORKSPACE
@@ -41,14 +41,13 @@ tf_configure(name = "local_config_tf")
 # 3. Request the new archive to be mirrored on mirror.bazel.build for more
 #    reliable downloads.
 
-_TENSORFLOW_GIT_COMMIT = "3f878cff5b698b82eea85db2b60d65a2e320850e"  # tf 2.8.0
-_TENSORFLOW_ARCHIVE_SHA256 = "21d919ad6d96fcc0477c8d4f7b1f7e4295aaec2986e035551ed263c2b1cd52ee"
+_TENSORFLOW_GIT_COMMIT = "8a20d54a3c1bfa38c03ea99a2ad3c1b0a45dfa95"  # tf 2.9.0
+_TENSORFLOW_ARCHIVE_SHA256 = "5b3cde72888d680e4b6162351d614d6ac2d3ef44032e91b5838a9647ecc387a6"
 
 http_archive(
     name = "org_tensorflow",
     sha256 = _TENSORFLOW_ARCHIVE_SHA256,
     urls = [
-        "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
         "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
     strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -19,18 +19,19 @@ version: '3.1'
 # We mount the struct2tensor project root at /struct2tensor (which is the WORKDIR of the image)
 # in the container.
 #
-# The expected environment variables to be set are PYTHON_VERSION and TF_VERSION
-# Valid PYTHON_VERSION are 35|36|37|27
+# We need to set PYTHON_VERSION in `build` command which is used to pick the
+# base TF SIG Build image.
+# The TF_VERSION environment variables is expected when run the pipeline
 # Valid TF_VERSION are NIGHTLY_TF|NIGHTLY_TF_2|RELEASED_TF|PRERELEASED_TF|RELEASED_TF_2|PRERELEASED_TF_2
 # Sample usage:
-# docker-compose build manylinux2010
-# docker-compose run -e PYTHON_VERSION=36 -e TF_VERSION=NIGHTLY_TF manylinux2010
+# docker-compose build --build-arg PYTHON_VERSION=3.7 manylinux2014
+# docker-compose run -e TF_VERSION=NIGHTLY_TF manylinux2014
 
 services:
-  manylinux2010:
-    image: struct2tensor-build:manylinux2010
+  manylinux2014:
+    image: struct2tensor-build:manylinux2014
     build:
       context: .
-      dockerfile: struct2tensor/tools/docker_build/Dockerfile.manylinux2010
+      dockerfile: struct2tensor/tools/docker_build/Dockerfile.manylinux2014
     volumes:
       - .:/struct2tensor:delegated
diff --git a/setup.py b/setup.py
@@ -75,7 +75,7 @@ def select_constraint(default, nightly=None, git_master=None):
     packages=find_packages(),
     install_requires=[
         'protobuf>=3.13,<4',
-        'tensorflow>=2.8.0,<2.9',
+        'tensorflow>=2.9.0,<2.10',
         'tensorflow-metadata' + select_constraint(
             default='>=1.8.0,<1.9.0',
             nightly='>=1.9.0.dev',
diff --git a/struct2tensor/struct2tensor.bzl b/struct2tensor/struct2tensor.bzl
@@ -71,8 +71,6 @@ def s2t_proto_library(
 
 DYNAMIC_COPTS = [
     "-pthread",
-    "-std=c++14",
-    "-D_GLIBCXX_USE_CXX11_ABI=0",
 ]
 
 DYNAMIC_DEPS = ["@local_config_tf//:libtensorflow_framework", "@local_config_tf//:tf_header_lib"]
diff --git a/struct2tensor/tools/docker_build/Dockerfile.manylinux2010 b/struct2tensor/tools/docker_build/Dockerfile.manylinux2010
diff --git a/struct2tensor/tools/docker_build/Dockerfile.manylinux2014 b/struct2tensor/tools/docker_build/Dockerfile.manylinux2014
@@ -0,0 +1,39 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for building a manylinux2014 struct2tensor wheel.
+ARG PYTHON_VERSION
+FROM tensorflow/build:latest-python${PYTHON_VERSION}
+WORKDIR /struct2tensor
+
+# Uses manylinux2014 compatible devtoolset. See below for the detail:
+# https://github.com/tensorflow/build/blob/master/tf_sig_build_dockerfiles/builder.devtoolset/build_devtoolset.sh
+ENV CC=/dt9/usr/bin/gcc
+
+# tensorflow/build images already contains nightly packages. Clean them up.
+RUN pip uninstall -y tf-nightly tb-nightly tf-estimator-nightly keras-nightly
+
+RUN pip install auditwheel
+
+# Update binutils to avoid linker(gold) issue. See b/227299577#comment9
+RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils_2.35.1-1ubuntu1_amd64.deb \
+ && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.35.1-1ubuntu1_amd64.deb \
+ && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils-common_2.35.1-1ubuntu1_amd64.deb \
+ && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/libbinutils_2.35.1-1ubuntu1_amd64.deb
+RUN dpkg -i binutils_2.35.1-1ubuntu1_amd64.deb \
+            binutils-x86-64-linux-gnu_2.35.1-1ubuntu1_amd64.deb \
+            binutils-common_2.35.1-1ubuntu1_amd64.deb \
+            libbinutils_2.35.1-1ubuntu1_amd64.deb
+
+CMD ["struct2tensor/tools/docker_build/build_manylinux.sh"]
diff --git a/struct2tensor/tools/docker_build/build_manylinux.sh b/struct2tensor/tools/docker_build/build_manylinux.sh
@@ -14,52 +14,34 @@
 # limitations under the License.
 
 # This script is expected to run in the docker container defined in
-# Dockerfile.manylinux2010
+# Dockerfile.manylinux2014
 # Assumptions:
-# - CentOS environment.
-# - devtoolset-8 is installed.
+# - Based on TF SIG Build Dockerfile.
+# - Will use manylinux2014 compatible devtoolset under /dt9
 # - $PWD is struct2tensor's project root.
-# - Python of different versions are installed at /opt/python/.
-# - patchelf, zip, bazel, rsync are installed and are in $PATH.
+# - Python/pip/wheel/auditwheel are available in $PATH.
+# - patchelf, zip, bazelisk, rsync are installed and are in $PATH.
 
 WORKING_DIR=$PWD
 
 function setup_environment() {
   set -x
-  source scl_source enable rh-python36
+
   # Since someone may run this twice from the same directory,
   # it is important to delete the dist directory.
   rm -rf dist
 
-  if [[ -z "${PYTHON_VERSION}" ]]; then
-    echo "Must set PYTHON_VERSION env to 37|38|39"; exit 1;
-  fi
-  # Bazel will use PYTHON_BIN_PATH to determine the right python library.
-  if [[ "${PYTHON_VERSION}" == 37 ]]; then
-    PYTHON_DIR=/opt/python/cp37-cp37m
-  elif [[ "${PYTHON_VERSION}" == 38 ]]; then
-    PYTHON_DIR=/opt/python/cp38-cp38
-  elif [[ "${PYTHON_VERSION}" == 39 ]]; then
-    PYTHON_DIR=/opt/python/cp39-cp39
-  else
-    echo "Must set PYTHON_VERSION env to 37|38|39"; exit 1;
-  fi
-
-  export PIP_BIN="${PYTHON_DIR}"/bin/pip || exit 1;
-  export PYTHON_BIN_PATH="${PYTHON_DIR}"/bin/python || exit 1;
-  echo "PYTHON_BIN_PATH=${PYTHON_BIN_PATH}" || exit 1;
-  export WHEEL_BIN="${PYTHON_DIR}"/bin/wheel || exit 1;
-  ${PIP_BIN} install --upgrade pip || exit 1;
-  ${PIP_BIN} install wheel --upgrade || exit 1;
-  # Auditwheel does not have a python2 version and auditwheel is just a binary.
-  pip3 install auditwheel || exit 1;
+  export PIP_BIN_PATH=$(which pip) || exit 1;
+  export PYTHON_BIN_PATH=$(which python) || exit 1;
+  ${PIP_BIN_PATH} install --upgrade pip || exit 1;
+  ${PIP_BIN_PATH} install wheel --upgrade || exit 1;
 }
 
 function bazel_build() {
   set -x
   local build_args=()
   build_args+=("--python_bin_path=${PYTHON_BIN_PATH}")
-  build_args+=("--pip_bin_path=${PIP_BIN}")
+  build_args+=("--pip_bin_path=${PIP_BIN_PATH}")
   build_args+=("--tf_version=${TF_VERSION}")
   if [[ -n ${SKIP_STATIC_LINK_TEST} ]]; then
     build_args+=("--skip_static_link_test")
@@ -83,7 +65,7 @@ LIBRARY_DIR="struct2tensor/ops/"
 # that struct2tensor depends on are treated incorrectly by auditwheel.
 # We have to do this trick to make auditwheel happily stamp on our wheel.
 # Note that even though auditwheel would reject the wheel produced in the end,
-# it's still manylinux2010 compliant according to the standard, because it only
+# it's still manylinux2014 compliant according to the standard, because it only
 # depends on the specified shared libraries, assuming pyarrow is installed.
 function stamp_wheel() {
   set -x
@@ -107,11 +89,10 @@ function stamp_wheel() {
     zip "${WHEEL_PATH}" "${SO_FILE_PATH}" || exit 1;
   done
   popd
-
-  auditwheel repair --plat manylinux2010_x86_64 -w "${WHEEL_DIR}" "${WHEEL_PATH}"  || exit 1;
+  auditwheel repair --plat manylinux2014_x86_64 -w "${WHEEL_DIR}" "${WHEEL_PATH}"  || exit 1;
   rm "${WHEEL_PATH}" || exit 1;
   MANY_LINUX_WHEEL_PATH=$(ls "${WHEEL_DIR}"/*manylinux*.whl)
-  # Unzip the manylinux2010 wheel and pack it again with the original .so file.
+  # Unzip the wheel and pack it again with the original .so file.
   # We need to use "wheel pack" in order to compute the file hashes again.
   TMP_DIR="$(mktemp -d)"
   unzip "${MANY_LINUX_WHEEL_PATH}" -d "${TMP_DIR}" || exit 1;
@@ -121,8 +102,8 @@ function stamp_wheel() {
   done
 
   rm "${MANY_LINUX_WHEEL_PATH}" || exit 1;
-  ${WHEEL_BIN} version || exit 1;
-  ${WHEEL_BIN} pack "${TMP_DIR}" --dest-dir "${WHEEL_DIR}" || exit 1;
+  wheel version || exit 1;
+  wheel pack "${TMP_DIR}" --dest-dir "${WHEEL_DIR}" || exit 1;
 }
 
 set -x
diff --git a/struct2tensor/workspace.bzl b/struct2tensor/workspace.bzl
@@ -28,16 +28,6 @@ def struct2tensor_workspace():
         url = "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.5/rules_pkg-0.2.5.tar.gz",
     )
 
-    http_archive(
-        name = "com_github_google_flatbuffers",
-        sha256 = "12a13686cab7ffaf8ea01711b8f55e1dbd3bf059b7c46a25fefa1250bdd9dd23",
-        strip_prefix = "flatbuffers-b99332efd732e6faf60bb7ce1ce5902ed65d5ba3",
-        urls = [
-            "https://mirror.bazel.build/github.com/google/flatbuffers/archive/b99332efd732e6faf60bb7ce1ce5902ed65d5ba3.tar.gz",
-            "https://github.com/google/flatbuffers/archive/b99332efd732e6faf60bb7ce1ce5902ed65d5ba3.tar.gz",
-        ],
-    )
-
     # LINT.IfChange(thrift_archive_version)
     http_archive(
         name = "thrift",
diff --git a/third_party/arrow.BUILD b/third_party/arrow.BUILD
@@ -15,7 +15,7 @@
 # Description:
 #   Apache arrow library
 
-load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
+load("@flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
 
 flatbuffer_cc_library(
     name = "arrow_format",

Original file line number	Diff line number	Diff line change
`@@ -71,8 +71,6 @@ def s2t_proto_library(`
`71`	`71`
`72`	`72`	`DYNAMIC_COPTS = [`
`73`	`73`	`"-pthread",`
`74`		`- "-std=c++14",`
`75`		`- "-D_GLIBCXX_USE_CXX11_ABI=0",`
`76`	`74`	`]`
`77`	`75`
`78`	`76`	`DYNAMIC_DEPS = ["@local_config_tf//:libtensorflow_framework", "@local_config_tf//:tf_header_lib"]`