Skip to content

Commit 3e4ef23

Browse files
terryheoterryheo
committed
Add scripts to build Python wheels for ARM systems
Usage: $ make -C tensorflow_lite_support/tools/pip_package/rpi docker-build Added ARM build toolchains of TensorFlow. * install_bazel.sh update_sources.sh with_the_same_user are copied from tensorflow/tools/ci_build. * Dockerfile.py3, Makefile are referred from tensorflow/lite/tools/pip_package.
1 parent ded908c commit 3e4ef23

File tree

9 files changed

+290
-0
lines changed

9 files changed

+290
-0
lines changed

‎.bazelrc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ build:ios_x86_64 --cpu=ios_x86_64
5656
build:ios_fat --config=ios
5757
build:ios_fat --ios_multi_cpus=armv7,arm64,x86_64
5858

59+
# TFLite build configs for generic embedded Linux
60+
build:elinux --crosstool_top=@local_config_embedded_arm//:toolchain
61+
build:elinux --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
62+
build:elinux_aarch64 --config=elinux
63+
build:elinux_aarch64 --cpu=aarch64
64+
build:elinux_aarch64 --distinct_host_configuration=true
65+
build:elinux_armhf --config=elinux
66+
build:elinux_armhf --cpu=armhf
67+
build:elinux_armhf --distinct_host_configuration=true
68+
5969
# By default, build TF in C++ 14 mode.
6070
build:android --cxxopt=-std=c++14
6171
build:android --host_cxxopt=-std=c++14

‎WORKSPACE‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,3 +475,23 @@ maven_install(
475475
fetch_sources = True,
476476
version_conflict_policy = "pinned",
477477
)
478+
479+
http_archive(
480+
name = "tf_toolchains",
481+
sha256 = "d72b2e52baf0592f5b94347b128ef75422fc22f63dfcf2d5fd46bc732cab052b",
482+
strip_prefix = "toolchains-1.3.0",
483+
urls = [
484+
"http://mirror.tensorflow.org/github.com/tensorflow/toolchains/archive/v1.3.0.tar.gz",
485+
"https://github.com/tensorflow/toolchains/archive/v1.3.0.tar.gz",
486+
],
487+
)
488+
489+
load("@tf_toolchains//toolchains/embedded/arm-linux:arm_linux_toolchain_configure.bzl", "arm_linux_toolchain_configure")
490+
491+
# TFLite crossbuild toolchain for embeddeds Linux
492+
arm_linux_toolchain_configure(
493+
name = "local_config_embedded_arm",
494+
build_file = "@tf_toolchains//toolchains/embedded/arm-linux:BUILD",
495+
aarch64_repo = "../aarch64_linux_toolchain",
496+
armhf_repo = "../armhf_linux_toolchain",
497+
)

‎tensorflow_lite_support/tools/pip_package/build_pip_package.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,16 @@ function main() {
211211
PKG_NAME_FLAG="--project_name tflite_support_nightly"
212212
fi
213213

214+
# Set package name flags for ARM builds.
215+
case "${TENSORFLOW_TARGET}" in
216+
armhf)
217+
PKG_NAME_FLAG="${PKG_NAME_FLAG} --plat-name=linux-armv7l"
218+
;;
219+
aarch64)
220+
PKG_NAME_FLAG="${PKG_NAME_FLAG} --plat-name=linux-aarch64"
221+
;;
222+
esac
223+
214224
if [[ ${NIGHTLY_BUILD} == "1" ]]; then
215225
# we use a script to update versions to avoid any tool differences on different platforms.
216226
if [[ ! -z ${VERSION} ]]; then

‎tensorflow_lite_support/tools/pip_package/rpi/Dockerfile.py3‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
ARG IMAGE
2+
FROM ${IMAGE}
3+
ARG PYTHON_VERSION
4+
5+
COPY update_sources.sh /
6+
RUN /update_sources.sh
7+
8+
RUN apt-get update && \
9+
apt-get install -y \
10+
build-essential \
11+
software-properties-common \
12+
zlib1g-dev \
13+
curl \
14+
unzip \
15+
git && \
16+
apt-get clean
17+
18+
# Install Python packages.
19+
RUN dpkg --add-architecture armhf
20+
RUN dpkg --add-architecture arm64
21+
RUN yes | add-apt-repository ppa:deadsnakes/ppa
22+
RUN apt-get update && \
23+
apt-get install -y \
24+
python$PYTHON_VERSION \
25+
python$PYTHON_VERSION-dev \
26+
python$PYTHON_VERSION-distutils \
27+
libpython$PYTHON_VERSION-dev \
28+
libpython$PYTHON_VERSION-dev:armhf \
29+
libpython$PYTHON_VERSION-dev:arm64
30+
RUN ln -sf /usr/bin/python$PYTHON_VERSION /usr/bin/python3
31+
RUN curl -OL https://bootstrap.pypa.io/get-pip.py
32+
RUN python3 get-pip.py
33+
RUN rm get-pip.py
34+
RUN pip3 install --upgrade pip
35+
RUN pip3 install numpy~=1.19.2 setuptools pybind11
36+
RUN ln -sf /usr/include/python$PYTHON_VERSION /usr/include/python3
37+
RUN ln -sf /usr/local/lib/python$PYTHON_VERSION/dist-packages/numpy/core/include/numpy /usr/include/python3/numpy
38+
RUN ln -sf /usr/bin/python3 /usr/bin/python
39+
40+
ENV CI_BUILD_PYTHON=python$PYTHON_VERSION
41+
ENV CROSSTOOL_PYTHON_INCLUDE_PATH=/usr/include/python$PYTHON_VERSION
42+
43+
COPY install_bazel.sh /
44+
RUN /install_bazel.sh
45+
46+
COPY with_the_same_user /

‎tensorflow_lite_support/tools/pip_package/rpi/Makefile‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Values: debian:<version>, ubuntu:<version>
2+
BASE_IMAGE ?= ubuntu:16.04
3+
PYTHON_VERSION ?= 3.9
4+
5+
MAKEFILE_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST))))
6+
WORKSPACE_DIR := $(MAKEFILE_DIR)/../../../..
7+
TAG_IMAGE := "tflite-runtime-builder-$(subst :,-,$(BASE_IMAGE))"
8+
9+
DOCKER_PARAMS := --pid=host \
10+
--env "CI_BUILD_USER=$(shell id -u -n)" \
11+
--env "CI_BUILD_UID=$(shell id -u)" \
12+
--env "CI_BUILD_GROUP=$(shell id -g -n)" \
13+
--env "CI_BUILD_GID=$(shell id -g)" \
14+
--env "CI_BUILD_HOME=$(WORKSPACE_DIR)/bazel-ci_build-cache" \
15+
--volume $(WORKSPACE_DIR)/bazel-ci_build-cache:$(WORKSPACE_DIR)/bazel-ci-build-cache \
16+
--volume $(WORKSPACE_DIR):/workspace \
17+
--workdir /workspace
18+
19+
.PHONY: help \
20+
docker-image \
21+
docker-shell \
22+
docker-build \
23+
clean
24+
25+
help:
26+
@echo "make docker-image -- build docker image"
27+
@echo "make docker-shell -- run shell inside the docker image"
28+
@echo "make docker-build -- build wheel and deb inside the docker image"
29+
@echo "make clean -- remove wheel and deb files"
30+
31+
docker-image:
32+
docker build -t $(TAG_IMAGE) --build-arg IMAGE=$(BASE_IMAGE) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -f $(MAKEFILE_DIR)/Dockerfile.py3 $(MAKEFILE_DIR)/.
33+
34+
docker-shell: docker-image
35+
mkdir -p $(WORKSPACE_DIR)/bazel-ci_build-cache
36+
docker run --rm --interactive --tty \
37+
$(DOCKER_PARAMS) \
38+
$(TAG_IMAGE) /with_the_same_user /bin/bash
39+
40+
docker-build: docker-image
41+
mkdir -p $(WORKSPACE_DIR)/bazel-ci_build-cache
42+
docker run \
43+
--rm --interactive $(shell tty -s && echo --tty) \
44+
$(DOCKER_PARAMS) \
45+
$(TAG_IMAGE) \
46+
/with_the_same_user /bin/bash -C tensorflow_lite_support/tools/pip_package/rpi/build_arm_pip_package.sh
47+
48+
clean:
49+
rm -rf $(CURDIR)/wheels

‎tensorflow_lite_support/tools/pip_package/rpi/build_arm_pip_package.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
17+
set -ex
18+
19+
bazel build -c opt --config=elinux_armhf tensorflow_lite_support/tools/pip_package:build_pip_package
20+
TENSORFLOW_TARGET=armhf ./bazel-bin/tensorflow_lite_support/tools/pip_package/build_pip_package --dst wheels --nightly_flag
21+
bazel build -c opt --config=elinux_aarch64 tensorflow_lite_support/tools/pip_package:build_pip_package
22+
TENSORFLOW_TARGET=aarch64 ./bazel-bin/tensorflow_lite_support/tools/pip_package/build_pip_package --dst wheels --nightly_flag

‎tensorflow_lite_support/tools/pip_package/rpi/install_bazel.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
17+
# Select bazel version.
18+
BAZEL_VERSION="3.7.2"
19+
20+
set +e
21+
local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
22+
23+
if [[ "$local_bazel_ver" == "$BAZEL_VERSION" ]]; then
24+
exit 0
25+
fi
26+
27+
set -e
28+
29+
# Install bazel.
30+
mkdir -p /bazel
31+
cd /bazel
32+
if [[ ! -f "bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" ]]; then
33+
curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
34+
fi
35+
chmod +x /bazel/bazel-*.sh
36+
/bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
37+
rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
38+
39+
# Enable bazel auto completion.
40+
echo "source /usr/local/lib/bazel/bin/bazel-complete.bash" >> ~/.bashrc

‎tensorflow_lite_support/tools/pip_package/rpi/update_sources.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
17+
#!/bin/bash
18+
. /etc/os-release
19+
20+
[[ "${NAME}" == "Ubuntu" ]] || exit 0
21+
22+
sed -i "s/deb\ /deb \[arch=amd64\]\ /g" /etc/apt/sources.list
23+
24+
cat <<EOT >> /etc/apt/sources.list
25+
deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME} main universe
26+
deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-updates main universe
27+
deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-security main universe
28+
EOT

‎tensorflow_lite_support/tools/pip_package/rpi/with_the_same_user‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
17+
# This script is a wrapper creating the same user inside container as the one
18+
# running the ci_build.sh outside the container. It also set the home directory
19+
# for the user inside container to match the same absolute path as the workspace
20+
# outside of container.
21+
# We do this so that the bazel running inside container generate symbolic links
22+
# and user permissions which makes sense outside of container.
23+
# Do not run this manually. It does not make sense. It is intended to be called
24+
# by ci_build.sh only.
25+
26+
set -e
27+
28+
COMMAND=("$@")
29+
30+
if ! touch /this_is_writable_file_system; then
31+
echo "You can't write to your filesystem!"
32+
echo "If you are in Docker you should check you do not have too many images" \
33+
"with too many files in them. Docker has some issue with it."
34+
exit 1
35+
else
36+
rm /this_is_writable_file_system
37+
fi
38+
39+
if [ -n "${CI_BUILD_USER_FORCE_BADNAME}" ]; then
40+
ADDUSER_OPTS="--force-badname"
41+
fi
42+
43+
apt-get install sudo
44+
45+
getent group "${CI_BUILD_GID}" || addgroup ${ADDUSER_OPTS} --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
46+
getent passwd "${CI_BUILD_UID}" || adduser ${ADDUSER_OPTS} \
47+
--gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \
48+
--gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
49+
--disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
50+
usermod -a -G sudo "${CI_BUILD_USER}"
51+
echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
52+
53+
if [[ "${TF_NEED_ROCM}" -eq 1 ]]; then
54+
# ROCm requires the video group in order to use the GPU for compute. If it
55+
# exists on the host, add it to the container.
56+
getent group video || addgroup video && adduser "${CI_BUILD_USER}" video
57+
fi
58+
59+
if [ -e /root/.bazelrc ]; then
60+
cp /root/.bazelrc "${CI_BUILD_HOME}/.bazelrc"
61+
chown "${CI_BUILD_UID}:${CI_BUILD_GID}" "${CI_BUILD_HOME}/.bazelrc"
62+
fi
63+
64+
sudo -u "#${CI_BUILD_UID}" --preserve-env "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" \
65+
"HOME=${CI_BUILD_HOME}" ${COMMAND[@]}

0 commit comments

Comments
 (0)