Skip to content

[buildbot] TFLite updates #343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[buildbot] TFLite updates #343

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 47 additions & 9 deletions buildbot/build_tflite.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,37 +16,49 @@

set -e

if [[ -z "${TFLITE_SYSROOT}" ]]; then
EXTRA_CMAKE_FLAGS=" "
else
EXTRA_CMAKE_FLAGS="-DCMAKE_SYSROOT=${TFLITE_SYSROOT} "
fi

echo extra cmake flags: ${EXTRA_CMAKE_FLAGS}

readonly CPUINFO_REPOSITORY="https://github.com/pytorch/cpuinfo"
readonly CPUINFO_TAG="5e63739504f0f8e18e941bd63b2d6d42536c7d90"
readonly CPUINFO_TAG="ef634603954d88d2643d5809011288b890ac126e"

readonly RUY_REPOSITORY="https://github.com/google/ruy"
readonly RUY_TAG="97ebb72aa0655c0af98896b317476a5d0dacad9c"
readonly RUY_TAG="3286a34cc8de6149ac6844107dfdffac91531e72"

readonly ABSEIL_REPOSITORY="https://github.com/abseil/abseil-cpp"
readonly ABSEIL_TAG="dc370a82467cb35066475537b797197aee3e5164"
readonly ABSEIL_TAG="fb3621f4f897824c0dbe0615fa94543df6192f30"

readonly EIGEN_REPOSITORY="https://gitlab.com/libeigen/eigen"
readonly EIGEN_TAG="0e187141679fdb91da33249d18cb79a011c0e2ea"
readonly EIGEN_TAG="aa6964bf3a34fd607837dd8123bc42465185c4f8"

readonly NEON_2_SSE_REPOSITORY="https://github.com/intel/ARM_NEON_2_x86_SSE"
readonly NEON_2_SSE_TAG="cef9501d1d1c47223466bf2b8cd43f3368c37773"
readonly NEON_2_SSE_TAG="a15b489e1222b2087007546b4912e21293ea86ff"

readonly FLATBUFFERS_REPOSITORY="https://github.com/google/flatbuffers"
readonly FLATBUFFERS_TAG="48da2389205ca5fbd0d1f40ad52d9c0b8685a076"
readonly FLATBUFFERS_TAG="7d6d99c6befa635780a4e944d37ebfd58e68a108"

readonly GEMMLOWP_REPOSITORY="https://github.com/google/gemmlowp"
readonly GEMMLOWP_TAG="08e4bb339e34017a0835269d4a37c4ea04d15a69"
readonly GEMMLOWP_TAG="16e8662c34917be0065110bfcd9cc27d30f52fdf"

readonly ML_DTYPES_REPOSITORY="https://github.com/jax-ml/ml_dtypes"
readonly ML_DTYPES_TAG="780b6d0ee01ffbfac45f7ec5418bc08f2b166483"
readonly ML_DTYPES_TAG="2ca30a2b3c0744625ae3d6988f5596740080bbd0"

readonly PTHREADPOOL_REPOSITORY="https://github.com/Maratyszcza/pthreadpool"
readonly PTHREADPOOL_TAG="4fe0e1e183925bf8cfa6aae24237e724a96479b8"

readonly TENSORFLOW_REPOSITORY="https://github.com/tensorflow/tensorflow"
readonly TENSORFLOW_TAG="b1bf09989871721a23f936617a97bb60b3d27a58"
readonly TENSORFLOW_TAG="v2.16.1"

# cpuinfo
git clone --filter=tree:0 --no-checkout ${CPUINFO_REPOSITORY} cpuinfo/src/cpuinfo
git -C cpuinfo/src/cpuinfo checkout ${CPUINFO_TAG}
cmake -GNinja -S cpuinfo/src/cpuinfo -B cpuinfo/src/cpuinfo-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/cpuinfo \
-DCMAKE_INSTALL_LIBDIR=lib \
Expand All @@ -60,6 +72,7 @@ ninja -C cpuinfo/src/cpuinfo-build install
git clone --filter=tree:0 --no-checkout ${RUY_REPOSITORY} ruy/src/ruy
git -C ruy/src/ruy checkout ${RUY_TAG}
cmake -GNinja -S ruy/src/ruy -B ruy/src/ruy-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/ruy \
-DCMAKE_INSTALL_LIBDIR=lib \
Expand All @@ -74,6 +87,7 @@ ninja -C ruy/src/ruy-build install
git clone --filter=tree:0 --no-checkout ${ABSEIL_REPOSITORY} abseil-cpp/src/abseil-cpp
git -C abseil-cpp/src/abseil-cpp checkout ${ABSEIL_TAG}
cmake -GNinja -S abseil-cpp/src/abseil-cpp -B abseil-cpp/src/abseil-cpp-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/abseil-cpp \
-DCMAKE_INSTALL_LIBDIR=lib \
Expand All @@ -86,6 +100,7 @@ ninja -C abseil-cpp/src/abseil-cpp-build install
git clone --filter=tree:0 --no-checkout ${EIGEN_REPOSITORY} eigen/src/eigen
git -C eigen/src/eigen checkout ${EIGEN_TAG}
cmake -GNinja -S eigen/src/eigen -B eigen/src/eigen-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/eigen \
-DCMAKE_INSTALL_LIBDIR=lib \
Expand All @@ -98,6 +113,7 @@ ninja -C eigen/src/eigen-build install
git clone --filter=tree:0 --no-checkout ${NEON_2_SSE_REPOSITORY} ARM_NEON_2_x86_SSE/src/ARM_NEON_2_x86_SSE
git -C ARM_NEON_2_x86_SSE/src/ARM_NEON_2_x86_SSE checkout ${NEON_2_SSE_TAG}
cmake -GNinja -S ARM_NEON_2_x86_SSE/src/ARM_NEON_2_x86_SSE -B ARM_NEON_2_x86_SSE/src/ARM_NEON_2_x86_SSE-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/ARM_NEON_2_x86_SSE \
-DCMAKE_INSTALL_LIBDIR=lib \
Expand All @@ -108,6 +124,7 @@ ninja -C ARM_NEON_2_x86_SSE/src/ARM_NEON_2_x86_SSE-build install
git clone --filter=tree:0 --no-checkout ${FLATBUFFERS_REPOSITORY} flatbuffers/src/flatbuffers
git -C flatbuffers/src/flatbuffers checkout ${FLATBUFFERS_TAG}
cmake -GNinja -S flatbuffers/src/flatbuffers -B flatbuffers/src/flatbuffers-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/flatbuffers \
-DCMAKE_INSTALL_LIBDIR=lib \
Expand All @@ -119,6 +136,7 @@ ninja -C flatbuffers/src/flatbuffers-build install
git clone --filter=tree:0 --no-checkout ${GEMMLOWP_REPOSITORY} gemmlowp/src/gemmlowp
git -C gemmlowp/src/gemmlowp checkout ${GEMMLOWP_TAG}
cmake -GNinja -S gemmlowp/src/gemmlowp/contrib -B gemmlowp/src/gemmlowp-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/gemmlowp \
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
ninja -C gemmlowp/src/gemmlowp-build install
Expand All @@ -127,10 +145,30 @@ ninja -C gemmlowp/src/gemmlowp-build install
git clone --filter=tree:0 --no-checkout ${ML_DTYPES_REPOSITORY} ml_dtypes/src/ml_dtypes
git -C ml_dtypes/src/ml_dtypes checkout ${ML_DTYPES_TAG}

# pthreadpool
# NOTE: currently a hack, because pthreadpool doesn't support find_package.
# we install in the default install dir, which really means the script must be run
# under sudo. Works for buildbots. Not ideal elsewhere.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would need doc updates probably?

I'm wondering if we should just install it to the sysroot if that's specified, but otherwise default to the system dependency.

if [[ -z "${TFLITE_SYSROOT}" ]]; then
PTHREADPOOL_INSTALL_PREFIX="/usr/local"
else
PTHREADPOOL_INSTALL_PREFIX="${TFLITE_SYSROOT}"
fi
git clone --filter=tree:0 --no-checkout ${PTHREADPOOL_REPOSITORY} pthreadpool/src/pthreadpool
git -C pthreadpool/src/pthreadpool checkout ${PTHREADPOOL_TAG}
cmake -GNinja -S pthreadpool/src/pthreadpool -B pthreadpool/src/pthreadpool-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_INSTALL_PREFIX:PATH=${PTHREADPOOL_INSTALL_PREFIX} \
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \
-DPTHREADPOOL_BUILD_TESTS:BOOL=OFF \
-DPTHREADPOOL_BUILD_BENCHMARKS:BOOL=OFF
ninja -C pthreadpool/src/pthreadpool-build install

# tflite
git clone --filter=tree:0 --no-checkout ${TENSORFLOW_REPOSITORY} tensorflow/src/tensorflow
git -C tensorflow/src/tensorflow checkout ${TENSORFLOW_TAG}
cmake -GNinja -S tensorflow/src/tensorflow/tensorflow/lite -B tensorflow/src/tensorflow-build \
${EXTRA_CMAKE_FLAGS} \
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${PWD}/tensorflow \
-DCMAKE_INSTALL_LIBDIR=lib \
Expand Down