From 41d7014293ed42b64d07c51430f39728a475d582 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Wed, 12 Mar 2025 21:18:16 -0700 Subject: [PATCH 1/3] Try to make build script more versatile --- .../collect_android_artifacts_for_upload.sh | 34 +++++++++++++ .github/workflows/_android.yml | 10 +++- .github/workflows/android-perf.yml | 9 +++- .../workflows/android-release-artifacts.yml | 9 +++- .gitignore | 1 + build/build_android_demo_app.sh | 30 +++++++++++ build/build_android_library.sh | 50 +------------------ 7 files changed, 90 insertions(+), 53 deletions(-) create mode 100644 .ci/scripts/collect_android_artifacts_for_upload.sh create mode 100644 build/build_android_demo_app.sh diff --git a/.ci/scripts/collect_android_artifacts_for_upload.sh b/.ci/scripts/collect_android_artifacts_for_upload.sh new file mode 100644 index 00000000000..f455f0d88c8 --- /dev/null +++ b/.ci/scripts/collect_android_artifacts_for_upload.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +collect_artifacts_to_be_uploaded() { + ARTIFACTS_DIR_NAME="$1" + DEMO_APP_DIR="${ARTIFACTS_DIR_NAME}/llm_demo" + # The app directory is named using its build flavor as a suffix. + mkdir -p "${DEMO_APP_DIR}" + # Collect the app and its test suite + cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/debug/*.apk "${DEMO_APP_DIR}" || true + cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/androidTest/debug/*.apk "${DEMO_APP_DIR}" || true + # Collect JAR and AAR + cp extension/android/build/libs/executorch.jar "${DEMO_APP_DIR}" + find "${BUILD_AAR_DIR}/" -name 'executorch*.aar' -exec cp {} "${DEMO_APP_DIR}" \; + # Collect MiniBench APK + MINIBENCH_APP_DIR="${ARTIFACTS_DIR_NAME}/minibench" + mkdir -p "${MINIBENCH_APP_DIR}" + cp extension/benchmark/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}" || true + cp extension/benchmark/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}" || true + # Collect Java library test + JAVA_LIBRARY_TEST_DIR="${ARTIFACTS_DIR_NAME}/library_test_dir" + mkdir -p "${JAVA_LIBRARY_TEST_DIR}" + cp extension/android_test/build/outputs/apk/debug/*.apk "${JAVA_LIBRARY_TEST_DIR}" || true + cp extension/android_test/build/outputs/apk/androidTest/debug/*.apk "${JAVA_LIBRARY_TEST_DIR}" || true +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + collect_artifacts_to_be_uploaded "$@" +fi diff --git a/.github/workflows/_android.yml b/.github/workflows/_android.yml index 82e49d6672e..37a18f0c9ee 100644 --- a/.github/workflows/_android.yml +++ b/.github/workflows/_android.yml @@ -28,8 +28,16 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2 export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded + source build/build_android_library.sh + build_jar + build_android_native_library "arm64-v8a" + build_aar # Build LLM Demo for Android - bash build/build_android_library.sh ${ARTIFACTS_DIR_NAME} + source build/build_android_demo_app.sh + build_android_llama_demo + build_android_test + # Upload + .ci/scripts/collect_android_artifacts_for_upload.sh $ARTIFACTS_DIR_NAME # Running Android emulator directly on the runner and not using Docker run-emulator: diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index f21ed849d03..5156588a2a3 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -362,8 +362,13 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh - export ANDROID_ABIS="arm64-v8a" - PYTHON_EXECUTABLE=python EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029 bash build/build_android_library.sh ${ARTIFACTS_DIR_NAME} + source build/build_android_library.sh + PYTHON_EXECUTABLE=python build_jar + PYTHON_EXECUTABLE=python EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029 build_android_native_library "arm64-v8a" + build_aar + source build/build_android_demo_app.sh + PYTHON_EXECUTABLE=python build_android_llama_demo + PYTHON_EXECUTABLE=python .ci/scripts/collect_android_artifacts_for_upload.sh $ARTIFACTS_DIR_NAME # Let's see how expensive this job is, we might want to tone it down by running it periodically benchmark-on-device: diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 26423e59233..a29320312f4 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -53,7 +53,14 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - bash build/build_android_library.sh ${ARTIFACTS_DIR_NAME} + source build/build_android_library.sh + build_jar + build_android_native_library "arm64-v8a" + build_android_native_library "x86_64" + build_aar + # Upload + .ci/scripts/collect_android_artifacts_for_upload.sh $ARTIFACTS_DIR_NAME + shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" diff --git a/.gitignore b/.gitignore index 7b8279f604d..022ea450573 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Build and tool-generated files +aar-out/ arm_test/ buck-out/ buck2-bin/ diff --git a/build/build_android_demo_app.sh b/build/build_android_demo_app.sh new file mode 100644 index 00000000000..a4ec5aa19b3 --- /dev/null +++ b/build/build_android_demo_app.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +build_android_llama_demo() { + mkdir -p examples/demo-apps/android/LlamaDemo/app/libs + cp ${BUILD_AAR_DIR}/executorch.aar examples/demo-apps/android/LlamaDemo/app/libs + pushd examples/demo-apps/android/LlamaDemo + ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest + popd +} + +build_android_benchmark_app() { + mkdir -p extension/benchmark/android/benchmark/app/libs + cp ${BUILD_AAR_DIR}/executorch.aar extension/benchmark/android/benchmark/app/libs + pushd extension/benchmark/android/benchmark + ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest + popd +} + +build_android_test() { + pushd extension/android_test + ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew testDebugUnitTest + ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest + popd +} diff --git a/build/build_android_library.sh b/build/build_android_library.sh index 01ea86bf830..bb5747eea78 100644 --- a/build/build_android_library.sh +++ b/build/build_android_library.sh @@ -137,51 +137,9 @@ build_aar() { popd } -build_android_demo_apps() { - mkdir -p examples/demo-apps/android/LlamaDemo/app/libs - cp ${BUILD_AAR_DIR}/executorch.aar examples/demo-apps/android/LlamaDemo/app/libs - pushd examples/demo-apps/android/LlamaDemo - ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest - popd - - mkdir -p extension/benchmark/android/benchmark/app/libs - cp ${BUILD_AAR_DIR}/executorch.aar extension/benchmark/android/benchmark/app/libs - pushd extension/benchmark/android/benchmark - ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest - popd - - pushd extension/android_test - ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew testDebugUnitTest - ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest - popd -} - -collect_artifacts_to_be_uploaded() { - ARTIFACTS_DIR_NAME="$1" - DEMO_APP_DIR="${ARTIFACTS_DIR_NAME}/llm_demo" - # The app directory is named using its build flavor as a suffix. - mkdir -p "${DEMO_APP_DIR}" - # Collect the app and its test suite - cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/debug/*.apk "${DEMO_APP_DIR}" - cp examples/demo-apps/android/LlamaDemo/app/build/outputs/apk/androidTest/debug/*.apk "${DEMO_APP_DIR}" - # Collect JAR and AAR - cp extension/android/build/libs/executorch.jar "${DEMO_APP_DIR}" - find "${BUILD_AAR_DIR}/" -name 'executorch*.aar' -exec cp {} "${DEMO_APP_DIR}" \; - # Collect MiniBench APK - MINIBENCH_APP_DIR="${ARTIFACTS_DIR_NAME}/minibench" - mkdir -p "${MINIBENCH_APP_DIR}" - cp extension/benchmark/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}" - cp extension/benchmark/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}" - # Collect Java library test - JAVA_LIBRARY_TEST_DIR="${ARTIFACTS_DIR_NAME}/library_test_dir" - mkdir -p "${JAVA_LIBRARY_TEST_DIR}" - cp extension/android_test/build/outputs/apk/debug/*.apk "${JAVA_LIBRARY_TEST_DIR}" - cp extension/android_test/build/outputs/apk/androidTest/debug/*.apk "${JAVA_LIBRARY_TEST_DIR}" -} - main() { if [[ -z "${BUILD_AAR_DIR:-}" ]]; then - BUILD_AAR_DIR="$(mktemp -d)" + BUILD_AAR_DIR="$(mkdir -p aar-out)" fi export BUILD_AAR_DIR if [ -z "$ANDROID_ABIS" ]; then @@ -189,17 +147,11 @@ main() { fi export ANDROID_ABIS - ARTIFACTS_DIR_NAME="$1" - build_jar for ANDROID_ABI in "${ANDROID_ABIS[@]}"; do build_android_native_library ${ANDROID_ABI} done build_aar - build_android_demo_apps - if [ -n "$ARTIFACTS_DIR_NAME" ]; then - collect_artifacts_to_be_uploaded ${ARTIFACTS_DIR_NAME} - fi } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then From 887b53662a27b12509417fa6d65d53d7c7e4a907 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Thu, 13 Mar 2025 10:41:26 -0700 Subject: [PATCH 2/3] Update --- .github/workflows/_android.yml | 1 + .github/workflows/android-perf.yml | 1 + .github/workflows/android-release-artifacts.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/_android.yml b/.github/workflows/_android.yml index 37a18f0c9ee..d4c26a56dea 100644 --- a/.github/workflows/_android.yml +++ b/.github/workflows/_android.yml @@ -28,6 +28,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2 export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded + export BUILD_AAR_DIR="$(mkdir -p aar-out)" source build/build_android_library.sh build_jar build_android_native_library "arm64-v8a" diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 5156588a2a3..02fb170cb86 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -362,6 +362,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh + export BUILD_AAR_DIR="$(mkdir -p aar-out)" source build/build_android_library.sh PYTHON_EXECUTABLE=python build_jar PYTHON_EXECUTABLE=python EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029 build_android_native_library "arm64-v8a" diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index a29320312f4..4390506fbf6 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -53,6 +53,7 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android + export BUILD_AAR_DIR="$(mkdir -p aar-out)" source build/build_android_library.sh build_jar build_android_native_library "arm64-v8a" From 4fff07006e648fa7b4e84e3b9246f02d6ae472ae Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Thu, 13 Mar 2025 15:18:07 -0700 Subject: [PATCH 3/3] Update --- build/build_android_demo_app.sh | 7 +++++++ build/build_android_library.sh | 3 --- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build/build_android_demo_app.sh b/build/build_android_demo_app.sh index a4ec5aa19b3..753734af22b 100644 --- a/build/build_android_demo_app.sh +++ b/build/build_android_demo_app.sh @@ -28,3 +28,10 @@ build_android_test() { ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest popd } + + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + build_android_llama_demo + build_android_benchmark_app + build_android_test +fi diff --git a/build/build_android_library.sh b/build/build_android_library.sh index bb5747eea78..88209aa1b74 100644 --- a/build/build_android_library.sh +++ b/build/build_android_library.sh @@ -126,9 +126,6 @@ build_aar() { \ \ \ > "${BUILD_AAR_DIR}/AndroidManifest.xml" pushd "${BUILD_AAR_DIR}" - # Rename libexecutorch_jni.so to libexecutorch.so for soname consistency - # between Java and JNI - find jni -type f -name "libexecutorch_jni.so" -exec bash -c 'mv "$1" "${1/_jni/}"' bash {} \; if [ "$EXECUTORCH_CMAKE_BUILD_TYPE" == "Release" ]; then find jni -type f -name "*.so" -exec "$ANDROID_NDK"/toolchains/llvm/prebuilt/*/bin/llvm-strip {} \; fi