Skip to content

Commit 68a1a0d

Browse files
larryliu0820facebook-github-bot
larryliu0820
authored and
facebook-github-bot
committed
Add selective build examples for buck2 (#152)
Summary: Pull Request resolved: #152 As titled. Adding 3 examples for how to use selective build in buck2 Reviewed By: iseeyuan Differential Revision: D48717520 fbshipit-source-id: 078ac77772228f66b95c61bd2599625120d1bf1f
1 parent 56fc73f commit 68a1a0d

File tree

5 files changed

+224
-0
lines changed

5 files changed

+224
-0
lines changed

‎.github/workflows/pull.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,3 +137,53 @@ jobs:
137137
# Build and test custom ops
138138
PYTHON_EXECUTABLE=python bash examples/custom_ops/test_custom_ops.sh "${BUILD_TOOL}"
139139
popd
140+
141+
test-selective-build-linux:
142+
name: test-selective-build-linux
143+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
144+
strategy:
145+
matrix:
146+
include:
147+
- build-tool: buck2
148+
# - build-tool: cmake (TODO)
149+
fail-fast: false
150+
with:
151+
runner: linux.2xlarge
152+
docker-image: executorch-ubuntu-22.04-clang12
153+
submodules: 'true'
154+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
155+
script: |
156+
PYTHON_VERSION=3.10
157+
# TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path
158+
# here, as it's there in the container
159+
export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
160+
161+
BUILD_TOOL=${{ matrix.build-tool }}
162+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
163+
# Test selective build
164+
PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
165+
166+
test-selective-build-macos:
167+
name: test-selective-build-macos
168+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
169+
strategy:
170+
matrix:
171+
include:
172+
- build-tool: buck2
173+
# - build-tool: cmake (TODO)
174+
fail-fast: false
175+
with:
176+
runner: macos-m1-12
177+
submodules: 'true'
178+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
179+
script: |
180+
WORKSPACE=$(pwd)
181+
182+
pushd "${WORKSPACE}/pytorch/executorch"
183+
184+
BUILD_TOOL=${{ matrix.build-tool }}
185+
# Setup MacOS dependencies as there is no Docker support on MacOS atm
186+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
187+
# Build and test selective build
188+
PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
189+
popd

‎examples/selective_build/README.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Selective Build Examples
2+
To optimize binary size of ExecuTorch runtime, selective build can be used. This folder contains examples to select only the operators needed for Executorch build. We provide APIs for both CMake build and buck2 build. This example will demonstrate both. You can find more information on how to use buck2 macros in [wiki](https://github.com/pytorch/executorch/blob/main/docs/website/docs/tutorials/selective_build.md).
3+
4+
## How to run
5+
6+
Prerequisite: finish the [setting up wiki](https://github.com/pytorch/executorch/blob/main/docs/website/docs/tutorials/00_setting_up_executorch.md).
7+
8+
Run:
9+
10+
```bash
11+
bash test_selective_build.sh
12+
```
13+
14+
## BUCK2 examples
15+
16+
Check out `targets.bzl` for demo of 3 selective build APIs:
17+
1. Select all ops from the dependency kernel libraries, register all of them into Executorch runtime.
18+
2. Only select ops from `ops` kwarg in `et_operator_library` macro.
19+
3. Only select from a yaml file from `ops_schema_yaml_target` kwarg in `et_operator_library` macro.
20+
21+
We have one more API incoming: only select from an exported model file (.pte).
22+
23+
## CMake examples
24+
25+
TODO

‎examples/selective_build/TARGETS‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Any targets that should be shared between fbcode and xplat must be defined in
2+
# targets.bzl. This file can contain fbcode-only targets.
3+
4+
load(":targets.bzl", "define_common_targets")
5+
6+
oncall("executorch")
7+
8+
define_common_targets()

‎examples/selective_build/targets.bzl‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_oss_build_kwargs", "runtime")
2+
load("@fbsource//xplat/executorch/codegen:codegen.bzl", "et_operator_library", "executorch_generated_lib")
3+
4+
def define_common_targets():
5+
"""Defines targets that should be shared between fbcode and xplat.
6+
7+
The directory containing this targets.bzl file should also contain both
8+
TARGETS and BUCK files that call this function.
9+
"""
10+
11+
# Select all ops: register all the ops in portable/functions.yaml
12+
et_operator_library(
13+
name = "select_all_ops",
14+
include_all_operators = True,
15+
)
16+
17+
executorch_generated_lib(
18+
name = "select_all_lib",
19+
functions_yaml_target = "//executorch/kernels/portable:functions.yaml",
20+
deps = [
21+
"//executorch/kernels/portable:operators",
22+
":select_all_ops",
23+
],
24+
)
25+
26+
# Select a list of operators: defined in `ops`
27+
et_operator_library(
28+
name = "select_ops_in_list",
29+
ops = [
30+
"aten::add.out",
31+
"aten::mm.out",
32+
],
33+
)
34+
35+
executorch_generated_lib(
36+
name = "select_ops_in_list_lib",
37+
functions_yaml_target = "//executorch/kernels/portable:functions.yaml",
38+
deps = [
39+
"//executorch/kernels/portable:operators",
40+
":select_ops_in_list",
41+
],
42+
)
43+
44+
# Select all ops from a yaml file
45+
et_operator_library(
46+
name = "select_ops_from_yaml",
47+
ops_schema_yaml_target = "//executorch/examples/custom_ops:custom_ops.yaml",
48+
)
49+
50+
executorch_generated_lib(
51+
name = "select_ops_from_yaml_lib",
52+
custom_ops_yaml_target = "//executorch/examples/custom_ops:custom_ops.yaml",
53+
deps = [
54+
"//executorch/examples/custom_ops:custom_ops_1",
55+
"//executorch/examples/custom_ops:custom_ops_2",
56+
":select_ops_from_yaml",
57+
],
58+
)
59+
60+
# Select all ops from a given model
61+
# TODO(larryliu0820): Add this
62+
63+
# ~~~ Test binary for selective build ~~~
64+
select_ops = native.read_config("executorch", "select_ops", None)
65+
lib = []
66+
if select_ops == "all":
67+
lib.append(":select_all_lib")
68+
elif select_ops == "list":
69+
lib.append(":select_ops_in_list_lib")
70+
elif select_ops == "yaml":
71+
lib.append(":select_ops_from_yaml_lib")
72+
runtime.cxx_binary(
73+
name = "selective_build_test",
74+
srcs = [],
75+
deps = [
76+
"//executorch/examples/executor_runner:executor_runner_lib",
77+
] + lib,
78+
define_static_target = True,
79+
**get_oss_build_kwargs()
80+
)

‎examples/selective_build/test_selective_build.sh‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
# Test the end-to-end flow of selective build, using 3 APIs:
9+
# 1. Select all ops
10+
# 2. Select from a list of ops
11+
# 3. Select from a yaml file
12+
# 4. (TODO) Select from a serialized model (.pte)
13+
set -e
14+
15+
test_buck2_select_all_ops() {
16+
echo "Exporting MobilenetV3"
17+
${PYTHON_EXECUTABLE} -m examples.export.export_example --model_name="mv3"
18+
19+
echo "Running executor_runner"
20+
buck2 run //examples/selective_build:selective_build_test \
21+
--config=executorch.select_ops=all -- --model_path=./mv3.pte
22+
23+
echo "Removing mv3.pte"
24+
rm "./mv3.pte"
25+
}
26+
27+
test_buck2_select_ops_in_list() {
28+
echo "Exporting add_mul"
29+
${PYTHON_EXECUTABLE} -m examples.export.export_example --model_name="add_mul"
30+
31+
echo "Running executor_runner"
32+
buck2 run //examples/selective_build:selective_build_test \
33+
--config=executorch.select_ops=list -- --model_path=./add_mul.pte
34+
35+
echo "Removing add_mul.pte"
36+
rm "./add_mul.pte"
37+
}
38+
39+
test_buck2_select_ops_from_yaml() {
40+
echo "Exporting custom_op_1"
41+
${PYTHON_EXECUTABLE} -m examples.custom_ops.custom_ops_1
42+
43+
echo "Running executor_runner"
44+
buck2 run //examples/selective_build:selective_build_test \
45+
--config=executorch.select_ops=yaml -- --model_path=./custom_ops_1.pte
46+
47+
echo "Removing custom_ops_1.pte"
48+
rm "./custom_ops_1.pte"
49+
}
50+
51+
# TODO(larryliu0820) Add example to select ops from model.
52+
53+
if [[ -z $PYTHON_EXECUTABLE ]];
54+
then
55+
PYTHON_EXECUTABLE=python3
56+
fi
57+
58+
59+
test_buck2_select_all_ops
60+
test_buck2_select_ops_in_list
61+
test_buck2_select_ops_from_yaml

0 commit comments

Comments
 (0)