Skip to content

Commit 25ec7ce

Browse files
jackzhxngjackzhxng
committed
Merge branch 'jz/tt-llama-rebased' into jz/native-runner-tt
2 parents b8ff8e2 + b1f6678 commit 25ec7ce

File tree

589 files changed

+16292
-7338
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

589 files changed

+16292
-7338
lines changed

‎.ci/docker/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ case "${IMAGE_NAME}" in
4141
LINTRUNNER=""
4242
CLANG_VERSION=12
4343
# From https://developer.android.com/ndk/downloads
44-
ANDROID_NDK_VERSION=r26c
44+
ANDROID_NDK_VERSION=r27b
4545
;;
4646
*)
4747
echo "Invalid image name ${IMAGE_NAME}"

‎.ci/docker/ci_commit_pins/pytorch.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
d1b87e26e5c4343f5b56bb1e6f89b479b389bfac
1+
export-D64151426

‎.ci/scripts/build_llama_android.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ install_executorch_and_backend_lib() {
1919
cmake -DBUCK2="${BUCK2}" \
2020
-DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
2121
-DANDROID_ABI="${ANDROID_ABI}" \
22-
-DANDROID_PLATFORM=android-23 \
2322
-DCMAKE_INSTALL_PREFIX=cmake-android-out \
2423
-DCMAKE_BUILD_TYPE=Release \
2524
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
@@ -41,16 +40,15 @@ build_llama_runner() {
4140
cmake -DBUCK2="${BUCK2}" \
4241
-DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK"/build/cmake/android.toolchain.cmake \
4342
-DANDROID_ABI="${ANDROID_ABI}" \
44-
-DANDROID_PLATFORM=android-23 \
4543
-DCMAKE_INSTALL_PREFIX=cmake-android-out \
4644
-DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE=python \
4745
-DEXECUTORCH_BUILD_XNNPACK=ON \
4846
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
4947
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
5048
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
51-
-Bcmake-android-out/examples/models/llama2 examples/models/llama2
49+
-Bcmake-android-out/examples/models/llama examples/models/llama
5250

53-
cmake --build cmake-android-out/examples/models/llama2 -j4 --config Release
51+
cmake --build cmake-android-out/examples/models/llama -j4 --config Release
5452
}
5553
install_flatc_from_source
5654
install_executorch_and_backend_lib

‎.ci/scripts/test_eval_llama_mmlu.sh‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
11+
PYTHON_EXECUTABLE=python3
12+
fi
13+
14+
# Download and prepare stories model artifacts
15+
prepare_model_artifacts() {
16+
echo "Preparing stories model artifacts"
17+
wget -O stories110M.pt "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt"
18+
wget -O tokenizer.model "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model"
19+
echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json
20+
}
21+
22+
run_and_verify() {
23+
NOW=$(date +"%H:%M:%S")
24+
echo "Starting to run eval_llama at ${NOW}"
25+
if [[ ! -f "stories110M.pt" ]]; then
26+
echo "stories110M.pt is missing."
27+
exit 1
28+
fi
29+
if [[ ! -f "tokenizer.model" ]]; then
30+
echo "tokenizer.model is missing."
31+
exit 1
32+
fi
33+
if [[ ! -f "params.json" ]]; then
34+
echo "params.json is missing."
35+
exit 1
36+
fi
37+
$PYTHON_EXECUTABLE -m examples.models.llama.eval_llama \
38+
--model llama2 \
39+
-c stories110M.pt \
40+
-p params.json \
41+
-t tokenizer.model \
42+
-kv \
43+
-d fp32 \
44+
--tasks mmlu \
45+
-f 5 \
46+
--max_seq_length 2048 \
47+
--limit 5 > result.txt
48+
49+
# Verify result.txt
50+
RESULT=$(cat result.txt)
51+
EXPECTED_TASK="mmlu"
52+
EXPECTED_RESULT="acc"
53+
if [[ "${RESULT}" == "${EXPECTED_TASK}: {"*"${EXPECTED_RESULT}"* ]]; then
54+
echo "Actual result: ${RESULT}"
55+
echo "Success"
56+
exit 0
57+
else
58+
echo "Actual result: ${RESULT}"
59+
echo "Failure; results not the same"
60+
exit 1
61+
fi
62+
}
63+
64+
prepare_model_artifacts
65+
run_and_verify

‎.ci/scripts/test_eval_llama_wikitext.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
11+
PYTHON_EXECUTABLE=python3
12+
fi
13+
14+
# Download and prepare stories model artifacts
15+
prepare_model_artifacts() {
16+
echo "Preparing stories model artifacts"
17+
wget -O stories110M.pt "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt"
18+
wget -O tokenizer.model "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model"
19+
echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json
20+
}
21+
22+
run_and_verify() {
23+
NOW=$(date +"%H:%M:%S")
24+
echo "Starting to run eval_llama at ${NOW}"
25+
if [[ ! -f "stories110M.pt" ]]; then
26+
echo "stories110M.pt is missing."
27+
exit 1
28+
fi
29+
if [[ ! -f "tokenizer.model" ]]; then
30+
echo "tokenizer.model is missing."
31+
exit 1
32+
fi
33+
if [[ ! -f "params.json" ]]; then
34+
echo "params.json is missing."
35+
exit 1
36+
fi
37+
$PYTHON_EXECUTABLE -m examples.models.llama.eval_llama \
38+
--model llama2 \
39+
-c stories110M.pt \
40+
-p params.json \
41+
-t tokenizer.model \
42+
-kv \
43+
-d fp32 \
44+
--max_seq_length 2048 \
45+
--limit 5 > result.txt
46+
47+
# Verify result.txt
48+
RESULT=$(cat result.txt)
49+
EXPECTED_TASK="wikitext"
50+
EXPECTED_RESULT="word_perplexity"
51+
if [[ "${RESULT}" == "${EXPECTED_TASK}: {"*"${EXPECTED_RESULT}"* ]]; then
52+
echo "Actual result: ${RESULT}"
53+
echo "Success"
54+
exit 0
55+
else
56+
echo "Actual result: ${RESULT}"
57+
echo "Failure; results not the same"
58+
exit 1
59+
fi
60+
}
61+
62+
prepare_model_artifacts
63+
run_and_verify

‎.ci/scripts/test_llama.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ cmake_install_executorch_libraries() {
125125

126126
cmake_build_llama_runner() {
127127
echo "Building llama runner"
128-
dir="examples/models/llama2"
128+
dir="examples/models/llama"
129129
retry cmake \
130130
-DCMAKE_INSTALL_PREFIX=cmake-out \
131131
-DCMAKE_BUILD_TYPE=Debug \
@@ -206,7 +206,7 @@ if [[ "${QNN}" == "ON" ]]; then
206206
EXPORT_ARGS="${EXPORT_ARGS} -kv -v --qnn --disable_dynamic_shape"
207207
fi
208208
# Add dynamically linked library location
209-
$PYTHON_EXECUTABLE -m examples.models.llama2.export_llama ${EXPORT_ARGS}
209+
$PYTHON_EXECUTABLE -m examples.models.llama.export_llama --model llama3 ${EXPORT_ARGS}
210210

211211
# Create tokenizer.bin.
212212
echo "Creating tokenizer.bin"
@@ -219,15 +219,15 @@ echo "Running ${EXPORTED_MODEL_NAME} in portable mode"
219219
if [[ "${BUILD_TOOL}" == "buck2" ]]; then
220220
# Run model.
221221
# shellcheck source=/dev/null
222-
$BUCK run examples/models/llama2:main -- ${RUNTIME_ARGS} > result.txt
222+
$BUCK run examples/models/llama:main -- ${RUNTIME_ARGS} > result.txt
223223
elif [[ "${BUILD_TOOL}" == "cmake" ]]; then
224224
cmake_install_executorch_libraries
225225
cmake_build_llama_runner
226226
# Run llama runner
227227
NOW=$(date +"%H:%M:%S")
228228
echo "Starting to run llama runner at ${NOW}"
229229
# shellcheck source=/dev/null
230-
cmake-out/examples/models/llama2/llama_main ${RUNTIME_ARGS} > result.txt
230+
cmake-out/examples/models/llama/llama_main ${RUNTIME_ARGS} > result.txt
231231
NOW=$(date +"%H:%M:%S")
232232
echo "Finished at ${NOW}"
233233
else

‎.ci/scripts/test_llama_runner_eager.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
11+
PYTHON_EXECUTABLE=python3
12+
fi
13+
14+
# Download and prepare stories model artifacts
15+
prepare_model_artifacts() {
16+
echo "Preparing stories model artifacts"
17+
wget -O stories110M.pt "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt"
18+
wget -O tokenizer.model "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model"
19+
echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json
20+
}
21+
22+
run_and_verify() {
23+
NOW=$(date +"%H:%M:%S")
24+
echo "Starting to run eval_llama at ${NOW}"
25+
if [[ ! -f "stories110M.pt" ]]; then
26+
echo "stories110M.pt is missing."
27+
exit 1
28+
fi
29+
if [[ ! -f "tokenizer.model" ]]; then
30+
echo "tokenizer.model is missing."
31+
exit 1
32+
fi
33+
if [[ ! -f "params.json" ]]; then
34+
echo "params.json is missing."
35+
exit 1
36+
fi
37+
$PYTHON_EXECUTABLE -m examples.models.llama.runner.eager \
38+
--model llama2 \
39+
-c stories110M.pt \
40+
-p params.json \
41+
-t tokenizer.model \
42+
-kv \
43+
-d fp32 \
44+
--max_seq_length 32 \
45+
--temperature 0 \
46+
--prompt "Once upon a time," > result.txt
47+
48+
# Verify result.txt
49+
RESULT=$(cat result.txt)
50+
EXPECTED_RESULT="there was a little girl"
51+
if [[ "${RESULT}" == *"${EXPECTED_RESULT}"* ]]; then
52+
echo "Actual result: ${RESULT}"
53+
echo "Success"
54+
exit 0
55+
else
56+
echo "Actual result: ${RESULT}"
57+
echo "Failure; results not the same"
58+
exit 1
59+
fi
60+
}
61+
62+
prepare_model_artifacts
63+
run_and_verify

‎.ci/scripts/test_llava.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ cmake_install_executorch_libraries_for_android() {
5656
cmake \
5757
-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
5858
-DANDROID_ABI=arm64-v8a \
59-
-DANDROID_PLATFORM=android-23 \
6059
${EXECUTORCH_COMMON_CMAKE_ARGS} \
6160
-B${BUILD_DIR} .
6261

@@ -93,7 +92,6 @@ cmake_build_llava_runner_for_android() {
9392
cmake \
9493
-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
9594
-DANDROID_ABI=arm64-v8a \
96-
-DANDROID_PLATFORM=android-23 \
9795
${LLAVA_COMMON_CMAKE_ARGS} \
9896
-DCMAKE_PREFIX_PATH="$python_lib" \
9997
-DLLAVA_RUNNER_NO_TORCH_DUMMY_IMAGE=ON \

‎.ci/scripts/test_model.sh‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,9 @@ run_portable_executor_runner() {
7575
test_model() {
7676
if [[ "${MODEL_NAME}" == "llama2" ]]; then
7777
# Install requirements for export_llama
78-
bash examples/models/llama2/install_requirements.sh
79-
# Test export_llama script: python3 -m examples.models.llama2.export_llama
80-
"${PYTHON_EXECUTABLE}" -m examples.models.llama2.export_llama -c examples/models/llama2/params/demo_rand_params.pth -p examples/models/llama2/params/demo_config.json
78+
bash examples/models/llama/install_requirements.sh
79+
# Test export_llama script: python3 -m examples.models.llama.export_llama
80+
"${PYTHON_EXECUTABLE}" -m examples.models.llama.export_llama --model llama2 -c examples/models/llama/params/demo_rand_params.pth -p examples/models/llama/params/demo_config.json
8181
run_portable_executor_runner
8282
rm "./${MODEL_NAME}.pte"
8383
fi
@@ -197,6 +197,11 @@ test_model_with_coreml() {
197197
fi
198198
}
199199

200+
test_model_with_mps() {
201+
"${PYTHON_EXECUTABLE}" -m examples.apple.mps.scripts.mps_example --model_name="${MODEL_NAME}" --use_fp16
202+
EXPORTED_MODEL=$(find "." -type f -name "${MODEL_NAME}*.pte" -print -quit)
203+
}
204+
200205
if [[ "${BACKEND}" == "portable" ]]; then
201206
echo "Testing ${MODEL_NAME} with portable kernels..."
202207
test_model
@@ -212,6 +217,12 @@ elif [[ "${BACKEND}" == "coreml" ]]; then
212217
if [[ $? -eq 0 ]]; then
213218
prepare_artifacts_upload
214219
fi
220+
elif [[ "${BACKEND}" == "mps" ]]; then
221+
echo "Testing ${MODEL_NAME} with mps..."
222+
test_model_with_mps
223+
if [[ $? -eq 0 ]]; then
224+
prepare_artifacts_upload
225+
fi
215226
elif [[ "${BACKEND}" == "xnnpack" ]]; then
216227
echo "Testing ${MODEL_NAME} with xnnpack..."
217228
WITH_QUANTIZATION=true

0 commit comments

Comments
 (0)