Skip to content

Commit 818d095

Browse files
SS-JIAssjia
SS-JIA
and
ssjia
authored
[ET-VK][testing] Add scripts to facilitate operator testiing (#13593)
Stack from [ghstack](https://github.com/ezyang/ghstack) (oldest at bottom): * #13597 * #13596 * #13595 * #13594 * __->__ #13593 * #13600 * #13599 * #13598 Differential Revision: [D80800081](https://our.internmc.facebook.com/intern/diff/D80800081) Co-authored-by: ssjia <[email protected]>
1 parent bd70337 commit 818d095

File tree

2 files changed

+268
-3
lines changed

2 files changed

+268
-3
lines changed

‎backends/vulkan/test/op_tests/CMakeLists.txt‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,20 @@ function(vulkan_op_test test_name test_src)
8888
endfunction()
8989

9090
if(TARGET vulkan_backend AND LIB_TORCH)
91+
add_library(test_utils ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.cpp)
92+
target_include_directories(test_utils PRIVATE ${COMMON_INCLUDES})
93+
target_link_libraries(
94+
test_utils PRIVATE vulkan_backend ${LIB_TORCH} ${LIB_TORCH_CPU}
95+
)
96+
9197
find_library(
9298
CUSTOM_OPS_LIB custom_ops_aot_lib
9399
HINTS ${CMAKE_INSTALL_PREFIX}/executorch/extension/llm/custom_ops
94100
)
95101
if(CUSTOM_OPS_LIB)
96102
vulkan_op_test(
97103
vulkan_sdpa_test ${CMAKE_CURRENT_SOURCE_DIR}/sdpa_test.cpp
98-
${CUSTOM_OPS_LIB}
104+
${CUSTOM_OPS_LIB} test_utils
99105
)
100106
else()
101107
message(
@@ -104,10 +110,11 @@ if(TARGET vulkan_backend AND LIB_TORCH)
104110
endif()
105111
vulkan_op_test(
106112
vulkan_rope_test ${CMAKE_CURRENT_SOURCE_DIR}/rotary_embedding_test.cpp
113+
test_utils
107114
)
108115
vulkan_op_test(
109-
vulkan_linear_weight_int4_test
110-
${CMAKE_CURRENT_SOURCE_DIR}/linear_weight_int4_test.cpp
116+
quantized_linear_test ${CMAKE_CURRENT_SOURCE_DIR}/quantized_linear_test.cpp
117+
test_utils
111118
)
112119

113120
# Only build generated op tests if a path to tags.yaml and

‎backends/vulkan/test/scripts/test_op.sh‎

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
# Initialize variables
11+
RUN_BUILD=false
12+
RUN_CLEAN=false
13+
RUN_CLEAN_TESTS=false
14+
RUN_RECOMPILE=false
15+
RUN_TESTS=false
16+
TEST_BINARY=""
17+
ATEN_OP=""
18+
19+
# Parse arguments
20+
SKIP_NEXT=false
21+
if [[ $# -eq 0 ]]; then
22+
# No arguments provided - run default test
23+
TEST_BINARY="vulkan_op_correctness_tests"
24+
RUN_TESTS=true
25+
else
26+
for i in $(seq 1 $#); do
27+
if [[ "$SKIP_NEXT" == true ]]; then
28+
SKIP_NEXT=false
29+
continue
30+
fi
31+
32+
arg="${!i}"
33+
case $arg in
34+
--build|-b)
35+
RUN_BUILD=true
36+
;;
37+
--clean|-c)
38+
RUN_CLEAN=true
39+
RUN_BUILD=true
40+
;;
41+
--clean_tests|-ct)
42+
RUN_CLEAN_TESTS=true
43+
;;
44+
--recompile|-rc)
45+
RUN_RECOMPILE=true
46+
;;
47+
--test|-t)
48+
RUN_TESTS=true
49+
;;
50+
--aten)
51+
next_i=$((i + 1))
52+
if [[ $next_i -le $# ]]; then
53+
ATEN_OP="${!next_i}"
54+
TEST_BINARY="vulkan_op_correctness_tests"
55+
RUN_TESTS=true
56+
SKIP_NEXT=true
57+
else
58+
echo "Error: --aten requires an operator name"
59+
exit 1
60+
fi
61+
;;
62+
--*|-*)
63+
echo "Unknown argument: $arg"
64+
exit 1
65+
;;
66+
*)
67+
if [[ -z "$TEST_BINARY" ]]; then
68+
TEST_BINARY="$arg"
69+
RUN_TESTS=true
70+
else
71+
echo "Multiple test binaries provided: $TEST_BINARY and $arg"
72+
exit 1
73+
fi
74+
;;
75+
esac
76+
done
77+
fi
78+
79+
# Determine execution mode based on parsed arguments
80+
if [[ "$RUN_BUILD" == true ]] && [[ -z "$TEST_BINARY" ]] && [[ "$RUN_TESTS" == false ]]; then
81+
# Build-only mode
82+
echo "Build-only mode"
83+
elif [[ "$RUN_BUILD" == true ]] && [[ -n "$TEST_BINARY" ]]; then
84+
# Build and test mode
85+
echo "Build and test mode for: $TEST_BINARY"
86+
elif [[ "$RUN_BUILD" == false ]] && [[ -n "$TEST_BINARY" ]]; then
87+
# Test-only mode
88+
echo "Test-only mode for: $TEST_BINARY"
89+
elif [[ "$RUN_TESTS" == true ]] && [[ -z "$TEST_BINARY" ]]; then
90+
# Run all available tests
91+
echo "Running all available operator tests"
92+
elif [[ $# -eq 0 ]]; then
93+
# No arguments provided - run default test
94+
TEST_BINARY="vulkan_op_correctness_tests"
95+
RUN_TESTS=true
96+
echo "No arguments provided, running default test: $TEST_BINARY"
97+
else
98+
echo "Invalid argument combination. Usage:"
99+
echo " $0 # Run default vulkan_op_correctness_tests"
100+
echo " $0 --build|-b [--clean|-c] [--clean_tests|-ct] [--recompile|-rc] # Build-only mode"
101+
echo " $0 [test_binary_name] [--build|-b] [--clean|-c] [--clean_tests|-ct] [--recompile|-rc] # Test mode or build+test mode"
102+
echo " $0 --test|-t [--build|-b] [--clean|-c] [--clean_tests|-ct] [--recompile|-rc] # Run all tests mode"
103+
echo " $0 --aten <operator_name> [--build|-b] [--clean|-c] [--clean_tests|-ct] [--recompile|-rc] # Run specific ATen operator test"
104+
echo " $0 --clean_tests|-ct # Clean and rebuild only operator tests"
105+
echo ""
106+
echo "Available test binaries:"
107+
echo " - vulkan_op_correctness_tests"
108+
echo " - vulkan_op_benchmarks"
109+
echo " - compute_graph_op_tests"
110+
echo " - sdpa_test"
111+
exit 1
112+
fi
113+
114+
if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
115+
PYTHON_EXECUTABLE=python3
116+
fi
117+
which "${PYTHON_EXECUTABLE}"
118+
119+
CMAKE_OUTPUT_DIR=cmake-out
120+
121+
clean_build_directory() {
122+
echo "Cleaning build directory: ${CMAKE_OUTPUT_DIR}"
123+
rm -rf ${CMAKE_OUTPUT_DIR}
124+
}
125+
126+
clean_test_directory() {
127+
echo "Cleaning test build directory: ${CMAKE_OUTPUT_DIR}/backends/vulkan/test/op_tests"
128+
rm -rf ${CMAKE_OUTPUT_DIR}/backends/vulkan/test/op_tests
129+
}
130+
131+
build_core_libraries() {
132+
cmake . \
133+
-DCMAKE_INSTALL_PREFIX=cmake-out \
134+
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
135+
-DEXECUTORCH_BUILD_KERNELS_CUSTOM_AOT=ON \
136+
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
137+
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
138+
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
139+
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
140+
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
141+
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
142+
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
143+
-DEXECUTORCH_BUILD_EXECUTOR_RUNNER=ON \
144+
-DEXECUTORCH_BUILD_DEVTOOLS=ON \
145+
-DEXECUTORCH_BUILD_VULKAN=ON \
146+
-DEXECUTORCH_BUILD_XNNPACK=ON \
147+
-DEXECUTORCH_BUILD_TESTS=ON \
148+
-Bcmake-out && \
149+
cmake --build cmake-out -j64 --target install
150+
}
151+
152+
build_operator_tests() {
153+
echo "Building Vulkan operator tests..."
154+
155+
# Check if TORCH_OPS_YAML_PATH is set, if not use default
156+
if [[ -z "${TORCH_OPS_YAML_PATH:-}" ]]; then
157+
TORCH_OPS_YAML_PATH="$HOME/Github/pytorch/aten/src/ATen/native"
158+
echo "Using default TORCH_OPS_YAML_PATH: $TORCH_OPS_YAML_PATH"
159+
fi
160+
161+
# Verify that TORCH_OPS_YAML_PATH exists
162+
if [[ ! -d "$TORCH_OPS_YAML_PATH" ]]; then
163+
echo "Error: TORCH_OPS_YAML_PATH directory does not exist: $TORCH_OPS_YAML_PATH"
164+
echo "Please set TORCH_OPS_YAML_PATH to a valid PyTorch native operations directory"
165+
echo "Example: export TORCH_OPS_YAML_PATH=/path/to/pytorch/aten/src/ATen/native"
166+
exit 1
167+
fi
168+
169+
# Verify required YAML files exist
170+
if [[ ! -f "$TORCH_OPS_YAML_PATH/native_functions.yaml" ]]; then
171+
echo "Error: Required file not found: $TORCH_OPS_YAML_PATH/native_functions.yaml"
172+
exit 1
173+
fi
174+
175+
if [[ ! -f "$TORCH_OPS_YAML_PATH/tags.yaml" ]]; then
176+
echo "Error: Required file not found: $TORCH_OPS_YAML_PATH/tags.yaml"
177+
exit 1
178+
fi
179+
180+
echo "Using TORCH_OPS_YAML_PATH: $TORCH_OPS_YAML_PATH"
181+
182+
# Build operator tests
183+
cmake backends/vulkan/test/op_tests \
184+
-DCMAKE_INSTALL_PREFIX=cmake-out \
185+
-DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \
186+
-DTORCH_OPS_YAML_PATH="$TORCH_OPS_YAML_PATH" \
187+
-DCMAKE_CXX_STANDARD=17 \
188+
-Bcmake-out/backends/vulkan/test/op_tests && \
189+
cmake --build cmake-out/backends/vulkan/test/op_tests -j16
190+
}
191+
192+
recompile() {
193+
echo "Recompiling..."
194+
cmake --build cmake-out -j64 --target install
195+
cmake --build cmake-out/backends/vulkan/test/op_tests -j16
196+
}
197+
198+
run_operator_test() {
199+
local test_name="$1"
200+
local test_binary_path=""
201+
202+
case "$test_name" in
203+
"aten")
204+
test_binary_path="${CMAKE_OUTPUT_DIR}/backends/vulkan/test/op_tests/vulkan_op_correctness_tests"
205+
;;
206+
*)
207+
# Try to find the binary directly
208+
test_binary_path="${CMAKE_OUTPUT_DIR}/backends/vulkan/test/op_tests/${test_name}"
209+
;;
210+
esac
211+
212+
if [[ -f "$test_binary_path" ]]; then
213+
echo "Running test binary: $test_binary_path"
214+
215+
# Add gtest filter if ATEN_OP is specified
216+
if [[ -n "$ATEN_OP" ]]; then
217+
echo "Filtering tests for ATen operator: $ATEN_OP"
218+
"$test_binary_path" --gtest_filter="*${ATEN_OP}*"
219+
else
220+
"$test_binary_path"
221+
fi
222+
else
223+
echo "Error: Test binary not found at $test_binary_path"
224+
echo "Available binaries in ${CMAKE_OUTPUT_DIR}/backends/vulkan/test/op_tests/:"
225+
ls -la "${CMAKE_OUTPUT_DIR}/backends/vulkan/test/op_tests/" 2>/dev/null || echo "Directory not found"
226+
exit 1
227+
fi
228+
}
229+
230+
# Main execution
231+
if [[ "${RUN_CLEAN_TESTS}" == true ]]; then
232+
clean_test_directory
233+
build_operator_tests
234+
fi
235+
236+
if [[ "${RUN_BUILD}" == true ]]; then
237+
if [[ "${RUN_CLEAN}" == true ]]; then
238+
clean_build_directory
239+
fi
240+
build_core_libraries
241+
build_operator_tests
242+
fi
243+
244+
if [[ "${RUN_RECOMPILE}" == true ]]; then
245+
recompile
246+
fi
247+
248+
if [[ "${RUN_TESTS}" == true ]]; then
249+
run_operator_test "$TEST_BINARY"
250+
251+
# Check if tests completed successfully
252+
if [[ $? -eq 0 ]]; then
253+
echo "Vulkan operator tests completed successfully!"
254+
else
255+
echo "Some Vulkan operator tests failed!"
256+
exit 1
257+
fi
258+
fi

0 commit comments

Comments
 (0)