Skip to content

Commit e56f899

Browse files
zackcamzackcam
committed
Improving build script to provide more options
Signed-off-by: zackcam <[email protected]>
1 parent 223d780 commit e56f899

File tree

2 files changed

+190
-105
lines changed

2 files changed

+190
-105
lines changed

‎README.md‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,17 +34,32 @@ valkey-server --loadmodule ./target/release/libvalkey_bloom.so
3434

3535
#### Local development script to build, run format checks, run unit / integration tests, and for cargo release:
3636
```
37-
# Builds the valkey-server (unstable) for integration testing.
37+
# Builds only the valkey-bloom module
3838
SERVER_VERSION=unstable
3939
./build.sh
40-
# Same as above, but uses valkey-server (8.0.0) for integration testing.
40+
41+
# Same as above, but uses valkey-server (8.0.0).
4142
SERVER_VERSION=8.0.0
4243
./build.sh
43-
# Build with asan, you may need to remove the old valkey binary if you have used ./build.sh before. You can do this by deleting the `.build` folder in the `tests` folder
44+
45+
# To view the available arguments, use:
46+
./build.sh --help
47+
48+
# Clean build artifacts
49+
./build.sh --clean
50+
51+
# Build and run the unit tests
52+
./build.sh --unit
53+
54+
# Run all integration tests
55+
./build.sh --integration
56+
57+
# Run a unique set of integration tests
58+
TEST_PATTERN=<test-function-or-file> ./build.sh --integration
59+
60+
# Build with asan, you may need to remove the old valkey binary if you have used ./build.sh before. You can do this by deleting the `.build` folder in the `tests` folder. This option can be used with any of the build script options.
4461
ASAN_BUILD=true
4562
./build.sh
46-
# Clean build artifacts
47-
./build.sh clean
4863
```
4964

5065
## Load the Module

‎build.sh‎

Lines changed: 170 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,73 @@
55
# Exit the script if any command fails
66
set -e
77

8+
function print_usage() {
9+
cat<<EOF
10+
Usage: build.sh [--release] [--unit] [--integration] [--clean]
11+
12+
--help | -h Print this help message and exit.
13+
--release Builds the release configuration.
14+
--unit Builds the unit tests configuration.
15+
--integration Builds the integration tests configuration.
16+
--clean Cleans the build artifacts.
17+
18+
Example usage:
19+
20+
# Build the release configuration,
21+
./build.sh --release
22+
23+
# Cleans the build artifacts,
24+
./build.sh --clean
25+
26+
EOF
27+
}
28+
29+
830
SCRIPT_DIR=$(pwd)
931
echo "Script Directory: $SCRIPT_DIR"
10-
11-
if [ "$1" = "clean" ]; then
32+
RUN_UNIT=0
33+
RUN_INTEGRATION=0
34+
BUILD_RELEASE=1
35+
CLEAN_BUILD=0
36+
37+
## Parse command line argument
38+
while [[ $# -gt 0 ]];
39+
do
40+
arg="$1"
41+
case $arg in
42+
--release)
43+
BUILD_RELEASE=1
44+
RUN_UNIT=0
45+
RUN_INTEGRATION=0
46+
;;
47+
--unit)
48+
RUN_UNIT=1
49+
RUN_INTEGRATION=0
50+
BUILD_RELEASE=0
51+
;;
52+
--integration)
53+
RUN_UNIT=0
54+
RUN_INTEGRATION=1
55+
;;
56+
--clean)
57+
CLEAN_BUILD=1
58+
RUN_UNIT=0
59+
RUN_INTEGRATION=0
60+
;;
61+
--help|-h)
62+
print_usage
63+
exit 0
64+
;;
65+
*)
66+
print_usage
67+
exit 1
68+
;;
69+
esac
70+
shift
71+
done
72+
73+
74+
if [ $CLEAN_BUILD -eq 1 ]; then
1275
echo "Cleaning build artifacts"
1376
rm -rf target/
1477
rm -rf tests/build/
@@ -21,10 +84,6 @@ echo "Running cargo and clippy format checks..."
2184
cargo fmt --check
2285
cargo clippy --profile release --all-targets -- -D clippy::all
2386

24-
25-
echo "Running unit tests..."
26-
cargo test --features enable-system-alloc
27-
2887
# Ensure SERVER_VERSION environment variable is set
2988
if [ -z "$SERVER_VERSION" ]; then
3089
echo "ERROR: SERVER_VERSION environment variable is not set. Defaulting to unstable."
@@ -36,117 +95,128 @@ if [ "$SERVER_VERSION" != "unstable" ] && [ "$SERVER_VERSION" != "8.0" ] && [ "$
3695
exit 1
3796
fi
3897

39-
echo "Running cargo build release..."
40-
if [ "$SERVER_VERSION" == "8.0" ] ; then
41-
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
42-
else
43-
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
98+
if [ $RUN_UNIT -eq 1 ]; then
99+
echo "Running unit tests..."
100+
cargo test --features enable-system-alloc
44101
fi
45102

46103

47-
REPO_URL="https://github.com/valkey-io/valkey.git"
48-
BINARY_PATH="tests/build/binaries/$SERVER_VERSION/valkey-server"
49-
CACHED_VALKEY_PATH="tests/build/valkey"
50-
if [ -f "$BINARY_PATH" ] && [ -x "$BINARY_PATH" ]; then
51-
echo "valkey-server binary '$BINARY_PATH' found."
52-
else
53-
echo "valkey-server binary '$BINARY_PATH' not found."
54-
mkdir -p "tests/build/binaries/$SERVER_VERSION"
55-
rm -rf $CACHED_VALKEY_PATH
56-
cd tests/build
57-
git clone "$REPO_URL"
58-
cd valkey
59-
git checkout "$SERVER_VERSION"
60-
make distclean
61-
if [ ! -z "${ASAN_BUILD}" ]; then
62-
make -j SANITIZER=address
104+
if [ $BUILD_RELEASE -eq 1 ]; then
105+
echo "Running cargo build release..."
106+
if [ "$SERVER_VERSION" == "8.0" ] ; then
107+
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
63108
else
64-
make -j
109+
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
65110
fi
66-
cp src/valkey-server ../binaries/$SERVER_VERSION/
67-
cd $SCRIPT_DIR
68-
rm -rf $CACHED_VALKEY_PATH
69111
fi
70112

71113

72-
TEST_FRAMEWORK_REPO="https://github.com/valkey-io/valkey-test-framework"
73-
TEST_FRAMEWORK_DIR="tests/build/valkeytestframework"
74114

75-
if [ -d "$TEST_FRAMEWORK_DIR" ]; then
76-
echo "valkeytestframework found."
77-
else
78-
echo "Cloning valkey-test-framework..."
79-
git clone "$TEST_FRAMEWORK_REPO"
80-
mkdir -p "$TEST_FRAMEWORK_DIR"
81-
mv "valkey-test-framework/src"/* "$TEST_FRAMEWORK_DIR/"
82-
rm -rf valkey-test-framework
83-
fi
84-
85-
REQUIREMENTS_FILE="requirements.txt"
86-
87-
# Check if pip is available
88-
if command -v pip > /dev/null 2>&1; then
89-
echo "Using pip to install packages..."
90-
pip install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
91-
# Check if pip3 is available
92-
elif command -v pip3 > /dev/null 2>&1; then
93-
echo "Using pip3 to install packages..."
94-
pip3 install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
95-
else
96-
echo "Error: Neither pip nor pip3 is available. Please install Python package installer."
97-
exit 1
98-
fi
115+
if [ $RUN_INTEGRATION -eq 1 ]; then
116+
REPO_URL="https://github.com/valkey-io/valkey.git"
117+
BINARY_PATH="tests/build/binaries/$SERVER_VERSION/valkey-server"
118+
CACHED_VALKEY_PATH="tests/build/valkey"
119+
if [ -f "$BINARY_PATH" ] && [ -x "$BINARY_PATH" ]; then
120+
echo "valkey-server binary '$BINARY_PATH' found."
121+
else
122+
echo "valkey-server binary '$BINARY_PATH' not found."
123+
mkdir -p "tests/build/binaries/$SERVER_VERSION"
124+
rm -rf $CACHED_VALKEY_PATH
125+
cd tests/build
126+
git clone "$REPO_URL"
127+
cd valkey
128+
git checkout "$SERVER_VERSION"
129+
make distclean
130+
if [ ! -z "${ASAN_BUILD}" ]; then
131+
make -j SANITIZER=address
132+
else
133+
make -j
134+
fi
135+
cp src/valkey-server ../binaries/$SERVER_VERSION/
136+
cd $SCRIPT_DIR
137+
rm -rf $CACHED_VALKEY_PATH
138+
fi
139+
TEST_FRAMEWORK_REPO="https://github.com/valkey-io/valkey-test-framework"
140+
TEST_FRAMEWORK_DIR="tests/build/valkeytestframework"
99141

100-
os_type=$(uname)
101-
MODULE_EXT=".so"
102-
if [[ "$os_type" == "Darwin" ]]; then
103-
MODULE_EXT=".dylib"
104-
elif [[ "$os_type" == "Linux" ]]; then
105-
MODULE_EXT=".so"
106-
else
107-
echo "Unsupported OS type: $os_type"
108-
exit 1
109-
fi
110-
export MODULE_PATH="$SCRIPT_DIR/target/release/libvalkey_bloom$MODULE_EXT"
111142

112-
echo "Running the integration tests..."
113-
if [ ! -z "${ASAN_BUILD}" ]; then
114-
# TEST_PATTERN can be used to run specific tests or test patterns.
115-
if [[ -n "$TEST_PATTERN" ]]; then
116-
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN 2>&1 | tee test_output.tmp
143+
if [ -d "$TEST_FRAMEWORK_DIR" ]; then
144+
echo "valkeytestframework found."
117145
else
118-
echo "TEST_PATTERN is not set. Running all integration tests."
119-
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" 2>&1 | tee test_output.tmp
146+
echo "Cloning valkey-test-framework..."
147+
git clone "$TEST_FRAMEWORK_REPO"
148+
mkdir -p "$TEST_FRAMEWORK_DIR"
149+
mv "valkey-test-framework/src"/* "$TEST_FRAMEWORK_DIR/"
150+
rm -rf valkey-test-framework
120151
fi
121152

122-
# Check for memory leaks in the output
123-
if grep -q "LeakSanitizer: detected memory leaks" test_output.tmp; then
124-
RED='\033[0;31m'
125-
echo -e "${RED}Memory leaks detected in the following tests:"
126-
LEAKING_TESTS=$(grep -B 2 "LeakSanitizer: detected memory leaks" test_output.tmp | \
127-
grep -v "LeakSanitizer" | \
128-
grep ".*\.py::")
129-
130-
LEAK_COUNT=$(echo "$LEAKING_TESTS" | wc -l)
131-
132-
# Output each leaking test
133-
echo "$LEAKING_TESTS" | while read -r line; do
134-
echo "::error::Test with leak: $line"
135-
done
136-
137-
echo -e "\n$LEAK_COUNT python integration tests have leaks detected in them"
138-
rm test_output.tmp
153+
REQUIREMENTS_FILE="requirements.txt"
154+
155+
# Check if pip is available
156+
if command -v pip > /dev/null 2>&1; then
157+
echo "Using pip to install packages..."
158+
pip install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
159+
# Check if pip3 is available
160+
elif command -v pip3 > /dev/null 2>&1; then
161+
echo "Using pip3 to install packages..."
162+
pip3 install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
163+
else
164+
echo "Error: Neither pip nor pip3 is available. Please install Python package installer."
139165
exit 1
140166
fi
141-
rm test_output.tmp
142-
else
143-
# TEST_PATTERN can be used to run specific tests or test patterns.
144-
if [[ -n "$TEST_PATTERN" ]]; then
145-
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN
167+
168+
os_type=$(uname)
169+
MODULE_EXT=".so"
170+
if [[ "$os_type" == "Darwin" ]]; then
171+
MODULE_EXT=".dylib"
172+
elif [[ "$os_type" == "Linux" ]]; then
173+
MODULE_EXT=".so"
146174
else
147-
echo "TEST_PATTERN is not set. Running all integration tests."
148-
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/"
175+
echo "Unsupported OS type: $os_type"
176+
exit 1
177+
fi
178+
export MODULE_PATH="$SCRIPT_DIR/target/release/libvalkey_bloom$MODULE_EXT"
179+
180+
echo "Running the integration tests..."
181+
if [ ! -z "${ASAN_BUILD}" ]; then
182+
# TEST_PATTERN can be used to run specific tests or test patterns.
183+
if [[ -n "$TEST_PATTERN" ]]; then
184+
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN -m "not skip_for_asan" 2>&1 | tee test_output.tmp
185+
else
186+
echo "TEST_PATTERN is not set. Running all integration tests."
187+
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" -m "not skip_for_asan" 2>&1 | tee test_output.tmp
188+
fi
189+
190+
# Check for memory leaks in the output
191+
if grep -q "LeakSanitizer: detected memory leaks" test_output.tmp; then
192+
RED='\033[0;31m'
193+
echo -e "${RED}Memory leaks detected in the following tests:"
194+
LEAKING_TESTS=$(grep -B 2 "LeakSanitizer: detected memory leaks" test_output.tmp | \
195+
grep -v "LeakSanitizer" | \
196+
grep ".*\.py::")
197+
198+
LEAK_COUNT=$(echo "$LEAKING_TESTS" | wc -l)
199+
200+
# Output each leaking test
201+
echo "$LEAKING_TESTS" | while read -r line; do
202+
echo "::error::Test with leak: $line"
203+
done
204+
205+
echo -e "\n$LEAK_COUNT python integration tests have leaks detected in them"
206+
rm test_output.tmp
207+
exit 1
208+
fi
209+
rm test_output.tmp
210+
else
211+
# TEST_PATTERN can be used to run specific tests or test patterns.
212+
if [[ -n "$TEST_PATTERN" ]]; then
213+
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN
214+
else
215+
echo "TEST_PATTERN is not set. Running all integration tests."
216+
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/"
217+
fi
149218
fi
150219
fi
151220

152-
echo "Build, Format Checks, Unit tests, and Integration Tests succeeded"
221+
222+
# echo "Build, Format Checks, Unit tests, and Integration Tests succeeded"

0 commit comments

Comments
 (0)