Skip to content

Commit 5379269

Browse files
zackcamzackcam
committed
Improving build script to provide more options
Signed-off-by: zackcam <[email protected]>
1 parent 223d780 commit 5379269

File tree

2 files changed

+203
-109
lines changed

2 files changed

+203
-109
lines changed

‎README.md‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,17 +34,32 @@ valkey-server --loadmodule ./target/release/libvalkey_bloom.so
3434

3535
#### Local development script to build, run format checks, run unit / integration tests, and for cargo release:
3636
```
37-
# Builds the valkey-server (unstable) for integration testing.
37+
# Builds only the valkey-bloom module
3838
SERVER_VERSION=unstable
3939
./build.sh
40-
# Same as above, but uses valkey-server (8.0.0) for integration testing.
40+
41+
# Same as above, but uses valkey-server (8.0.0).
4142
SERVER_VERSION=8.0.0
4243
./build.sh
43-
# Build with asan, you may need to remove the old valkey binary if you have used ./build.sh before. You can do this by deleting the `.build` folder in the `tests` folder
44+
45+
# To view the available arguments, use:
46+
./build.sh --help
47+
48+
# Clean build artifacts
49+
./build.sh --clean
50+
51+
# Build and run the unit tests
52+
./build.sh --unit
53+
54+
# Run all integration tests
55+
./build.sh --integration
56+
57+
# Run a unique set of integration tests
58+
TEST_PATTERN=<test-function-or-file> ./build.sh --integration
59+
60+
# Build with asan, you may need to remove the old valkey binary if you have used ./build.sh before. You can do this by deleting the `.build` folder in the `tests` folder. This option can be used with any of the build script options.
4461
ASAN_BUILD=true
4562
./build.sh
46-
# Clean build artifacts
47-
./build.sh clean
4863
```
4964

5065
## Load the Module

‎build.sh‎

Lines changed: 183 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,81 @@
55
# Exit the script if any command fails
66
set -e
77

8+
function print_usage() {
9+
cat<<EOF
10+
Usage: build.sh [--release] [--unit] [--integration] [--clean]
11+
12+
--help | -h Print this help message and exit.
13+
--release Builds the release configuration.
14+
--unit Builds the unit tests configuration.
15+
--integration Builds the integration tests configuration.
16+
--clean Cleans the build artifacts.
17+
--no-clippy Skips clippy formatting checks.
18+
19+
Example usage:
20+
21+
# Build the release configuration,
22+
./build.sh --release
23+
24+
# Cleans the build artifacts,
25+
./build.sh --clean
26+
27+
EOF
28+
}
29+
30+
831
SCRIPT_DIR=$(pwd)
932
echo "Script Directory: $SCRIPT_DIR"
10-
11-
if [ "$1" = "clean" ]; then
33+
RUN_UNIT=0
34+
RUN_INTEGRATION=0
35+
BUILD_RELEASE=1
36+
CLEAN_BUILD=0
37+
NO_CLIPPY=0
38+
39+
## Parse command line argument
40+
while [[ $# -gt 0 ]];
41+
do
42+
arg="$1"
43+
case $arg in
44+
--release)
45+
BUILD_RELEASE=1
46+
RUN_UNIT=0
47+
RUN_INTEGRATION=0
48+
;;
49+
--unit)
50+
RUN_UNIT=1
51+
RUN_INTEGRATION=0
52+
BUILD_RELEASE=0
53+
;;
54+
--integration)
55+
RUN_UNIT=0
56+
RUN_INTEGRATION=1
57+
;;
58+
--clean)
59+
CLEAN_BUILD=1
60+
RUN_UNIT=0
61+
RUN_INTEGRATION=0
62+
;;
63+
--help|-h)
64+
print_usage
65+
exit 0
66+
;;
67+
--no-clippy)
68+
RUN_UNIT=1
69+
RUN_INTEGRATION=1
70+
BUILD_RELEASE=1
71+
NO_CLIPPY=1
72+
;;
73+
*)
74+
print_usage
75+
exit 1
76+
;;
77+
esac
78+
shift
79+
done
80+
81+
82+
if [ $CLEAN_BUILD -eq 1 ]; then
1283
echo "Cleaning build artifacts"
1384
rm -rf target/
1485
rm -rf tests/build/
@@ -17,14 +88,11 @@ if [ "$1" = "clean" ]; then
1788
exit 0
1889
fi
1990

20-
echo "Running cargo and clippy format checks..."
21-
cargo fmt --check
22-
cargo clippy --profile release --all-targets -- -D clippy::all
23-
24-
25-
echo "Running unit tests..."
26-
cargo test --features enable-system-alloc
27-
91+
if [ $NO_CLIPPY -eq 0 ]; then
92+
echo "Running cargo and clippy format checks..."
93+
cargo fmt --check
94+
cargo clippy --profile release --all-targets -- -D clippy::all
95+
fi
2896
# Ensure SERVER_VERSION environment variable is set
2997
if [ -z "$SERVER_VERSION" ]; then
3098
echo "ERROR: SERVER_VERSION environment variable is not set. Defaulting to unstable."
@@ -36,117 +104,128 @@ if [ "$SERVER_VERSION" != "unstable" ] && [ "$SERVER_VERSION" != "8.0" ] && [ "$
36104
exit 1
37105
fi
38106

39-
echo "Running cargo build release..."
40-
if [ "$SERVER_VERSION" == "8.0" ] ; then
41-
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
42-
else
43-
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
107+
if [ $RUN_UNIT -eq 1 ]; then
108+
echo "Running unit tests..."
109+
cargo test --features enable-system-alloc
44110
fi
45111

46112

47-
REPO_URL="https://github.com/valkey-io/valkey.git"
48-
BINARY_PATH="tests/build/binaries/$SERVER_VERSION/valkey-server"
49-
CACHED_VALKEY_PATH="tests/build/valkey"
50-
if [ -f "$BINARY_PATH" ] && [ -x "$BINARY_PATH" ]; then
51-
echo "valkey-server binary '$BINARY_PATH' found."
52-
else
53-
echo "valkey-server binary '$BINARY_PATH' not found."
54-
mkdir -p "tests/build/binaries/$SERVER_VERSION"
55-
rm -rf $CACHED_VALKEY_PATH
56-
cd tests/build
57-
git clone "$REPO_URL"
58-
cd valkey
59-
git checkout "$SERVER_VERSION"
60-
make distclean
61-
if [ ! -z "${ASAN_BUILD}" ]; then
62-
make -j SANITIZER=address
113+
if [ $BUILD_RELEASE -eq 1 ]; then
114+
echo "Running cargo build release..."
115+
if [ "$SERVER_VERSION" == "8.0" ] ; then
116+
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
63117
else
64-
make -j
118+
RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
65119
fi
66-
cp src/valkey-server ../binaries/$SERVER_VERSION/
67-
cd $SCRIPT_DIR
68-
rm -rf $CACHED_VALKEY_PATH
69120
fi
70121

71122

72-
TEST_FRAMEWORK_REPO="https://github.com/valkey-io/valkey-test-framework"
73-
TEST_FRAMEWORK_DIR="tests/build/valkeytestframework"
74-
75-
if [ -d "$TEST_FRAMEWORK_DIR" ]; then
76-
echo "valkeytestframework found."
77-
else
78-
echo "Cloning valkey-test-framework..."
79-
git clone "$TEST_FRAMEWORK_REPO"
80-
mkdir -p "$TEST_FRAMEWORK_DIR"
81-
mv "valkey-test-framework/src"/* "$TEST_FRAMEWORK_DIR/"
82-
rm -rf valkey-test-framework
83-
fi
84123

85-
REQUIREMENTS_FILE="requirements.txt"
86-
87-
# Check if pip is available
88-
if command -v pip > /dev/null 2>&1; then
89-
echo "Using pip to install packages..."
90-
pip install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
91-
# Check if pip3 is available
92-
elif command -v pip3 > /dev/null 2>&1; then
93-
echo "Using pip3 to install packages..."
94-
pip3 install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
95-
else
96-
echo "Error: Neither pip nor pip3 is available. Please install Python package installer."
97-
exit 1
98-
fi
124+
if [ $RUN_INTEGRATION -eq 1 ]; then
125+
REPO_URL="https://github.com/valkey-io/valkey.git"
126+
BINARY_PATH="tests/build/binaries/$SERVER_VERSION/valkey-server"
127+
CACHED_VALKEY_PATH="tests/build/valkey"
128+
if [ -f "$BINARY_PATH" ] && [ -x "$BINARY_PATH" ]; then
129+
echo "valkey-server binary '$BINARY_PATH' found."
130+
else
131+
echo "valkey-server binary '$BINARY_PATH' not found."
132+
mkdir -p "tests/build/binaries/$SERVER_VERSION"
133+
rm -rf $CACHED_VALKEY_PATH
134+
cd tests/build
135+
git clone "$REPO_URL"
136+
cd valkey
137+
git checkout "$SERVER_VERSION"
138+
make distclean
139+
if [ ! -z "${ASAN_BUILD}" ]; then
140+
make -j SANITIZER=address
141+
else
142+
make -j
143+
fi
144+
cp src/valkey-server ../binaries/$SERVER_VERSION/
145+
cd $SCRIPT_DIR
146+
rm -rf $CACHED_VALKEY_PATH
147+
fi
148+
TEST_FRAMEWORK_REPO="https://github.com/valkey-io/valkey-test-framework"
149+
TEST_FRAMEWORK_DIR="tests/build/valkeytestframework"
99150

100-
os_type=$(uname)
101-
MODULE_EXT=".so"
102-
if [[ "$os_type" == "Darwin" ]]; then
103-
MODULE_EXT=".dylib"
104-
elif [[ "$os_type" == "Linux" ]]; then
105-
MODULE_EXT=".so"
106-
else
107-
echo "Unsupported OS type: $os_type"
108-
exit 1
109-
fi
110-
export MODULE_PATH="$SCRIPT_DIR/target/release/libvalkey_bloom$MODULE_EXT"
111151

112-
echo "Running the integration tests..."
113-
if [ ! -z "${ASAN_BUILD}" ]; then
114-
# TEST_PATTERN can be used to run specific tests or test patterns.
115-
if [[ -n "$TEST_PATTERN" ]]; then
116-
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN 2>&1 | tee test_output.tmp
152+
if [ -d "$TEST_FRAMEWORK_DIR" ]; then
153+
echo "valkeytestframework found."
117154
else
118-
echo "TEST_PATTERN is not set. Running all integration tests."
119-
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" 2>&1 | tee test_output.tmp
155+
echo "Cloning valkey-test-framework..."
156+
git clone "$TEST_FRAMEWORK_REPO"
157+
mkdir -p "$TEST_FRAMEWORK_DIR"
158+
mv "valkey-test-framework/src"/* "$TEST_FRAMEWORK_DIR/"
159+
rm -rf valkey-test-framework
120160
fi
121161

122-
# Check for memory leaks in the output
123-
if grep -q "LeakSanitizer: detected memory leaks" test_output.tmp; then
124-
RED='\033[0;31m'
125-
echo -e "${RED}Memory leaks detected in the following tests:"
126-
LEAKING_TESTS=$(grep -B 2 "LeakSanitizer: detected memory leaks" test_output.tmp | \
127-
grep -v "LeakSanitizer" | \
128-
grep ".*\.py::")
129-
130-
LEAK_COUNT=$(echo "$LEAKING_TESTS" | wc -l)
131-
132-
# Output each leaking test
133-
echo "$LEAKING_TESTS" | while read -r line; do
134-
echo "::error::Test with leak: $line"
135-
done
136-
137-
echo -e "\n$LEAK_COUNT python integration tests have leaks detected in them"
138-
rm test_output.tmp
162+
REQUIREMENTS_FILE="requirements.txt"
163+
164+
# Check if pip is available
165+
if command -v pip > /dev/null 2>&1; then
166+
echo "Using pip to install packages..."
167+
pip install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
168+
# Check if pip3 is available
169+
elif command -v pip3 > /dev/null 2>&1; then
170+
echo "Using pip3 to install packages..."
171+
pip3 install -r "$SCRIPT_DIR/$REQUIREMENTS_FILE"
172+
else
173+
echo "Error: Neither pip nor pip3 is available. Please install Python package installer."
139174
exit 1
140175
fi
141-
rm test_output.tmp
142-
else
143-
# TEST_PATTERN can be used to run specific tests or test patterns.
144-
if [[ -n "$TEST_PATTERN" ]]; then
145-
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN
176+
177+
os_type=$(uname)
178+
MODULE_EXT=".so"
179+
if [[ "$os_type" == "Darwin" ]]; then
180+
MODULE_EXT=".dylib"
181+
elif [[ "$os_type" == "Linux" ]]; then
182+
MODULE_EXT=".so"
146183
else
147-
echo "TEST_PATTERN is not set. Running all integration tests."
148-
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/"
184+
echo "Unsupported OS type: $os_type"
185+
exit 1
186+
fi
187+
export MODULE_PATH="$SCRIPT_DIR/target/release/libvalkey_bloom$MODULE_EXT"
188+
189+
echo "Running the integration tests..."
190+
if [ ! -z "${ASAN_BUILD}" ]; then
191+
# TEST_PATTERN can be used to run specific tests or test patterns.
192+
if [[ -n "$TEST_PATTERN" ]]; then
193+
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN -m "not skip_for_asan" 2>&1 | tee test_output.tmp
194+
else
195+
echo "TEST_PATTERN is not set. Running all integration tests."
196+
python3 -m pytest --capture=sys --cache-clear -v "$SCRIPT_DIR/tests/" -m "not skip_for_asan" 2>&1 | tee test_output.tmp
197+
fi
198+
199+
# Check for memory leaks in the output
200+
if grep -q "LeakSanitizer: detected memory leaks" test_output.tmp; then
201+
RED='\033[0;31m'
202+
echo -e "${RED}Memory leaks detected in the following tests:"
203+
LEAKING_TESTS=$(grep -B 2 "LeakSanitizer: detected memory leaks" test_output.tmp | \
204+
grep -v "LeakSanitizer" | \
205+
grep ".*\.py::")
206+
207+
LEAK_COUNT=$(echo "$LEAKING_TESTS" | wc -l)
208+
209+
# Output each leaking test
210+
echo "$LEAKING_TESTS" | while read -r line; do
211+
echo "::error::Test with leak: $line"
212+
done
213+
214+
echo -e "\n$LEAK_COUNT python integration tests have leaks detected in them"
215+
rm test_output.tmp
216+
exit 1
217+
fi
218+
rm test_output.tmp
219+
else
220+
# TEST_PATTERN can be used to run specific tests or test patterns.
221+
if [[ -n "$TEST_PATTERN" ]]; then
222+
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN
223+
else
224+
echo "TEST_PATTERN is not set. Running all integration tests."
225+
python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/"
226+
fi
149227
fi
150228
fi
151229

152-
echo "Build, Format Checks, Unit tests, and Integration Tests succeeded"
230+
231+
# echo "Build, Format Checks, Unit tests, and Integration Tests succeeded"

0 commit comments

Comments
 (0)