Skip to content

Commit 704d56e

Browse files
authored
[CI] Build zstd for release build container (#20445)
We enabled using zstd for release builds, but it requires zstd static library. zstd and libzstd-devel packages don't contain libzstd.a (note: it's rocky linux 8.10 / RHEL 8.10), therefore building from source. Container build: https://github.com/intel/llvm/actions/runs/18755423984 SYCL release nightly: https://github.com/intel/llvm/actions/runs/18755907952
1 parent 1e0c1f4 commit 704d56e

File tree

6 files changed

+131
-5
lines changed

6 files changed

+131
-5
lines changed

devops/containers/release_build.Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ RUN dnf -y install https://repo.radeon.com/amdgpu-install/6.4.1/rhel/8.10/amdgpu
2020
dnf -y install rocm && \
2121
dnf clean all && rm -rf /var/cache/dnf
2222

23+
# Build zstd static library from sources
24+
COPY scripts/build_zstd.sh /build_zstd.sh
25+
RUN /build_zstd.sh
26+
2327
COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh
2428

2529
USER sycl

devops/containers/ubuntu2404_base.Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ RUN /install.sh
1717
# This causes linking errors when building SYCL runtime.
1818
# Bug: https://github.com/intel/llvm/issues/15935
1919
# Workaround: build zstd from sources with -fPIC flag.
20-
COPY scripts/build_zstd_1_5_6_ub24.sh /build_zstd_1_5_6_ub24.sh
21-
RUN /build_zstd_1_5_6_ub24.sh
20+
COPY scripts/build_zstd.sh /build_zstd.sh
21+
RUN /build_zstd.sh
2222

2323
COPY scripts/create-sycl-user.sh /user-setup.sh
2424
RUN /user-setup.sh

devops/containers/ubuntu2404_build.Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ RUN /install.sh
1212
# This causes linking errors when building SYCL runtime.
1313
# Bug: https://github.com/intel/llvm/issues/15935
1414
# Workaround: build zstd from sources with -fPIC flag.
15-
COPY scripts/build_zstd_1_5_6_ub24.sh /build_zstd_1_5_6_ub24.sh
16-
RUN /build_zstd_1_5_6_ub24.sh
15+
COPY scripts/build_zstd.sh /build_zstd.sh
16+
RUN /build_zstd.sh
1717

1818
SHELL ["/bin/bash", "-ec"]
1919

devops/scripts/build_zstd.sh

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/bin/bash
2+
3+
# Script to build and install zstd on Ubuntu 24, with -fPIC flag.
4+
# The default installation of zstd on Ubuntu 24 does not have -fPIC flag
5+
# enabled, which is required for building DPC++ in shared libraries mode.
6+
7+
# OR on Rocky Linux 8.10 (used for nightly release builds). There is no static
8+
# library (libzstd.a) in available packages, therefore it is necessary to build
9+
# it from source.
10+
11+
# Function to check OS
12+
check_os() {
13+
local expected_name="$1"
14+
local expected_version="$2"
15+
. /etc/os-release
16+
if [[ "$NAME" == "$expected_name" && "$VERSION_ID" == "$expected_version" ]]; then
17+
return 0
18+
else
19+
return 1
20+
fi
21+
}
22+
23+
# Function to install packages with or without sudo
24+
install_packages() {
25+
if [ "$USE_SUDO" = true ]; then
26+
sudo apt-get update
27+
sudo apt-get install -y build-essential wget
28+
else
29+
apt-get update
30+
apt-get install -y build-essential wget
31+
fi
32+
}
33+
34+
# Function to uninstall libzstd-dev if installed
35+
uninstall_libzstd_dev() {
36+
if dpkg -l | grep -q libzstd-dev; then
37+
if [ "$USE_SUDO" = true ]; then
38+
sudo apt-get remove -y libzstd-dev
39+
else
40+
apt-get remove -y libzstd-dev
41+
fi
42+
fi
43+
}
44+
45+
# Function to build a shared library by linking zstd static lib.
46+
# This is used to verify that zstd is built correctly, with -fPIC flag.
47+
build_test_program() {
48+
cat <<EOF > test_zstd.c
49+
#include <zstd.h>
50+
int main() {
51+
ZSTD_CCtx* cctx = ZSTD_createCCtx();
52+
ZSTD_freeCCtx(cctx);
53+
return 0;
54+
}
55+
EOF
56+
57+
# Try to use zstd's static library with -fPIC
58+
gcc test_zstd.c -lzstd -fPIC -shared
59+
if [ $? -ne 0 ]; then
60+
echo "zstd installation verification failed."
61+
else
62+
echo "zstd installation verification passed."
63+
fi
64+
65+
# There won't be a.out file if verification failed.
66+
rm test_zstd.c a.out || true
67+
}
68+
69+
# Check the OS
70+
if ! check_os "Ubuntu" "24.04" && ! check_os "Rocky Linux" "8.10"; then
71+
echo "Warning: This script has only been tested with Ubuntu 24.04 and Rocky Linux 8.10."
72+
fi
73+
74+
# Set USE_SUDO to true or false based on your preference
75+
USE_SUDO=true
76+
77+
# Install necessary build tools & uninstall libzstd-dev package if installed
78+
if check_os "Ubuntu" "24.04"; then
79+
install_packages
80+
uninstall_libzstd_dev
81+
fi
82+
83+
# Define the version and URL for zstd
84+
ZSTD_VERSION="1.5.7"
85+
ZSTD_URL="https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz"
86+
87+
# Create a directory for the source code
88+
mkdir -p zstd_build
89+
cd zstd_build
90+
91+
# Download and extract zstd source code
92+
wget $ZSTD_URL
93+
tar -xzf zstd-$ZSTD_VERSION.tar.gz
94+
cd zstd-$ZSTD_VERSION
95+
96+
# Build zstd with -fPIC flag.
97+
CFLAGS="-fPIC" CXXFLAGS="-fPIC" make
98+
if [ $? -ne 0 ]; then
99+
echo "Error: make failed."
100+
exit 1
101+
fi
102+
103+
# Install zstd.
104+
if [ "$USE_SUDO" = true ]; then
105+
sudo make install
106+
else
107+
make install
108+
fi
109+
if [ $? -ne 0 ]; then
110+
echo "Error: make install failed."
111+
exit 1
112+
fi
113+
114+
# Verify zstd installation.
115+
build_test_program
116+
117+
# Clean up
118+
rm -rf zstd_build

devops/scripts/build_zstd_1_5_6_ub24.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
#!/bin/bash
22

3+
# The actual version of this script is build_zstd.sh.
4+
# This one is needed for jenkins precommit to pass, since jenkins still uses the
5+
# old name. Will be removed when Jenkins is updated.
6+
37
# Script to build and install zstd 1.5.6 on Ubuntu 24, with -fPIC flag.
48
# The default installation of zstd on Ubuntu 24 does not have -fPIC flag
59
# enabled, which is required for building DPC++ in shared libraries mode.

sycl/doc/GetStartedGuide.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ You can install zstd using the package manager of your distribution. For example
351351
```sh
352352
sudo apt-get install libzstd-dev
353353
```
354-
Note that the libzstd-dev package provided on Ubuntu 24.04 has a bug ([link](https://bugs.launchpad.net/ubuntu/+source/libzstd/+bug/2086543)) and the zstd static library is not built with the `-fPIC` flag. Linking to this library will result in a build failure. For example: [Issue#15935](https://github.com/intel/llvm/issues/15935). As an alternative, zstd can be built from source either manually or by using the [build_zstd_1_5_6_ub24.sh](https://github.com/intel/llvm/blob/sycl/devops/scripts/build_zstd_1_5_6_ub24.sh) script.
354+
Note that the libzstd-dev package provided on Ubuntu 24.04 has a bug ([link](https://bugs.launchpad.net/ubuntu/+source/libzstd/+bug/2086543)) and the zstd static library is not built with the `-fPIC` flag. Linking to this library will result in a build failure. For example: [Issue#15935](https://github.com/intel/llvm/issues/15935). As an alternative, zstd can be built from source either manually or by using the [build_zstd.sh](https://github.com/intel/llvm/blob/sycl/devops/scripts/build_zstd.sh) script (it works on Rocky Linux 8.10 / RHEL 8.10 as well).
355355

356356
**Windows**
357357

0 commit comments

Comments
 (0)