Skip to content

Commit 84f89d4

Browse files
committed
[LDC] Set up CI / package generation with Azure Pipelines and Shippable
Azure: Windows x86 & x64, macOS x64, Linux x64, Android ARMv7-A & AArch64 Shippable: Linux AArch64 2 packages per platform (enabled/disabled assertions), except for cross- compiled Android. Tagged builds are uploaded to the corresponding GitHub release, untagged builds to the GitHub 'CI' release.
1 parent 9a5b2d6 commit 84f89d4

File tree

4 files changed

+452
-0
lines changed

4 files changed

+452
-0
lines changed

.azure-pipelines/posix.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Each step starts in the checked-out source directory,
2+
# environment variables aren't persisted across steps.
3+
4+
steps:
5+
- script: |
6+
set -ex
7+
cd ..
8+
if [ "$CI_OS" = "osx" ]; then export PATH="$PWD/ninja:$PATH"; fi
9+
cmake --version
10+
ninja --version
11+
mkdir build
12+
cd build
13+
cmake -G Ninja $BUILD_SOURCESDIRECTORY/llvm \
14+
-DCMAKE_BUILD_TYPE=Release \
15+
-DCMAKE_INSTALL_PREFIX=$PWD/../install \
16+
-DLLVM_ENABLE_PROJECTS="$LLVM_PROJECTS_TO_ENABLE" \
17+
-DLLVM_TARGETS_TO_BUILD="$LLVM_TARGETS_TO_BUILD" \
18+
-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="$LLVM_EXPERIMENTAL_TARGETS_TO_BUILD" \
19+
-DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS \
20+
$BASE_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS_ANDROID
21+
ninja install
22+
displayName: Build & install LLVM incl. LLD, compiler-rt, MLIR and the Khronos SPIRV-LLVM-Translator
23+
- script: |
24+
set -ex
25+
cd ..
26+
if [ "${BUILD_SOURCEBRANCH:0:10}" = "refs/tags/" ]; then
27+
ARTIFACT_ID=${BUILD_SOURCEBRANCH:15}
28+
else
29+
ARTIFACT_ID=${BUILD_SOURCEVERSION:0:8}
30+
fi
31+
ASSERTS_SUFFIX=""
32+
if [ "$LLVM_ENABLE_ASSERTIONS" = "ON" ]; then
33+
ASSERTS_SUFFIX="-withAsserts"
34+
fi
35+
artifactName=llvm-$ARTIFACT_ID-$CI_OS-$ARCH$ASSERTS_SUFFIX
36+
mv install $artifactName
37+
mkdir artifacts
38+
chmod -R go=rX $artifactName
39+
if [ "$CI_OS" = "osx" ]; then
40+
sudo chown -R root:wheel $artifactName
41+
tar -cJf artifacts/$artifactName.tar.xz --options='compression-level=9' $artifactName
42+
else
43+
tar -cf - --owner=0 --group=0 $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9
44+
fi
45+
# Persist the ARTIFACT_ID and ASSERTS_SUFFIX variables for future steps
46+
set +x
47+
echo "##vso[task.setvariable variable=ARTIFACT_ID]$ARTIFACT_ID"
48+
echo "##vso[task.setvariable variable=ASSERTS_SUFFIX]$ASSERTS_SUFFIX"
49+
displayName: Pack installation dir
50+
- script: |
51+
set -ex
52+
artifactName=llvm-$ARTIFACT_ID.src
53+
# just the LLVM src dir (already incl. LLVM-SPIRV-Translator) + compiler-rt + lld + mlir
54+
mv compiler-rt mlir llvm/projects
55+
mv lld llvm/tools
56+
chmod -R go=rX llvm
57+
tar -cf - --exclude-vcs --owner=0 --group=0 --transform=s,${BUILD_SOURCESDIRECTORY:1}/llvm,$artifactName, $BUILD_SOURCESDIRECTORY/llvm | 7za a ../artifacts/$artifactName.tar.xz -si -txz -mx9
58+
displayName: Pack source dir
59+
condition: and(succeeded(), and(eq(variables['CI_OS'], 'linux'), eq(variables['LLVM_ENABLE_ASSERTIONS'], 'OFF')))
60+
- publish: ../artifacts
61+
artifact: $(CI_OS)-$(ARCH)$(ASSERTS_SUFFIX)

.azure-pipelines/windows.yml

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Each step starts in the checked-out source directory,
2+
# environment variables aren't persisted across steps.
3+
4+
steps:
5+
- script: |
6+
set
7+
displayName: Print environment variables
8+
- checkout: self
9+
submodules: true
10+
fetchDepth: 50
11+
- script: |
12+
echo on
13+
cd ..
14+
:: Download & extract Ninja
15+
curl -L -o ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-win.zip 2>&1
16+
mkdir ninja
17+
cd ninja
18+
7z x ../ninja.zip > nul
19+
cd ..
20+
:: Download & install clang
21+
curl -L -o clang.exe https://github.com/llvm/llvm-project/releases/download/llvmorg-%CLANG_VERSION%/LLVM-%CLANG_VERSION%-win64.exe 2>&1
22+
clang.exe /S
23+
displayName: Install prerequisites
24+
- script: |
25+
echo on
26+
cd ..
27+
set PATH=C:\Program Files\LLVM\bin;%CD%\ninja;%PATH%
28+
call "%VSINSTALLDIR%Common7\Tools\VsDevCmd.bat" -arch=%ARCH%
29+
echo on
30+
cmake --version
31+
ninja --version
32+
clang-cl --version
33+
mkdir build
34+
cd build
35+
cmake -G Ninja %BUILD_SOURCESDIRECTORY%\llvm -DCMAKE_C_COMPILER:PATH=clang-cl.exe -DCMAKE_CXX_COMPILER:PATH=clang-cl.exe -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%CD%/../install -DLLVM_ENABLE_PROJECTS=%LLVM_PROJECTS_TO_ENABLE% -DLLVM_TARGETS_TO_BUILD=%LLVM_TARGETS_TO_BUILD% -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=%LLVM_EXPERIMENTAL_TARGETS_TO_BUILD% -DLLVM_ENABLE_ASSERTIONS=%LLVM_ENABLE_ASSERTIONS% -DLLVM_DEFAULT_TARGET_TRIPLE=%LLVM_DEFAULT_TARGET_TRIPLE% %BASE_CMAKE_FLAGS% %EXTRA_CMAKE_FLAGS%
36+
ninja install
37+
displayName: Build & install LLVM incl. LLD, compiler-rt, MLIR and the Khronos SPIRV-LLVM-Translator
38+
- script: |
39+
echo on
40+
cd ..
41+
:: On Windows, the lld symlinks are copies
42+
rm install/bin/ld.lld.exe
43+
rm install/bin/ld64.lld.exe
44+
rm install/bin/lld.exe
45+
rm install/bin/wasm-ld.exe
46+
if "%BUILD_SOURCEBRANCH:~0,10%" == "refs/tags/" (
47+
set ARTIFACT_ID=%BUILD_SOURCEBRANCH:~15%
48+
) else (
49+
set ARTIFACT_ID=%BUILD_SOURCEVERSION:~0,8%
50+
)
51+
set ASSERTS_SUFFIX=
52+
if "%LLVM_ENABLE_ASSERTIONS%" == "ON" (
53+
set ASSERTS_SUFFIX=-withAsserts
54+
)
55+
set ARTIFACT_NAME=llvm-%ARTIFACT_ID%-windows-%ARCH%%ASSERTS_SUFFIX%
56+
mv install %ARTIFACT_NAME%
57+
mkdir artifacts
58+
cd %ARTIFACT_NAME%
59+
7z a -mx=9 ../artifacts/%ARTIFACT_NAME%.7z * > nul
60+
:: Persist the ASSERTS_SUFFIX env variable for future steps
61+
echo off
62+
echo ##vso[task.setvariable variable=ASSERTS_SUFFIX]%ASSERTS_SUFFIX%
63+
displayName: Pack installation dir
64+
- publish: ../artifacts
65+
artifact: windows-$(ARCH)$(ASSERTS_SUFFIX)

azure-pipelines.yml

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
variables:
2+
# Version of clang host compiler.
3+
CLANG_VERSION: 10.0.1
4+
LLVM_PROJECTS_TO_ENABLE: compiler-rt;lld;mlir
5+
LLVM_TARGETS_TO_BUILD: AArch64;AMDGPU;ARM;Mips;MSP430;NVPTX;PowerPC;RISCV;WebAssembly;X86
6+
LLVM_EXPERIMENTAL_TARGETS_TO_BUILD: AVR
7+
# Prevent dependencies on some purely optional libraries, and a dependency on clang
8+
# for compiler-rt and SPIRV-LLVM-Translator tests.
9+
BASE_CMAKE_FLAGS: -DLLVM_ENABLE_UNWIND_TABLES=OFF -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_LIBEDIT=OFF -DCOMPILER_RT_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_TESTS=OFF -DCOMPILER_RT_USE_LIBCXX=OFF
10+
11+
# Although tags are supposed to trigger builds by default, they don't.
12+
trigger:
13+
- '*'
14+
- refs/tags/ldc-v*
15+
16+
jobs:
17+
- job: Windows
18+
timeoutInMinutes: 180
19+
pool:
20+
vmImage: 'windows-2019'
21+
variables:
22+
CLANG_VERSION: 10.0.0
23+
VSINSTALLDIR: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\
24+
EXTRA_CMAKE_FLAGS: -DLLVM_USE_CRT_RELEASE=MT
25+
strategy:
26+
matrix:
27+
x64:
28+
ARCH: x64
29+
LLVM_DEFAULT_TARGET_TRIPLE: x86_64-pc-windows-msvc
30+
LLVM_ENABLE_ASSERTIONS: OFF
31+
x64_withAsserts:
32+
ARCH: x64
33+
LLVM_DEFAULT_TARGET_TRIPLE: x86_64-pc-windows-msvc
34+
LLVM_ENABLE_ASSERTIONS: ON
35+
x86:
36+
ARCH: x86
37+
LLVM_DEFAULT_TARGET_TRIPLE: i686-pc-windows-msvc
38+
LLVM_ENABLE_ASSERTIONS: OFF
39+
# Let CMake configure 64-bit clang-cl for 32-bit code emission.
40+
CFLAGS: -m32
41+
CXXFLAGS: -m32
42+
ASMFLAGS: -m32
43+
x86_withAsserts:
44+
ARCH: x86
45+
LLVM_DEFAULT_TARGET_TRIPLE: i686-pc-windows-msvc
46+
LLVM_ENABLE_ASSERTIONS: ON
47+
CFLAGS: -m32
48+
CXXFLAGS: -m32
49+
ASMFLAGS: -m32
50+
steps:
51+
- template: .azure-pipelines/windows.yml
52+
53+
- job: Linux
54+
timeoutInMinutes: 180
55+
pool:
56+
vmImage: 'ubuntu-16.04'
57+
variables:
58+
CI_OS: linux
59+
ARCH: x86_64
60+
# To improve portability of the generated binaries, link the C++ standard library statically.
61+
EXTRA_CMAKE_FLAGS: -DLLVM_BINUTILS_INCDIR=/usr/include -DLLVM_STATIC_LINK_CXX_STDLIB=ON
62+
strategy:
63+
matrix:
64+
x64:
65+
LLVM_ENABLE_ASSERTIONS: OFF
66+
x64_withAsserts:
67+
LLVM_ENABLE_ASSERTIONS: ON
68+
steps:
69+
- checkout: self
70+
submodules: true
71+
fetchDepth: 50
72+
- script: |
73+
set -ex
74+
cd ..
75+
export DEBIAN_FRONTEND=noninteractive
76+
sudo apt-get -q update
77+
sudo apt-get -yq install \
78+
git-core cmake ninja-build binutils-dev python p7zip-full
79+
# Download & extract clang
80+
curl -L -o clang.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-$CLANG_VERSION/clang+llvm-$CLANG_VERSION-x86_64-linux-gnu-ubuntu-16.04.tar.xz
81+
mkdir clang
82+
tar -xf clang.tar.xz --strip 1 -C clang
83+
# Set CC and CXX to clang[++] for future steps
84+
set +x
85+
echo "##vso[task.setvariable variable=CC]$PWD/clang/bin/clang"
86+
echo "##vso[task.setvariable variable=CXX]$PWD/clang/bin/clang++"
87+
displayName: Install prerequisites
88+
- template: .azure-pipelines/posix.yml
89+
90+
- job: macOS
91+
timeoutInMinutes: 180
92+
pool:
93+
vmImage: 'macOS-10.15'
94+
variables:
95+
CI_OS: osx
96+
ARCH: x86_64
97+
EXTRA_CMAKE_FLAGS: -DHAVE_FUTIMENS=0
98+
MACOSX_DEPLOYMENT_TARGET: 10.9
99+
strategy:
100+
matrix:
101+
x64:
102+
LLVM_ENABLE_ASSERTIONS: OFF
103+
x64_withAsserts:
104+
LLVM_ENABLE_ASSERTIONS: ON
105+
steps:
106+
- checkout: self
107+
submodules: true
108+
fetchDepth: 50
109+
- script: |
110+
set -ex
111+
cd ..
112+
# Download & extract Ninja
113+
curl -L -o ninja-mac.zip https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-mac.zip
114+
mkdir ninja
115+
tar -xf ninja-mac.zip -C ninja
116+
displayName: Install prerequisites
117+
- template: .azure-pipelines/posix.yml
118+
119+
- job: Android
120+
timeoutInMinutes: 180
121+
pool:
122+
vmImage: 'ubuntu-16.04'
123+
variables:
124+
CI_OS: android
125+
LLVM_PROJECTS_TO_ENABLE: compiler-rt;lld
126+
LLVM_TARGETS_TO_BUILD: AArch64;ARM;WebAssembly;X86
127+
LLVM_EXPERIMENTAL_TARGETS_TO_BUILD: ''
128+
LLVM_ENABLE_ASSERTIONS: OFF
129+
ANDROID_NDK_VERSION: r21d
130+
strategy:
131+
matrix:
132+
armv7a:
133+
ARCH: armv7a
134+
EXTRA_CMAKE_FLAGS: -DANDROID_ABI=armeabi-v7a -DANDROID_NATIVE_API_LEVEL=21 -DANDROID_STL=c++_static -DCMAKE_CROSSCOMPILING=True -DLLVM_TARGET_ARCH=ARM -DLLVM_DEFAULT_TARGET_TRIPLE=armv7a--linux-androideabi -DLLVM_ENABLE_PLUGINS=OFF
135+
aarch64:
136+
ARCH: aarch64
137+
EXTRA_CMAKE_FLAGS: -DANDROID_ABI=arm64-v8a -DANDROID_NATIVE_API_LEVEL=21 -DANDROID_STL=c++_static -DCMAKE_CROSSCOMPILING=True -DLLVM_TARGET_ARCH=AArch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64--linux-android -DLLVM_ENABLE_PLUGINS=OFF -DSANITIZER_CXX_ABI=libc++ -DCMAKE_CXX_FLAGS=-U_FORTIFY_SOURCE
138+
steps:
139+
- checkout: self
140+
submodules: true
141+
fetchDepth: 50
142+
- script: |
143+
set -ex
144+
cd ..
145+
export DEBIAN_FRONTEND=noninteractive
146+
sudo apt-get -q update
147+
sudo apt-get -yq install \
148+
git-core cmake ninja-build g++ curl python p7zip-full unzip
149+
displayName: Install prerequisites
150+
- script: |
151+
set -ex
152+
cd ..
153+
cmake --version
154+
ninja --version
155+
mkdir build-native
156+
cd build-native
157+
cmake -G Ninja $BUILD_SOURCESDIRECTORY/llvm \
158+
-DCMAKE_BUILD_TYPE=Release \
159+
-DLLVM_TARGETS_TO_BUILD="$LLVM_TARGETS_TO_BUILD" \
160+
-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="$LLVM_EXPERIMENTAL_TARGETS_TO_BUILD" \
161+
-DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS \
162+
$BASE_CMAKE_FLAGS
163+
ninja llvm-tblgen
164+
displayName: Build native llvm-tblgen
165+
- script: |
166+
set -ex
167+
cd ..
168+
# Download & extract Android NDK
169+
curl -OL https://dl.google.com/android/repository/android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip
170+
unzip android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip > /dev/null
171+
rm android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip
172+
# The NDK toolchain file enforces `-g` as base C[XX] flag - remove it to
173+
# *significantly* reduce executable sizes
174+
TOOLCHAIN_FILE=$PWD/android-ndk-$ANDROID_NDK_VERSION/build/cmake/android.toolchain.cmake
175+
sed -i 's|^ -g$||' $TOOLCHAIN_FILE
176+
# Set EXTRA_CMAKE_FLAGS_ANDROID for future steps
177+
set +x
178+
echo "##vso[task.setvariable variable=EXTRA_CMAKE_FLAGS_ANDROID]-DLLVM_TABLEGEN=$PWD/build-native/bin/llvm-tblgen -DLLVM_CONFIG_PATH=$PWD/build-native/bin/llvm-config -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN_FILE"
179+
displayName: Set up cross-compilation
180+
- template: .azure-pipelines/posix.yml
181+
182+
# Uploads the packages to a GitHub release.
183+
- job: Upload_to_GitHub
184+
timeoutInMinutes: 30
185+
pool:
186+
vmImage: 'ubuntu-16.04'
187+
dependsOn:
188+
- Windows
189+
- Linux
190+
- macOS
191+
- Android
192+
# Only run for branches and tags, no PRs.
193+
condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/'), startsWith(variables['Build.SourceBranch'], 'refs/tags/')))
194+
steps:
195+
- checkout: none
196+
- task: DownloadPipelineArtifact@2
197+
inputs:
198+
targetPath: artifacts
199+
- bash: |
200+
set -ex
201+
cd artifacts
202+
mv */*.* .
203+
find . -type d -empty -delete
204+
ls -l
205+
sha256sum * > sha256sums.txt
206+
displayName: Move artifact files out of their subdirs and compute hashes
207+
- bash: |
208+
set -ex
209+
if [ "${BUILD_SOURCEBRANCH:0:10}" != "refs/tags/" ]; then
210+
releaseTag=CI
211+
isPreRelease=true
212+
artifactID=${BUILD_SOURCEVERSION:0:8}
213+
else
214+
releaseTag=${BUILD_SOURCEBRANCH:10}
215+
isPreRelease=false
216+
artifactID=${BUILD_SOURCEBRANCH:15}
217+
fi
218+
mv artifacts/sha256sums.txt artifacts/llvm-$artifactID.sha256sums.txt
219+
set +x
220+
echo "##vso[task.setvariable variable=GITHUB_RELEASE_TAG]$releaseTag"
221+
echo "##vso[task.setvariable variable=GITHUB_RELEASE_ISPRERELEASE]$isPreRelease"
222+
displayName: Set GITHUB_RELEASE_{TAG,ISPRERELEASE} env variables
223+
- task: GitHubRelease@0
224+
inputs:
225+
gitHubConnection: kinke
226+
repositoryName: ldc-developers/llvm-project
227+
action: edit
228+
tag: $(GITHUB_RELEASE_TAG)
229+
assets: artifacts/*
230+
assetUploadMode: replace
231+
isPreRelease: $(GITHUB_RELEASE_ISPRERELEASE)
232+
addChangeLog: false
233+
displayName: Upload artifacts to GitHub release

0 commit comments

Comments
 (0)