Skip to content

Commit 0bd0759

Browse files
committed
[LDC] Set up CI / package generation with Azure Pipelines and Shippable
Azure: Windows x86 & x64, macOS x64, Linux x64, Android ARMv7-A & AArch64 Shippable: Linux AArch64 2 packages per platform (enabled/disabled assertions), except for cross- compiled Android. Tagged builds are uploaded to the corresponding GitHub release, untagged builds to the GitHub 'CI' release.
1 parent 4c7aeb6 commit 0bd0759

File tree

4 files changed

+449
-0
lines changed

4 files changed

+449
-0
lines changed

.azure-pipelines/posix.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Each step starts in the checked-out source directory,
2+
# environment variables aren't persisted across steps.
3+
4+
steps:
5+
- script: |
6+
set -ex
7+
cd ..
8+
if [ "$CI_OS" = "osx" ]; then export PATH="$PWD/ninja:$PATH"; fi
9+
cmake --version
10+
ninja --version
11+
mkdir build
12+
cd build
13+
cmake -G Ninja $BUILD_SOURCESDIRECTORY/llvm \
14+
-DCMAKE_BUILD_TYPE=Release \
15+
-DCMAKE_INSTALL_PREFIX=$PWD/../install \
16+
-DLLVM_ENABLE_PROJECTS="$LLVM_PROJECTS_TO_ENABLE" \
17+
-DLLVM_TARGETS_TO_BUILD="$LLVM_TARGETS_TO_BUILD" \
18+
-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="$LLVM_EXPERIMENTAL_TARGETS_TO_BUILD" \
19+
-DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS \
20+
$BASE_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS_ANDROID
21+
ninja install
22+
displayName: Build & install LLVM incl. LLD, compiler-rt, MLIR and the Khronos SPIRV-LLVM-Translator
23+
- script: |
24+
set -ex
25+
cd ..
26+
if [ "${BUILD_SOURCEBRANCH:0:10}" = "refs/tags/" ]; then
27+
ARTIFACT_ID=${BUILD_SOURCEBRANCH:15}
28+
else
29+
ARTIFACT_ID=${BUILD_SOURCEVERSION:0:8}
30+
fi
31+
ASSERTS_SUFFIX=""
32+
if [ "$LLVM_ENABLE_ASSERTIONS" = "ON" ]; then
33+
ASSERTS_SUFFIX="-withAsserts"
34+
fi
35+
artifactName=llvm-$ARTIFACT_ID-$CI_OS-$ARCH$ASSERTS_SUFFIX
36+
mv install $artifactName
37+
mkdir artifacts
38+
chmod -R go=rX $artifactName
39+
if [ "$CI_OS" = "osx" ]; then
40+
sudo chown -R root:wheel $artifactName
41+
tar -cJf artifacts/$artifactName.tar.xz --options='compression-level=9' $artifactName
42+
else
43+
tar -cf - --owner=0 --group=0 $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9
44+
fi
45+
# Persist the ARTIFACT_ID and ASSERTS_SUFFIX variables for future steps
46+
set +x
47+
echo "##vso[task.setvariable variable=ARTIFACT_ID]$ARTIFACT_ID"
48+
echo "##vso[task.setvariable variable=ASSERTS_SUFFIX]$ASSERTS_SUFFIX"
49+
displayName: Pack installation dir
50+
- script: |
51+
set -ex
52+
artifactName=llvm-$ARTIFACT_ID.src
53+
# just the LLVM src dir (already incl. LLVM-SPIRV-Translator) + compiler-rt + lld + mlir
54+
mv compiler-rt mlir llvm/projects
55+
mv lld llvm/tools
56+
chmod -R go=rX llvm
57+
tar -cf - --exclude-vcs --owner=0 --group=0 --transform=s,${BUILD_SOURCESDIRECTORY:1}/llvm,$artifactName, $BUILD_SOURCESDIRECTORY/llvm | 7za a ../artifacts/$artifactName.tar.xz -si -txz -mx9
58+
displayName: Pack source dir
59+
condition: and(succeeded(), and(eq(variables['CI_OS'], 'linux'), eq(variables['LLVM_ENABLE_ASSERTIONS'], 'OFF')))
60+
- publish: ../artifacts
61+
artifact: $(CI_OS)-$(ARCH)$(ASSERTS_SUFFIX)

.azure-pipelines/windows.yml

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Each step starts in the checked-out source directory,
2+
# environment variables aren't persisted across steps.
3+
4+
steps:
5+
- script: |
6+
set
7+
displayName: Print environment variables
8+
- checkout: self
9+
submodules: true
10+
fetchDepth: 50
11+
- script: |
12+
echo on
13+
cd ..
14+
:: Download & extract Ninja
15+
curl -L -o ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip 2>&1
16+
mkdir ninja
17+
cd ninja
18+
7z x ../ninja.zip > nul
19+
cd ..
20+
:: Download & install clang
21+
curl -L -o clang.exe http://releases.llvm.org/%CLANG_VERSION%/LLVM-%CLANG_VERSION%-win64.exe 2>&1
22+
clang.exe /S
23+
displayName: Install prerequisites
24+
- script: |
25+
echo on
26+
cd ..
27+
set PATH=C:\Program Files\LLVM\bin;%CD%\ninja;%PATH%
28+
call "%VSINSTALLDIR%Common7\Tools\VsDevCmd.bat" -arch=%ARCH%
29+
echo on
30+
cmake --version
31+
ninja --version
32+
clang-cl --version
33+
mkdir build
34+
cd build
35+
cmake -G Ninja %BUILD_SOURCESDIRECTORY%\llvm -DCMAKE_C_COMPILER:PATH=clang-cl.exe -DCMAKE_CXX_COMPILER:PATH=clang-cl.exe -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%CD%/../install -DLLVM_ENABLE_PROJECTS=%LLVM_PROJECTS_TO_ENABLE% -DLLVM_TARGETS_TO_BUILD=%LLVM_TARGETS_TO_BUILD% -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=%LLVM_EXPERIMENTAL_TARGETS_TO_BUILD% -DLLVM_ENABLE_ASSERTIONS=%LLVM_ENABLE_ASSERTIONS% -DLLVM_DEFAULT_TARGET_TRIPLE=%LLVM_DEFAULT_TARGET_TRIPLE% %BASE_CMAKE_FLAGS% %EXTRA_CMAKE_FLAGS%
36+
ninja install
37+
displayName: Build & install LLVM incl. LLD, compiler-rt, MLIR and the Khronos SPIRV-LLVM-Translator
38+
- script: |
39+
echo on
40+
cd ..
41+
:: On Windows, the lld symlinks are copies
42+
rm install/bin/ld.lld.exe
43+
rm install/bin/ld64.lld.exe
44+
rm install/bin/lld.exe
45+
rm install/bin/wasm-ld.exe
46+
if "%BUILD_SOURCEBRANCH:~0,10%" == "refs/tags/" (
47+
set ARTIFACT_ID=%BUILD_SOURCEBRANCH:~15%
48+
) else (
49+
set ARTIFACT_ID=%BUILD_SOURCEVERSION:~0,8%
50+
)
51+
set ASSERTS_SUFFIX=
52+
if "%LLVM_ENABLE_ASSERTIONS%" == "ON" (
53+
set ASSERTS_SUFFIX=-withAsserts
54+
)
55+
set ARTIFACT_NAME=llvm-%ARTIFACT_ID%-windows-%ARCH%%ASSERTS_SUFFIX%
56+
mv install %ARTIFACT_NAME%
57+
mkdir artifacts
58+
cd %ARTIFACT_NAME%
59+
7z a -mx=9 ../artifacts/%ARTIFACT_NAME%.7z * > nul
60+
:: Persist the ASSERTS_SUFFIX env variable for future steps
61+
echo off
62+
echo ##vso[task.setvariable variable=ASSERTS_SUFFIX]%ASSERTS_SUFFIX%
63+
displayName: Pack installation dir
64+
- publish: ../artifacts
65+
artifact: windows-$(ARCH)$(ASSERTS_SUFFIX)

azure-pipelines.yml

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
variables:
2+
# Version of clang host compiler.
3+
CLANG_VERSION: 9.0.0
4+
LLVM_PROJECTS_TO_ENABLE: compiler-rt;lld;mlir
5+
LLVM_TARGETS_TO_BUILD: AArch64;ARM;Mips;MSP430;NVPTX;PowerPC;RISCV;WebAssembly;X86
6+
LLVM_EXPERIMENTAL_TARGETS_TO_BUILD: AVR
7+
# Prevent dependencies on some purely optional libraries, and a dependency on clang
8+
# for compiler-rt and SPIRV-LLVM-Translator tests.
9+
BASE_CMAKE_FLAGS: -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_LIBEDIT=OFF -DCOMPILER_RT_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_TESTS=OFF -DCOMPILER_RT_USE_LIBCXX=OFF
10+
11+
# Although tags are supposed to trigger builds by default, they don't.
12+
trigger:
13+
- '*'
14+
- refs/tags/ldc-v*
15+
16+
jobs:
17+
- job: Windows
18+
timeoutInMinutes: 180
19+
pool:
20+
vmImage: 'windows-2019'
21+
variables:
22+
VSINSTALLDIR: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\
23+
EXTRA_CMAKE_FLAGS: -DLLVM_USE_CRT_RELEASE=MT
24+
strategy:
25+
matrix:
26+
x64:
27+
ARCH: x64
28+
LLVM_DEFAULT_TARGET_TRIPLE: x86_64-pc-windows-msvc
29+
LLVM_ENABLE_ASSERTIONS: OFF
30+
x64_withAsserts:
31+
ARCH: x64
32+
LLVM_DEFAULT_TARGET_TRIPLE: x86_64-pc-windows-msvc
33+
LLVM_ENABLE_ASSERTIONS: ON
34+
x86:
35+
ARCH: x86
36+
LLVM_DEFAULT_TARGET_TRIPLE: i686-pc-windows-msvc
37+
LLVM_ENABLE_ASSERTIONS: OFF
38+
# Let CMake configure 64-bit clang-cl for 32-bit code emission.
39+
CFLAGS: -m32
40+
CXXFLAGS: -m32
41+
ASMFLAGS: -m32
42+
x86_withAsserts:
43+
ARCH: x86
44+
LLVM_DEFAULT_TARGET_TRIPLE: i686-pc-windows-msvc
45+
LLVM_ENABLE_ASSERTIONS: ON
46+
CFLAGS: -m32
47+
CXXFLAGS: -m32
48+
ASMFLAGS: -m32
49+
steps:
50+
- template: .azure-pipelines/windows.yml
51+
52+
- job: Linux
53+
timeoutInMinutes: 180
54+
pool:
55+
vmImage: 'ubuntu-16.04'
56+
variables:
57+
CI_OS: linux
58+
ARCH: x86_64
59+
# To improve portability of the generated binaries, link the C++ standard library statically.
60+
EXTRA_CMAKE_FLAGS: -DLLVM_BINUTILS_INCDIR=/usr/include -DLLVM_STATIC_LINK_CXX_STDLIB=ON
61+
strategy:
62+
matrix:
63+
x64:
64+
LLVM_ENABLE_ASSERTIONS: OFF
65+
x64_withAsserts:
66+
LLVM_ENABLE_ASSERTIONS: ON
67+
steps:
68+
- checkout: self
69+
submodules: true
70+
fetchDepth: 50
71+
- script: |
72+
set -ex
73+
cd ..
74+
export DEBIAN_FRONTEND=noninteractive
75+
sudo apt-get -q update
76+
sudo apt-get -yq install \
77+
git-core cmake ninja-build binutils-dev python p7zip-full
78+
# Download & extract clang
79+
curl -L -o clang.tar.xz http://releases.llvm.org/$CLANG_VERSION/clang+llvm-$CLANG_VERSION-x86_64-linux-gnu-ubuntu-16.04.tar.xz
80+
mkdir clang
81+
tar -xf clang.tar.xz --strip 1 -C clang
82+
# Set CC and CXX to clang[++] for future steps
83+
set +x
84+
echo "##vso[task.setvariable variable=CC]$PWD/clang/bin/clang"
85+
echo "##vso[task.setvariable variable=CXX]$PWD/clang/bin/clang++"
86+
displayName: Install prerequisites
87+
- template: .azure-pipelines/posix.yml
88+
89+
- job: macOS
90+
timeoutInMinutes: 180
91+
pool:
92+
vmImage: 'macOS-10.14'
93+
variables:
94+
CI_OS: osx
95+
ARCH: x86_64
96+
EXTRA_CMAKE_FLAGS: -DHAVE_FUTIMENS=0
97+
MACOSX_DEPLOYMENT_TARGET: 10.9
98+
strategy:
99+
matrix:
100+
x64:
101+
LLVM_ENABLE_ASSERTIONS: OFF
102+
x64_withAsserts:
103+
LLVM_ENABLE_ASSERTIONS: ON
104+
steps:
105+
- checkout: self
106+
submodules: true
107+
fetchDepth: 50
108+
- script: |
109+
set -ex
110+
cd ..
111+
# Download & extract Ninja
112+
curl -L -o ninja-mac.zip https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-mac.zip
113+
mkdir ninja
114+
tar -xf ninja-mac.zip -C ninja
115+
displayName: Install prerequisites
116+
- template: .azure-pipelines/posix.yml
117+
118+
- job: Android
119+
timeoutInMinutes: 180
120+
pool:
121+
vmImage: 'ubuntu-16.04'
122+
variables:
123+
CI_OS: android
124+
LLVM_PROJECTS_TO_ENABLE: compiler-rt;lld
125+
LLVM_TARGETS_TO_BUILD: AArch64;ARM;WebAssembly;X86
126+
LLVM_EXPERIMENTAL_TARGETS_TO_BUILD: ''
127+
LLVM_ENABLE_ASSERTIONS: OFF
128+
strategy:
129+
matrix:
130+
armv7a:
131+
ARCH: armv7a
132+
EXTRA_CMAKE_FLAGS: -DANDROID_ABI=armeabi-v7a -DANDROID_NATIVE_API_LEVEL=21 -DANDROID_STL=c++_static -DCMAKE_CROSSCOMPILING=True -DLLVM_TARGET_ARCH=ARM -DLLVM_DEFAULT_TARGET_TRIPLE=armv7a--linux-androideabi -DLLVM_ENABLE_PLUGINS=OFF
133+
aarch64:
134+
ARCH: aarch64
135+
EXTRA_CMAKE_FLAGS: -DANDROID_ABI=arm64-v8a -DANDROID_NATIVE_API_LEVEL=21 -DANDROID_STL=c++_static -DCMAKE_CROSSCOMPILING=True -DLLVM_TARGET_ARCH=AArch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64--linux-android -DLLVM_ENABLE_PLUGINS=OFF -DSANITIZER_CXX_ABI=libc++ -DCMAKE_CXX_FLAGS=-U_FORTIFY_SOURCE
136+
steps:
137+
- checkout: self
138+
submodules: true
139+
fetchDepth: 50
140+
- script: |
141+
set -ex
142+
cd ..
143+
export DEBIAN_FRONTEND=noninteractive
144+
sudo apt-get -q update
145+
sudo apt-get -yq install \
146+
git-core cmake ninja-build g++ curl python p7zip-full unzip
147+
displayName: Install prerequisites
148+
- script: |
149+
set -ex
150+
cd ..
151+
cmake --version
152+
ninja --version
153+
mkdir build-native
154+
cd build-native
155+
cmake -G Ninja $BUILD_SOURCESDIRECTORY/llvm \
156+
-DCMAKE_BUILD_TYPE=Release \
157+
-DLLVM_TARGETS_TO_BUILD="$LLVM_TARGETS_TO_BUILD" \
158+
-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="$LLVM_EXPERIMENTAL_TARGETS_TO_BUILD" \
159+
-DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS \
160+
$BASE_CMAKE_FLAGS
161+
ninja llvm-tblgen
162+
displayName: Build native llvm-tblgen
163+
- script: |
164+
set -ex
165+
cd ..
166+
# Download & extract Android NDK
167+
curl -OL https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip
168+
unzip android-ndk-r21-linux-x86_64.zip > /dev/null
169+
rm android-ndk-r21-linux-x86_64.zip
170+
# The NDK toolchain file enforces `-g` as base C[XX] flag - remove it to
171+
# *significantly* reduce executable sizes
172+
TOOLCHAIN_FILE=$PWD/android-ndk-r21/build/cmake/android.toolchain.cmake
173+
sed -i 's|^ -g$||' $TOOLCHAIN_FILE
174+
# Set EXTRA_CMAKE_FLAGS_ANDROID for future steps
175+
set +x
176+
echo "##vso[task.setvariable variable=EXTRA_CMAKE_FLAGS_ANDROID]-DLLVM_TABLEGEN=$PWD/build-native/bin/llvm-tblgen -DLLVM_CONFIG_PATH=$PWD/build-native/bin/llvm-config -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN_FILE"
177+
displayName: Set up cross-compilation
178+
- template: .azure-pipelines/posix.yml
179+
180+
# Uploads the packages to a GitHub release.
181+
- job: Upload_to_GitHub
182+
timeoutInMinutes: 30
183+
pool:
184+
vmImage: 'ubuntu-16.04'
185+
dependsOn:
186+
- Windows
187+
- Linux
188+
- macOS
189+
- Android
190+
# Only run for branches and tags, no PRs.
191+
condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/'), startsWith(variables['Build.SourceBranch'], 'refs/tags/')))
192+
steps:
193+
- checkout: none
194+
- task: DownloadPipelineArtifact@2
195+
inputs:
196+
targetPath: artifacts
197+
- bash: |
198+
set -ex
199+
cd artifacts
200+
mv */*.* .
201+
find . -type d -empty -delete
202+
ls -l
203+
sha256sum * > sha256sums.txt
204+
displayName: Move artifact files out of their subdirs and compute hashes
205+
- bash: |
206+
set -ex
207+
if [ "${BUILD_SOURCEBRANCH:0:10}" != "refs/tags/" ]; then
208+
releaseTag=CI
209+
isPreRelease=true
210+
artifactID=${BUILD_SOURCEVERSION:0:8}
211+
else
212+
releaseTag=${BUILD_SOURCEBRANCH:10}
213+
isPreRelease=false
214+
artifactID=${BUILD_SOURCEBRANCH:15}
215+
fi
216+
mv artifacts/sha256sums.txt artifacts/llvm-$artifactID.sha256sums.txt
217+
set +x
218+
echo "##vso[task.setvariable variable=GITHUB_RELEASE_TAG]$releaseTag"
219+
echo "##vso[task.setvariable variable=GITHUB_RELEASE_ISPRERELEASE]$isPreRelease"
220+
displayName: Set GITHUB_RELEASE_{TAG,ISPRERELEASE} env variables
221+
- task: GitHubRelease@0
222+
inputs:
223+
gitHubConnection: kinke
224+
repositoryName: ldc-developers/llvm-project
225+
action: edit
226+
tag: $(GITHUB_RELEASE_TAG)
227+
assets: artifacts/*
228+
assetUploadMode: replace
229+
isPreRelease: $(GITHUB_RELEASE_ISPRERELEASE)
230+
addChangeLog: false
231+
displayName: Upload artifacts to GitHub release

0 commit comments

Comments
 (0)